Merge branch 'cassandra-4.1' into trunk
diff --git a/.asf.yaml b/.asf.yaml
new file mode 100644
index 0000000..c701eee
--- /dev/null
+++ b/.asf.yaml
@@ -0,0 +1,8 @@
+github:
+ enabled_merge_buttons:
+ squash: false
+ merge: false
+ rebase: true
+
+notifications:
+ jira_options: link worklog
diff --git a/.build/build-git.xml b/.build/build-git.xml
new file mode 100644
index 0000000..03b5ca7
--- /dev/null
+++ b/.build/build-git.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project basedir="." name="apache-cassandra-git-tasks"
+ xmlns:if="ant:if">
+ <target name="get-git-sha">
+ <exec executable="git" osfamily="unix" dir="${basedir}" logError="false" failonerror="false" failifexecutionfails="false" resultproperty="git.is-available.exit-code">
+ <arg value="rev-parse"/>
+ <arg value="--is-inside-work-tree"/>
+ <redirector outputproperty="git.is-available.output"/>
+ </exec>
+ <condition property="git.is-available" else="false">
+ <equals arg1="${git.is-available.exit-code}" arg2="0"/>
+ </condition>
+ <echo message="git.is-available=${git.is-available}"/>
+
+ <exec if:true="${git.is-available}" executable="git" osfamily="unix" dir="${basedir}" logError="true" failonerror="false" failifexecutionfails="false">
+ <arg value="describe"/>
+ <arg value="--match=''"/>
+ <arg value="--always"/>
+ <arg value="--abbrev=0"/>
+ <arg value="--dirty"/>
+ <redirector outputproperty="git.sha"/>
+ </exec>
+ <property name="git.sha" value="Unknown"/>
+ <echo level="info">git.sha=${git.sha}</echo>
+
+ <exec if:true="${git.is-available}" executable="git" osfamily="unix" dir="${basedir}" logError="true" failonerror="false" failifexecutionfails="false">
+ <arg value="diff"/>
+ <arg value="--stat"/>
+ <redirector outputproperty="git.diffstat"/>
+ </exec>
+ <condition property="is-dirty">
+ <contains string="${git.sha}" substring="-dirty"/>
+ </condition>
+ <echo level="warning" if:true="${is-dirty}">Repository state is dirty</echo>
+ <echo level="warning" if:true="${is-dirty}">${git.diffstat}</echo>
+ </target>
+</project>
diff --git a/.build/build-rat.xml b/.build/build-rat.xml
index 49d20cc..f261a23 100644
--- a/.build/build-rat.xml
+++ b/.build/build-rat.xml
@@ -52,13 +52,14 @@
<fileset dir="." includesfile="${build.dir}/.ratinclude">
<!-- Config files with not much creativity -->
<exclude name="**/.asf.yaml"/>
+ <exclude name="**/.github/pull_request_template.md"/>
<exclude name="**/ide/**/*"/>
<exclude name="**/metrics-reporter-config-sample.yaml"/>
<exclude name="**/cassandra.yaml"/>
<exclude name="**/cassandra-murmur.yaml"/>
<exclude name="**/cassandra-seeds.yaml"/>
<exclude name="**/harry-generic.yaml"/>
- <exclude NAME="**/doc/antora.yml"/>
+ <exclude name="**/doc/antora.yml"/>
<exclude name="**/test/conf/cassandra.yaml"/>
<exclude name="**/test/conf/cassandra-old.yaml"/>
<exclude name="**/test/conf/cassandra-converters-special-cases-old-names.yaml"/>
@@ -83,12 +84,12 @@
<exclude name="**/tools/cqlstress-insanity-example.yaml"/>
<exclude name="**/tools/cqlstress-lwt-example.yaml"/>
<!-- Documentation files -->
- <exclude NAME="**/doc/modules/**/*"/>
- <exclude NAME="**/src/java/**/*.md"/>
+ <exclude name="**/doc/modules/**/*"/>
+ <exclude name="**/src/java/**/*.md"/>
<!-- NOTICE files -->
- <exclude NAME="**/NOTICE.md"/>
+ <exclude name="**/NOTICE.md"/>
<!-- LICENSE files -->
- <exclude NAME="**/LICENSE.md"/>
+ <exclude name="**/LICENSE.md"/>
</fileset>
</rat:report>
<exec executable="grep" outputproperty="rat.failed.files" failifexecutionfails="false">
diff --git a/.build/build-resolver.xml b/.build/build-resolver.xml
index c38a272..e78bbf0 100644
--- a/.build/build-resolver.xml
+++ b/.build/build-resolver.xml
@@ -126,6 +126,12 @@
</dependencies>
<path refid="jflex.classpath" classpath="runtime"/>
</resolve>
+ <resolve>
+ <dependencies>
+ <dependency groupId="com.puppycrawl.tools" artifactId="checkstyle" version="${checkstyle.version}" />
+ </dependencies>
+ <path refid="checkstyle.classpath" classpath="runtime"/>
+ </resolve>
<macrodef name="install">
<attribute name="pomFile"/>
@@ -175,7 +181,8 @@
</resolve>
<resolve>
<dependencies pomRef="pom-deps"/>
- <files dir="${test.lib}/jars" layout="{artifactId}-{version}-{classifier}.{extension}" scopes="test,!provide,!system"/>
+ <!-- Needed to include compile here, so ant _build-test would not fail on missing jimfs dependency -->
+ <files dir="${test.lib}/jars" layout="{artifactId}-{version}-{classifier}.{extension}" scopes="compile,test,!provide,!system"/>
</resolve>
@@ -188,7 +195,7 @@
</unzip>
</target>
- <target name="resolver-dist-lib" depends="resolver-retrieve-build,write-poms">
+ <target name="resolver-dist-lib" depends="resolver-retrieve-build">
<resolvepom file="${build.dir}/${final.name}.pom" id="all-pom" />
<resolve failOnMissingAttachments="true">
@@ -198,10 +205,11 @@
<mkdir dir="${local.repository}/org/apache/cassandra/deps/sigar-bin"/>
<mkdir dir="${build.lib}/sigar-bin"/>
- <!-- files.pythonhosted.org -->
+ <!-- files.pythonhosted.org/packages -->
<get src="${artifact.python.pypi}/59/a0/cf4cd997e1750f0c2d91c6ea5abea218251c43c3581bcc2f118b00baf5cf/futures-2.1.6-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/futures-2.1.6-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
<get src="${artifact.python.pypi}/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/six-1.12.0-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
<get src="${artifact.python.pypi}/37/b2/ef1124540ee2c0b417be8d0f74667957e6aa084a3f26621aa67e2e77f3fb/pure_sasl-0.6.2-py2-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/pure_sasl-0.6.2-py2-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
+ <get src="${artifact.python.pypi}/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.5-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
<!-- apache/cassandra/lib -->
<get src="${lib.download.base.url}/lib/geomet-0.1.0.zip" dest="${local.repository}/org/apache/cassandra/deps/geomet-0.1.0.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
@@ -233,6 +241,7 @@
<file file="${local.repository}/org/apache/cassandra/deps/six-1.12.0-py2.py3-none-any.zip"/>
<file file="${local.repository}/org/apache/cassandra/deps/geomet-0.1.0.zip"/>
<file file="${local.repository}/org/apache/cassandra/deps/pure_sasl-0.6.2-py2-none-any.zip"/>
+ <file file="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.5-py2.py3-none-any.zip"/>
</copy>
<copy todir="${build.lib}/sigar-bin/" quiet="true">
<file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/libsigar-amd64-freebsd-6.so"/>
diff --git a/.build/cassandra-build-deps-template.xml b/.build/cassandra-build-deps-template.xml
new file mode 100644
index 0000000..cbf2f8c
--- /dev/null
+++ b/.build/cassandra-build-deps-template.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>cassandra-parent</artifactId>
+ <groupId>org.apache.cassandra</groupId>
+ <version>@version@</version>
+ <relativePath>@final.name@-parent.pom</relativePath>
+ </parent>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-build-deps</artifactId>
+ <version>@version@</version>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-inline</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-tree</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-commons</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-util</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.jimfs</groupId>
+ <artifactId>jimfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.quicktheories</groupId>
+ <artifactId>quicktheories</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.java-allocation-instrumenter</groupId>
+ <artifactId>java-allocation-instrumenter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>dtest-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.ju-n.compile-command-annotations</groupId>
+ <artifactId>compile-command-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>harry-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.junit</groupId>
+ <artifactId>junit-bom</artifactId>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jacoco</groupId>
+ <artifactId>org.jacoco.agent</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jacoco</groupId>
+ <artifactId>org.jacoco.ant</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/.build/cassandra-deps-template.xml b/.build/cassandra-deps-template.xml
new file mode 100644
index 0000000..e907bff
--- /dev/null
+++ b/.build/cassandra-deps-template.xml
@@ -0,0 +1,380 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>cassandra-parent</artifactId>
+ <groupId>org.apache.cassandra</groupId>
+ <version>@version@</version>
+ <relativePath>@final.name@-parent.pom</relativePath>
+ </parent>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-all</artifactId>
+ <version>@version@</version>
+ <name>Apache Cassandra</name>
+ <description>The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.</description>
+ <url>https://cassandra.apache.org</url>
+ <inceptionYear>2009</inceptionYear>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+ <scm>
+ <connection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</connection>
+ <developerConnection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</developerConnection>
+ <url>https://gitbox.apache.org/repos/asf?p=cassandra.git;a=tree</url>
+ </scm>
+ <dependencies>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.ning</groupId>
+ <artifactId>compress-lzf</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>ST4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.boundary</groupId>
+ <artifactId>high-scale-lib</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mindrot</groupId>
+ <artifactId>jbcrypt</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>airline</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-logback</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.addthis.metrics</groupId>
+ <artifactId>reporter-config3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.clearspring.analytics</groupId>
+ <artifactId>stream</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <classifier>shaded</classifier>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-bom</artifactId>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </dependency>
+
+ <!-- chronicle queue, and fixed transitive dependencies -->
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-queue</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-bytes</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-wire</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-threads</artifactId>
+ </dependency>
+ <dependency>
+ <!-- transitive to chronicle-core -->
+ <groupId>net.openhft</groupId>
+ <artifactId>posix</artifactId>
+ </dependency>
+ <dependency>
+ <!-- transitive to posix to chronicle-core -->
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ </dependency>
+ <dependency>
+ <!-- transitive to posix to chronicle-core -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jnr-ffi</artifactId>
+ </dependency>
+ <dependency>
+ <!-- transitive to jnr-ffi to posix to chronicle-core -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jffi</artifactId>
+ </dependency>
+ <dependency>
+ <!-- transitive to jnr-ffi to posix to chronicle-core -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jffi</artifactId>
+ <classifier>native</classifier>
+ </dependency>
+ <dependency>
+ <!-- transitive to jnr-ffi to chronicle-core -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jnr-constants</artifactId>
+ </dependency>
+ <dependency>
+ <!-- transitive to chronicle-threads -->
+ <groupId>net.openhft</groupId>
+ <artifactId>affinity</artifactId>
+ </dependency>
+ <!-- end of chronicle-queue -->
+
+ <dependency>
+ <groupId>org.fusesource</groupId>
+ <artifactId>sigar</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jdt.core.compiler</groupId>
+ <artifactId>ecj</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.caffinitas.ohc</groupId>
+ <artifactId>ohc-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.caffinitas.ohc</groupId>
+ <artifactId>ohc-core-j8</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jctools</groupId>
+ <artifactId>jctools-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-cli</artifactId>
+ <version>0.14</version>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-stacktrace</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>mxdump</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.lab</groupId>
+ <artifactId>jvm-attach-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.psjava</groupId>
+ <artifactId>psjava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.inject</groupId>
+ <artifactId>javax.inject</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.j2objc</groupId>
+ <artifactId>j2objc-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hdrhistogram</groupId>
+ <artifactId>HdrHistogram</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>de.jflex</groupId>
+ <artifactId>jflex</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.rholder</groupId>
+ <artifactId>snowball-stemmer</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.concurrent-trees</groupId>
+ <artifactId>concurrent-trees</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.ju-n.compile-command-annotations</groupId>
+ <artifactId>compile-command-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman-install</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman-submit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman-bmunit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.seancfoley</groupId>
+ <artifactId>ipaddress</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.obermuhlner</groupId>
+ <artifactId>big-math</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.agrona</groupId>
+ <artifactId>agrona</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml
new file mode 100644
index 0000000..7200bb4
--- /dev/null
+++ b/.build/parent-pom-template.xml
@@ -0,0 +1,1052 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache</artifactId>
+ <groupId>org.apache</groupId>
+ <version>22</version>
+ </parent>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-parent</artifactId>
+ <version>@version@</version>
+ <packaging>pom</packaging>
+ <name>Apache Cassandra</name>
+ <description>The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.</description>
+ <url>https://cassandra.apache.org</url>
+ <inceptionYear>2009</inceptionYear>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+ <properties>
+ <bytebuddy.version>1.12.13</bytebuddy.version>
+ <byteman.version>4.0.20</byteman.version>
+ <ohc.version>0.5.1</ohc.version>
+
+ <!-- These are referenced in build.xml, so need to be propagated from there -->
+ <asm.version>@asm.version@</asm.version>
+ <jamm.version>@jamm.version@</jamm.version>
+ <allocation-instrumenter.version>@allocation-instrumenter.version@</allocation-instrumenter.version>
+ <ecj.version>@ecj.version@</ecj.version>
+ <jacoco.version>@jacoco.version@</jacoco.version>
+ <jflex.version>@jflex.version@</jflex.version>
+ </properties>
+ <developers>
+ <developer>
+ <id>adelapena</id>
+ <name>Andres de la Peña</name>
+ </developer>
+ <developer>
+ <id>alakshman</id>
+ <name>Avinash Lakshman</name>
+ </developer>
+ <developer>
+ <id>aleksey</id>
+ <name>Aleksey Yeschenko</name>
+ </developer>
+ <developer>
+ <id>amorton</id>
+ <name>Aaron Morton</name>
+ </developer>
+ <developer>
+ <id>aweisberg</id>
+ <name>Ariel Weisberg</name>
+ </developer>
+ <developer>
+ <id>bdeggleston</id>
+ <name>Blake Eggleston</name>
+ </developer>
+ <developer>
+ <id>benedict</id>
+ <name>Benedict Elliott Smith</name>
+ </developer>
+ <developer>
+ <id>benjamin</id>
+ <name>Benjamin Lerer</name>
+ </developer>
+ <developer>
+ <id>blambov</id>
+ <name>Branimir Lambov</name>
+ </developer>
+ <developer>
+ <id>brandonwilliams</id>
+ <name>Brandon Williams</name>
+ </developer>
+ <developer>
+ <id>carl</id>
+ <name>Carl Yeksigian</name>
+ </developer>
+ <developer>
+ <id>dbrosius</id>
+ <name>David Brosiusd</name>
+ </developer>
+ <developer>
+ <id>dikang</id>
+ <name>Dikang Gu</name>
+ </developer>
+ <developer>
+ <id>eevans</id>
+ <name>Eric Evans</name>
+ </developer>
+ <developer>
+ <id>edimitrova</id>
+ <name>Ekaterina Dimitrova</name>
+ </developer>
+ <developer>
+ <id>gdusbabek</id>
+ <name>Gary Dusbabek</name>
+ </developer>
+ <developer>
+ <id>goffinet</id>
+ <name>Chris Goffinet</name>
+ </developer>
+ <developer>
+ <id>ifesdjeen</id>
+ <name>Alex Petrov</name>
+ </developer>
+ <developer>
+ <id>jaakko</id>
+ <name>Laine Jaakko Olavi</name>
+ </developer>
+ <developer>
+ <id>jake</id>
+ <name>T Jake Luciani</name>
+ </developer>
+ <developer>
+ <id>jasonbrown</id>
+ <name>Jason Brown</name>
+ </developer>
+ <developer>
+ <id>jbellis</id>
+ <name>Jonathan Ellis</name>
+ </developer>
+ <developer>
+ <id>jfarrell</id>
+ <name>Jake Farrell</name>
+ </developer>
+ <developer>
+ <id>jjirsa</id>
+ <name>Jeff Jirsa</name>
+ </developer>
+ <developer>
+ <id>jkni</id>
+ <name>Joel Knighton</name>
+ </developer>
+ <developer>
+ <id>jmckenzie</id>
+ <name>Josh McKenzie</name>
+ </developer>
+ <developer>
+ <id>johan</id>
+ <name>Johan Oskarsson</name>
+ </developer>
+ <developer>
+ <id>junrao</id>
+ <name>Jun Rao</name>
+ </developer>
+ <developer>
+ <id>jzhuang</id>
+ <name>Jay Zhuang</name>
+ </developer>
+ <developer>
+ <id>kohlisankalp</id>
+ <name>Sankalp Kohli</name>
+ </developer>
+ <developer>
+ <id>marcuse</id>
+ <name>Marcus Eriksson</name>
+ </developer>
+ <developer>
+ <id>mck</id>
+ <name>Michael Semb Wever</name>
+ </developer>
+ <developer>
+ <id>mishail</id>
+ <name>Mikhail Stepura</name>
+ </developer>
+ <developer>
+ <id>mshuler</id>
+ <name>Michael Shuler</name>
+ </developer>
+ <developer>
+ <id>paulo</id>
+ <name>Paulo Motta</name>
+ </developer>
+ <developer>
+ <id>pmalik</id>
+ <name>Prashant Malik</name>
+ </developer>
+ <developer>
+ <id>rstupp</id>
+ <name>Robert Stupp</name>
+ </developer>
+ <developer>
+ <id>scode</id>
+ <name>Peter Schuller</name>
+ </developer>
+ <developer>
+ <id>beobal</id>
+ <name>Sam Tunnicliffe</name>
+ </developer>
+ <developer>
+ <id>slebresne</id>
+ <name>Sylvain Lebresne</name>
+ </developer>
+ <developer>
+ <id>stefania</id>
+ <name>Stefania Alborghetti</name>
+ </developer>
+ <developer>
+ <id>tylerhobbs</id>
+ <name>Tyler Hobbs</name>
+ </developer>
+ <developer>
+ <id>vijay</id>
+ <name>Vijay Parthasarathy</name>
+ </developer>
+ <developer>
+ <id>xedin</id>
+ <name>Pavel Yaskevich</name>
+ </developer>
+ <developer>
+ <id>yukim</id>
+ <name>Yuki Morishita</name>
+ </developer>
+ <developer>
+ <id>zznate</id>
+ <name>Nate McCall</name>
+ </developer>
+ </developers>
+ <scm>
+ <connection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</connection>
+ <developerConnection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</developerConnection>
+ <url>https://gitbox.apache.org/repos/asf?p=cassandra.git;a=tree</url>
+ </scm>
+ <dependencyManagement>
+ <!--
+ Dependency metadata is specified here (version, scope, exclusions, etc.), then referenced in child POMs by groupId and
+ artifactId.
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>1.1.8.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ <version>1.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.ning</groupId>
+ <artifactId>compress-lzf</artifactId>
+ <version>0.8.4</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ <version>1.5.5-1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.0-jre</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jsr305</artifactId>
+ <groupId>com.google.code.findbugs</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>animal-sniffer-annotations</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>listenablefuture</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>failureaccess</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>checker-qual</artifactId>
+ <groupId>org.checkerframework</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>error_prone_annotations</artifactId>
+ <groupId>com.google.errorprone</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.jimfs</groupId>
+ <artifactId>jimfs</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.hdrhistogram</groupId>
+ <artifactId>HdrHistogram</artifactId>
+ <version>2.1.9</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.9</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.11.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.11</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr</artifactId>
+ <version>3.5.2</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>stringtemplate</artifactId>
+ <groupId>org.antlr</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>ST4</artifactId>
+ <version>4.0.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ <version>3.5.2</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>stringtemplate</artifactId>
+ <groupId>org.antlr</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.25</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>1.7.25</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <version>1.7.25</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>1.2.9</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.2.9</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.13.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.13.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.13.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <version>2.13.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>2.13.2</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>snakeyaml</artifactId>
+ <groupId>org.yaml</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.boundary</groupId>
+ <artifactId>high-scale-lib</artifactId>
+ <version>1.0.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <version>${jamm.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.26</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hamcrest-core</artifactId>
+ <groupId>org.hamcrest</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>4.7.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-inline</artifactId>
+ <version>4.7.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.quicktheories</groupId>
+ <artifactId>quicktheories</artifactId>
+ <version>0.26</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.java-allocation-instrumenter</groupId>
+ <artifactId>java-allocation-instrumenter</artifactId>
+ <version>${allocation-instrumenter.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>guava</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>harry-core</artifactId>
+ <version>0.0.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.10.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>dtest-api</artifactId>
+ <version>0.0.13</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>5.13.0</version>
+ </dependency>
+
+ <dependency>
+ <!-- transitive to posix to chronicle-core, declared explicit to use newer version -->
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>5.13.0</version>
+ </dependency>
+ <dependency>
+ <!-- transitive to posix to chronicle-core, declared explicit to use newer version -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jnr-ffi</artifactId>
+ <version>2.2.13</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-analysis</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-commons</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-tree</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- transitive to jnr-ffi to posix to chronicle-core, declared explicit to use newer version -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jffi</artifactId>
+ <version>1.3.11</version>
+ </dependency>
+ <dependency>
+ <!-- transitive to jnr-ffi to posix to chronicle-core, declared explicit to use newer version -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jffi</artifactId>
+ <classifier>native</classifier>
+ <version>1.3.11</version>
+ </dependency>
+ <dependency>
+ <!-- transitive to posix to chronicle-core, declared explicit to use newer version -->
+ <groupId>com.github.jnr</groupId>
+ <artifactId>jnr-constants</artifactId>
+ <version>0.10.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jacoco</groupId>
+ <artifactId>org.jacoco.agent</artifactId>
+ <version>${jacoco.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jacoco</groupId>
+ <artifactId>org.jacoco.ant</artifactId>
+ <version>${jacoco.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman-install</artifactId>
+ <version>${byteman.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman</artifactId>
+ <version>${byteman.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman-submit</artifactId>
+ <version>${byteman.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.byteman</groupId>
+ <artifactId>byteman-bmunit</artifactId>
+ <version>${byteman.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ <version>${bytebuddy.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy-agent</artifactId>
+ <version>${bytebuddy.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>1.21</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>1.21</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-junit</artifactId>
+ <version>1.10.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-all</artifactId>
+ <version>4.1-alpha2-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>3.1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-logback</artifactId>
+ <version>3.1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>com.addthis.metrics</groupId>
+ <artifactId>reporter-config3</artifactId>
+ <version>3.0.3</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>hibernate-validator</artifactId>
+ <groupId>org.hibernate</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mindrot</groupId>
+ <artifactId>jbcrypt</artifactId>
+ <version>0.4</version>
+ </dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>airline</artifactId>
+ <version>0.8</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jsr305</artifactId>
+ <groupId>com.google.code.findbugs</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-bom</artifactId>
+ <version>4.1.58.Final</version>
+ <type>pom</type>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.58.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>2.0.36.Final</version>
+ </dependency>
+
+ <!-- chronicle-queue deps -->
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-queue</artifactId>
+ <version>5.23.37</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>tools</artifactId>
+ <groupId>com.sun</groupId>
+ </exclusion>
+ <exclusion>
+ <!-- pulls in affinity-3.23ea1 which pulls in third-party-bom-3.22.4-SNAPSHOT -->
+ <groupId>net.openhft</groupId>
+ <artifactId>affinity</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-core</artifactId>
+ <version>2.23.36</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>chronicle-analytics</artifactId>
+ <groupId>net.openhft</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>annotations</artifactId>
+ <groupId>org.jetbrains</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-bytes</artifactId>
+ <version>2.23.33</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>annotations</artifactId>
+ <groupId>org.jetbrains</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-wire</artifactId>
+ <version>2.23.39</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>compiler</artifactId>
+ <groupId>net.openhft</groupId>
+ </exclusion>
+ <exclusion>
+ <!-- pulls in affinity-3.23ea1 which pulls in third-party-bom-3.22.4-SNAPSHOT -->
+ <groupId>net.openhft</groupId>
+ <artifactId>affinity</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>chronicle-threads</artifactId>
+ <version>2.23.25</version>
+ <exclusions>
+ <exclusion>
+ <!-- pulls in affinity-3.23ea1 which pulls in third-party-bom-3.22.4-SNAPSHOT -->
+ <groupId>net.openhft</groupId>
+ <artifactId>affinity</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- transitive to chronicle-queue, declared explicit to use newer version -->
+ <groupId>net.openhft</groupId>
+ <artifactId>affinity</artifactId>
+ <version>3.23.3</version>
+ </dependency>
+ <dependency>
+ <!-- transitive to chronicle-queue, declared explicit to use newer version -->
+ <groupId>net.openhft</groupId>
+ <artifactId>posix</artifactId>
+ <version>2.24ea4</version>
+ </dependency>
+ <!-- end of chronicle-queue -->
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>2.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.clearspring.analytics</groupId>
+ <artifactId>stream</artifactId>
+ <version>2.5.2</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>fastutil</artifactId>
+ <groupId>it.unimi.dsi</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>3.11.0</version>
+ <classifier>shaded</classifier>
+ <exclusions>
+ <exclusion>
+ <artifactId>netty-buffer</artifactId>
+ <groupId>io.netty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>netty-codec</artifactId>
+ <groupId>io.netty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>netty-handler</artifactId>
+ <groupId>io.netty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>netty-transport</artifactId>
+ <groupId>io.netty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jnr-ffi</artifactId>
+ <groupId>com.github.jnr</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jnr-posix</artifactId>
+ <groupId>com.github.jnr</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jdt.core.compiler</groupId>
+ <artifactId>ecj</artifactId>
+ <version>${ecj.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.caffinitas.ohc</groupId>
+ <artifactId>ohc-core</artifactId>
+ <version>${ohc.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.caffinitas.ohc</groupId>
+ <artifactId>ohc-core-j8</artifactId>
+ <version>${ohc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.ju-n.compile-command-annotations</groupId>
+ <artifactId>compile-command-annotations</artifactId>
+ <version>1.2.0</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.fusesource</groupId>
+ <artifactId>sigar</artifactId>
+ <version>1.6.4</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>0.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>de.jflex</groupId>
+ <artifactId>jflex</artifactId>
+ <version>${jflex.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>ant</artifactId>
+ <groupId>org.apache.ant</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.github.rholder</groupId>
+ <artifactId>snowball-stemmer</artifactId>
+ <version>1.3.0.581.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.concurrent-trees</groupId>
+ <artifactId>concurrent-trees</artifactId>
+ <version>2.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>2.9.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jctools</groupId>
+ <artifactId>jctools-core</artifactId>
+ <version>3.1.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ <version>${asm.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-analysis</artifactId>
+ <version>${asm.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-tree</artifactId>
+ <version>${asm.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-commons</artifactId>
+ <version>${asm.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-util</artifactId>
+ <version>${asm.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-cli</artifactId>
+ <version>0.14</version>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-core</artifactId>
+ <version>0.14</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>sjk-hflame</artifactId>
+ <groupId>org.gridkit.jvmtool</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjk-hflame</artifactId>
+ <groupId>org.perfkit.sjk.parsers</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjk-jfr-standalone</artifactId>
+ <groupId>org.perfkit.sjk.parsers</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjk-nps</artifactId>
+ <groupId>org.perfkit.sjk.parsers</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjk-jfr5</artifactId>
+ <groupId>org.perfkit.sjk.parsers</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjk-jfr6</artifactId>
+ <groupId>org.perfkit.sjk.parsers</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-stacktrace</artifactId>
+ <version>0.14</version>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>mxdump</artifactId>
+ <version>0.14</version>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.lab</groupId>
+ <artifactId>jvm-attach-api</artifactId>
+ <version>1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.gridkit.jvmtool</groupId>
+ <artifactId>sjk-json</artifactId>
+ <version>0.14</version>
+ </dependency>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>1.30</version>
+ </dependency>
+ <dependency>
+ <groupId>org.psjava</groupId>
+ <artifactId>psjava</artifactId>
+ <version>0.1.19</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.inject</groupId>
+ <artifactId>javax.inject</artifactId>
+ <version>1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.j2objc</groupId>
+ <artifactId>j2objc-annotations</artifactId>
+ <version>1.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.junit</groupId>
+ <artifactId>junit-bom</artifactId>
+ <version>5.6.0</version>
+ <type>pom</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>3.15.0</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>4.0.3</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hamcrest</artifactId>
+ <groupId>org.hamcrest</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest</artifactId>
+ <version>2.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.seancfoley</groupId>
+ <artifactId>ipaddress</artifactId>
+ <version>5.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.agrona</groupId>
+ <artifactId>agrona</artifactId>
+ <version>1.17.1</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.obermuhlner</groupId>
+ <artifactId>big-math</artifactId>
+ <version>2.3.0</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
diff --git a/.circleci/config.yml b/.circleci/config.yml
index b009cd1..74e5d21 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -95,8 +95,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -172,7 +170,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -186,8 +184,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -252,8 +248,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -363,8 +357,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -519,8 +511,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -606,8 +596,6 @@
destination: dtest_j8_large_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -725,8 +713,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -799,8 +785,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -876,7 +860,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -890,8 +874,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1001,8 +983,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1044,6 +1024,95 @@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@ -1121,8 +1190,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1198,7 +1265,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -1212,8 +1279,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1331,8 +1396,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1442,8 +1505,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1552,8 +1613,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1630,7 +1689,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -1644,8 +1703,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1760,6 +1817,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -1840,8 +1898,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1927,8 +1983,6 @@
destination: dtest_j11_large_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2062,8 +2116,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2173,8 +2225,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2283,8 +2333,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2370,8 +2418,6 @@
destination: dtest_j11_large_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2448,7 +2494,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -2462,8 +2508,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2573,8 +2617,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2683,8 +2725,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2749,8 +2789,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2868,8 +2906,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2942,8 +2978,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3062,8 +3096,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3196,8 +3228,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3330,8 +3360,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3465,8 +3493,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3584,8 +3610,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3700,8 +3724,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3819,8 +3841,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3892,8 +3912,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3970,7 +3988,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -3984,8 +4002,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4070,8 +4086,6 @@
destination: dtest_j8_large_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4112,6 +4126,120 @@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_simulator_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_utests_stress:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@ -4143,8 +4271,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4254,8 +4380,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4388,8 +4512,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4465,7 +4587,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4479,8 +4601,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4636,8 +4756,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4714,7 +4832,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4728,8 +4846,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4805,7 +4921,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4819,8 +4935,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4938,8 +5052,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5049,8 +5161,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5159,8 +5269,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5237,7 +5345,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5251,8 +5359,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5361,8 +5467,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5439,7 +5543,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5453,8 +5557,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5495,6 +5597,124 @@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@ -5587,8 +5807,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5630,6 +5848,123 @@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@ -5698,8 +6033,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5775,7 +6108,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5789,8 +6122,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5923,8 +6254,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6033,8 +6362,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6120,8 +6447,6 @@
destination: dtest_j8_upgradetests_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6228,8 +6553,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6306,7 +6629,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6320,8 +6643,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6430,8 +6751,186 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6508,7 +7007,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6522,8 +7021,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6633,8 +7130,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6711,7 +7206,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6725,8 +7220,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6811,8 +7304,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6930,8 +7421,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7049,8 +7538,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7127,7 +7614,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -7141,8 +7628,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7250,8 +7735,6 @@
- .m2
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7337,8 +7820,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7423,8 +7904,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7487,8 +7966,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7565,7 +8042,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -7579,8 +8056,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7687,8 +8162,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7803,6 +8276,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -7883,8 +8357,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7956,8 +8428,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8075,8 +8545,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8185,8 +8653,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8319,8 +8785,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8438,8 +8902,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8546,8 +9008,6 @@
- .m2
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8656,8 +9116,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8733,7 +9191,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -8747,8 +9205,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8821,8 +9277,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8885,8 +9339,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8993,8 +9445,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9071,7 +9521,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -9085,8 +9535,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9219,8 +9667,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9296,7 +9742,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -9310,8 +9756,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9416,8 +9860,6 @@
- dtest_jars
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9569,6 +10011,18 @@
requires:
- start_j11_utests_compression
- j8_build
+ - start_j8_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_j8_utests_trie
+ - j8_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j8_build
- start_j8_utests_stress:
type: approval
- j8_utests_stress:
@@ -9833,6 +10287,16 @@
requires:
- start_utests_compression
- j8_build
+ - start_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
- start_utests_stress:
type: approval
- j8_utests_stress:
@@ -10010,6 +10474,12 @@
requires:
- start_j11_jvm_dtests_vnode
- j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@@ -10110,6 +10580,12 @@
requires:
- start_j11_utests_compression
- j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
- start_j11_utests_stress:
type: approval
- j11_utests_stress:
@@ -10144,6 +10620,9 @@
- j11_jvm_dtests_vnode:
requires:
- j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
- j11_cqlshlib_tests:
requires:
- j11_build
@@ -10222,6 +10701,12 @@
requires:
- start_utests_compression
- j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
- start_utests_stress:
type: approval
- j11_utests_stress:
diff --git a/.circleci/config.yml.FREE b/.circleci/config.yml.FREE
index b009cd1..74e5d21 100644
--- a/.circleci/config.yml.FREE
+++ b/.circleci/config.yml.FREE
@@ -95,8 +95,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -172,7 +170,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -186,8 +184,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -252,8 +248,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -363,8 +357,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -519,8 +511,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -606,8 +596,6 @@
destination: dtest_j8_large_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -725,8 +713,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -799,8 +785,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -876,7 +860,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -890,8 +874,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1001,8 +983,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1044,6 +1024,95 @@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@ -1121,8 +1190,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1198,7 +1265,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -1212,8 +1279,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1331,8 +1396,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1442,8 +1505,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1552,8 +1613,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1630,7 +1689,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -1644,8 +1703,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1760,6 +1817,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -1840,8 +1898,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1927,8 +1983,6 @@
destination: dtest_j11_large_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2062,8 +2116,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2173,8 +2225,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2283,8 +2333,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2370,8 +2418,6 @@
destination: dtest_j11_large_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2448,7 +2494,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -2462,8 +2508,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2573,8 +2617,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2683,8 +2725,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2749,8 +2789,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2868,8 +2906,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2942,8 +2978,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3062,8 +3096,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3196,8 +3228,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3330,8 +3360,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3465,8 +3493,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3584,8 +3610,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3700,8 +3724,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3819,8 +3841,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3892,8 +3912,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3970,7 +3988,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -3984,8 +4002,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4070,8 +4086,6 @@
destination: dtest_j8_large_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4112,6 +4126,120 @@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_simulator_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_utests_stress:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@ -4143,8 +4271,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4254,8 +4380,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4388,8 +4512,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4465,7 +4587,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4479,8 +4601,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4636,8 +4756,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4714,7 +4832,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4728,8 +4846,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4805,7 +4921,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4819,8 +4935,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4938,8 +5052,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5049,8 +5161,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5159,8 +5269,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5237,7 +5345,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5251,8 +5359,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5361,8 +5467,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5439,7 +5543,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5453,8 +5557,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5495,6 +5597,124 @@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@ -5587,8 +5807,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5630,6 +5848,123 @@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@ -5698,8 +6033,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5775,7 +6108,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5789,8 +6122,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5923,8 +6254,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6033,8 +6362,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6120,8 +6447,6 @@
destination: dtest_j8_upgradetests_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6228,8 +6553,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6306,7 +6629,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6320,8 +6643,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6430,8 +6751,186 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6508,7 +7007,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6522,8 +7021,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6633,8 +7130,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6711,7 +7206,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6725,8 +7220,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6811,8 +7304,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6930,8 +7421,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7049,8 +7538,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7127,7 +7614,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -7141,8 +7628,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7250,8 +7735,6 @@
- .m2
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7337,8 +7820,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7423,8 +7904,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7487,8 +7966,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7565,7 +8042,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -7579,8 +8056,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7687,8 +8162,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7803,6 +8276,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -7883,8 +8357,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7956,8 +8428,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8075,8 +8545,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8185,8 +8653,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8319,8 +8785,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8438,8 +8902,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8546,8 +9008,6 @@
- .m2
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8656,8 +9116,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8733,7 +9191,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -8747,8 +9205,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8821,8 +9277,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8885,8 +9339,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8993,8 +9445,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9071,7 +9521,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -9085,8 +9535,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9219,8 +9667,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9296,7 +9742,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -9310,8 +9756,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9416,8 +9860,6 @@
- dtest_jars
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9569,6 +10011,18 @@
requires:
- start_j11_utests_compression
- j8_build
+ - start_j8_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_j8_utests_trie
+ - j8_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j8_build
- start_j8_utests_stress:
type: approval
- j8_utests_stress:
@@ -9833,6 +10287,16 @@
requires:
- start_utests_compression
- j8_build
+ - start_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
- start_utests_stress:
type: approval
- j8_utests_stress:
@@ -10010,6 +10474,12 @@
requires:
- start_j11_jvm_dtests_vnode
- j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@@ -10110,6 +10580,12 @@
requires:
- start_j11_utests_compression
- j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
- start_j11_utests_stress:
type: approval
- j11_utests_stress:
@@ -10144,6 +10620,9 @@
- j11_jvm_dtests_vnode:
requires:
- j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
- j11_cqlshlib_tests:
requires:
- j11_build
@@ -10222,6 +10701,12 @@
requires:
- start_utests_compression
- j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
- start_utests_stress:
type: approval
- j11_utests_stress:
diff --git a/.circleci/config.yml.PAID b/.circleci/config.yml.PAID
index 3118f37..e70174d 100644
--- a/.circleci/config.yml.PAID
+++ b/.circleci/config.yml.PAID
@@ -95,8 +95,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -172,7 +170,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -186,8 +184,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -252,8 +248,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -363,8 +357,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -519,8 +511,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -606,8 +596,6 @@
destination: dtest_j8_large_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -725,8 +713,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -799,8 +785,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -876,7 +860,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -890,8 +874,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1001,8 +983,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1044,6 +1024,95 @@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@ -1121,8 +1190,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1198,7 +1265,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -1212,8 +1279,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1331,8 +1396,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1442,8 +1505,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1552,8 +1613,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1630,7 +1689,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -1644,8 +1703,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1760,6 +1817,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -1840,8 +1898,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -1927,8 +1983,6 @@
destination: dtest_j11_large_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2062,8 +2116,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2173,8 +2225,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2283,8 +2333,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2370,8 +2418,6 @@
destination: dtest_j11_large_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2448,7 +2494,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -2462,8 +2508,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2573,8 +2617,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2683,8 +2725,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2749,8 +2789,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2868,8 +2906,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -2942,8 +2978,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3062,8 +3096,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3196,8 +3228,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3330,8 +3360,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3465,8 +3493,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3584,8 +3610,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3700,8 +3724,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3819,8 +3841,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3892,8 +3912,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -3970,7 +3988,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -3984,8 +4002,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4070,8 +4086,6 @@
destination: dtest_j8_large_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4112,6 +4126,120 @@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_simulator_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_utests_stress:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@ -4143,8 +4271,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4254,8 +4380,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4388,8 +4512,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4465,7 +4587,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4479,8 +4601,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4636,8 +4756,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4714,7 +4832,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4728,8 +4846,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4805,7 +4921,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -4819,8 +4935,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -4938,8 +5052,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5049,8 +5161,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5159,8 +5269,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5237,7 +5345,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5251,8 +5359,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5361,8 +5467,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5439,7 +5543,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5453,8 +5557,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5495,6 +5597,124 @@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@ -5587,8 +5807,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5630,6 +5848,123 @@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@ -5698,8 +6033,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5775,7 +6108,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -5789,8 +6122,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -5923,8 +6254,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6033,8 +6362,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6079,7 +6406,7 @@
j8_upgrade_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
+ resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
@@ -6120,8 +6447,6 @@
destination: dtest_j8_upgradetests_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6228,8 +6553,6 @@
destination: dtest_j11_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6306,7 +6629,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6320,8 +6643,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6430,8 +6751,186 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6508,7 +7007,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6522,8 +7021,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6633,8 +7130,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6711,7 +7206,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -6725,8 +7220,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6811,8 +7304,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -6930,8 +7421,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7049,8 +7538,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7127,7 +7614,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -7141,8 +7628,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7250,8 +7735,6 @@
- .m2
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7337,8 +7820,6 @@
destination: dtest_j8_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7423,8 +7904,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7487,8 +7966,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7565,7 +8042,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -7579,8 +8056,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7687,8 +8162,6 @@
destination: dtest_j11_without_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7803,6 +8276,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -7883,8 +8357,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -7956,8 +8428,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8075,8 +8545,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8185,8 +8653,6 @@
destination: dtest_j8_dtests_offheap_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8319,8 +8785,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8438,8 +8902,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8546,8 +9008,6 @@
- .m2
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8656,8 +9116,6 @@
destination: dtest_j8_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8733,7 +9191,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -8747,8 +9205,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8821,8 +9277,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8885,8 +9339,6 @@
path: /tmp/cassandra/pylib
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -8993,8 +9445,6 @@
destination: dtest_j11_with_vnodes_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9071,7 +9521,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -9085,8 +9535,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9219,8 +9667,6 @@
destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9296,7 +9742,7 @@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@ -9310,8 +9756,6 @@
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9416,8 +9860,6 @@
- dtest_jars
environment:
- ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -9569,6 +10011,18 @@
requires:
- start_j11_utests_compression
- j8_build
+ - start_j8_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_j8_utests_trie
+ - j8_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j8_build
- start_j8_utests_stress:
type: approval
- j8_utests_stress:
@@ -9833,6 +10287,16 @@
requires:
- start_utests_compression
- j8_build
+ - start_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
- start_utests_stress:
type: approval
- j8_utests_stress:
@@ -10010,6 +10474,12 @@
requires:
- start_j11_jvm_dtests_vnode
- j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@@ -10110,6 +10580,12 @@
requires:
- start_j11_utests_compression
- j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
- start_j11_utests_stress:
type: approval
- j11_utests_stress:
@@ -10144,6 +10620,9 @@
- j11_jvm_dtests_vnode:
requires:
- j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
- j11_cqlshlib_tests:
requires:
- j11_build
@@ -10222,6 +10701,12 @@
requires:
- start_utests_compression
- j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
- start_utests_stress:
type: approval
- j11_utests_stress:
diff --git a/.circleci/config_11_and_17.yml b/.circleci/config_11_and_17.yml
new file mode 100644
index 0000000..2d0e77e
--- /dev/null
+++ b/.circleci/config_11_and_17.yml
@@ -0,0 +1,9561 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: 2
+jobs:
+ j17_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlshlib_cython_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py311_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlshlib_cython_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py3_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_offheap_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py311:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_dtests_large_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py311:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if ${REPEATED_ANT_TEST_VNODES}; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_with_vnodes_raw /tmp/all_dtest_tests_j11_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_with_vnodes_raw > /tmp/all_dtest_tests_j11_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_with_vnodes > /tmp/split_dtest_tests_j11_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_large_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_without_vnodes_raw /tmp/all_dtest_tests_j11_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_without_vnodes_raw > /tmp/all_dtest_tests_j11_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_without_vnodes > /tmp/split_dtest_tests_j11_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if ${REPEATED_ANT_TEST_VNODES}; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_offheap_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_jvm_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_simulator_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlsh_dtests_py311_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_large_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_with_vnodes_raw /tmp/all_dtest_tests_j17_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_with_vnodes_raw > /tmp/all_dtest_tests_j17_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_with_vnodes > /tmp/split_dtest_tests_j17_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_large_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_large_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_large_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py311_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py311_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_without_vnodes_raw /tmp/all_dtest_tests_j17_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_without_vnodes_raw > /tmp/all_dtest_tests_j17_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_without_vnodes > /tmp/split_dtest_tests_j17_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+workflows:
+ version: 2
+ java11_separate_tests:
+ jobs:
+ - start_j11_build:
+ type: approval
+ - j11_build:
+ requires:
+ - start_j11_build
+ - start_j11_unit_tests:
+ type: approval
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_build
+ - start_j11_jvm_dtests:
+ type: approval
+ - j11_jvm_dtests:
+ requires:
+ - start_j11_jvm_dtests
+ - j11_build
+ - start_j11_jvm_dtests_vnode:
+ type: approval
+ - j11_jvm_dtests_vnode:
+ requires:
+ - start_j11_jvm_dtests_vnode
+ - j11_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j11_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
+ - start_j11_cqlshlib_tests:
+ type: approval
+ - j11_cqlshlib_tests:
+ requires:
+ - start_j11_cqlshlib_tests
+ - j11_build
+ - start_j11_cqlshlib_cython_tests:
+ type: approval
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - start_j11_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j11_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j11_build
+ - start_j11_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_j11_utests_long
+ - j11_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j11_build
+ - start_j11_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_j11_utests_cdc
+ - j11_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j11_build
+ - start_j11_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_j11_utests_compression
+ - j11_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j11_build
+ - start_j11_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_j11_utests_stress
+ - j11_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j11_build
+ - start_j11_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_j11_utests_fqltool
+ - j11_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j11_build
+ - start_j11_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - start_j11_utests_system_keyspace_directory
+ - j11_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ - j11_dtests:
+ requires:
+ - start_j11_dtests
+ - j11_build
+ - start_j11_dtests_vnode:
+ type: approval
+ - j11_dtests_vnode:
+ requires:
+ - start_j11_dtests_vnode
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j11_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j11_dtests_large_vnode:
+ type: approval
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large_vnode
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j11_build
+ - start_j11_cqlsh_tests:
+ type: approval
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - start_j11_cqlsh_tests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - start_j17_cqlsh_tests_offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ java11_pre-commit_tests:
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j11_build:
+ requires:
+ - start_pre-commit_tests
+ - j11_unit_tests:
+ requires:
+ - j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j17_jvm_dtests:
+ requires:
+ - j11_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j11_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_unit_tests:
+ requires:
+ - j11_build
+ - start_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - start_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - start_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - start_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - start_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - j11_build
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
+ - j11_dtests:
+ requires:
+ - j11_build
+ - j11_dtests_vnode:
+ requires:
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - j17_dtests:
+ requires:
+ - j11_build
+ - j17_dtests_vnode:
+ requires:
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ - start_j11_cqlsh_dtests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ java17_separate_tests:
+ jobs:
+ - start_j17_build:
+ type: approval
+ - j17_build:
+ requires:
+ - start_j17_build
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j17_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j17_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j17_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j17_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j17_build
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j17_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j17_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j17_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j17_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j17_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j17_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j17_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j17_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j17_build
+ java17_pre-commit_tests:
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j17_build:
+ requires:
+ - start_pre-commit_tests
+ - j17_unit_tests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j17_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j17_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j17_build
+ - j17_dtests:
+ requires:
+ - j17_build
+ - j17_dtests_vnode:
+ requires:
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - start_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j17_build
+ - start_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j17_build
+ - start_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j17_build
+ - start_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j17_build
+ - start_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j17_build
+ - start_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j17_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j17_build
diff --git a/.circleci/config_11_and_17.yml.FREE b/.circleci/config_11_and_17.yml.FREE
new file mode 100644
index 0000000..2d0e77e
--- /dev/null
+++ b/.circleci/config_11_and_17.yml.FREE
@@ -0,0 +1,9561 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: 2
+jobs:
+ j17_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlshlib_cython_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py311_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlshlib_cython_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py3_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_offheap_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py311:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_dtests_large_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py311:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if ${REPEATED_ANT_TEST_VNODES}; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_with_vnodes_raw /tmp/all_dtest_tests_j11_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_with_vnodes_raw > /tmp/all_dtest_tests_j11_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_with_vnodes > /tmp/split_dtest_tests_j11_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_large_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_without_vnodes_raw /tmp/all_dtest_tests_j11_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_without_vnodes_raw > /tmp/all_dtest_tests_j11_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_without_vnodes > /tmp/split_dtest_tests_j11_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if ${REPEATED_ANT_TEST_VNODES}; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_offheap_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_jvm_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_simulator_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlsh_dtests_py311_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_large_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_with_vnodes_raw /tmp/all_dtest_tests_j17_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_with_vnodes_raw > /tmp/all_dtest_tests_j17_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_with_vnodes > /tmp/split_dtest_tests_j17_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_large_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_large_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_large_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py311_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py311_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_without_vnodes_raw /tmp/all_dtest_tests_j17_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_without_vnodes_raw > /tmp/all_dtest_tests_j17_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_without_vnodes > /tmp/split_dtest_tests_j17_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+workflows:
+ version: 2
+ java11_separate_tests:
+ jobs:
+ - start_j11_build:
+ type: approval
+ - j11_build:
+ requires:
+ - start_j11_build
+ - start_j11_unit_tests:
+ type: approval
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_build
+ - start_j11_jvm_dtests:
+ type: approval
+ - j11_jvm_dtests:
+ requires:
+ - start_j11_jvm_dtests
+ - j11_build
+ - start_j11_jvm_dtests_vnode:
+ type: approval
+ - j11_jvm_dtests_vnode:
+ requires:
+ - start_j11_jvm_dtests_vnode
+ - j11_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j11_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
+ - start_j11_cqlshlib_tests:
+ type: approval
+ - j11_cqlshlib_tests:
+ requires:
+ - start_j11_cqlshlib_tests
+ - j11_build
+ - start_j11_cqlshlib_cython_tests:
+ type: approval
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - start_j11_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j11_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j11_build
+ - start_j11_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_j11_utests_long
+ - j11_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j11_build
+ - start_j11_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_j11_utests_cdc
+ - j11_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j11_build
+ - start_j11_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_j11_utests_compression
+ - j11_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j11_build
+ - start_j11_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_j11_utests_stress
+ - j11_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j11_build
+ - start_j11_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_j11_utests_fqltool
+ - j11_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j11_build
+ - start_j11_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - start_j11_utests_system_keyspace_directory
+ - j11_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ - j11_dtests:
+ requires:
+ - start_j11_dtests
+ - j11_build
+ - start_j11_dtests_vnode:
+ type: approval
+ - j11_dtests_vnode:
+ requires:
+ - start_j11_dtests_vnode
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j11_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j11_dtests_large_vnode:
+ type: approval
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large_vnode
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j11_build
+ - start_j11_cqlsh_tests:
+ type: approval
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - start_j11_cqlsh_tests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - start_j17_cqlsh_tests_offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ java11_pre-commit_tests:
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j11_build:
+ requires:
+ - start_pre-commit_tests
+ - j11_unit_tests:
+ requires:
+ - j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j17_jvm_dtests:
+ requires:
+ - j11_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j11_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_unit_tests:
+ requires:
+ - j11_build
+ - start_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - start_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - start_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - start_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - start_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - j11_build
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
+ - j11_dtests:
+ requires:
+ - j11_build
+ - j11_dtests_vnode:
+ requires:
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - j17_dtests:
+ requires:
+ - j11_build
+ - j17_dtests_vnode:
+ requires:
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ - start_j11_cqlsh_dtests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ java17_separate_tests:
+ jobs:
+ - start_j17_build:
+ type: approval
+ - j17_build:
+ requires:
+ - start_j17_build
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j17_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j17_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j17_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j17_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j17_build
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j17_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j17_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j17_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j17_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j17_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j17_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j17_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j17_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j17_build
+ java17_pre-commit_tests:
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j17_build:
+ requires:
+ - start_pre-commit_tests
+ - j17_unit_tests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j17_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j17_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j17_build
+ - j17_dtests:
+ requires:
+ - j17_build
+ - j17_dtests_vnode:
+ requires:
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - start_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j17_build
+ - start_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j17_build
+ - start_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j17_build
+ - start_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j17_build
+ - start_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j17_build
+ - start_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j17_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j17_build
diff --git a/.circleci/config_11_and_17.yml.PAID b/.circleci/config_11_and_17.yml.PAID
new file mode 100644
index 0000000..ab0958b
--- /dev/null
+++ b/.circleci/config_11_and_17.yml.PAID
@@ -0,0 +1,9561 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: 2
+jobs:
+ j17_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlshlib_cython_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py311_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlshlib_cython_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py3_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_offheap_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py311:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_dtests_large_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py311:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if ${REPEATED_ANT_TEST_VNODES}; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_with_vnodes_raw /tmp/all_dtest_tests_j11_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_with_vnodes_raw > /tmp/all_dtest_tests_j11_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_with_vnodes > /tmp/split_dtest_tests_j11_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_large_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_without_vnodes_raw /tmp/all_dtest_tests_j11_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_without_vnodes_raw > /tmp/all_dtest_tests_j11_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_without_vnodes > /tmp/split_dtest_tests_j11_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 10
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if ${REPEATED_ANT_TEST_VNODES}; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_offheap_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_jvm_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 10
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_simulator_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlsh_dtests_py311_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_large_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_with_vnodes_raw /tmp/all_dtest_tests_j17_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_with_vnodes_raw > /tmp/all_dtest_tests_j17_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_with_vnodes > /tmp/split_dtest_tests_j17_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_large_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_large_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_large_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-trie\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_cqlsh_dtests_py311_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py311_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
+
+ source ~/env3.11/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.11
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 10
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j11_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 10
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_without_vnodes_raw /tmp/all_dtest_tests_j17_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_without_vnodes_raw > /tmp/all_dtest_tests_j17_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_without_vnodes > /tmp/split_dtest_tests_j17_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j17_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ j17_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j17_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_offheap_raw /tmp/all_dtest_tests_j17_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_offheap_raw > /tmp/all_dtest_tests_j17_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_offheap > /tmp/split_dtest_tests_j17_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n"
+ - run:
+ name: Run dtests (j17_dtests_offheap)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j17_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j17_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j17_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ j11_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+workflows:
+ version: 2
+ java11_separate_tests:
+ jobs:
+ - start_j11_build:
+ type: approval
+ - j11_build:
+ requires:
+ - start_j11_build
+ - start_j11_unit_tests:
+ type: approval
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_build
+ - start_j11_jvm_dtests:
+ type: approval
+ - j11_jvm_dtests:
+ requires:
+ - start_j11_jvm_dtests
+ - j11_build
+ - start_j11_jvm_dtests_vnode:
+ type: approval
+ - j11_jvm_dtests_vnode:
+ requires:
+ - start_j11_jvm_dtests_vnode
+ - j11_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j11_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
+ - start_j11_cqlshlib_tests:
+ type: approval
+ - j11_cqlshlib_tests:
+ requires:
+ - start_j11_cqlshlib_tests
+ - j11_build
+ - start_j11_cqlshlib_cython_tests:
+ type: approval
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - start_j11_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j11_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j11_build
+ - start_j11_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_j11_utests_long
+ - j11_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j11_build
+ - start_j11_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_j11_utests_cdc
+ - j11_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j11_build
+ - start_j11_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_j11_utests_compression
+ - j11_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j11_build
+ - start_j11_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_j11_utests_stress
+ - j11_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j11_build
+ - start_j11_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_j11_utests_fqltool
+ - j11_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j11_build
+ - start_j11_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - start_j11_utests_system_keyspace_directory
+ - j11_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j11_build
+ - start_j11_dtests:
+ type: approval
+ - j11_dtests:
+ requires:
+ - start_j11_dtests
+ - j11_build
+ - start_j11_dtests_vnode:
+ type: approval
+ - j11_dtests_vnode:
+ requires:
+ - start_j11_dtests_vnode
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j11_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j11_dtests_large_vnode:
+ type: approval
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large_vnode
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j11_build
+ - start_j11_cqlsh_tests:
+ type: approval
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - start_j11_cqlsh_tests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - start_j17_cqlsh_tests_offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ java11_pre-commit_tests:
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j11_build:
+ requires:
+ - start_pre-commit_tests
+ - j11_unit_tests:
+ requires:
+ - j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j17_jvm_dtests:
+ requires:
+ - j11_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j11_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_unit_tests:
+ requires:
+ - j11_build
+ - start_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - start_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - start_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - start_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - start_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - j11_build
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
+ - j11_dtests:
+ requires:
+ - j11_build
+ - j11_dtests_vnode:
+ requires:
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - j17_dtests:
+ requires:
+ - j11_build
+ - j17_dtests_vnode:
+ requires:
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ - start_j11_cqlsh_dtests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ java17_separate_tests:
+ jobs:
+ - start_j17_build:
+ type: approval
+ - j17_build:
+ requires:
+ - start_j17_build
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j17_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j17_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j17_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j17_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j17_build
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j17_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j17_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j17_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j17_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j17_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j17_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j17_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j17_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j17_build
+ java17_pre-commit_tests:
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j17_build:
+ requires:
+ - start_pre-commit_tests
+ - j17_unit_tests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j17_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j17_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j17_build
+ - j17_dtests:
+ requires:
+ - j17_build
+ - j17_dtests_vnode:
+ requires:
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - start_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j17_build
+ - start_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j17_build
+ - start_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j17_build
+ - start_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j17_build
+ - start_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j17_build
+ - start_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j17_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j17_build
diff --git a/.circleci/config_template.yml b/.circleci/config_template.yml
index 4367bdc..29b6ee0 100644
--- a/.circleci/config_template.yml
+++ b/.circleci/config_template.yml
@@ -28,8 +28,6 @@
# please remember to modify the generate.sh script and the documentation accordingly.
ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
LANG: en_US.UTF-8
KEEP_TEST_DIR: true
DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@ -136,6 +134,7 @@
# REPEATED_ANT_TEST_TARGET: test-jvm-dtest-some
# REPEATED_ANT_TEST_TARGET: test-cdc
# REPEATED_ANT_TEST_TARGET: test-compression
+ # REPEATED_ANT_TEST_TARGET: test-trie
# REPEATED_ANT_TEST_TARGET: test-system-keyspace-directory
REPEATED_ANT_TEST_TARGET: testsome
# The name of JUnit class to be run multiple times, for example:
@@ -434,6 +433,30 @@
requires:
- start_j11_utests_compression_repeat
- j8_build
+ - start_j8_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_j8_utests_trie
+ - j8_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j8_build
+ - start_j8_utests_trie_repeat:
+ type: approval
+ - j8_utests_trie_repeat:
+ requires:
+ - start_j8_utests_trie_repeat
+ - j8_build
+ - start_j11_utests_trie_repeat:
+ type: approval
+ - j11_utests_trie_repeat:
+ requires:
+ - start_j11_utests_trie_repeat
+ - j8_build
- start_j8_utests_stress:
type: approval
- j8_utests_stress:
@@ -882,6 +905,24 @@
requires:
- start_utests_compression
- j8_build
+ - start_utests_trie:
+ type: approval
+ - j8_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j8_build
+ - j8_utests_trie_repeat:
+ requires:
+ - start_utests_trie
+ - j8_build
+ - j11_utests_trie_repeat:
+ requires:
+ - start_utests_trie
+ - j8_build
- start_utests_stress:
type: approval
- j8_utests_stress:
@@ -1153,6 +1194,18 @@
requires:
- start_j11_jvm_dtests_vnode_repeat
- j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
+ - start_j11_simulator_dtests_repeat:
+ type: approval
+ - j11_simulator_dtests_repeat:
+ requires:
+ - start_j11_simulator_dtests_repeat
+ - j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@@ -1291,6 +1344,18 @@
requires:
- start_j11_utests_compression_repeat
- j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
+ - start_j11_utests_trie_repeat:
+ type: approval
+ - j11_utests_trie_repeat:
+ requires:
+ - start_j11_utests_trie_repeat
+ - j11_build
- start_j11_utests_stress:
type: approval
- j11_utests_stress:
@@ -1373,6 +1438,12 @@
- j11_jvm_dtests_vnode_repeat:
requires:
- j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
+ - j11_simulator_dtests_repeat:
+ requires:
+ - j11_build
- j11_cqlshlib_tests:
requires:
- j11_build
@@ -1484,6 +1555,16 @@
requires:
- start_utests_compression
- j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j11_utests_trie_repeat:
+ requires:
+ - start_utests_trie
+ - j11_build
- start_utests_stress:
type: approval
- j11_utests_stress:
@@ -1606,6 +1687,15 @@
- log_environment
- run_simulator_tests
+ j11_simulator_dtests:
+ <<: *j11_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_simulator_tests
+
j8_jvm_dtests:
<<: *j8_small_par_executor
steps:
@@ -1766,6 +1856,26 @@
- run_parallel_junit_tests:
target: testclasslist-compression
+ j8_utests_trie:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-trie
+
+ j11_utests_trie:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-trie
+
j8_utests_stress:
<<: *j8_seq_executor
steps:
@@ -2329,6 +2439,22 @@
- log_environment
- run_utests_compression_repeat
+ j8_utests_trie_repeat:
+ <<: *j8_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_trie_repeat
+
+ j11_utests_trie_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_trie_repeat
+
j8_utests_system_keyspace_directory_repeat:
<<: *j8_repeated_utest_executor
steps:
@@ -2417,6 +2543,14 @@
- log_environment
- run_simulator_dtests_repeat
+ j11_simulator_dtests_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_simulator_dtests_repeat
+
j8_jvm_upgrade_dtests_repeat:
<<: *j8_repeated_jvm_upgrade_dtest_executor
steps:
@@ -3053,6 +3187,14 @@
count: ${REPEATED_UTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ run_utests_trie_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-trie
+ tests: ${REPEATED_UTESTS}
+ count: ${REPEATED_UTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
run_utests_system_keyspace_directory_repeat:
steps:
- run_repeated_utests:
@@ -3155,7 +3297,7 @@
# Put manually specified tests and automatically detected tests together, removing duplicates
tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
-
+
# Prepare the JVM dtests vnodes argument, which is optional.
vnodes=<<parameters.vnodes>>
vnodes_args=""
@@ -3172,6 +3314,8 @@
testtag="compression"
elif [[ $target == "test-system-keyspace-directory" ]]; then
testtag="system_keyspace_directory"
+ elif [[ $target == "test-trie" ]]; then
+ testtag="trie"
fi
# Run each test class as many times as requested.
@@ -3192,6 +3336,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
@@ -3318,6 +3463,7 @@
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-trie" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "long-test" || \
diff --git a/.circleci/config_template.yml.PAID.patch b/.circleci/config_template.yml.PAID.patch
index 098ccd2..daaad2f 100644
--- a/.circleci/config_template.yml.PAID.patch
+++ b/.circleci/config_template.yml.PAID.patch
@@ -1,6 +1,6 @@
---- config-2_1.yml 2023-02-02 21:24:39.000000000 -0500
-+++ config-2_1.yml.MIDRES 2023-02-02 21:25:05.000000000 -0500
-@@ -157,14 +157,14 @@
+--- config_template.yml 2023-03-10 09:47:05.552165036 -0600
++++ config_template.yml.PAID 2023-03-10 09:51:21.174071576 -0600
+@@ -156,14 +156,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@ -19,7 +19,7 @@
j8_small_executor: &j8_small_executor
executor:
-@@ -172,29 +172,41 @@
+@@ -171,29 +171,41 @@
exec_resource_class: medium
parallelism: 1
@@ -32,7 +32,7 @@
+j8_very_large_par_executor: &j8_very_large_par_executor
+ executor:
+ name: java8-executor
-+ exec_resource_class: xlarge
++ exec_resource_class: large
+ parallelism: 100
+
j8_medium_par_executor: &j8_medium_par_executor
@@ -68,7 +68,7 @@
j11_small_executor: &j11_small_executor
executor:
-@@ -205,44 +217,56 @@
+@@ -204,44 +216,56 @@
j11_medium_par_executor: &j11_medium_par_executor
executor:
name: java11-executor
@@ -134,7 +134,7 @@
j8_separate_jobs: &j8_separate_jobs
jobs:
-@@ -1819,7 +1843,7 @@
+@@ -1929,7 +1953,7 @@
target: testclasslist-system-keyspace-directory
j8_dtests_vnode:
@@ -143,7 +143,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1833,7 +1857,7 @@
+@@ -1943,7 +1967,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j8_dtests_offheap:
@@ -152,7 +152,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1847,7 +1871,7 @@
+@@ -1957,7 +1981,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j11_dtests_vnode:
@@ -161,7 +161,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1862,7 +1886,7 @@
+@@ -1972,7 +1996,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j11_dtests_offheap:
@@ -170,7 +170,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1877,7 +1901,7 @@
+@@ -1987,7 +2011,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j8_dtests:
@@ -179,7 +179,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1891,7 +1915,7 @@
+@@ -2001,7 +2025,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j11_dtests:
@@ -188,7 +188,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1906,7 +1930,7 @@
+@@ -2016,7 +2040,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_upgrade_dtests:
@@ -197,7 +197,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1920,7 +1944,7 @@
+@@ -2030,7 +2054,7 @@
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
j8_cqlsh_dtests_py3_vnode:
@@ -206,7 +206,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1935,7 +1959,7 @@
+@@ -2045,7 +2069,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py3_offheap:
@@ -215,7 +215,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1950,7 +1974,7 @@
+@@ -2060,7 +2084,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38_vnode:
@@ -224,7 +224,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1968,7 +1992,7 @@
+@@ -2078,7 +2102,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py311_vnode:
@@ -233,7 +233,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -1986,7 +2010,7 @@
+@@ -2096,7 +2120,7 @@
python_version: '3.11'
j8_cqlsh_dtests_py38_offheap:
@@ -242,7 +242,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2004,7 +2028,7 @@
+@@ -2114,7 +2138,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py311_offheap:
@@ -251,7 +251,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2022,7 +2046,7 @@
+@@ -2132,7 +2156,7 @@
python_version: '3.11'
j8_cqlsh_dtests_py3:
@@ -260,7 +260,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2037,7 +2061,7 @@
+@@ -2147,7 +2171,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38:
@@ -269,7 +269,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2055,7 +2079,7 @@
+@@ -2165,7 +2189,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py311:
@@ -278,7 +278,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2073,7 +2097,7 @@
+@@ -2183,7 +2207,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py3_vnode:
@@ -287,7 +287,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2088,7 +2112,7 @@
+@@ -2198,7 +2222,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py3_offheap:
@@ -296,7 +296,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2103,7 +2127,7 @@
+@@ -2213,7 +2237,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38_vnode:
@@ -305,7 +305,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2121,7 +2145,7 @@
+@@ -2231,7 +2255,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_vnode:
@@ -314,7 +314,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2139,7 +2163,7 @@
+@@ -2249,7 +2273,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py38_offheap:
@@ -323,7 +323,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2157,7 +2181,7 @@
+@@ -2267,7 +2291,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_offheap:
@@ -332,7 +332,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2175,7 +2199,7 @@
+@@ -2285,7 +2309,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py3:
@@ -341,7 +341,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2190,7 +2214,7 @@
+@@ -2300,7 +2324,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38:
@@ -350,7 +350,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2208,7 +2232,7 @@
+@@ -2318,7 +2342,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311:
@@ -359,7 +359,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2226,7 +2250,7 @@
+@@ -2336,7 +2360,7 @@
python_version: '3.11'
j8_dtests_large_vnode:
@@ -368,7 +368,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2240,7 +2264,7 @@
+@@ -2350,7 +2374,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
@@ -377,7 +377,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2254,7 +2278,7 @@
+@@ -2364,7 +2388,7 @@
pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large_vnode:
@@ -386,7 +386,7 @@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2268,7 +2292,7 @@
+@@ -2378,7 +2402,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large:
diff --git a/.circleci/config_template_11_and_17.yml b/.circleci/config_template_11_and_17.yml
new file mode 100644
index 0000000..f9ad91c
--- /dev/null
+++ b/.circleci/config_template_11_and_17.yml
@@ -0,0 +1,3451 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: 2.1
+
+default_env_vars: &default_env_vars
+
+ # The values of some of these environment variables are meant to be frequently changed by developers.
+ # The generate.sh script contains a list of accepted environment variables that should contain some of
+ # these variables. Also, some variables are mentioned in the documentation, at least in
+ # .circleci/readme.md and in doc/source/development/testing.rst.
+ # If you modify these variables, or if you add new variables whose values are meant to be changed frequently,
+ # please remember to modify the generate.sh script and the documentation accordingly.
+
+ ANT_HOME: /usr/share/ant
+ LANG: en_US.UTF-8
+ KEEP_TEST_DIR: true
+ DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ PYTHONIOENCODING: utf-8
+ PYTHONUNBUFFERED: true
+ CASS_DRIVER_NO_EXTENSIONS: true
+ CASS_DRIVER_NO_CYTHON: true
+ #Skip all syncing to disk to avoid performance issues in flaky CI environments
+ CASSANDRA_SKIP_SYNC: true
+ DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ DTEST_BRANCH: trunk
+ CCM_MAX_HEAP_SIZE: 1024M
+ CCM_HEAP_NEWSIZE: 256M
+
+ # Whether the repeated test iterations should stop on the first failure by default.
+ REPEATED_TESTS_STOP_ON_FAILURE: false
+
+ # Comma-separated list of tests that should be included in the repeated run for regular unit tests,
+ # in addition to automatically detected new and modified tests. For example:
+ # REPEATED_UTESTS: org.apache.cassandra.cql3.ViewTest
+ # REPEATED_UTESTS: org.apache.cassandra.cql3.ViewTest#testCountersTable
+ # REPEATED_UTESTS: org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.cql3.functions.TimeFctsTest
+ REPEATED_UTESTS:
+ # The number of times that new, modified or manually specified unit tests should be run.
+ REPEATED_UTESTS_COUNT: 500
+
+ # Comma-separated list of tests that should be included in the repeated run for fqltool unit tests,
+ # in addition to automatically detected new and modified tests. For example:
+ # REPEATED_UTESTS_FQLTOOL: org.apache.cassandra.fqltool.FQLCompareTest
+ # REPEATED_UTESTS_FQLTOOL: org.apache.cassandra.fqltool.FQLCompareTest#endToEnd
+ # REPEATED_UTESTS_FQLTOOL: org.apache.cassandra.fqltool.FQLCompareTest,org.apache.cassandra.fqltool.FQLReplayTest
+ REPEATED_UTESTS_FQLTOOL:
+ # The number of times that new, modified or manually specified fqltool unit tests should be run.
+ REPEATED_UTESTS_FQLTOOL_COUNT: 500
+
+ # Comma-separated list of tests that should be included in the repeated run for long unit tests,
+ # in addition to automatically detected new and modified tests. For example:
+ # REPEATED_UTESTS_LONG: org.apache.cassandra.db.commitlog.CommitLogStressTest
+ # REPEATED_UTESTS_LONG: org.apache.cassandra.db.commitlog.CommitLogStressTest#testRandomSize
+ REPEATED_UTESTS_LONG:
+ # The number of times that new, modified or manually specified long unit tests should be run.
+ REPEATED_UTESTS_LONG_COUNT: 100
+
+ # Comma-separated list of tests that should be included in the repeated run for stress unit tests,
+ # in addition to automatically detected new and modified tests. For example:
+ # REPEATED_UTESTS_STRESS: org.apache.cassandra.stress.generate.DistributionGaussianTest
+ # REPEATED_UTESTS_STRESS: org.apache.cassandra.stress.generate.DistributionGaussianTest#simpleGaussian
+ REPEATED_UTESTS_STRESS:
+ # The number of times that new, modified or manually specified stress unit tests should be run.
+ REPEATED_UTESTS_STRESS_COUNT: 500
+
+ # Comma-separated list of tests that should be included in the repeated run for simulator dtests,
+ # in addition to automatically detected new and modified tests. For example:
+ # REPEATED_SIMULATOR_DTESTS: org.apache.cassandra.simulator.test.TrivialSimulationTest
+ # REPEATED_SIMULATOR_DTESTS: org.apache.cassandra.simulator.test.TrivialSimulationTest#trivialTest
+ REPEATED_SIMULATOR_DTESTS:
+ # The number of times that new, modified or manually specified simulator dtests should be run.
+ REPEATED_SIMULATOR_DTESTS_COUNT: 500
+
+ # Comma-separated list of tests that should be included in the repeated run for JVM dtests,
+ # in addition to automatically detected new and modified tests. For example:
+ # REPEATED_JVM_DTESTS: org.apache.cassandra.distributed.test.PagingTest
+ # REPEATED_JVM_DTESTS: org.apache.cassandra.distributed.test.PagingTest#testPaging
+ REPEATED_JVM_DTESTS:
+ # The number of times that new, modified or manually specified JVM dtests should be run.
+ REPEATED_JVM_DTESTS_COUNT: 500
+
+ # Comma-separated list of Python dtests that should be repeatedly run, for example:
+ # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py
+ # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke
+ # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke::test_create_index
+ # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py,consistency_test.py
+ REPEATED_DTESTS:
+ # The number of times that the manually specified Python dtests should be run.
+ REPEATED_DTESTS_COUNT: 500
+
+ # Comma-separated list of Python large dtests that should be repeatedly run, for example:
+ # REPEATED_LARGE_DTESTS: replace_address_test.py
+ # REPEATED_LARGE_DTESTS: replace_address_test.py::TestReplaceAddress
+ # REPEATED_LARGE_DTESTS: replace_address_test.py::TestReplaceAddress::test_replace_stopped_node
+ # REPEATED_LARGE_DTESTS: replace_address_test.py,materialized_views_test.py
+ REPEATED_LARGE_DTESTS:
+ # The number of times that the manually specified Python large dtests should be run.
+ REPEATED_LARGE_DTESTS_COUNT: 100
+
+ # The Ant test target to run, for example:
+ # REPEATED_ANT_TEST_TARGET: testsome
+ # REPEATED_ANT_TEST_TARGET: test-jvm-dtest-some
+ # REPEATED_ANT_TEST_TARGET: test-cdc
+ # REPEATED_ANT_TEST_TARGET: test-compression
+ # REPEATED_ANT_TEST_TARGET: test-trie
+ # REPEATED_ANT_TEST_TARGET: test-system-keyspace-directory
+ REPEATED_ANT_TEST_TARGET: testsome
+ # The name of JUnit class to be run multiple times, for example:
+ # REPEATED_ANT_TEST_CLASS: org.apache.cassandra.cql3.ViewTest
+ # REPEATED_ANT_TEST_CLASS: org.apache.cassandra.distributed.test.PagingTest
+ REPEATED_ANT_TEST_CLASS:
+ # The optional specific methods within REPEATED_ANT_TEST_CLASS to be run, for example:
+ # REPEATED_ANT_TEST_METHODS: testCompoundPartitionKey
+ # REPEATED_ANT_TEST_METHODS: testCompoundPartitionKey,testStaticTable
+ # Please note that some Ant targets will ignore the -Dtest.methods argument produced by this.
+ REPEATED_ANT_TEST_METHODS:
+ # Whether the test iteration should use vnodes for JVM dtests (-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16').
+ # This will only be applied as a default to JVM dtests that don't provide their own initial tokens or token count,
+ # in the same way that it's done for *_jvm_dtests_vnode jobs. Ant targets other than JVM dtests will ignore this.
+ REPEATED_ANT_TEST_VNODES: false
+ # The number of times that the repeated JUnit test should be run.
+ REPEATED_ANT_TEST_COUNT: 500
+
+j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 4
+
+j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
+j11_small_executor: &j11_small_executor
+ executor:
+ name: java11-executor
+ exec_resource_class: medium
+ parallelism: 1
+
+j11_medium_par_executor: &j11_medium_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
+j11_seq_executor: &j11_seq_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ exec_resource_class: medium
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+j17_par_executor: &j17_par_executor
+ executor:
+ name: java17-executor
+ #exec_resource_class: xlarge
+ parallelism: 4
+
+j17_small_par_executor: &j17_small_par_executor
+ executor:
+ name: java17-executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
+j17_small_executor: &j17_small_executor
+ executor:
+ name: java17-executor
+ #exec_resource_class: medium
+ parallelism: 1
+
+j17_medium_par_executor: &j17_medium_par_executor
+ executor:
+ name: java17-executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
+j17_seq_executor: &j17_seq_executor
+ executor:
+ name: java17-executor
+ #exec_resource_class: xlarge
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+j11_repeated_utest_executor: &j11_repeated_utest_executor
+ executor:
+ name: java11-executor
+ parallelism: 4
+
+j11_repeated_dtest_executor: &j11_repeated_dtest_executor
+ executor:
+ name: java11-executor
+ parallelism: 4
+
+j17_repeated_utest_executor: &j17_repeated_utest_executor
+ executor:
+ name: java17-executor
+ parallelism: 4
+
+j17_repeated_dtest_executor: &j17_repeated_dtest_executor
+ executor:
+ name: java17-executor
+ parallelism: 4
+
+j11_separate_jobs: &j11_separate_jobs
+ jobs:
+ - start_j11_build:
+ type: approval
+ - j11_build:
+ requires:
+ - start_j11_build
+ # Java 11 unit tests
+ - start_j11_unit_tests:
+ type: approval
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_build
+ - start_j11_unit_tests_repeat:
+ type: approval
+ - j11_unit_tests_repeat:
+ requires:
+ - start_j11_unit_tests_repeat
+ - j11_build
+ - start_j11_jvm_dtests:
+ type: approval
+ - j11_jvm_dtests:
+ requires:
+ - start_j11_jvm_dtests
+ - j11_build
+ - start_j11_jvm_dtests_vnode:
+ type: approval
+ - j11_jvm_dtests_vnode:
+ requires:
+ - start_j11_jvm_dtests_vnode
+ - j11_build
+ - start_j11_jvm_dtests_repeat:
+ type: approval
+ - j11_jvm_dtests_repeat:
+ requires:
+ - start_j11_jvm_dtests_repeat
+ - j11_build
+ - start_j11_jvm_dtests_vnode_repeat:
+ type: approval
+ - j11_jvm_dtests_vnode_repeat:
+ requires:
+ - start_j11_jvm_dtests_vnode_repeat
+ - j11_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j11_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j11_build
+ - start_j17_jvm_dtests_repeat:
+ type: approval
+ - j17_jvm_dtests_repeat:
+ requires:
+ - start_j17_jvm_dtests_repeat
+ - j11_build
+ - start_j17_jvm_dtests_vnode_repeat:
+ type: approval
+ - j17_jvm_dtests_vnode_repeat:
+ requires:
+ - start_j17_jvm_dtests_vnode_repeat
+ - j11_build
+ - start_j11_simulator_dtests:
+ type: approval
+ - j11_simulator_dtests:
+ requires:
+ - start_j11_simulator_dtests
+ - j11_build
+ - start_j11_simulator_dtests_repeat:
+ type: approval
+ - j11_simulator_dtests_repeat:
+ requires:
+ - start_j11_simulator_dtests_repeat
+ - j11_build
+ - start_j11_cqlshlib_tests:
+ type: approval
+ - j11_cqlshlib_tests:
+ requires:
+ - start_j11_cqlshlib_tests
+ - j11_build
+ - start_j11_cqlshlib_cython_tests:
+ type: approval
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - start_j11_cqlshlib_cython_tests
+ - j11_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j11_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j11_build
+ # Java 17 unit tests
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j11_build
+ - start_j17_unit_tests_repeat:
+ type: approval
+ - j17_unit_tests_repeat:
+ requires:
+ - start_j17_unit_tests_repeat
+ - j11_build
+ # specialized unit tests (all run on request)
+ - start_j11_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_j11_utests_long
+ - j11_build
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j11_build
+ - start_j11_utests_long_repeat:
+ type: approval
+ - j11_utests_long_repeat:
+ requires:
+ - start_j11_utests_long_repeat
+ - j11_build
+ - start_j17_utests_long_repeat:
+ type: approval
+ - j17_utests_long_repeat:
+ requires:
+ - start_j17_utests_long_repeat
+ - j11_build
+ - start_j11_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_j11_utests_cdc
+ - j11_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j11_build
+ - start_j11_utests_cdc_repeat:
+ type: approval
+ - j11_utests_cdc_repeat:
+ requires:
+ - start_j11_utests_cdc_repeat
+ - j11_build
+ - start_j17_utests_cdc_repeat:
+ type: approval
+ - j17_utests_cdc_repeat:
+ requires:
+ - start_j17_utests_cdc_repeat
+ - j11_build
+ - start_j11_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_j11_utests_compression
+ - j11_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j11_build
+ - start_j11_utests_compression_repeat:
+ type: approval
+ - j11_utests_compression_repeat:
+ requires:
+ - start_j11_utests_compression_repeat
+ - j11_build
+ - start_j17_utests_compression_repeat:
+ type: approval
+ - j17_utests_compression_repeat:
+ requires:
+ - start_j17_utests_compression_repeat
+ - j11_build
+ - start_j11_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_j11_utests_trie
+ - j11_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j11_build
+ - start_j11_utests_trie_repeat:
+ type: approval
+ - j11_utests_trie_repeat:
+ requires:
+ - start_j11_utests_trie_repeat
+ - j11_build
+ - start_j17_utests_trie_repeat:
+ type: approval
+ - j17_utests_trie_repeat:
+ requires:
+ - start_j17_utests_trie_repeat
+ - j11_build
+ - start_j11_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_j11_utests_stress
+ - j11_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j11_build
+ - start_j11_utests_stress_repeat:
+ type: approval
+ - j11_utests_stress_repeat:
+ requires:
+ - start_j11_utests_stress_repeat
+ - j11_build
+ - start_j17_utests_stress_repeat:
+ type: approval
+ - j17_utests_stress_repeat:
+ requires:
+ - start_j17_utests_stress_repeat
+ - j11_build
+ - start_j11_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_j11_utests_fqltool
+ - j11_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j11_build
+ - start_j11_utests_fqltool_repeat:
+ type: approval
+ - j11_utests_fqltool_repeat:
+ requires:
+ - start_j11_utests_fqltool_repeat
+ - j11_build
+ - start_j17_utests_fqltool_repeat:
+ type: approval
+ - j17_utests_fqltool_repeat:
+ requires:
+ - start_j17_utests_fqltool_repeat
+ - j11_build
+ - start_j11_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - start_j11_utests_system_keyspace_directory
+ - j11_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j11_build
+ - start_j11_utests_system_keyspace_directory_repeat:
+ type: approval
+ - j11_utests_system_keyspace_directory_repeat:
+ requires:
+ - start_j11_utests_system_keyspace_directory_repeat
+ - j11_build
+ - start_j17_utests_system_keyspace_directory_repeat:
+ type: approval
+ - j17_utests_system_keyspace_directory_repeat:
+ requires:
+ - start_j17_utests_system_keyspace_directory_repeat
+ - j11_build
+ # Python DTests
+ - start_j11_dtests:
+ type: approval
+ - j11_dtests:
+ requires:
+ - start_j11_dtests
+ - j11_build
+ - start_j11_dtests_vnode:
+ type: approval
+ - j11_dtests_vnode:
+ requires:
+ - start_j11_dtests_vnode
+ - j11_build
+ # Java 11 off-heap dtests
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - start_j11_dtests_offheap_repeat:
+ type: approval
+ - j11_dtests_offheap_repeat:
+ requires:
+ - start_j11_dtests_offheap_repeat
+ - j11_build
+ # Java 17 dtests
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j11_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j11_build
+ # Java 17 off-heap dtests
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - start_j17_dtests_offheap_repeat:
+ type: approval
+ - j17_dtests_offheap_repeat:
+ requires:
+ - start_j17_dtests_offheap_repeat
+ - j11_build
+ # Python large DTests
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j11_dtests_large_repeat:
+ type: approval
+ - j11_dtests_large_repeat:
+ requires:
+ - start_j11_dtests_large_repeat
+ - j11_build
+ - start_j11_dtests_large_vnode:
+ type: approval
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large_vnode
+ - j11_build
+ - start_j11_dtests_large_vnode_repeat:
+ type: approval
+ - j11_dtests_large_vnode_repeat:
+ requires:
+ - start_j11_dtests_large_vnode_repeat
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - start_j17_dtests_large_repeat:
+ type: approval
+ - j17_dtests_large_repeat:
+ requires:
+ - start_j17_dtests_large_repeat
+ - j11_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j11_build
+ - start_j17_dtests_large_vnode_repeat:
+ type: approval
+ - j17_dtests_large_vnode_repeat:
+ requires:
+ - start_j17_dtests_large_vnode_repeat
+ - j11_build
+ # Java 11 cqlsh dtests
+ - start_j11_cqlsh_tests:
+ type: approval
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ # Java 11 cqlsh offheap dtests offheap
+ - start_j11_cqlsh_tests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_tests_offheap
+ - j11_build
+ # Java 17 cqlsh dtests
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j11_build
+ # Java 17 cqlsh dtests off-heap
+ - start_j17_cqlsh_tests_offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh_tests_offheap
+ - j11_build
+ # Java 11 repeated utest
+ - start_j11_repeated_ant_test:
+ type: approval
+ - j11_repeated_ant_test:
+ requires:
+ - start_j11_repeated_ant_test
+ - j11_build
+ # Java 17 repeated utest
+ - start_j17_repeated_ant_test:
+ type: approval
+ - j17_repeated_ant_test:
+ requires:
+ - start_j17_repeated_ant_test
+ - j11_build
+ # Java 11 repeated dtest
+ - start_j11_dtests_repeat:
+ type: approval
+ - j11_dtests_repeat:
+ requires:
+ - start_j11_dtests_repeat
+ - j11_build
+ - start_j11_dtests_vnode_repeat:
+ type: approval
+ - j11_dtests_vnode_repeat:
+ requires:
+ - start_j11_dtests_vnode_repeat
+ - j11_build
+ # Java 17 repeated dtest
+ - start_j17_dtests_repeat:
+ type: approval
+ - j17_dtests_repeat:
+ requires:
+ - start_j17_dtests_repeat
+ - j11_build
+ - start_j17_dtests_vnode_repeat:
+ type: approval
+ - j17_dtests_vnode_repeat:
+ requires:
+ - start_j17_dtests_vnode_repeat
+ - j11_build
+
+j11_pre-commit_jobs: &j11_pre-commit_jobs
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j11_build:
+ requires:
+ - start_pre-commit_tests
+ # Java 11 unit tests
+ - j11_unit_tests:
+ requires:
+ - j11_build
+ - j11_unit_tests_repeat:
+ requires:
+ - j11_build
+ - j11_simulator_dtests:
+ requires:
+ - j11_build
+ - j11_simulator_dtests_repeat:
+ requires:
+ - j11_build
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_repeat:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_vnode_repeat:
+ requires:
+ - j11_build
+ - j17_jvm_dtests:
+ requires:
+ - j11_build
+ - j17_jvm_dtests_repeat:
+ requires:
+ - j11_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j11_build
+ - j17_jvm_dtests_vnode_repeat:
+ requires:
+ - j11_build
+ - j11_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j11_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j11_build
+ # Java 17 unit tests
+ - j17_unit_tests:
+ requires:
+ - j11_build
+ - j17_unit_tests_repeat:
+ requires:
+ - j11_build
+ # specialized unit tests (all run on request)
+ - start_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j11_utests_long_repeat:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j17_utests_long_repeat:
+ requires:
+ - start_utests_long
+ - j11_build
+ - start_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j11_utests_cdc_repeat:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j17_utests_cdc_repeat:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - start_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j11_utests_compression_repeat:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j17_utests_compression_repeat:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - start_utests_trie:
+ type: approval
+ - j11_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j11_utests_trie_repeat:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - j17_utests_trie_repeat:
+ requires:
+ - start_utests_trie
+ - j11_build
+ - start_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j11_utests_stress_repeat:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j17_utests_stress_repeat:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - start_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j11_utests_fqltool_repeat:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j17_utests_fqltool_repeat:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - j11_build
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
+ - j11_utests_system_keyspace_directory_repeat:
+ requires:
+ - j11_build
+ - j17_utests_system_keyspace_directory_repeat:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
+ # Python DTests
+ - j11_dtests:
+ requires:
+ - j11_build
+ - j11_dtests_repeat:
+ requires:
+ - j11_build
+ - j11_dtests_vnode:
+ requires:
+ - j11_build
+ - j11_dtests_vnode_repeat:
+ requires:
+ - j11_build
+ - start_j11_dtests_offheap:
+ type: approval
+ - j11_dtests_offheap:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ - j11_dtests_offheap_repeat:
+ requires:
+ - start_j11_dtests_offheap
+ - j11_build
+ # Java 17 dtests
+ - j17_dtests:
+ requires:
+ - j11_build
+ - j17_dtests_repeat:
+ requires:
+ - j11_build
+ - j17_dtests_vnode:
+ requires:
+ - j11_build
+ - j17_dtests_vnode_repeat:
+ requires:
+ - j11_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ - j17_dtests_offheap_repeat:
+ requires:
+ - start_j17_dtests_offheap
+ - j11_build
+ # Large Python DTests
+ - start_j11_dtests_large:
+ type: approval
+ - j11_dtests_large:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - j11_dtests_large_repeat:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - j11_dtests_large_vnode:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - j11_dtests_large_vnode_repeat:
+ requires:
+ - start_j11_dtests_large
+ - j11_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j17_dtests_large_repeat:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ - j17_dtests_large_vnode_repeat:
+ requires:
+ - start_j17_dtests_large
+ - j11_build
+ # Java 11 cqlsh dtests
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ # Java 11 cqlsh dtests offheap
+ - start_j11_cqlsh_dtests_offheap:
+ type: approval
+ - j11_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ - j11_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j11_cqlsh_dtests_offheap
+ - j11_build
+ # Java 17 cqlsh dtests
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j11_build
+ # Java 17 cqlsh dtests off-heap
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j11_build
+
+j17_separate_jobs: &j17_separate_jobs
+ jobs:
+ - start_j17_build:
+ type: approval
+ - j17_build:
+ requires:
+ - start_j17_build
+ # Java 17 unit tests
+ - start_j17_unit_tests:
+ type: approval
+ - j17_unit_tests:
+ requires:
+ - start_j17_unit_tests
+ - j17_build
+ - start_j17_unit_tests_repeat:
+ type: approval
+ - j17_unit_tests_repeat:
+ requires:
+ - start_j17_unit_tests_repeat
+ - j17_build
+ - start_j17_jvm_dtests:
+ type: approval
+ - j17_jvm_dtests:
+ requires:
+ - start_j17_jvm_dtests
+ - j17_build
+ - start_j17_jvm_dtests_vnode:
+ type: approval
+ - j17_jvm_dtests_vnode:
+ requires:
+ - start_j17_jvm_dtests_vnode
+ - j17_build
+ - start_j17_jvm_dtests_repeat:
+ type: approval
+ - j17_jvm_dtests_repeat:
+ requires:
+ - start_j17_jvm_dtests_repeat
+ - j17_build
+ - start_j17_jvm_dtests_vnode_repeat:
+ type: approval
+ - j17_jvm_dtests_vnode_repeat:
+ requires:
+ - start_j17_jvm_dtests_vnode_repeat
+ - j17_build
+ - start_j17_cqlshlib_tests:
+ type: approval
+ - j17_cqlshlib_tests:
+ requires:
+ - start_j17_cqlshlib_tests
+ - j17_build
+ - start_j17_cqlshlib_cython_tests:
+ type: approval
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - start_j17_cqlshlib_cython_tests
+ - j17_build
+ # Java 17 dtests
+ - start_j17_dtests:
+ type: approval
+ - j17_dtests:
+ requires:
+ - start_j17_dtests
+ - j17_build
+ - start_j17_dtests_vnode:
+ type: approval
+ - j17_dtests_vnode:
+ requires:
+ - start_j17_dtests_vnode
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_offheap_repeat:
+ type: approval
+ - j17_dtests_offheap_repeat:
+ requires:
+ - start_j17_dtests_offheap_repeat
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - start_j17_dtests_large_repeat:
+ type: approval
+ - j17_dtests_large_repeat:
+ requires:
+ - start_j17_dtests_large_repeat
+ - j17_build
+ - start_j17_dtests_large_vnode:
+ type: approval
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large_vnode
+ - j17_build
+ - start_j17_dtests_large_vnode_repeat:
+ type: approval
+ - j17_dtests_large_vnode_repeat:
+ requires:
+ - start_j17_dtests_large_vnode_repeat
+ - j17_build
+ - start_j17_cqlsh_tests:
+ type: approval
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - start_j17_cqlsh_tests
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ # specialized unit tests (all run on request)
+ - start_j17_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_j17_utests_long
+ - j17_build
+ - start_j17_utests_long_repeat:
+ type: approval
+ - j17_utests_long_repeat:
+ requires:
+ - start_j17_utests_long_repeat
+ - j17_build
+ - start_j17_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_j17_utests_cdc
+ - j17_build
+ - start_j17_utests_cdc_repeat:
+ type: approval
+ - j17_utests_cdc_repeat:
+ requires:
+ - start_j17_utests_cdc_repeat
+ - j17_build
+ - start_j17_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_j17_utests_compression
+ - j17_build
+ - start_j17_utests_compression_repeat:
+ type: approval
+ - j17_utests_compression_repeat:
+ requires:
+ - start_j17_utests_compression_repeat
+ - j17_build
+ - start_j17_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_j17_utests_trie
+ - j17_build
+ - start_j17_utests_trie_repeat:
+ type: approval
+ - j17_utests_trie_repeat:
+ requires:
+ - start_j17_utests_trie_repeat
+ - j17_build
+ - start_j17_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_j17_utests_stress
+ - j17_build
+ - start_j17_utests_stress_repeat:
+ type: approval
+ - j17_utests_stress_repeat:
+ requires:
+ - start_j17_utests_stress_repeat
+ - j17_build
+ - start_j17_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_j17_utests_fqltool
+ - j17_build
+ - start_j17_utests_fqltool_repeat:
+ type: approval
+ - j17_utests_fqltool_repeat:
+ requires:
+ - start_j17_utests_fqltool_repeat
+ - j17_build
+ - start_j17_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_j17_utests_system_keyspace_directory
+ - j17_build
+ - start_j17_utests_system_keyspace_directory_repeat:
+ type: approval
+ - j17_utests_system_keyspace_directory_repeat:
+ requires:
+ - start_j17_utests_system_keyspace_directory_repeat
+ - j17_build
+ # Java 17 repeated utest
+ - start_j17_repeated_ant_test:
+ type: approval
+ - j17_repeated_ant_test:
+ requires:
+ - start_j17_repeated_ant_test
+ - j17_build
+ # Java 17 repeated dtest
+ - start_j17_dtests_repeat:
+ type: approval
+ - j17_dtests_repeat:
+ requires:
+ - start_j17_dtests_repeat
+ - j17_build
+ - start_j17_dtests_vnode_repeat:
+ type: approval
+ - j17_dtests_vnode_repeat:
+ requires:
+ - start_j17_dtests_vnode_repeat
+ - j17_build
+
+j17_pre-commit_jobs: &j17_pre-commit_jobs
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j17_build:
+ requires:
+ - start_pre-commit_tests
+ - j17_unit_tests:
+ requires:
+ - j17_build
+ - j17_unit_tests_repeat:
+ requires:
+ - j17_build
+ - j17_jvm_dtests:
+ requires:
+ - j17_build
+ - j17_jvm_dtests_repeat:
+ requires:
+ - j17_build
+ - j17_jvm_dtests_vnode:
+ requires:
+ - j17_build
+ - j17_jvm_dtests_vnode_repeat:
+ requires:
+ - j17_build
+ - j17_cqlshlib_tests:
+ requires:
+ - j17_build
+ - j17_cqlshlib_cython_tests:
+ requires:
+ - j17_build
+ - j17_dtests:
+ requires:
+ - j17_build
+ - j17_dtests_repeat:
+ requires:
+ - j17_build
+ - j17_dtests_vnode:
+ requires:
+ - j17_build
+ - j17_dtests_vnode_repeat:
+ requires:
+ - j17_build
+ - start_j17_dtests_offheap:
+ type: approval
+ - j17_dtests_offheap:
+ requires:
+ - start_j17_dtests_offheap
+ - j17_build
+ - start_j17_dtests_offheap_repeat:
+ type: approval
+ - j17_dtests_offheap_repeat:
+ requires:
+ - start_j17_dtests_offheap_repeat
+ - j17_build
+ - start_j17_dtests_large:
+ type: approval
+ - j17_dtests_large:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_dtests_large_repeat:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_dtests_large_vnode:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_dtests_large_vnode_repeat:
+ requires:
+ - start_j17_dtests_large
+ - j17_build
+ - j17_cqlsh_dtests_py3:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py3_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py38_vnode:
+ requires:
+ - j17_build
+ - j17_cqlsh_dtests_py311_vnode:
+ requires:
+ - j17_build
+ - start_j17_cqlsh-dtests-offheap:
+ type: approval
+ - j17_cqlsh_dtests_py3_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py38_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ - j17_cqlsh_dtests_py311_offheap:
+ requires:
+ - start_j17_cqlsh-dtests-offheap
+ - j17_build
+ # specialized unit tests (all run on request)
+ - start_utests_long:
+ type: approval
+ - j17_utests_long:
+ requires:
+ - start_utests_long
+ - j17_build
+ - j17_utests_long_repeat:
+ requires:
+ - start_utests_long
+ - j17_build
+ - start_utests_cdc:
+ type: approval
+ - j17_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j17_build
+ - j17_utests_cdc_repeat:
+ requires:
+ - start_utests_cdc
+ - j17_build
+ - start_utests_compression:
+ type: approval
+ - j17_utests_compression:
+ requires:
+ - start_utests_compression
+ - j17_build
+ - j17_utests_compression_repeat:
+ requires:
+ - start_utests_compression
+ - j17_build
+ - start_utests_trie:
+ type: approval
+ - j17_utests_trie:
+ requires:
+ - start_utests_trie
+ - j17_build
+ - j17_utests_trie_repeat:
+ requires:
+ - start_utests_trie
+ - j17_build
+ - start_utests_stress:
+ type: approval
+ - j17_utests_stress:
+ requires:
+ - start_utests_stress
+ - j17_build
+ - j17_utests_stress_repeat:
+ requires:
+ - start_utests_stress
+ - j17_build
+ - start_utests_fqltool:
+ type: approval
+ - j17_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j17_build
+ - j17_utests_fqltool_repeat:
+ requires:
+ - start_utests_fqltool
+ - j17_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j17_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j17_build
+ - j17_utests_system_keyspace_directory_repeat:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j17_build
+
+workflows:
+ version: 2
+ java11_separate_tests: *j11_separate_jobs
+ java11_pre-commit_tests: *j11_pre-commit_jobs
+ java17_separate_tests: *j17_separate_jobs
+ java17_pre-commit_tests: *j17_pre-commit_jobs
+
+executors:
+ java11-executor:
+ parameters:
+ exec_resource_class:
+ type: string
+ default: medium
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: << parameters.exec_resource_class >>
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ environment:
+ <<: *default_env_vars
+ JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+
+ java17-executor:
+ parameters:
+ exec_resource_class:
+ type: string
+ default: medium
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: << parameters.exec_resource_class >>
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ environment:
+ <<: *default_env_vars
+ JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+ JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+
+build_common: &build_common
+ parallelism: 1 # This job doesn't benefit from parallelism
+ steps:
+ - log_environment
+ - clone_cassandra
+ - build_cassandra
+ - run_eclipse_warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+
+jobs:
+ j11_build:
+ executor: java11-executor
+ <<: *build_common
+
+ j17_build:
+ executor: java17-executor
+ <<: *build_common
+
+ j11_dtest_jars_build:
+ executor: java11-executor
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - build_cassandra_dtest_jars
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
+
+ j11_unit_tests:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests
+
+ j11_simulator_dtests:
+ <<: *j11_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_simulator_tests
+
+ j11_jvm_dtests:
+ <<: *j11_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+
+ j11_jvm_dtests_vnode:
+ <<: *j11_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+ arguments: "-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+
+ j17_jvm_dtests:
+ <<: *j17_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+
+ j17_jvm_dtests_vnode:
+ <<: *j17_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+ arguments: "-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+
+ j17_unit_tests:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests
+
+ j11_cqlshlib_tests:
+ <<: *j11_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_cqlshlib_tests
+
+ j11_cqlshlib_cython_tests:
+ <<: *j11_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_cqlshlib_cython_tests
+
+ j17_cqlshlib_tests:
+ <<: *j17_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_cqlshlib_tests
+
+ j17_cqlshlib_cython_tests:
+ <<: *j17_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_cqlshlib_cython_tests
+
+ j11_utests_long:
+ <<: *j11_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: long-test
+
+ j17_utests_long:
+ <<: *j17_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: long-test
+
+ j11_utests_cdc:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-cdc
+
+ j17_utests_cdc:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-cdc
+
+ j11_utests_compression:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-compression
+
+ j17_utests_compression:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-compression
+
+ j11_utests_trie:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-trie
+
+ j17_utests_trie:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-trie
+
+ j11_utests_stress:
+ <<: *j11_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: stress-test
+
+ j17_utests_stress:
+ <<: *j17_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: stress-test
+
+ j11_utests_fqltool:
+ <<: *j11_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: fqltool-test
+
+ j17_utests_fqltool:
+ <<: *j17_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: fqltool-test
+
+ j11_utests_system_keyspace_directory:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-system-keyspace-directory
+
+ j17_utests_system_keyspace_directory:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-system-keyspace-directory
+
+ j11_dtests_vnode:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j11_dtests_offheap:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+
+ j17_dtests_vnode:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j17_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j17_dtests_offheap:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j17_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+
+ j11_dtests:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j17_dtests:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j17_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j11_cqlsh_dtests_py3_vnode:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py3_offheap:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py38_vnode:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j11_cqlsh_dtests_py311_vnode:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.11'
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.11'
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11'
+ python_version: '3.11'
+
+ j11_cqlsh_dtests_py38_offheap:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j11_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j11_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j11_cqlsh_dtests_py311_offheap:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.11'
+ - create_dtest_containers:
+ file_tag: j11_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.11'
+ - run_dtests:
+ file_tag: j11_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11'
+ python_version: '3.11'
+
+ j11_cqlsh_dtests_py3:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py38:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j11_cqlsh_dtests_py311:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.11'
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.11'
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11'
+ python_version: '3.11'
+
+ j17_cqlsh_dtests_py3_vnode:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j17_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j17_cqlsh_dtests_py3_offheap:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j17_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j17_cqlsh_dtests_py38_vnode:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j17_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j17_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j17_cqlsh_dtests_py311_vnode:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.11'
+ - create_dtest_containers:
+ file_tag: j17_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.11'
+ - run_dtests:
+ file_tag: j17_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11'
+ python_version: '3.11'
+
+ j17_cqlsh_dtests_py38_offheap:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j17_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j17_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j17_cqlsh_dtests_py311_offheap:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.11'
+ - create_dtest_containers:
+ file_tag: j17_dtests_offheap
+ run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.11'
+ - run_dtests:
+ file_tag: j17_dtests_offheap
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11'
+ python_version: '3.11'
+
+ j17_cqlsh_dtests_py3:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ - run_dtests:
+ file_tag: j17_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j17_cqlsh_dtests_py38:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.8'
+ - create_dtest_containers:
+ file_tag: j17_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.8'
+ - run_dtests:
+ file_tag: j17_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
+ python_version: '3.8'
+
+ j17_cqlsh_dtests_py311:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv:
+ python_version: '3.11'
+ - create_dtest_containers:
+ file_tag: j17_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
+ python_version: '3.11'
+ - run_dtests:
+ file_tag: j17_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11'
+ python_version: '3.11'
+
+ j11_dtests_large_vnode:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_large_with_vnodes
+ run_dtests_extra_args: '--use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests'
+ - run_dtests:
+ file_tag: j11_large_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_dtests_large:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_large_without_vnodes
+ run_dtests_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+ - run_dtests:
+ file_tag: j11_large_without_vnodes
+ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j17_dtests_large_vnode:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_large_with_vnodes
+ run_dtests_extra_args: '--use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests'
+ - run_dtests:
+ file_tag: j17_large_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j17_dtests_large:
+ <<: *j17_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j17_large_without_vnodes
+ run_dtests_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+ - run_dtests:
+ file_tag: j17_large_without_vnodes
+ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_unit_tests_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_unit_tests_repeat
+
+ j17_unit_tests_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_unit_tests_repeat
+
+ j11_utests_cdc_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_cdc_repeat
+
+ j17_utests_cdc_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_cdc_repeat
+
+ j11_utests_compression_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_compression_repeat
+
+ j17_utests_compression_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_compression_repeat
+
+ j11_utests_trie_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_trie_repeat
+
+ j17_utests_trie_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_trie_repeat
+
+ j11_utests_system_keyspace_directory_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_system_keyspace_directory_repeat
+
+ j17_utests_system_keyspace_directory_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_system_keyspace_directory_repeat
+
+ j11_utests_fqltool_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_fqltool_repeat
+
+ j17_utests_fqltool_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_fqltool_repeat
+
+ j11_utests_long_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_long_repeat
+
+ j17_utests_long_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_long_repeat
+
+ j11_utests_stress_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_stress_repeat
+
+ j17_utests_stress_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_stress_repeat
+
+ j11_jvm_dtests_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_jvm_dtests_repeat
+
+ j11_jvm_dtests_vnode_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_jvm_dtests_vnode_repeat
+
+ j11_simulator_dtests_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_simulator_dtests_repeat
+
+ j17_jvm_dtests_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_jvm_dtests_repeat
+
+ j17_jvm_dtests_vnode_repeat:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_jvm_dtests_vnode_repeat
+
+ j11_repeated_ant_test:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_repeated_utest:
+ target: ${REPEATED_ANT_TEST_TARGET}
+ class: ${REPEATED_ANT_TEST_CLASS}
+ methods: ${REPEATED_ANT_TEST_METHODS}
+ vnodes: ${REPEATED_ANT_TEST_VNODES}
+ count: ${REPEATED_ANT_TEST_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ j17_repeated_ant_test:
+ <<: *j17_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_repeated_utest:
+ target: ${REPEATED_ANT_TEST_TARGET}
+ class: ${REPEATED_ANT_TEST_CLASS}
+ methods: ${REPEATED_ANT_TEST_METHODS}
+ vnodes: ${REPEATED_ANT_TEST_VNODES}
+ count: ${REPEATED_ANT_TEST_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ j11_dtests_repeat:
+ <<: *j11_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTESTS}
+ vnodes: "false"
+ upgrade: "false"
+ count: ${REPEATED_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ j11_dtests_vnode_repeat:
+ <<: *j11_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTESTS}
+ vnodes: "true"
+ upgrade: "false"
+ count: ${REPEATED_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ j11_dtests_offheap_repeat:
+ <<: *j11_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTESTS}
+ vnodes: "true"
+ upgrade: "false"
+ count: ${REPEATED_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ extra_dtest_args: "--use-off-heap-memtables --skip-resource-intensive-tests"
+
+ j11_dtests_large_repeat:
+ <<: *j11_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_LARGE_DTESTS}
+ vnodes: "false"
+ upgrade: "false"
+ count: ${REPEATED_LARGE_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
+ j11_dtests_large_vnode_repeat:
+ <<: *j11_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_LARGE_DTESTS}
+ vnodes: "true"
+ upgrade: "false"
+ count: ${REPEATED_LARGE_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
+ j17_dtests_repeat:
+ <<: *j17_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTESTS}
+ vnodes: "false"
+ upgrade: "false"
+ count: ${REPEATED_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ j17_dtests_vnode_repeat:
+ <<: *j17_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTESTS}
+ vnodes: "true"
+ upgrade: "false"
+ count: ${REPEATED_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ j17_dtests_offheap_repeat:
+ <<: *j17_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTESTS}
+ vnodes: "true"
+ upgrade: "false"
+ count: ${REPEATED_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ extra_dtest_args: "--use-off-heap-memtables --skip-resource-intensive-tests"
+
+ j17_dtests_large_repeat:
+ <<: *j17_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_LARGE_DTESTS}
+ vnodes: "false"
+ upgrade: "false"
+ count: ${REPEATED_LARGE_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
+ j17_dtests_large_vnode_repeat:
+ <<: *j17_repeated_dtest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_LARGE_DTESTS}
+ vnodes: "true"
+ upgrade: "false"
+ count: ${REPEATED_LARGE_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+ extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
+commands:
+ log_environment:
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+
+ clone_cassandra:
+ steps:
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+
+ clone_dtest:
+ steps:
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+
+ build_cassandra:
+ steps:
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+
+ build_cassandra_dtest_jars:
+ steps:
+ - run:
+ name: Build Cassandra DTest jars
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ mkdir ~/dtest_jars
+ git remote add apache https://github.com/apache/cassandra.git
+ for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do
+ # check out the correct cassandra version:
+ git remote set-branches --add apache '$branch'
+ git fetch --depth 1 apache $branch
+ git checkout $branch
+ git clean -fd
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ cp build/dtest*.jar ~/dtest_jars
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+ # and build the dtest-jar for the branch under test
+ ${ANT_HOME}/bin/ant realclean
+ git checkout origin/$CIRCLE_BRANCH
+ git clean -fd
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ cp build/dtest*.jar ~/dtest_jars
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ ls -l ~/dtest_jars
+ no_output_timeout: 15m
+
+ run_eclipse_warnings:
+ steps:
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+
+ create_junit_containers:
+ parameters:
+ classlistprefix:
+ type: string
+ default: unit
+ extra_filters:
+ type: string
+ default: ""
+ steps:
+ - run:
+ name: Determine <<parameters.classlistprefix>> Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/<<parameters.classlistprefix>>/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/<<parameters.classlistprefix>>/;;g" | grep "Test\.java$" <<parameters.extra_filters>> > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+
+ no_output_timeout: 15m
+
+ run_simulator_tests:
+ parameters:
+ no_output_timeout:
+ type: string
+ default: 30m
+ steps:
+ - run:
+ name: Run Simulator Tests
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant test-simulator-dtest -Dno-build-test=true
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+
+ run_junit_tests:
+ parameters:
+ target:
+ type: string
+ no_output_timeout:
+ type: string
+ default: 15m
+ steps:
+ - run:
+ name: Run Unit Tests (<<parameters.target>>)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant <<parameters.target>> -Dno-build-test=true
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+
+ run_cqlshlib_tests:
+ parameters:
+ no_output_timeout:
+ type: string
+ default: 15m
+ steps:
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+
+ run_cqlshlib_cython_tests:
+ parameters:
+ no_output_timeout:
+ type: string
+ default: 15m
+ steps:
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ export cython="yes"
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/
+ ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+
+ run_parallel_junit_tests:
+ parameters:
+ target:
+ type: string
+ default: testclasslist
+ no_output_timeout:
+ type: string
+ default: 15m
+ classlistprefix:
+ type: string
+ default: unit
+ arguments:
+ type: string
+ default: " "
+ steps:
+ - run:
+ name: Run Unit Tests (<<parameters.target>>)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.<<parameters.classlistprefix>>.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant <<parameters.target>> <<parameters.arguments>> -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>> -Dno-build-test=true
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+
+ create_venv:
+ parameters:
+ python_version:
+ type: enum
+ default: "3.6"
+ enum: ["3.6", "3.7", "3.8", "3.11"]
+ steps:
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env<<parameters.python_version>>/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+
+ create_dtest_containers:
+ parameters:
+ file_tag:
+ type: string
+ run_dtests_extra_args:
+ type: string
+ default: ''
+ extra_env_args:
+ type: string
+ default: ''
+ tests_filter_pattern:
+ type: string
+ default: ''
+ python_version:
+ type: enum
+ default: "3.6"
+ enum: ["3.6", "3.7", "3.8", "3.11"]
+ steps:
+ - run:
+ name: Determine Tests to Run (<<parameters.file_tag>>)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env<<parameters.python_version>>/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n '<<parameters.extra_env_args>>' ]; then
+ export <<parameters.extra_env_args>>
+ fi
+
+ echo "***Collected DTests (<<parameters.file_tag>>)***"
+ set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
+ if [ -z '<<parameters.tests_filter_pattern>>' ]; then
+ mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
+ else
+ grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
+ cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+
+ run_dtests:
+ parameters:
+ file_tag:
+ type: string
+ pytest_extra_args:
+ type: string
+ default: ''
+ extra_env_args:
+ type: string
+ default: ''
+ python_version:
+ type: enum
+ default: "3.6"
+ enum: ["3.6", "3.7", "3.8", "3.11"]
+ steps:
+ - run:
+ name: Run dtests (<<parameters.file_tag>>)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt"
+ cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+
+ source ~/env<<parameters.python_version>>/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n '<<parameters.extra_env_args>>' ]; then
+ export <<parameters.extra_env_args>>
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_<<parameters.file_tag>>
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_<<parameters.file_tag>>_logs
+
+ run_unit_tests_repeat:
+ steps:
+ - run_repeated_utests:
+ target: testsome
+ tests: ${REPEATED_UTESTS}
+ count: ${REPEATED_UTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_cdc_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-cdc
+ tests: ${REPEATED_UTESTS}
+ count: ${REPEATED_UTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_compression_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-compression
+ tests: ${REPEATED_UTESTS}
+ count: ${REPEATED_UTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_trie_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-trie
+ tests: ${REPEATED_UTESTS}
+ count: ${REPEATED_UTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_system_keyspace_directory_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-system-keyspace-directory
+ tests: ${REPEATED_UTESTS}
+ count: ${REPEATED_UTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_long_repeat:
+ steps:
+ - run_repeated_utests:
+ target: long-testsome
+ tests: ${REPEATED_UTESTS_LONG}
+ count: ${REPEATED_UTESTS_LONG_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_fqltool_repeat:
+ steps:
+ - run_repeated_utests:
+ target: fqltool-test
+ tests: ${REPEATED_UTESTS_FQLTOOL}
+ count: ${REPEATED_UTESTS_FQLTOOL_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_utests_stress_repeat:
+ steps:
+ - run_repeated_utests:
+ target: stress-test-some
+ tests: ${REPEATED_UTESTS_STRESS}
+ count: ${REPEATED_UTESTS_STRESS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_jvm_dtests_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-jvm-dtest-some
+ tests: ${REPEATED_JVM_DTESTS}
+ count: ${REPEATED_JVM_DTESTS_COUNT}
+ vnodes: false
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_simulator_dtests_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-simulator-dtest
+ tests: ${REPEATED_SIMULATOR_DTESTS}
+ count: ${REPEATED_SIMULATOR_DTESTS_COUNT}
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_jvm_dtests_vnode_repeat:
+ steps:
+ - run_repeated_utests:
+ target: test-jvm-dtest-some
+ tests: ${REPEATED_JVM_DTESTS}
+ count: ${REPEATED_JVM_DTESTS_COUNT}
+ vnodes: true
+ stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
+ run_repeated_utests:
+ parameters:
+ target:
+ type: string
+ tests:
+ type: string
+ count:
+ type: string
+ vnodes:
+ type: boolean
+ default: false
+ stop_on_failure:
+ type: string
+ steps:
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((<<parameters.count>> / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (<<parameters.count>> % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the JVM dtests vnodes argument, which is optional.
+ vnodes=<<parameters.vnodes>>
+ vnodes_args=""
+ if [ "$vnodes" = true ] ; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=<<parameters.target>>
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ elif [[ $target == "test-trie" ]]; then
+ testtag="trie"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant <<parameters.target>> $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ <<parameters.stop_on_failure>> = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+
+ run_repeated_utest:
+ parameters:
+ target:
+ type: string
+ class:
+ type: string
+ methods:
+ type: string
+ vnodes:
+ type: string
+ count:
+ type: string
+ stop_on_failure:
+ type: string
+ steps:
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "<<parameters.class>>" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "<<parameters.count>>" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "<<parameters.count>>" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((<<parameters.count>> / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (<<parameters.count>> % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running <<parameters.target>> <<parameters.class>> <<parameters.methods>> <<parameters.count>> times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=<<parameters.target>>
+ class_path=<<parameters.class>>
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-trie" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" || \
+ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "<<parameters.methods>>" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=<<parameters.methods>>"
+ fi
+
+ # Prepare the JVM dtests vnodes argument, which is optional
+ vnodes_args=""
+ if <<parameters.vnodes>>; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/<<parameters.target>>-<<parameters.class>>.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ <<parameters.stop_on_failure>> = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+
+ run_repeated_dtest:
+ parameters:
+ tests:
+ type: string
+ vnodes:
+ type: string
+ upgrade:
+ type: string
+ count:
+ type: string
+ stop_on_failure:
+ type: string
+ extra_dtest_args:
+ type: string
+ default: ""
+ steps:
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "<<parameters.tests>>" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "<<parameters.count>>" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "<<parameters.count>>" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((<<parameters.count>> / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (<<parameters.count>> % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running <<parameters.tests>> $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo <<parameters.tests>> | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if <<parameters.stop_on_failure>>; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if <<parameters.vnodes>>; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if <<parameters.upgrade>>; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir <<parameters.extra_dtest_args>> $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
diff --git a/.circleci/config_template_11_and_17.yml.PAID.patch b/.circleci/config_template_11_and_17.yml.PAID.patch
new file mode 100644
index 0000000..c0c6560
--- /dev/null
+++ b/.circleci/config_template_11_and_17.yml.PAID.patch
@@ -0,0 +1,374 @@
+--- config_template_11_and_17.yml 2023-03-15 21:34:57.000000000 -0400
++++ config_template_11_and_17.yml.PAID 2023-03-15 21:37:25.000000000 -0400
+@@ -140,14 +140,14 @@
+ j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ exec_resource_class: large
++ parallelism: 10
+
+ j11_small_executor: &j11_small_executor
+ executor:
+@@ -155,30 +155,41 @@
+ exec_resource_class: medium
+ parallelism: 1
+
++j11_large_par_executor: &j11_large_par_executor
++ executor:
++ name: java11-executor
++ exec_resource_class: large
++ parallelism: 50
++
++j11_very_large_par_executor: &j11_very_large_par_executor
++ executor:
++ name: java11-executor
++ exec_resource_class: large
++ parallelism: 100
++
+ j11_medium_par_executor: &j11_medium_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ exec_resource_class: xlarge
++ parallelism: 4
+
+ j11_seq_executor: &j11_seq_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+ exec_resource_class: medium
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+ j17_par_executor: &j17_par_executor
+ executor:
+ name: java17-executor
+- #exec_resource_class: xlarge
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j17_small_par_executor: &j17_small_par_executor
+ executor:
+ name: java17-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ exec_resource_class: large
++ parallelism: 10
+
+ j17_small_executor: &j17_small_executor
+ executor:
+@@ -189,34 +200,44 @@
+ j17_medium_par_executor: &j17_medium_par_executor
+ executor:
+ name: java17-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ exec_resource_class: xlarge
++ parallelism: 4
++
++j17_large_par_executor: &j17_large_par_executor
++ executor:
++ name: java17-executor
++ exec_resource_class: large
++ parallelism: 50
+
+ j17_seq_executor: &j17_seq_executor
+ executor:
+ name: java17-executor
+- #exec_resource_class: xlarge
++ exec_resource_class: medium
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+ j11_repeated_utest_executor: &j11_repeated_utest_executor
+ executor:
+ name: java11-executor
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j11_repeated_dtest_executor: &j11_repeated_dtest_executor
+ executor:
+ name: java11-executor
+- parallelism: 4
++ exec_resource_class: large
++ parallelism: 25
+
+ j17_repeated_utest_executor: &j17_repeated_utest_executor
+ executor:
+ name: java17-executor
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j17_repeated_dtest_executor: &j17_repeated_dtest_executor
+ executor:
+ name: java17-executor
+- parallelism: 4
++ exec_resource_class: large
++ parallelism: 25
+
+ j11_separate_jobs: &j11_separate_jobs
+ jobs:
+@@ -1808,7 +1829,7 @@
+ target: testclasslist-system-keyspace-directory
+
+ j11_dtests_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1822,7 +1843,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j11_dtests_offheap:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1836,7 +1857,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+
+ j17_dtests_vnode:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1851,7 +1872,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j17_dtests_offheap:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1866,7 +1887,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
+
+ j11_dtests:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1880,7 +1901,7 @@
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j17_dtests:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1895,7 +1916,7 @@
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j11_cqlsh_dtests_py3_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1910,7 +1931,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py3_offheap:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1925,7 +1946,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py38_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1943,7 +1964,7 @@
+ python_version: '3.8'
+
+ j11_cqlsh_dtests_py311_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1961,7 +1982,7 @@
+ python_version: '3.11'
+
+ j11_cqlsh_dtests_py38_offheap:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1979,7 +2000,7 @@
+ python_version: '3.8'
+
+ j11_cqlsh_dtests_py311_offheap:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -1997,7 +2018,7 @@
+ python_version: '3.11'
+
+ j11_cqlsh_dtests_py3:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2012,7 +2033,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py38:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2030,7 +2051,7 @@
+ python_version: '3.8'
+
+ j11_cqlsh_dtests_py311:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2048,7 +2069,7 @@
+ python_version: '3.11'
+
+ j17_cqlsh_dtests_py3_vnode:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2063,7 +2084,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j17_cqlsh_dtests_py3_offheap:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2078,7 +2099,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j17_cqlsh_dtests_py38_vnode:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2096,7 +2117,7 @@
+ python_version: '3.8'
+
+ j17_cqlsh_dtests_py311_vnode:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2114,7 +2135,7 @@
+ python_version: '3.11'
+
+ j17_cqlsh_dtests_py38_offheap:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2132,7 +2153,7 @@
+ python_version: '3.8'
+
+ j17_cqlsh_dtests_py311_offheap:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2150,7 +2171,7 @@
+ python_version: '3.11'
+
+ j17_cqlsh_dtests_py3:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2165,7 +2186,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j17_cqlsh_dtests_py38:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2183,7 +2204,7 @@
+ python_version: '3.8'
+
+ j17_cqlsh_dtests_py311:
+- <<: *j17_par_executor
++ <<: *j17_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2201,7 +2222,7 @@
+ python_version: '3.11'
+
+ j11_dtests_large_vnode:
+- <<: *j11_par_executor
++ <<: *j11_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2215,7 +2236,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_dtests_large:
+- <<: *j11_par_executor
++ <<: *j11_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2229,7 +2250,7 @@
+ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j17_dtests_large_vnode:
+- <<: *j17_par_executor
++ <<: *j17_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+@@ -2243,7 +2264,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j17_dtests_large:
+- <<: *j17_par_executor
++ <<: *j17_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
diff --git a/.circleci/generate.sh b/.circleci/generate.sh
index 29f66a2..e540d6f 100755
--- a/.circleci/generate.sh
+++ b/.circleci/generate.sh
@@ -18,7 +18,7 @@
#
BASEDIR=`dirname $0`
-BASE_BRANCH=cassandra-4.1
+BASE_BRANCH=trunk
set -e
die ()
@@ -258,6 +258,8 @@
delete_job "$1" "j11_utests_cdc_repeat"
delete_job "$1" "j8_utests_compression_repeat"
delete_job "$1" "j11_utests_compression_repeat"
+ delete_job "$1" "j8_utests_trie_repeat"
+ delete_job "$1" "j11_utests_trie_repeat"
delete_job "$1" "j8_utests_system_keyspace_directory_repeat"
delete_job "$1" "j11_utests_system_keyspace_directory_repeat"
fi
diff --git a/.circleci/generate_11_and_17.sh b/.circleci/generate_11_and_17.sh
new file mode 100755
index 0000000..c09723a
--- /dev/null
+++ b/.circleci/generate_11_and_17.sh
@@ -0,0 +1,306 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BASEDIR=`dirname $0`
+BASE_BRANCH=trunk
+set -e
+
+die ()
+{
+ echo "ERROR: $*"
+ print_help
+ exit 1
+}
+
+print_help()
+{
+ echo "Usage: $0 [-f|-p|-a|-e|-i]"
+ echo " -a Generate the config_11_and_17.yml, config_11_and_17.yml.FREE and config_11_and_17.yml.PAID expanded configuration"
+ echo " files from the main config_template.yml reusable configuration file."
+ echo " Use this for permanent changes in config_11_and_17.yml that will be committed to the main repo."
+ echo " -f Generate config.yml for tests compatible with the CircleCI free tier resources"
+ echo " -p Generate config.yml for tests compatible with the CircleCI paid tier resources"
+ echo " -e <key=value> Environment variables to be used in the generated config_11_and_17.yml, e.g.:"
+ echo " -e DTEST_BRANCH=CASSANDRA-8272"
+ echo " -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git"
+ echo " -e REPEATED_TESTS_STOP_ON_FAILURE=false"
+ echo " -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest#testCountersTable"
+ echo " -e REPEATED_UTESTS_COUNT=500"
+ echo " -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest"
+ echo " -e REPEATED_UTESTS_FQLTOOL_COUNT=500"
+ echo " -e REPEATED_UTESTS_LONG=org.apache.cassandra.db.commitlog.CommitLogStressTest"
+ echo " -e REPEATED_UTESTS_LONG_COUNT=100"
+ echo " -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest"
+ echo " -e REPEATED_UTESTS_STRESS_COUNT=500"
+ echo " -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.TrivialSimulationTest"
+ echo " -e REPEATED_SIMULATOR_DTESTS_COUNT=500"
+ echo " -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest"
+ echo " -e REPEATED_JVM_DTESTS_COUNT=500"
+ echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
+ echo " -e REPEATED_DTESTS_COUNT=500"
+ echo " -e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node"
+ echo " -e REPEATED_LARGE_DTESTS=100"
+ echo " -e REPEATED_ANT_TEST_TARGET=testsome"
+ echo " -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest"
+ echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable"
+ echo " -e REPEATED_ANT_TEST_VNODES=false"
+ echo " -e REPEATED_ANT_TEST_COUNT=500"
+ echo " For the complete list of environment variables, please check the"
+ echo " list of examples in config_template_11_and_17.yml and/or the documentation."
+ echo " If you want to specify multiple environment variables simply add"
+ echo " multiple -e options. The flags -f/-p should be used when using -e."
+ echo " -i Ignore unknown environment variables"
+}
+
+all=false
+free=false
+paid=false
+env_vars=""
+has_env_vars=false
+check_env_vars=true
+while getopts "e:afpi" opt; do
+ case $opt in
+ a ) all=true
+ ;;
+ f ) free=true
+ ;;
+ p ) paid=true
+ ;;
+ e ) if (! ($has_env_vars)); then
+ env_vars="$OPTARG"
+ else
+ env_vars="$env_vars|$OPTARG"
+ fi
+ has_env_vars=true
+ ;;
+ i ) check_env_vars=false
+ ;;
+ \?) die "Invalid option: -$OPTARG"
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+if [ "$#" -ne 0 ]; then
+ die "Unexpected arguments"
+fi
+
+# validate environment variables
+if $has_env_vars && $check_env_vars; then
+ for entry in $(echo $env_vars | tr "|" "\n"); do
+ key=$(echo $entry | tr "=" "\n" | head -n 1)
+ if [ "$key" != "DTEST_REPO" ] &&
+ [ "$key" != "DTEST_BRANCH" ] &&
+ [ "$key" != "REPEATED_TESTS_STOP_ON_FAILURE" ] &&
+ [ "$key" != "REPEATED_UTESTS" ] &&
+ [ "$key" != "REPEATED_UTESTS_COUNT" ] &&
+ [ "$key" != "REPEATED_UTESTS_FQLTOOL" ] &&
+ [ "$key" != "REPEATED_UTESTS_FQLTOOL_COUNT" ] &&
+ [ "$key" != "REPEATED_UTESTS_LONG" ] &&
+ [ "$key" != "REPEATED_UTESTS_LONG_COUNT" ] &&
+ [ "$key" != "REPEATED_UTESTS_STRESS" ] &&
+ [ "$key" != "REPEATED_UTESTS_STRESS_COUNT" ] &&
+ [ "$key" != "REPEATED_SIMULATOR_DTESTS" ] &&
+ [ "$key" != "REPEATED_SIMULATOR_DTESTS_COUNT" ] &&
+ [ "$key" != "REPEATED_JVM_DTESTS" ] &&
+ [ "$key" != "REPEATED_JVM_DTESTS_COUNT" ] &&
+ [ "$key" != "REPEATED_DTESTS" ] &&
+ [ "$key" != "REPEATED_DTESTS_COUNT" ] &&
+ [ "$key" != "REPEATED_LARGE_DTESTS" ] &&
+ [ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] &&
+ [ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
+ [ "$key" != "REPEATED_ANT_TEST_CLASS" ] &&
+ [ "$key" != "REPEATED_ANT_TEST_METHODS" ] &&
+ [ "$key" != "REPEATED_ANT_TEST_VNODES" ] &&
+ [ "$key" != "REPEATED_ANT_TEST_COUNT" ]; then
+ die "Unrecognised environment variable name: $key"
+ fi
+ done
+fi
+
+if $free; then
+ ($all || $paid) && die "Cannot use option -f with options -a or -p"
+ echo "Generating new config.yml file for free tier from config_template_11_and_17.yml"
+ circleci config process $BASEDIR/config_template_11_and_17.yml > $BASEDIR/config_11_and_17.yml.FREE.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.FREE.tmp > $BASEDIR/config.yml
+ cp $BASEDIR/config.yml $BASEDIR/config_11_and_17.yml
+ rm $BASEDIR/config_11_and_17.yml.FREE.tmp
+
+elif $paid; then
+ ($all || $free) && die "Cannot use option -p with options -a or -f"
+ echo "Generating new config.yml file for paid tier from config_template_11_and_17.yml"
+ patch -o $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_template_11_and_17.yml $BASEDIR/config_template_11_and_17.yml.PAID.patch
+ circleci config process $BASEDIR/config_template_11_and_17.yml.PAID > $BASEDIR/config_11_and_17.yml.PAID.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.PAID.tmp > $BASEDIR/config.yml
+ cp $BASEDIR/config.yml $BASEDIR/config_11_and_17.yml
+ rm $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_11_and_17.yml.PAID.tmp
+
+elif $all; then
+ ($free || $paid || $has_env_vars) && die "Cannot use option -a with options -f, -p or -e"
+ echo "Generating new default config_11_and_17.yml file for free tier and FREE/PAID templates from config_template_11_and_17.yml."
+ echo "Make sure you commit the newly generated config_11_and_17.yml, config_11_and_17.yml.FREE and config_11_and_17.yml.PAID files"
+ echo "after running this command if you want them to persist."
+
+ # setup config for free tier
+ circleci config process $BASEDIR/config_template_11_and_17.yml > $BASEDIR/config_11_and_17.yml.FREE.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.FREE.tmp > $BASEDIR/config_11_and_17.yml.FREE
+ rm $BASEDIR/config_11_and_17.yml.FREE.tmp
+
+ # setup config for paid tier
+ patch -o $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_template_11_and_17.yml $BASEDIR/config_template_11_and_17.yml.PAID.patch
+ circleci config process $BASEDIR/config_template_11_and_17.yml.PAID > $BASEDIR/config_11_and_17.yml.PAID.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.PAID.tmp > $BASEDIR/config_11_and_17.yml.PAID
+ rm $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_11_and_17.yml.PAID.tmp
+
+ # copy free tier into config_11_and_17.yml to make sure this gets updated
+ cp $BASEDIR/config_11_and_17.yml.FREE $BASEDIR/config_11_and_17.yml
+
+elif (! ($has_env_vars)); then
+ print_help
+ exit 0
+fi
+
+# add new or modified tests to the sets of tests to be repeated
+if (! ($all)); then
+ add_diff_tests ()
+ {
+ dir="${BASEDIR}/../${2}"
+ diff=$(git --no-pager diff --name-only --diff-filter=AMR ${BASE_BRANCH}...HEAD ${dir})
+ tests=$( echo "$diff" \
+ | grep "Test\\.java" \
+ | sed -e "s/\\.java//" \
+ | sed -e "s,^${2},," \
+ | tr '/' '.' \
+ | grep ${3} )\
+ || : # avoid execution interruptions due to grep return codes and set -e
+ for test in $tests; do
+ echo " $test"
+ has_env_vars=true
+ if echo "$env_vars" | grep -q "${1}="; then
+ env_vars=$(echo "$env_vars" | sed -e "s/${1}=/${1}=${test},/")
+ elif [ -z "$env_vars" ]; then
+ env_vars="${1}=${test}"
+ else
+ env_vars="$env_vars|${1}=${test}"
+ fi
+ done
+ }
+
+ echo
+ echo "Detecting new or modified tests with git diff --diff-filter=AMR ${BASE_BRANCH}...HEAD:"
+ add_diff_tests "REPEATED_UTESTS" "test/unit/" "org.apache.cassandra"
+ add_diff_tests "REPEATED_UTESTS_LONG" "test/long/" "org.apache.cassandra"
+ add_diff_tests "REPEATED_UTESTS_STRESS" "tools/stress/test/unit/" "org.apache.cassandra.stress"
+ add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool"
+ add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test"
+ add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test"
+fi
+
+# replace environment variables
+if $has_env_vars; then
+ echo
+ echo "Setting environment variables:"
+ IFS='='
+ echo "$env_vars" | tr '|' '\n' | while read entry; do
+ set -- $entry
+ key=$1
+ val=$2
+ echo " $key: $val"
+ sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml
+ cp $BASEDIR/config.yml $BASEDIR/config_11_and_17.yml
+ done
+ unset IFS
+fi
+
+# Define function to remove unneeded jobs.
+# The first argument is the file name, and the second arguemnt is the job name.
+delete_job()
+{
+ delete_yaml_block()
+ {
+ sed -Ei.bak "/^ - ${2}/,/^ [^[:space:]]+|^ [^[:space:]]+/{//!d;}" "$1"
+ sed -Ei.bak "/^ - ${2}/d" "$1"
+ }
+ file="$BASEDIR/$1"
+ delete_yaml_block "$file" "${2}"
+ delete_yaml_block "$file" "start_${2}"
+}
+
+# Define function to remove any unneeded repeated jobs.
+# The first and only argument is the file name.
+delete_repeated_jobs()
+{
+ if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS=" )); then
+ delete_job "$1" "j11_unit_tests_repeat"
+ delete_job "$1" "j17_unit_tests_repeat"
+ delete_job "$1" "j11_utests_cdc_repeat"
+ delete_job "$1" "j17_utests_cdc_repeat"
+ delete_job "$1" "j11_utests_compression_repeat"
+ delete_job "$1" "j17_utests_compression_repeat"
+ delete_job "$1" "j11_utests_trie_repeat"
+ delete_job "$1" "j17_utests_trie_repeat"
+ delete_job "$1" "j11_utests_system_keyspace_directory_repeat"
+ delete_job "$1" "j17_utests_system_keyspace_directory_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_LONG=")); then
+ delete_job "$1" "j11_utests_long_repeat"
+ delete_job "$1" "j17_utests_long_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_STRESS=")); then
+ delete_job "$1" "j11_utests_stress_repeat"
+ delete_job "$1" "j17_utests_stress_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_FQLTOOL=")); then
+ delete_job "$1" "j11_utests_fqltool_repeat"
+ delete_job "$1" "j17_utests_fqltool_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_SIMULATOR_DTESTS=")); then
+ delete_job "$1" "j11_simulator_dtests_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_JVM_DTESTS=")); then
+ delete_job "$1" "j11_jvm_dtests_repeat"
+ delete_job "$1" "j11_jvm_dtests_vnode_repeat"
+ delete_job "$1" "j17_jvm_dtests_repeat"
+ delete_job "$1" "j17_jvm_dtests_vnode_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then
+ delete_job "$1" "j11_dtests_repeat"
+ delete_job "$1" "j11_dtests_vnode_repeat"
+ delete_job "$1" "j11_dtests_offheap_repeat"
+ delete_job "$1" "j17_dtests_repeat"
+ delete_job "$1" "j17_dtests_vnode_repeat"
+ delete_job "$1" "j17_dtests_offheap_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_LARGE_DTESTS=")); then
+ delete_job "$1" "j11_dtests_large_repeat"
+ delete_job "$1" "j11_dtests_large_vnode_repeat"
+ delete_job "$1" "j17_dtests_large_repeat"
+ delete_job "$1" "j17_dtests_large_vnode_repeat"
+ fi
+ if (! (echo "$env_vars" | grep -q "REPEATED_ANT_TEST_CLASS=")); then
+ delete_job "$1" "j11_repeated_ant_test"
+ delete_job "$1" "j17_repeated_ant_test"
+ fi
+}
+
+delete_repeated_jobs "config.yml"
+delete_repeated_jobs "config_11_and_17.yml"
+if $all; then
+ delete_repeated_jobs "config_11_and_17.yml.FREE"
+ delete_repeated_jobs "config_11_and_17.yml.PAID"
+fi
diff --git a/.circleci/readme.md b/.circleci/readme.md
index d5389f6..cd6a736 100644
--- a/.circleci/readme.md
+++ b/.circleci/readme.md
@@ -23,12 +23,18 @@
This directory contains the configuration for CircleCI continous integration platform.
The file `config.yml` is the configuration file that is read by CircleCI. This file is
automatically generated by the `generate.sh` script from the `config_template.yml` file.
+Experimental JDK17 configuration is added for test purposes to enable easier testing
+while working on JDK17 related issues. The file `config_11_and_17.yml` is the configuration
+file that we can copy over `config.yml` to test JDK11+17 workflows. This file is automatically
+generated by the `generate_11_and_17.sh` script from the `config_template_11_and_17.yml` file.
The provided `config.yml` file uses low resources so users of the CircleCI free tier can
use it. Additionally, there are two versions of this file using different resources so
users who have access to premium CircleCI resources can use larger instances and more
parallelism. These files are `config.yml.FREE` and `config.yml.PAID`.
The default `config.yml` file is just a copy of `config.yml.FREE`.
+For the JDK11+17 workflows we have equivalent files - `config_11_and_17.yml.FREE` and
+`config_11_and_17.yml.PAID`.
## Switching to higher resource settings
This directory contains generated files for free and paid resource settings.
@@ -37,29 +43,41 @@
`cp .circleci/config.yml.PAID .circleci/config.yml`
+respectively for JDK11+17:
+
+`cp .circleci/config_11_and_17.yml.PAID .circleci/config.yml`
+
And for using lower resources comaptible with CircleCI's free tier:
`cp .circleci/config.yml.FREE .circleci/config.yml`
+respectively for JDK11+17:
+
+`cp .circleci/config_11_and_17.yml.FREE .circleci/config.yml`
+
Alternatively, you can run the `generate.sh` script with the flags `-f`/`-p`
to regenerate the `config.yml` file from `config_template.yml` using free or paid resources.
-This script validates and applies any changes to the `config_template.yml` file, and it
-requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be
-installed.
+For JDK11 and 17 you can run the `generate_11_and_17.sh` script with the flags
+`-f`/`-p` to regenerate the `config.yml` file from `config_template_11_and_17.yml`
+using free or paid resources. The two scripts validate and apply any changes to the
+`config_template.yml`and `config_template_11_and_17.yml` files, and they require the
+[CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be installed.
## Setting environment variables
Both `config_template.yml` and `config.yml` files contain a set of environment variables
defining things like what dtest repo and branch to use, what tests could be repeatedly
-run, etc.
+run, etc. Same applies for `config_template_11_and_17.yml` and `config_11_and_17.yml`.
These environment variables can be directly edited in the `config.yml` file, although if
you do this you should take into account that the entire set of env vars is repeated on
every job.
-A probably better approach is editing them in `config_template.yml` and then regenerate the
-`config.yml` file using the `generate.sh` script. You can also directly pass environment
-variable values to the `generate.sh` script with the `-e` flag. For example, to set the
-dtest repo and branch with MIDRES config you can run:
+A probably better approach is editing them in `config_template.yml` or respectively
+`config_template_11_and_17.yml` and then regenerate the `config.yml` and `config_11_and_17.yml`
+files using the `generate.sh` or respectively the `generate_11_and_17.yml` script. You can also
+directly pass environment variable values to the `generate.sh` and `generate_11_and_17.sh` scripts
+with the `-e` flag. For example, to set the dtest repo and branch with PAID config
+you can run:
```
generate.sh -p \
@@ -69,10 +87,10 @@
```
## Running tests in a loop
-Running the `generate.sh` script with use `git diff` to find the new or modified tests.
-The script will then create jobs to run each of these new or modified tests for a certain
-number of times, to verify that they are stable. You can use environment variables to
-specify the number of iterations of each type of test:
+Running the `generate.sh` or `generate_11_and_17.sh` script will use `git diff` to find the
+new or modified tests. The scripts will then create jobs to run each of these new or modified
+tests for a certain number of times, to verify that they are stable. You can use environment
+variables to specify the number of iterations of each type of test:
```
generate.sh -p \
-e REPEATED_UTESTS_COUNT=500 \
@@ -157,16 +175,18 @@
regenerate the `config.yml`, `config.yml.FREE` and `config.yml.PAID`
files by runnining the `generate.sh` script with `-a` flag. For using this script you
need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install).
+Same applies for the equivalent JDK11+17 config files and `generate_11_and_17.sh`.
As for temporal changes done while working in a patch, such as pointing to you dtest repo or
running a test repeatedly, you can either directly edit `config.yml` or edit `config_template.yml`
and then regenerate `config.yml` with the `generate.sh` script using a `-f`/`-p` flag.
-When this flag is used only the `config.yml` will be generated.
+When this flag is used only the `config.yml` will be generated. Same workflow applies to the respective
+JDK11+17 CircleCI configuration files.
Please note that any previous swapping or edition of the generated files will be overriden
-by running `generate.sh` with `-a` argument, returning `config.yml` to the default FREE. So if
-you previously swapped your `config.yml` to MIDRES you would need to either swap it
-again or use the `-f`/`-p` script flags.
+by running `generate.sh`/`generate_11_and_17.sh` with `-a` argument, returning `config.yml`
+to the default FREE. So if you previously swapped your `config.yml` to PAID you would need to
+either swap it again or use the `-f`/`-p` script flags.
Read below for details how to generate the files manually without the `generate.sh` script:
@@ -188,3 +208,5 @@
7. add the Apache license header to the newly created PAID file:
`cat license.yml config.yml.PAID > config.yml.PAID.new && mv config.yml.PAID.new config.yml.PAID`
8. finally, remember to update the config.yml
+
+The process is equivalent for `generate_11_and_17.sh` and the respective JDK11+17 configuration files.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 0000000..abcd216
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,25 @@
+Thanks for sending a pull request! Here are some tips if you're new here:
+
+ * Ensure you have added or run the [appropriate tests](https://cassandra.apache.org/_/development/testing.html) for your PR.
+ * Be sure to keep the PR description updated to reflect all changes.
+ * Write your PR title to summarize what this PR proposes.
+ * If possible, provide a concise example to reproduce the issue for a faster review.
+ * Read our [contributor guidelines](https://cassandra.apache.org/_/development/index.html)
+ * If you're making a documentation change, see our [guide to documentation contribution](https://cassandra.apache.org/_/development/documentation.html)
+
+Commit messages should follow the following format:
+
+```
+<One sentence description, usually Jira title or CHANGES.txt summary>
+
+<Optional lengthier description (context on patch)>
+
+patch by <Authors>; reviewed by <Reviewers> for CASSANDRA-#####
+
+Co-authored-by: Name1 <email1>
+Co-authored-by: Name2 <email2>
+
+```
+
+The [Cassandra Jira](https://issues.apache.org/jira/projects/CASSANDRA/issues/)
+
diff --git a/CHANGES.txt b/CHANGES.txt
index 246ba58..99a5340 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,10 +1,158 @@
-4.1.3
+5.0
+ * Print header and statistics for cassandra-stress output with arbitrary frequency (CASSANDRA-12972)
+ * CEP-25: Trie-indexed SSTable format (CASSANDRA-18398)
+ * Make cassandra-stress able to read all credentials from a file (CASSANDRA-18544)
+ * Add guardrail for partition size and deprecate compaction_large_partition_warning_threshold (CASSANDRA-18500)
+ * Add HISTORY command for CQLSH (CASSANDRA-15046)
+ * Fix sstable formats configuration (CASSANDRA-18441)
+ * Add guardrail to bound timestamps (CASSANDRA-18352)
+ * Add keyspace_name column to system_views.clients (CASSANDRA-18525)
+ * Moved system properties and envs to CassandraRelevantProperties and CassandraRelevantEnv respectively (CASSANDRA-17797)
+ * Add sstablepartitions offline tool to find large partitions in sstables (CASSANDRA-8720)
+ * Replace usages of json-simple dependency by Jackson (CASSANDRA-16855)
+ * When decommissioning should set Severity to limit traffic (CASSANDRA-18430)
+ * For Java11 and Java17 remove -XX:-UseBiasedLocking as it is the default already (CASSANDRA-17869)
+ * Upgrade commons-io to 2.11.0 (CASSANDRA-17364)
+ * Node Draining Should Abort All Current SSTables Imports (CASSANDRA-18373)
+ * Use snake case for the names of CQL native functions (CASSANDRA-18037)
+ * Use jdk-dependent checkstyle version to check the source code (CASSANDRA-18262)
+ * Provide summary of failed SessionInfo's in StreamResultFuture (CASSANDRA-17199)
+ * CEP-20: Dynamic Data Masking (CASSANDRA-17940)
+ * Add system_views.snapshots virtual table (CASSANDRA-18102)
+ * Update OpenHFT dependencies (chronicle-queue, chronicle-core, chronicle-bytes, chronicle-wire, chronicle-threads) (CASSANDRA-18049)
+ * Remove org.apache.cassandra.hadoop code (CASSANDRA-18323)
+ * Remove deprecated CQL functions dateOf and unixTimestampOf (CASSANDRA-18328)
+ * Remove DateTieredCompactionStrategy (CASSANDRA-18043)
+ * Add system_views.max_sstable_size and system_views.max_sstable_duration tables (CASSANDRA-18333)
+ * Extend implicit allow-filtering for virtual tables to clustering columns (CASSANDRA-18331)
+ * Upgrade maven-shade-plugin to 3.4.1 to fix shaded dtest JAR build (CASSANDRA-18136)
+ * Upgrade to Opcodes.ASM9 (CASSANDRA-17971)
+ * Add MaxSSTableSize and MaxSSTableDuration metrics and propagate them together with local read/write ratio to tablestats (CASSANDRA-18283)
+ * Add more logging around CompactionManager operations (CASSANDRA-18268)
+ * Reduce memory allocations of calls to ByteBufer.duplicate() made in org.apache.cassandra.transport.CBUtil#writeValue (CASSANDRA-18212)
+ * CEP-17: SSTable API (CASSANDRA-17056)
+ * Gossip stateMapOrdering does not have correct ordering when both EndpointState are in the bootstrapping set (CASSANDRA-18292)
+ * Snapshot only sstables containing mismatching ranges on preview repair mismatch (CASSANDRA-17561)
+ * More accurate skipping of sstables in read path (CASSANDRA-18134)
+ * Prepare for JDK17 experimental support (CASSANDRA-18179, CASSANDRA-18258)
+ * Remove Scripted UDFs internals; hooks to be added later in CASSANDRA-17281 (CASSANDRA-18252)
+ * Update JNA to 5.13.0 (CASSANDRA-18050)
+ * Make virtual tables decide if they implicitly enable ALLOW FILTERING (CASSANDRA-18238)
+ * Add row, tombstone, and sstable count to nodetool profileload (CASSANDRA-18022)
+ * Coordinator level metrics for read response and mutation row and column counts (CASSANDRA-18155)
+ * Add CQL functions for dynamic data masking (CASSANDRA-17941)
+ * Print friendly error when nodetool attempts to connect to uninitialized server (CASSANDRA-11537)
+ * Use G1GC by default, and update default G1GC settings (CASSANDRA-18027)
+ * SimpleSeedProvider can resolve multiple IP addresses per DNS record (CASSANDRA-14361)
+ * Remove mocking in InternalNodeProbe spying on StorageServiceMBean (CASSANDRA-18152)
+ * Add compaction_properties column to system.compaction_history table and nodetool compactionhistory command (CASSANDRA-18061)
+ * Remove ProtocolVersion entirely from the CollectionSerializer ecosystem (CASSANDRA-18114)
+ * Fix serialization error in new getsstables --show-levels option (CASSANDRA-18140)
+ * Use checked casts when reading vints as ints (CASSANDRA-18099)
+ * Add Mutation Serialization Caching (CASSANDRA-17998)
+ * Only reload compaction strategies if disk boundaries change (CASSANDRA-17874)
+ * CEP-10: Simulator Java11 Support (CASSANDRA-17178)
+ * Set the major compaction type correctly for compactionstats (CASSANDRA-18055)
+ * Print exception message without stacktrace when nodetool commands fail on probe.getOwnershipWithPort() (CASSANDRA-18079)
+ * Add option to print level in nodetool getsstables output (CASSANDRA-18023)
+ * Implement a guardrail for not having zero default_time_to_live on tables with TWCS (CASSANDRA-18042)
+ * Add CQL scalar functions for collection aggregation (CASSANDRA-18060)
+ * Make cassandra.replayList property for CommitLogReplayer possible to react on keyspaces only (CASSANDRA-18044)
+ * Add Mathematical functions (CASSANDRA-17221)
+ * Make incremental backup configurable per table (CASSANDRA-15402)
+ * Change shebangs of Python scripts to resolve Python 3 from env command (CASSANDRA-17832)
+ * Add reasons to guardrail messages and consider guardrails in the error message for needed ALLOW FILTERING (CASSANDRA-17967)
+ * Add support for CQL functions on collections, tuples and UDTs (CASSANDRA-17811)
+ * Add flag to exclude nodes from local DC when running nodetool rebuild (CASSANDRA-17870)
+ * Adding endpoint verification option to client_encryption_options (CASSANDRA-18034)
+ * Replace 'wcwidth.py' with pypi module (CASSANDRA-17287)
+ * Add nodetool forcecompact to remove tombstoned or ttl'd data ignoring GC grace for given table and partition keys (CASSANDRA-17711)
+ * Offer IF (NOT) EXISTS in cqlsh completion for CREATE TYPE, DROP TYPE, CREATE ROLE and DROP ROLE (CASSANDRA-16640)
+ * Nodetool bootstrap resume will now return an error if the operation fails (CASSANDRA-16491)
+ * Disable resumable bootstrap by default (CASSANDRA-17679)
+ * Include Git SHA in --verbose flag for nodetool version (CASSANDRA-17753)
+ * Update Byteman to 4.0.20 and Jacoco to 0.8.8 (CASSANDRA-16413)
+ * Add memtable option among possible tab completions for a table (CASSANDRA-17982)
+ * Adds a trie-based memtable implementation (CASSANDRA-17240)
+ * Further improves precision of memtable heap tracking (CASSANDRA-17240)
+ * Fix formatting of metrics documentation (CASSANDRA-17961)
+ * Keep sstable level when streaming for decommission and move (CASSANDRA-17969)
+ * Add Unavailables metric for CASWrite in the docs (CASSANDRA-16357)
+ * Make Cassandra logs able to be viewed in the virtual table system_views.system_logs (CASSANDRA-17946)
+ * IllegalArgumentException in Gossiper#order due to concurrent mutations to elements being applied (CASSANDRA-17908)
+ * Include estimated active compaction remaining write size when starting a new compaction (CASSANDRA-17931)
+ * Mixed mode support for internode authentication during TLS upgrades (CASSANDRA-17923)
+ * Revert Mockito downgrade from CASSANDRA-17750 (CASSANDRA-17496)
+ * Add --older-than and --older-than-timestamp options for nodetool clearsnapshots (CASSANDRA-16860)
+ * Fix "open RT bound as its last item" exception (CASSANDRA-17810)
+ * Fix leak of non-standard Java types in JMX MBeans `org.apache.cassandra.db:type=StorageService`
+ and `org.apache.cassandra.db:type=RepairService` as clients using JMX cannot handle them. More details in NEWS.txt (CASSANDRA-17668)
+ * Deprecate Throwables.propagate usage (CASSANDRA-14218)
+ * Allow disabling hotness persistence for high sstable counts (CASSANDRA-17868)
+ * Prevent NullPointerException when changing neverPurgeTombstones from true to false (CASSANDRA-17897)
+ * Add metrics around storage usage and compression (CASSANDRA-17898)
+ * Remove usage of deprecated javax certificate classes (CASSANDRA-17867)
+ * Make sure preview repairs don't optimise streams unless configured to (CASSANDRA-17865)
+ * Optionally avoid hint transfer during decommission (CASSANDRA-17808)
+ * Make disabling auto snapshot on selected tables possible (CASSANDRA-10383)
+ * Introduce compaction priorities to prevent upgrade compaction inability to finish (CASSANDRA-17851)
+ * Prevent a user from manually removing ephemeral snapshots (CASSANDRA-17757)
+ * Remove dependency on Maven Ant Tasks (CASSANDRA-17750)
+ * Update ASM(9.1 to 9.3), Mockito(1.10.10 to 1.12.13) and ByteBuddy(3.2.4 to 4.7.0) (CASSANDRA-17835)
+ * Add the ability for operators to loosen the definition of "empty" for edge cases (CASSANDRA-17842)
+ * Fix potential out of range exception on column index downsampling (CASSANDRA-17839)
+ * Introduce target directory to vtable output for sstable_tasks and for compactionstats (CASSANDRA-13010)
+ * Read/Write/Truncate throw RequestFailure in a race condition with callback timeouts, should return Timeout instead (CASSANDRA-17828)
+ * Add ability to log load profiles at fixed intervals (CASSANDRA-17821)
+ * Protect against Gossip backing up due to a quarantined endpoint without version information (CASSANDRA-17830)
+ * NPE in org.apache.cassandra.cql3.Attributes.getTimeToLive (CASSANDRA-17822)
+ * Add guardrail for column size (CASSANDRA-17151)
+ * When doing a host replacement, we need to check that the node is a live node before failing with "Cannot replace a live node..." (CASSANDRA-17805)
+ * Add support to generate a One-Shot heap dump on unhandled exceptions (CASSANDRA-17795)
+ * Rate-limit new client connection auth setup to avoid overwhelming bcrypt (CASSANDRA-17812)
+ * DataOutputBuffer#scratchBuffer can use off-heap or on-heap memory as a means to control memory allocations (CASSANDRA-16471)
+ * Add ability to read the TTLs and write times of the elements of a collection and/or UDT (CASSANDRA-8877)
+ * Removed Python < 2.7 support from formatting.py (CASSANDRA-17694)
+ * Cleanup pylint issues with pylexotron.py (CASSANDRA-17779)
+ * NPE bug in streaming checking if SSTable is being repaired (CASSANDRA-17801)
+ * Users of NativeLibrary should handle lack of JNA appropriately when running in client mode (CASSANDRA-17794)
+ * Warn on unknown directories found in system keyspace directory rather than kill node during startup checks (CASSANDRA-17777)
+ * Log duplicate rows sharing a partition key found in verify and scrub (CASSANDRA-17789)
+ * Add separate thread pool for Secondary Index building so it doesn't block compactions (CASSANDRA-17781)
+ * Added JMX call to getSSTableCountPerTWCSBucket for TWCS (CASSANDRA-17774)
+ * When doing a host replacement, -Dcassandra.broadcast_interval_ms is used to know when to check the ring but checks that the ring wasn't changed in -Dcassandra.ring_delay_ms, changes to ring delay should not depend on when we publish load stats (CASSANDRA-17776)
+ * When bootstrap fails, CassandraRoleManager may attempt to do read queries that fail with "Cannot read from a bootstrapping node", and increments unavailables counters (CASSANDRA-17754)
+ * Add guardrail to disallow DROP KEYSPACE commands (CASSANDRA-17767)
+ * Remove ephemeral snapshot marker file and introduce a flag to SnapshotManifest (CASSANDRA-16911)
+ * Add a virtual table that exposes currently running queries (CASSANDRA-15241)
+ * Allow sstableloader to specify table without relying on path (CASSANDRA-16584)
+ * Fix TestGossipingPropertyFileSnitch.test_prefer_local_reconnect_on_listen_address (CASSANDRA-17700)
+ * Add ByteComparable API (CASSANDRA-6936)
+ * Add guardrail for maximum replication factor (CASSANDRA-17500)
+ * Increment CQLSH to version 6.2.0 for release 4.2 (CASSANDRA-17646)
+ * Adding support to perform certificate based internode authentication (CASSANDRA-17661)
+ * Option to disable CDC writes of repaired data (CASSANDRA-17666)
+ * When a node is bootstrapping it gets the whole gossip state but applies in random order causing some cases where StorageService will fail causing an instance to not show up in TokenMetadata (CASSANDRA-17676)
+ * Add CQLSH command SHOW REPLICAS (CASSANDRA-17577)
+ * Add guardrail to allow disabling of SimpleStrategy (CASSANDRA-17647)
+ * Change default directory permission to 750 in packaging (CASSANDRA-17470)
+ * Adding support for TLS client authentication for internode communication (CASSANDRA-17513)
+ * Add new CQL function maxWritetime (CASSANDRA-17425)
+ * Add guardrail for ALTER TABLE ADD / DROP / REMOVE column operations (CASSANDRA-17495)
+ * Rename DisableFlag class to EnableFlag on guardrails (CASSANDRA-17544)
Merged from 4.0:
+ * Partial compaction can resurrect deleted data (CASSANDRA-18507)
+
+
+4.1.3
+Merged from 4.0:
* Remove unnecessary shuffling of GossipDigests in Gossiper#makeRandomGossipDigest (CASSANDRA-18546)
Merged from 3.11:
Merged from 3.0:
+
4.1.2
+ * NPE when deserializing malformed collections from client (CASSANDRA-18505)
* Allow keystore and trustrore passwords to be nullable (CASSANDRA-18124)
* Return snapshots with dots in their name in nodetool listsnapshots (CASSANDRA-18371)
* Fix NPE when loading snapshots and data directory is one directory from root (CASSANDRA-18359)
@@ -17,7 +165,6 @@
* Partial compaction can resurrect deleted data (CASSANDRA-18507)
* Allow internal address to change with reconnecting snitches (CASSANDRA-16718)
* Fix quoting in toCqlString methods of UDTs and aggregates (CASSANDRA-17918)
- * NPE when deserializing malformed collections from client (CASSANDRA-18505)
* Improve 'Not enough space for compaction' logging messages (CASSANDRA-18260)
* Incremental repairs fail on mixed IPv4/v6 addresses serializing SyncRequest (CASSANDRA-18474)
* Deadlock updating sstable metadata if disk boundaries need reloading (CASSANDRA-18443)
@@ -33,6 +180,7 @@
* Do not remove SSTables when cause of FSReadError is OutOfMemoryError while using best_effort disk failure policy (CASSANDRA-18336)
* Do not remove truncated_at entry in system.local while dropping an index (CASSANDRA-18105)
+
4.0.9
* Update zstd-jni library to version 1.5.5 (CASSANDRA-18429)
* Backport CASSANDRA-17205 to 4.0 branch - Remove self-reference in SSTableTidier (CASSANDRA-18332)
@@ -50,6 +198,7 @@
* Do not remove truncated_at entry in system.local while dropping an index (CASSANDRA-18105)
* Save host id to system.local and flush immediately after startup (CASSANDRA-18153)
+
4.1.1
* Deprecate org.apache.cassandra.hadoop code (CASSANDRA-16984)
* Fix too early schema version change in sysem local table (CASSANDRA-18291)
@@ -93,6 +242,7 @@
* Avoid anticompaction mixing data from two different time windows with TWCS (CASSANDRA-17970)
* Do not spam the logs with MigrationCoordinator not being able to pull schemas (CASSANDRA-18096)
+
4.1.0
* Fix ContentionStrategy backoff and Clock.waitUntil (CASSANDRA-18086)
Merged from 4.0:
@@ -129,6 +279,7 @@
* Suppress CVE-2019-2684 (CASSANDRA-17965)
* Fix auto-completing "WITH" when creating a materialized view (CASSANDRA-17879)
+
4.1-beta1
* We should not emit deprecation warning on startup for `key_cache_save_period`, `row_cache_save_period`, `counter_cache_save_period` (CASSANDRA-17904)
* upsert with adder support is not consistent with numbers and strings in LWT (CASSANDRA-17857)
@@ -191,6 +342,7 @@
* Fix scrubber falling into infinite loop when the last partition is broken (CASSANDRA-17862)
* Fix resetting schema (CASSANDRA-17819)
+
4.1-alpha1
* Handle config parameters upper bound on startup; Fix auto_snapshot_ttl and paxos_purge_grace_period min unit validations (CASSANDRA-17571)
* Fix leak of non-standard Java types in our Exceptions as clients using JMX are unable to handle them.
@@ -409,6 +561,7 @@
* Lazy transaction log replica creation allows incorrect replica content divergence during anticompaction (CASSANDRA-17273)
* LeveledCompactionStrategy disk space check improvements (CASSANDRA-17272)
+
4.0.3
* Deprecate otc_coalescing_strategy, otc_coalescing_window_us, otc_coalescing_enough_coalesced_messages,
otc_backlog_expiration_interval_ms (CASSANDRA-17377)
@@ -519,6 +672,7 @@
* Ensure java executable is on the path (CASSANDRA-14325)
* Clean transaction log leftovers at the beginning of sstablelevelreset and sstableofflinerelevel (CASSANDRA-12519)
+
4.0.0
* Avoid signaling DigestResolver until the minimum number of responses are guaranteed to be visible (CASSANDRA-16807)
* Fix pre-4.0 FWD_FRM parameter serializer (CASSANDRA-16808)
@@ -526,6 +680,7 @@
* Fix CassandraVersion::compareTo (CASSANDRA-16794)
* BinLog does not close chronicle queue leaving this to GC to cleanup (CASSANDRA-16774)
+
4.0-rc2
* Improved password obfuscation (CASSANDRA-16801)
* Avoid memoizing the wrong min cluster version during upgrades (CASSANDRA-16759)
@@ -561,6 +716,7 @@
* Prevent loss of commit log data when moving sstables between nodes (CASSANDRA-16619)
* Fix materialized view builders inserting truncated data (CASSANDRA-16567)
+
4.0-rc1
* Allow for setting buffer max capacity to increase it dynamically as needed (CASSANDRA-16524)
* Harden internode message resource limit accounting against serialization failures (CASSANDRA-16616)
@@ -642,6 +798,7 @@
* Fix centos packaging for arm64, >=4.0 rpm's now require python3 (CASSANDRA-16477)
* Make TokenMetadata's ring version increments atomic (CASSANDRA-16286)
+
4.0-beta4
* DROP COMPACT STORAGE should invalidate prepared statements still using CompactTableMetadata (CASSANDRA-16361)
* Update default num_tokens to 16 and allocate_tokens_for_local_replication_factor to 3 (CASSANDRA-13701)
@@ -689,6 +846,7 @@
Merged from 2.2:
* Fix the histogram merge of the table metrics (CASSANDRA-16259)
+
4.0-beta3
* Segregate Network and Chunk Cache BufferPools and Recirculate Partially Freed Chunks (CASSANDRA-15229)
* Fail truncation requests when they fail on a replica (CASSANDRA-16208)
@@ -731,6 +889,7 @@
* Automatically drop compact storage on tables for which it is safe (CASSANDRA-16048)
* Fixed NullPointerException for COMPACT STORAGE tables with null clustering (CASSANDRA-16241)
+
4.0-beta2
* Add addition incremental repair visibility to nodetool repair_admin (CASSANDRA-14939)
* Always access system properties and environment variables via the new CassandraRelevantProperties and CassandraRelevantEnv classes (CASSANDRA-15876)
@@ -769,6 +928,7 @@
Merged from 2.1:
* Only allow strings to be passed to JMX authentication (CASSANDRA-16077)
+
4.0-beta1
* Remove BackPressureStrategy (CASSANDRA-15375)
* Improve messaging on indexing frozen collections (CASSANDRA-15908)
@@ -858,6 +1018,7 @@
Merged from 2.1:
* Fix writing of snapshot manifest when the table has table-backed secondary indexes (CASSANDRA-10968)
+
4.0-alpha4
* Add client request size server metrics (CASSANDRA-15704)
* Add additional logging around FileUtils and compaction leftover cleanup (CASSANDRA-15705)
@@ -914,6 +1075,7 @@
* Fix Red Hat init script on newer systemd versions (CASSANDRA-15273)
* Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567)
+
4.0-alpha3
* Restore monotonic read consistency guarantees for blocking read repair (CASSANDRA-14740)
* Separate exceptions for CAS write timeout exceptions caused by contention and unkown result (CASSANDRA-15350)
@@ -965,6 +1127,7 @@
* In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429)
* Added data modeling documentation (CASSANDRA-15443)
+
4.0-alpha2
* Fix SASI non-literal string comparisons (range operators) (CASSANDRA-15169)
* Upgrade Guava to 27, and to java-driver 3.6.0 (from 3.4.0-SNAPSHOT) (CASSANDRA-14655)
@@ -1356,6 +1519,7 @@
* Multi-version in-JVM dtests (CASSANDRA-14937)
* Allow instance class loaders to be garbage collected for inJVM dtest (CASSANDRA-15170)
+
3.11.6
* Fix bad UDT sstable metadata serialization headers written by C* 3.0 on upgrade and in sstablescrub (CASSANDRA-15035)
* Fix nodetool compactionstats showing extra pending task for TWCS - patch implemented (CASSANDRA-15409)
@@ -1386,6 +1550,7 @@
* In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429)
* Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426)
+
3.11.5
* Fix cassandra-env.sh to use $CASSANDRA_CONF to find cassandra-jaas.config (CASSANDRA-14305)
* Fixed nodetool cfstats printing index name twice (CASSANDRA-14903)
@@ -2522,7 +2687,6 @@
* Sane default (200Mbps) for inter-DC streaming througput (CASSANDRA-8708)
-
3.2
* Make sure tokens don't exist in several data directories (CASSANDRA-6696)
* Add requireAuthorization method to IAuthorizer (CASSANDRA-10852)
@@ -7555,7 +7719,6 @@
- Similarly, merged batch_insert_super into batch_insert.
-
0.4.0 beta
* On-disk data format has changed to allow billions of keys/rows per
node instead of only millions
@@ -7592,7 +7755,6 @@
* Rename configuration "table" to "keyspace"
* Moved to crash-only design; no more shutdown (just kill the process)
* Lots of bug fixes
-
Full list of issues resolved in 0.4 is at https://issues.apache.org/jira/secure/IssueNavigator.jspa?reset=true&&pid=12310865&fixfor=12313862&resolution=1&sorter/field=issuekey&sorter/order=DESC
@@ -7623,3 +7785,5 @@
* Combined blocking and non-blocking versions of insert APIs
* Added FlushPeriodInMinutes configuration parameter to force
flushing of infrequently-updated ColumnFamilies
+
+
diff --git a/NEWS.txt b/NEWS.txt
index 0930ac0..2129ead 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -52,28 +52,147 @@
using the provided 'sstableupgrade' tool.
-4.1.1
-=====
+5.0
+===
-G1GC Recommended
-----------------
- - The G1 settings in jvm8-server.options and jvm11-server.options are updated according to broad feedback
- and testing. The G1 settings remain commented out by default in 4.1.x. It is recommended to switch
- to G1 for performance and for simpler GC tuning. CMS is already deprecated in JDK9, and the next major
- release of Cassandra makes G1 the default configuration.
+New features
+------------
+ - Added new Mathematical CQL functions: abs, exp, log, log10 and round.
+ - Added a trie-based memtable implementation, which improves memory use, garbage collection efficiency and lookup
+ performance. The new memtable is implemented by the TrieMemtable class and can be selected using the memtable
+ API, see src/java/org/apache/cassandra/db/memtable/Memtable_API.md.
+ - Added a new trie-indexed SSTable format with better lookup efficiency and size. The new format removes the index
+ summary component and does not require key caching. Additionally, it is able to efficiently search in partitions
+ spanning thousands or millions of rows.
+ The format is applied by setting "bti" as the selected sstable format in cassandra.yaml's sstables option.
+ - Added a new configuration cdc_on_repair_enabled to toggle whether CDC mutations are replayed through the
+ write path on streaming, e.g. repair. When enabled, CDC data streamed to the destination node will be written into
+ commit log first. When disabled, the streamed CDC data is written into SSTables just the same as normal streaming.
+ If this is set to false, streaming will be considerably faster however it's possible that, in extreme situations
+ (losing > quorum # nodes in a replica set), you may have data in your SSTables that never makes it to the CDC log.
+ The default is true/enabled. The configuration can be altered via JMX.
+ - Added support for reading the write times and TTLs of the elements of collections and UDTs, regardless of being
+ frozen or not. The CQL functions writetime, maxwritetime and ttl can now be applied to entire collections/UDTs,
+ single collection/UDT elements and slices of collection/UDT elements.
+ - Added a new CQL function, maxwritetime. It shows the largest unix timestamp that the data was written, similar to
+ its sibling CQL function, writetime.
+ - New Guardrails added:
+ - Whether ALTER TABLE commands are allowed to mutate columns
+ - Whether SimpleStrategy is allowed on keyspace creation or alteration
+ - Maximum replication factor
+ - Whether DROP KEYSPACE commands are allowed.
+ - Column value size
+ - Partition size
+ - It is possible to list ephemeral snapshots by nodetool listsnaphots command when flag "-e" is specified.
+ - Added a new flag to `nodetool profileload` and JMX endpoint to set up recurring profile load generation on specified
+ intervals (see CASSANDRA-17821)
+ - Added a new property, gossiper.loose_empty_enabled, to allow for a looser definition of "empty" when
+ considering the heartbeat state of another node in Gossip. This should only be used by knowledgeable
+ operators in the following scenarios:
+
+ Currently "empty" w/regards to heartbeat state in Gossip is very specific to a single edge case (i.e. in
+ isEmptyWithoutStatus() our usage of hbState() + applicationState), however there are other failure cases which
+ block host replacements and require intrusive workarounds and human intervention to recover from when you
+ have something in hbState() you don't expect. See CASSANDRA-17842 for further details.
+ - Added new CQL table property 'allow_auto_snapshot' which is by default true. When set to false and 'auto_snapshot: true'
+ in cassandra.yaml, there will be no snapshot taken when a table is truncated or dropped. When auto_snapshot in
+ casandra.yaml is set to false, the newly added table property does not have any effect.
+ - Changed default on resumable bootstrap to be disabled. Resumable bootstrap has edge cases with potential correctness
+ violations or data loss scenarios if nodes go down during bootstrap, tombstones are written, and operations race with
+ repair. As streaming is considerably faster in the 4.0+ era (as well as with zero copy streaming), the risks of
+ having these edge cases during a failed and resumed bootstrap are no longer deemed acceptable.
+ To re-enable this feature, use the -Dcassandra.reset_bootstrap_progress=false environment flag.
+ - Added --older-than and --older-than-timestamp options to nodetool clearsnapshot command. It is possible to
+ clear snapshots which are older than some period for example, "--older-than 5h" to remove
+ snapshots older than 5 hours and it is possible to clear all snapshots older than some timestamp, for example
+ --older-than-timestamp 2022-12-03T10:15:30Z.
+ - Cassandra logs can be viewed in the virtual table system_views.system_logs.
+ Please uncomment the respective appender in logback.xml file to make logs flow into this table. This feature is turned off by default.
+ - Added new CQL table property 'incremental_backups' which is by default true. When 'incremental_backups' property in cassandra.yaml
+ is set to true and table property is set to false, incremental backups for that specific table will not be done.
+ When 'incremental_backups' in casandra.yaml is set to false, the newly added table property does not have any effect.
+ Both properties have to be set to true (cassandra.yaml and table property) in order to make incremental backups.
+ - Added new CQL native scalar functions for collections. The new functions are mostly analogous to the existing
+ aggregation functions, but they operate on the elements of collection columns. The new functions are `map_keys`,
+ `map_values`, `collection_count`, `collection_min`, `collection_max`, `collection_sum` and `collection_avg`.
+ - Added compaction_properties column to system.compaction_history table and nodetool compactionhistory command
+ - SimpleSeedProvider can resolve multiple IP addresses per DNS record. SimpleSeedProvider reacts on
+ the paramater called `resolve_multiple_ip_addresses_per_dns_record` which value is meant to be boolean and by
+ default it is set to false. When set to true, SimpleSeedProvider will resolve all IP addresses per DNS record,
+ based on the configured name service on the system.
+ - Added new native CQL functions for data masking, allowing to replace or obscure sensitive data. The functions are:
+ - `mask_null` replaces the column value by null.
+ - `mask_default` replaces the data by a fixed default value of the same type.
+ - `mask_replace` replaces the data by a custom value.
+ - `mask_inner` replaces every character but the first and last ones by a fixed character.
+ - `mask_outer` replaces the first and last characters by a fixed character.
+ - `mask_hash` replaces the data by its hash, according to the specified algorithm.
+ - On virtual tables, it is not strictly necessary to specify `ALLOW FILTERING` for select statements which would
+ normally require it, except `system_views.system_logs`.
+ - More accurate skipping of sstables in read path due to better handling of min/max clustering and lower bound;
+ SSTable format has been bumped to 'nc' because there are new fields in stats metadata\
+ - Added MaxSSTableSize and MaxSSTableDuration metrics to TableMetrics. The former returns the size of the biggest
+ SSTable of a table or 0 when there is not any SSTable. The latter returns the maximum duration, computed as
+ `maxTimestamp - minTimestamp`, effectively non-zero for SSTables produced by TimeWindowCompactionStrategy.
+ - Added local read/write ratio to tablestats.
+ - Added system_views.max_sstable_size and system_views.max_sstable_duration tables.
+ - Added virtual table system_views.snapshots to see all snapshots from CQL shell.
+ - Added support for attaching CQL dynamic data masking functions to table columns on the schema. These masking
+ functions can be attached to or dettached from columns with CREATE/ALTER TABLE statements. The functions obscure
+ the masked data during queries, but they don't change the stored data.
+ - Added new UNMASK permission. It allows to see the clear data of columns with an attached mask. Superusers have it
+ by default, whereas regular users don't have it by default.
+ - Added new SELECT_MASKED permission. It allows to run SELECT queries selecting the clear values of masked columns.
+ Superusers have it by default, whereas regular users don't have it by default.
+ - Added support for using UDFs as masking functions attached to table columns on the schema.
+ - Added `sstablepartitions` offline tool to find large partitions in sstables.
+ - `cassandra-stress` has a new option called '-jmx' which enables a user to pass username and password to JMX (CASSANDRA-18544)
+ - It is possible to read all credentials for `cassandra-stress` from a file via option `-credentials-file` (CASSANDRA-18544)
Upgrading
---------
+ - Ephemeral marker files for snapshots done by repairs are not created anymore,
+ there is a dedicated flag in snapshot manifest instead. On upgrade of a node to this version, on node's start, in case there
+ are such ephemeral snapshots on disk, they will be deleted (same behaviour as before) and any new ephemeral snapshots
+ will stop to create ephemeral marker files as flag in a snapshot manifest was introduced instead.
+ - There were new table properties introduced called 'allow_auto_snapshot' and 'incremental_backups' (see section 'New features'). Hence, upgraded
+ node will be on a new schema version. Please do a rolling upgrade of nodes of a cluster to converge to one schema version.
- All previous versions of 4.x contained a mistake on the implementation of the old CQL native protocol v3. That
mistake produced issues when paging over tables with compact storage and a single clustering column during rolling
upgrades involving 3.x and 4.x nodes. The fix for that issue makes it can now appear during rolling upgrades from
4.1.0 or 4.0.0-4.0.7. If that is your case, please use protocol v4 or higher in your driver. See CASSANDRA-17507
for further details.
+ - Added API for alternative sstable implementations. For details, see src/java/org/apache/cassandra/io/sstable/SSTable_API.md
+ - DateTieredCompactionStrategy was removed. Please change the compaction strategy for the tables using this strategy
+ to TimeWindowCompactionStrategy before upgrading to this version.
+ - The deprecated functions `dateOf` and `unixTimestampOf` have been removed. They were deprecated and replaced by
+ `toTimestamp` and `toUnixTimestamp` in Cassandra 2.2.
+ - Hadoop integration is no longer available (CASSANDRA-18323). If you want to process Cassandra data by big data frameworks,
+ please upgrade your infrastructure to use Cassandra Spark connector.
+ - Keystore/truststore password configurations are nullable now and the code defaults of those passwords to 'cassandra' are
+ removed. Any deployments that depend upon the code default to this password value without explicitly specifying
+ it in cassandra.yaml will fail on upgrade. Please specify your keystore_password and truststore_password elements in cassandra.yaml with appropriate
+ values to prevent this failure.
Deprecation
-----------
- - Hadoop integration in package org.apache.cassandra.hadoop is deprecated and no longer actively maintained.
- This code is scheduled to be removed in the next major version of Cassandra.
+ - In the JMX MBean `org.apache.cassandra.db:type=RepairService` (CASSANDRA-17668):
+ - deprecate the getter/setter methods `getRepairSessionSpaceInMebibytes` and `setRepairSessionSpaceInMebibytes`
+ in favor of `getRepairSessionSpaceInMiB` and `setRepairSessionSpaceInMiB` respectively
+ - In the JMX MBean `org.apache.cassandra.db:type=StorageService` (CASSANDRA-17668):
+ - deprecate the getter/setter methods `getRepairSessionMaxTreeDepth` and `setRepairSessionMaxTreeDepth`
+ in favor of `getRepairSessionMaximumTreeDepth` and `setRepairSessionMaximumTreeDepth`
+ - deprecate the setter method `setColumnIndexSize` in favor of `setColumnIndexSizeInKiB`
+ - deprecate the getter/setter methods `getColumnIndexCacheSize` and `setColumnIndexCacheSize` in favor of
+ `getColumnIndexCacheSizeInKiB` and `setColumnIndexCacheSizeInKiB` respectively
+ - deprecate the getter/setter methods `getBatchSizeWarnThreshold` and `setBatchSizeWarnThreshold` in favor of
+ `getBatchSizeWarnThresholdInKiB` and `setBatchSizeWarnThresholdInKiB` respectively
+ - All native CQL functions names that don't use the snake case names are deprecated in favour of equivalent names
+ using snake casing. Thus, `totimestamp` is deprecated in favour of `to_timestamp`, `intasblob` in favour
+ of `int_as_blob`, `castAsInt` in favour of `cast_as_int`, etc.
+ - The config property `compaction_large_partition_warning_threshold` has been deprecated in favour of the new
+ guardrail for partition size. That guardrail is based on the properties `partition_size_warn_threshold` and
+ `partition_size_fail_threshold`. The warn threshold has a very similar behaviour to the old config property.
4.1
===
@@ -416,7 +535,7 @@
- Native protocol v5 is promoted from beta in this release. The wire format has changed
significantly and users should take care to ensure client drivers are upgraded to a version
with support for the final v5 format, if currently connecting over v5-beta. (CASSANDRA-15299, CASSANDRA-14973)
- - Cassandra removed support for the OldNetworkTopologyStrategy. Before upgrading you will need to change the
+ - Cassandra removed support for the OldNetworkTopologyStrategy. Before upgrading you will need to change the
replication strategy for the keyspaces using this strategy to the NetworkTopologyStrategy. (CASSANDRA-13990)
- Sstables for tables using with a frozen UDT written by C* 3.0 appear as corrupted.
@@ -559,7 +678,7 @@
the node will not start. See CASSANDRA-14477 for details.
- CASSANDRA-13701 To give a better out of the box experience, the default 'num_tokens'
value has been changed from 256 to 16 for reasons described in
- https://cassandra.apache.org/doc/latest/getting_started/production.html#tokens
+ https://cassandra.apache.org/doc/latest/getting-started/production.html#tokens
'allocate_tokens_for_local_replication_factor' is also uncommented and set to 3.
Please note when upgrading that if the 'num_tokens' value is different than what you have
configured, the upgraded node will refuse to start. Also note that if a new node joining
@@ -614,7 +733,7 @@
reason, a opt-in system property has been added to disable the fix:
-Dcassandra.unsafe.disable-serial-reads-linearizability=true
Use this flag at your own risk as it revert SERIAL reads to the incorrect behavior of
- previous versions. See CASSANDRA-12126 for details.
+ previous versions. See CASSANDRA-12126 for details.
- SASI's `max_compaction_flush_memory_in_mb` setting was previously getting interpreted in bytes. From 3.11.8
it is correctly interpreted in megabytes, but prior to 3.11.10 previous configurations of this setting will
lead to nodes OOM during compaction. From 3.11.10 previous configurations will be detected as incorrect,
@@ -711,7 +830,7 @@
Starting version 5.0, COMPACT STORAGE will no longer be supported.
'ALTER ... DROP COMPACT STORAGE' statement makes Compact Tables CQL-compatible,
exposing internal structure of Thrift/Compact Tables. You can find more details
- on exposed internal structure under:
+ on exposed internal structure under:
http://cassandra.apache.org/doc/latest/cql/appendices.html#appendix-c-dropping-compact-storage
For uninterrupted cluster upgrades, drivers now support 'NO_COMPACT' startup option.
diff --git a/README.asc b/README.asc
index f484aa2..16d5e09 100644
--- a/README.asc
+++ b/README.asc
@@ -9,6 +9,8 @@
For more information, see http://cassandra.apache.org/[the Apache Cassandra web site].
+Issues should be reported on https://issues.apache.org/jira/projects/CASSANDRA/issues/[The Cassandra Jira].
+
Requirements
------------
. Java >= 1.8 (OpenJDK and Oracle JVMS have been tested)
@@ -18,7 +20,7 @@
---------------
This short guide will walk you through getting a basic one node cluster up
-and running, and demonstrate some simple reads and writes. For a more-complete guide, please see the Apache Cassandra website's http://cassandra.apache.org/doc/latest/getting_started/[Getting Started Guide].
+and running, and demonstrate some simple reads and writes. For a more-complete guide, please see the Apache Cassandra website's http://cassandra.apache.org/doc/latest/getting-started/[Getting Started Guide].
First, we'll unpack our archive:
@@ -39,7 +41,7 @@
----
Connected to Test Cluster at localhost:9160.
-[cqlsh 6.0.0 | Cassandra 4.1 | CQL spec 3.4.6 | Native protocol v5]
+[cqlsh 6.2.0 | Cassandra 5.0-SNAPSHOT | CQL spec 3.4.7 | Native protocol v5]
Use HELP for help.
cqlsh>
----
@@ -76,8 +78,10 @@
Wondering where to go from here?
- * Join us in #cassandra on the https://s.apache.org/slack-invite[ASF Slack] and ask questions
+ * Join us in #cassandra on the https://s.apache.org/slack-invite[ASF Slack] and ask questions.
* Subscribe to the Users mailing list by sending a mail to
- user-subscribe@cassandra.apache.org
+ user-subscribe@cassandra.apache.org.
+ * Subscribe to the Developer mailing list by sending a mail to
+ dev-subscribe@cassandra.apache.org.
* Visit the http://cassandra.apache.org/community/[community section] of the Cassandra website for more information on getting involved.
* Visit the http://cassandra.apache.org/doc/latest/development/index.html[development section] of the Cassandra website for more information on how to contribute.
diff --git a/TESTING.md b/TESTING.md
index 0f25743..b9c5c7a 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -364,14 +364,14 @@
**Example, alternative**
-```javayy
+```java
class SomeVerbHandler implements IVerbHandler<SomeMessage>
{
@VisibleForTesting
protected boolean isAlive(InetAddress addr) { return FailureDetector.instance.isAlive(msg.payload.otherNode); }
@VisibleForTesting
- protected void streamSomethind(InetAddress to) { new StreamPlan(to).requestRanges(someRanges).execute(); }
+ protected void streamSomething(InetAddress to) { new StreamPlan(to).requestRanges(someRanges).execute(); }
@VisibleForTesting
protected void compactSomething(ColumnFamilyStore cfs ) { CompactionManager.instance.submitBackground(); }
@@ -404,7 +404,7 @@
protected boolean isAlive(InetAddress addr) { return alive; }
@Override
- protected void streamSomethind(InetAddress to) { streamCalled = true; }
+ protected void streamSomething(InetAddress to) { streamCalled = true; }
@Override
protected void compactSomething(ColumnFamilyStore cfs ) { compactCalled = true; }
diff --git a/bin/cassandra.in.sh b/bin/cassandra.in.sh
index dcbd12d..dfbb959 100644
--- a/bin/cassandra.in.sh
+++ b/bin/cassandra.in.sh
@@ -114,8 +114,9 @@
java_ver_output=`"${JAVA:-java}" -version 2>&1`
jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
JVM_VERSION=${jvmver%_*}
+short=$(echo "${jvmver}" | cut -c1-2)
-JAVA_VERSION=11
+JAVA_VERSION=17
if [ "$JVM_VERSION" = "1.8.0" ] ; then
JVM_PATCH_VERSION=${jvmver#*_}
if [ "$JVM_VERSION" \< "1.8" ] || [ "$JVM_VERSION" \> "1.8.2" ] ; then
@@ -130,6 +131,11 @@
elif [ "$JVM_VERSION" \< "11" ] ; then
echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
exit 1;
+elif [ "$short" = "11" ] ; then
+ JAVA_VERSION=11
+elif [ "$JVM_VERSION" \< "17" ] ; then
+ echo "Cassandra 5.0 requires Java 11 or Java 17."
+ exit 1;
fi
jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
@@ -153,7 +159,9 @@
# Read user-defined JVM options from jvm-server.options file
JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
-if [ $JAVA_VERSION -ge 11 ] ; then
+if [ $JAVA_VERSION -ge 17 ] ; then
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm17${jvmoptions_variant:--clients}.options
+elif [ $JAVA_VERSION -ge 11 ] ; then
JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
else
JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm8${jvmoptions_variant:--clients}.options
diff --git a/bin/cqlsh.py b/bin/cqlsh.py
index 6c1e7bd..0b0ee39 100755
--- a/bin/cqlsh.py
+++ b/bin/cqlsh.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,26 +16,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import cmd
-import codecs
-import configparser
-import csv
-import errno
-import getpass
-import optparse
import os
import platform
-import re
-import stat
-import subprocess
import sys
-import traceback
-import warnings
-import webbrowser
-from contextlib import contextmanager
from glob import glob
-from io import StringIO
-from uuid import UUID
if sys.version_info < (3, 6):
sys.exit("\ncqlsh requires Python 3.6+\n")
@@ -44,50 +28,9 @@
if platform.python_implementation().startswith('Jython'):
sys.exit("\nCQL Shell does not run on Jython\n")
-UTF8 = 'utf-8'
-
-description = "CQL Shell for Apache Cassandra"
-version = "6.1.0"
-
-readline = None
-try:
- # check if tty first, cause readline doesn't check, and only cares
- # about $TERM. we don't want the funky escape code stuff to be
- # output if not a tty.
- if sys.stdin.isatty():
- import readline
-except ImportError:
- pass
-
CQL_LIB_PREFIX = 'cassandra-driver-internal-only-'
CASSANDRA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
-CASSANDRA_CQL_HTML_FALLBACK = 'https://cassandra.apache.org/doc/latest/cql/index.html'
-
-# default location of local CQL.html
-if os.path.exists(CASSANDRA_PATH + '/doc/cql3/CQL.html'):
- # default location of local CQL.html
- CASSANDRA_CQL_HTML = 'file://' + CASSANDRA_PATH + '/doc/cql3/CQL.html'
-elif os.path.exists('/usr/share/doc/cassandra/CQL.html'):
- # fallback to package file
- CASSANDRA_CQL_HTML = 'file:///usr/share/doc/cassandra/CQL.html'
-else:
- # fallback to online version
- CASSANDRA_CQL_HTML = CASSANDRA_CQL_HTML_FALLBACK
-
-# On Linux, the Python webbrowser module uses the 'xdg-open' executable
-# to open a file/URL. But that only works, if the current session has been
-# opened from _within_ a desktop environment. I.e. 'xdg-open' will fail,
-# if the session's been opened via ssh to a remote box.
-#
-try:
- webbrowser.register_standard_browsers() # registration is otherwise lazy in Python3
-except AttributeError:
- pass
-if webbrowser._tryorder and webbrowser._tryorder[0] == 'xdg-open' and os.environ.get('XDG_DATA_DIRS', '') == '':
- # only on Linux (some OS with xdg-open)
- webbrowser._tryorder.remove('xdg-open')
- webbrowser._tryorder.append('xdg-open')
# use bundled lib for python-cql if available. if there
# is a ../lib dir, use bundled libs there preferentially.
@@ -113,14 +56,13 @@
sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
# the driver needs dependencies
-third_parties = ('six-', 'pure_sasl-')
+third_parties = ('six-', 'pure_sasl-', 'wcwidth-')
for lib in third_parties:
lib_zip = find_zip(lib)
if lib_zip:
sys.path.insert(0, lib_zip)
-warnings.filterwarnings("ignore", r".*blist.*")
try:
import cassandra
except ImportError as e:
@@ -130,14 +72,6 @@
'Module load path: %r\n\n'
'Error: %s\n' % (sys.executable, sys.path, e))
-from cassandra.auth import PlainTextAuthProvider
-from cassandra.cluster import Cluster
-from cassandra.cqltypes import cql_typename
-from cassandra.marshal import int64_unpack
-from cassandra.metadata import (ColumnMetadata, KeyspaceMetadata, TableMetadata)
-from cassandra.policies import WhiteListRoundRobinPolicy
-from cassandra.query import SimpleStatement, ordered_dict_factory, TraceUnavailable
-from cassandra.util import datetime_from_timestamp
# cqlsh should run correctly when run out of a Cassandra source tree,
# out of an unpacked Cassandra tarball, and after a proper package install.
@@ -145,2245 +79,10 @@
if os.path.isdir(cqlshlibdir):
sys.path.insert(0, cqlshlibdir)
-from cqlshlib import cql3handling, pylexotron, sslhandling, cqlshhandling, authproviderhandling
-from cqlshlib.copyutil import ExportTask, ImportTask
-from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
- RED, WHITE, FormattedValue, colorme)
-from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
- DEFAULT_TIMESTAMP_FORMAT, CqlType, DateTimeFormat,
- format_by_type)
-from cqlshlib.tracing import print_trace, print_trace_session
-from cqlshlib.util import get_file_encoding_bomsize
-from cqlshlib.util import is_file_secure
-
-
-DEFAULT_HOST = '127.0.0.1'
-DEFAULT_PORT = 9042
-DEFAULT_SSL = False
-DEFAULT_CONNECT_TIMEOUT_SECONDS = 5
-DEFAULT_REQUEST_TIMEOUT_SECONDS = 10
-
-DEFAULT_FLOAT_PRECISION = 5
-DEFAULT_DOUBLE_PRECISION = 5
-DEFAULT_MAX_TRACE_WAIT = 10
-
-if readline is not None and readline.__doc__ is not None and 'libedit' in readline.__doc__:
- DEFAULT_COMPLETEKEY = '\t'
-else:
- DEFAULT_COMPLETEKEY = 'tab'
-
-cqldocs = None
-cqlruleset = None
-
-epilog = """Connects to %(DEFAULT_HOST)s:%(DEFAULT_PORT)d by default. These
-defaults can be changed by setting $CQLSH_HOST and/or $CQLSH_PORT. When a
-host (and optional port number) are given on the command line, they take
-precedence over any defaults.""" % globals()
-
-parser = optparse.OptionParser(description=description, epilog=epilog,
- usage="Usage: %prog [options] [host [port]]",
- version='cqlsh ' + version)
-parser.add_option("-C", "--color", action='store_true', dest='color',
- help='Always use color output')
-parser.add_option("--no-color", action='store_false', dest='color',
- help='Never use color output')
-parser.add_option("--browser", dest='browser', help="""The browser to use to display CQL help, where BROWSER can be:
- - one of the supported browsers in https://docs.python.org/3/library/webbrowser.html.
- - browser path followed by %s, example: /usr/bin/google-chrome-stable %s""")
-parser.add_option('--ssl', action='store_true', help='Use SSL', default=False)
-parser.add_option("-u", "--username", help="Authenticate as user.")
-parser.add_option("-p", "--password", help="Authenticate using password.")
-parser.add_option('-k', '--keyspace', help='Authenticate to the given keyspace.')
-parser.add_option("-f", "--file", help="Execute commands from FILE, then exit")
-parser.add_option('--debug', action='store_true',
- help='Show additional debugging information')
-parser.add_option('--coverage', action='store_true',
- help='Collect coverage data')
-parser.add_option("--encoding", help="Specify a non-default encoding for output."
- + " (Default: %s)" % (UTF8,))
-parser.add_option("--cqlshrc", help="Specify an alternative cqlshrc file location.")
-parser.add_option("--credentials", help="Specify an alternative credentials file location.")
-parser.add_option('--cqlversion', default=None,
- help='Specify a particular CQL version, '
- 'by default the highest version supported by the server will be used.'
- ' Examples: "3.0.3", "3.1.0"')
-parser.add_option("--protocol-version", type="int", default=None,
- help='Specify a specific protcol version otherwise the client will default and downgrade as necessary')
-
-parser.add_option("-e", "--execute", help='Execute the statement and quit.')
-parser.add_option("--connect-timeout", default=DEFAULT_CONNECT_TIMEOUT_SECONDS, dest='connect_timeout',
- help='Specify the connection timeout in seconds (default: %default seconds).')
-parser.add_option("--request-timeout", default=DEFAULT_REQUEST_TIMEOUT_SECONDS, dest='request_timeout',
- help='Specify the default request timeout in seconds (default: %default seconds).')
-parser.add_option("-t", "--tty", action='store_true', dest='tty',
- help='Force tty mode (command prompt).')
-parser.add_option('-v', action="version", help='Print the current version of cqlsh.')
-
-# This is a hidden option to suppress the warning when the -p/--password command line option is used.
-# Power users may use this option if they know no other people has access to the system where cqlsh is run or don't care about security.
-# Use of this option in scripting is discouraged. Please use a (temporary) credentials file where possible.
-# The Cassandra distributed tests (dtests) also use this option in some tests when a well-known password is supplied via the command line.
-parser.add_option("--insecure-password-without-warning", action='store_true', dest='insecure_password_without_warning',
- help=optparse.SUPPRESS_HELP)
-
-opt_values = optparse.Values()
-(options, arguments) = parser.parse_args(sys.argv[1:], values=opt_values)
-
-# BEGIN history/config definition
-
-
-def mkdirp(path):
- """Creates all parent directories up to path parameter or fails when path exists, but it is not a directory."""
-
- try:
- os.makedirs(path)
- except OSError:
- if not os.path.isdir(path):
- raise
-
-
-def resolve_cql_history_file():
- default_cql_history = os.path.expanduser(os.path.join('~', '.cassandra', 'cqlsh_history'))
- if 'CQL_HISTORY' in os.environ:
- return os.environ['CQL_HISTORY']
- else:
- return default_cql_history
-
-
-HISTORY = resolve_cql_history_file()
-HISTORY_DIR = os.path.dirname(HISTORY)
-
-try:
- mkdirp(HISTORY_DIR)
-except OSError:
- print('\nWarning: Cannot create directory at `%s`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.\n' % HISTORY_DIR)
-
-DEFAULT_CQLSHRC = os.path.expanduser(os.path.join('~', '.cassandra', 'cqlshrc'))
-
-if hasattr(options, 'cqlshrc'):
- CONFIG_FILE = os.path.expanduser(options.cqlshrc)
- if not os.path.exists(CONFIG_FILE):
- print('\nWarning: Specified cqlshrc location `%s` does not exist. Using `%s` instead.\n' % (CONFIG_FILE, DEFAULT_CQLSHRC))
- CONFIG_FILE = DEFAULT_CQLSHRC
-else:
- CONFIG_FILE = DEFAULT_CQLSHRC
-
-CQL_DIR = os.path.dirname(CONFIG_FILE)
-
-CQL_ERRORS = (
- cassandra.AlreadyExists, cassandra.AuthenticationFailed, cassandra.CoordinationFailure,
- cassandra.InvalidRequest, cassandra.Timeout, cassandra.Unauthorized, cassandra.OperationTimedOut,
- cassandra.cluster.NoHostAvailable,
- cassandra.connection.ConnectionBusy, cassandra.connection.ProtocolError, cassandra.connection.ConnectionException,
- cassandra.protocol.ErrorMessage, cassandra.protocol.InternalError, cassandra.query.TraceUnavailable
-)
-
-debug_completion = bool(os.environ.get('CQLSH_DEBUG_COMPLETION', '') == 'YES')
-
-
-class NoKeyspaceError(Exception):
- pass
-
-
-class KeyspaceNotFound(Exception):
- pass
-
-
-class ColumnFamilyNotFound(Exception):
- pass
-
-
-class IndexNotFound(Exception):
- pass
-
-
-class MaterializedViewNotFound(Exception):
- pass
-
-
-class ObjectNotFound(Exception):
- pass
-
-
-class VersionNotSupported(Exception):
- pass
-
-
-class UserTypeNotFound(Exception):
- pass
-
-
-class FunctionNotFound(Exception):
- pass
-
-
-class AggregateNotFound(Exception):
- pass
-
-
-class DecodeError(Exception):
- verb = 'decode'
-
- def __init__(self, thebytes, err, colname=None):
- self.thebytes = thebytes
- self.err = err
- self.colname = colname
-
- def __str__(self):
- return str(self.thebytes)
-
- def message(self):
- what = 'value %r' % (self.thebytes,)
- if self.colname is not None:
- what = 'value %r (for column %r)' % (self.thebytes, self.colname)
- return 'Failed to %s %s : %s' \
- % (self.verb, what, self.err)
-
- def __repr__(self):
- return '<%s %s>' % (self.__class__.__name__, self.message())
-
-
-def maybe_ensure_text(val):
- return str(val) if val else val
-
-
-class FormatError(DecodeError):
- verb = 'format'
-
-
-def full_cql_version(ver):
- while ver.count('.') < 2:
- ver += '.0'
- ver_parts = ver.split('-', 1) + ['']
- vertuple = tuple(list(map(int, ver_parts[0].split('.'))) + [ver_parts[1]])
- return ver, vertuple
-
-
-def format_value(val, cqltype, encoding, addcolor=False, date_time_format=None,
- float_precision=None, colormap=None, nullval=None):
- if isinstance(val, DecodeError):
- if addcolor:
- return colorme(repr(val.thebytes), colormap, 'error')
- else:
- return FormattedValue(repr(val.thebytes))
- return format_by_type(val, cqltype=cqltype, encoding=encoding, colormap=colormap,
- addcolor=addcolor, nullval=nullval, date_time_format=date_time_format,
- float_precision=float_precision)
-
-
-def show_warning_without_quoting_line(message, category, filename, lineno, file=None, line=None):
- if file is None:
- file = sys.stderr
- try:
- file.write(warnings.formatwarning(message, category, filename, lineno, line=''))
- except IOError:
- pass
-
-
-warnings.showwarning = show_warning_without_quoting_line
-warnings.filterwarnings('always', category=cql3handling.UnexpectedTableStructure)
-
-
-def insert_driver_hooks():
-
- class DateOverFlowWarning(RuntimeWarning):
- pass
-
- # Native datetime types blow up outside of datetime.[MIN|MAX]_YEAR. We will fall back to an int timestamp
- def deserialize_date_fallback_int(byts, protocol_version):
- timestamp_ms = int64_unpack(byts)
- try:
- return datetime_from_timestamp(timestamp_ms / 1000.0)
- except OverflowError:
- warnings.warn(DateOverFlowWarning("Some timestamps are larger than Python datetime can represent. "
- "Timestamps are displayed in milliseconds from epoch."))
- return timestamp_ms
-
- cassandra.cqltypes.DateType.deserialize = staticmethod(deserialize_date_fallback_int)
-
- if hasattr(cassandra, 'deserializers'):
- del cassandra.deserializers.DesDateType
-
- # Return cassandra.cqltypes.EMPTY instead of None for empty values
- cassandra.cqltypes.CassandraType.support_empty_values = True
-
-
-class Shell(cmd.Cmd):
- custom_prompt = os.getenv('CQLSH_PROMPT', '')
- if custom_prompt != '':
- custom_prompt += "\n"
- default_prompt = custom_prompt + "cqlsh> "
- continue_prompt = " ... "
- keyspace_prompt = custom_prompt + "cqlsh:{}> "
- keyspace_continue_prompt = "{} ... "
- show_line_nums = False
- debug = False
- coverage = False
- coveragerc_path = None
- stop = False
- last_hist = None
- shunted_query_out = None
- use_paging = True
-
- default_page_size = 100
-
- def __init__(self, hostname, port, color=False,
- username=None, encoding=None, stdin=None, tty=True,
- completekey=DEFAULT_COMPLETEKEY, browser=None, use_conn=None,
- cqlver=None, keyspace=None,
- tracing_enabled=False, expand_enabled=False,
- display_nanotime_format=DEFAULT_NANOTIME_FORMAT,
- display_timestamp_format=DEFAULT_TIMESTAMP_FORMAT,
- display_date_format=DEFAULT_DATE_FORMAT,
- display_float_precision=DEFAULT_FLOAT_PRECISION,
- display_double_precision=DEFAULT_DOUBLE_PRECISION,
- display_timezone=None,
- max_trace_wait=DEFAULT_MAX_TRACE_WAIT,
- ssl=False,
- single_statement=None,
- request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,
- protocol_version=None,
- connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
- is_subshell=False,
- auth_provider=None):
- cmd.Cmd.__init__(self, completekey=completekey)
- self.hostname = hostname
- self.port = port
- self.auth_provider = auth_provider
- self.username = username
-
- if isinstance(auth_provider, PlainTextAuthProvider):
- self.username = auth_provider.username
- if not auth_provider.password:
- # if no password is provided, we need to query the user to get one.
- password = getpass.getpass()
- self.auth_provider = PlainTextAuthProvider(username=auth_provider.username, password=password)
-
- self.keyspace = keyspace
- self.ssl = ssl
- self.tracing_enabled = tracing_enabled
- self.page_size = self.default_page_size
- self.expand_enabled = expand_enabled
- if use_conn:
- self.conn = use_conn
- else:
- kwargs = {}
- if protocol_version is not None:
- kwargs['protocol_version'] = protocol_version
- self.conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=cqlver,
- auth_provider=self.auth_provider,
- ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None,
- load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
- control_connection_timeout=connect_timeout,
- connect_timeout=connect_timeout,
- **kwargs)
- self.owns_connection = not use_conn
-
- if keyspace:
- self.session = self.conn.connect(keyspace)
- else:
- self.session = self.conn.connect()
-
- if browser == "":
- browser = None
- self.browser = browser
- self.color = color
-
- self.display_nanotime_format = display_nanotime_format
- self.display_timestamp_format = display_timestamp_format
- self.display_date_format = display_date_format
-
- self.display_float_precision = display_float_precision
- self.display_double_precision = display_double_precision
-
- self.display_timezone = display_timezone
-
- self.session.default_timeout = request_timeout
- self.session.row_factory = ordered_dict_factory
- self.session.default_consistency_level = cassandra.ConsistencyLevel.ONE
- self.get_connection_versions()
- self.set_expanded_cql_version(self.connection_versions['cql'])
-
- self.current_keyspace = keyspace
-
- self.max_trace_wait = max_trace_wait
- self.session.max_trace_wait = max_trace_wait
-
- self.tty = tty
- self.encoding = encoding
-
- self.output_codec = codecs.lookup(encoding)
-
- self.statement = StringIO()
- self.lineno = 1
- self.in_comment = False
-
- self.prompt = ''
- if stdin is None:
- stdin = sys.stdin
-
- if tty:
- self.reset_prompt()
- self.report_connection()
- print('Use HELP for help.')
- else:
- self.show_line_nums = True
- self.stdin = stdin
- self.query_out = sys.stdout
- self.consistency_level = cassandra.ConsistencyLevel.ONE
- self.serial_consistency_level = cassandra.ConsistencyLevel.SERIAL
-
- self.empty_lines = 0
- self.statement_error = False
- self.single_statement = single_statement
- self.is_subshell = is_subshell
-
- @property
- def batch_mode(self):
- return not self.tty
-
- def set_expanded_cql_version(self, ver):
- ver, vertuple = full_cql_version(ver)
- self.cql_version = ver
- self.cql_ver_tuple = vertuple
-
- def cqlver_atleast(self, major, minor=0, patch=0):
- return self.cql_ver_tuple[:3] >= (major, minor, patch)
-
- def myformat_value(self, val, cqltype=None, **kwargs):
- if isinstance(val, DecodeError):
- self.decoding_errors.append(val)
- try:
- dtformats = DateTimeFormat(timestamp_format=self.display_timestamp_format,
- date_format=self.display_date_format, nanotime_format=self.display_nanotime_format,
- timezone=self.display_timezone)
- precision = self.display_double_precision if cqltype is not None and cqltype.type_name == 'double' \
- else self.display_float_precision
- return format_value(val, cqltype=cqltype, encoding=self.output_codec.name,
- addcolor=self.color, date_time_format=dtformats,
- float_precision=precision, **kwargs)
- except Exception as e:
- err = FormatError(val, e)
- self.decoding_errors.append(err)
- return format_value(err, cqltype=cqltype, encoding=self.output_codec.name, addcolor=self.color)
-
- def myformat_colname(self, name, table_meta=None):
- column_colors = COLUMN_NAME_COLORS.copy()
- # check column role and color appropriately
- if table_meta:
- if name in [col.name for col in table_meta.partition_key]:
- column_colors.default_factory = lambda: RED
- elif name in [col.name for col in table_meta.clustering_key]:
- column_colors.default_factory = lambda: CYAN
- elif name in table_meta.columns and table_meta.columns[name].is_static:
- column_colors.default_factory = lambda: WHITE
- return self.myformat_value(name, colormap=column_colors)
-
- def report_connection(self):
- self.show_host()
- self.show_version()
-
- def show_host(self):
- print("Connected to {0} at {1}:{2}"
- .format(self.applycolor(self.get_cluster_name(), BLUE),
- self.hostname,
- self.port))
-
- def show_version(self):
- vers = self.connection_versions.copy()
- vers['shver'] = version
- # system.Versions['cql'] apparently does not reflect changes with
- # set_cql_version.
- vers['cql'] = self.cql_version
- print("[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers)
-
- def show_session(self, sessionid, partial_session=False):
- print_trace_session(self, self.session, sessionid, partial_session)
-
- def get_connection_versions(self):
- result, = self.session.execute("select * from system.local where key = 'local'")
- vers = {
- 'build': result['release_version'],
- 'protocol': self.conn.protocol_version,
- 'cql': result['cql_version'],
- }
- self.connection_versions = vers
-
- def get_keyspace_names(self):
- return list(self.conn.metadata.keyspaces)
-
- def get_columnfamily_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return list(self.get_keyspace_meta(ksname).tables)
-
- def get_materialized_view_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return list(self.get_keyspace_meta(ksname).views)
-
- def get_index_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return list(self.get_keyspace_meta(ksname).indexes)
-
- def get_column_names(self, ksname, cfname):
- if ksname is None:
- ksname = self.current_keyspace
- layout = self.get_table_meta(ksname, cfname)
- return list(layout.columns)
-
- def get_usertype_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return list(self.get_keyspace_meta(ksname).user_types)
-
- def get_usertype_layout(self, ksname, typename):
- if ksname is None:
- ksname = self.current_keyspace
-
- ks_meta = self.get_keyspace_meta(ksname)
-
- try:
- user_type = ks_meta.user_types[typename]
- except KeyError:
- raise UserTypeNotFound("User type {!r} not found".format(typename))
-
- return list(zip(user_type.field_names, user_type.field_types))
-
- def get_userfunction_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return [f.name for f in list(self.get_keyspace_meta(ksname).functions.values())]
-
- def get_useraggregate_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return [f.name for f in list(self.get_keyspace_meta(ksname).aggregates.values())]
-
- def get_cluster_name(self):
- return self.conn.metadata.cluster_name
-
- def get_partitioner(self):
- return self.conn.metadata.partitioner
-
- def get_keyspace_meta(self, ksname):
- if ksname in self.conn.metadata.keyspaces:
- return self.conn.metadata.keyspaces[ksname]
-
- raise KeyspaceNotFound('Keyspace %r not found.' % ksname)
-
- def get_keyspaces(self):
- return list(self.conn.metadata.keyspaces.values())
-
- def get_ring(self, ks):
- self.conn.metadata.token_map.rebuild_keyspace(ks, build_if_absent=True)
- return self.conn.metadata.token_map.tokens_to_hosts_by_ks[ks]
-
- def get_table_meta(self, ksname, tablename):
- if ksname is None:
- ksname = self.current_keyspace
- ksmeta = self.get_keyspace_meta(ksname)
- if tablename not in ksmeta.tables:
- if ksname == 'system_auth' and tablename in ['roles', 'role_permissions']:
- self.get_fake_auth_table_meta(ksname, tablename)
- else:
- raise ColumnFamilyNotFound("Column family {} not found".format(tablename))
- else:
- return ksmeta.tables[tablename]
-
- def get_fake_auth_table_meta(self, ksname, tablename):
- # may be using external auth implementation so internal tables
- # aren't actually defined in schema. In this case, we'll fake
- # them up
- if tablename == 'roles':
- ks_meta = KeyspaceMetadata(ksname, True, None, None)
- table_meta = TableMetadata(ks_meta, 'roles')
- table_meta.columns['role'] = ColumnMetadata(table_meta, 'role', cassandra.cqltypes.UTF8Type)
- table_meta.columns['is_superuser'] = ColumnMetadata(table_meta, 'is_superuser', cassandra.cqltypes.BooleanType)
- table_meta.columns['can_login'] = ColumnMetadata(table_meta, 'can_login', cassandra.cqltypes.BooleanType)
- elif tablename == 'role_permissions':
- ks_meta = KeyspaceMetadata(ksname, True, None, None)
- table_meta = TableMetadata(ks_meta, 'role_permissions')
- table_meta.columns['role'] = ColumnMetadata(table_meta, 'role', cassandra.cqltypes.UTF8Type)
- table_meta.columns['resource'] = ColumnMetadata(table_meta, 'resource', cassandra.cqltypes.UTF8Type)
- table_meta.columns['permission'] = ColumnMetadata(table_meta, 'permission', cassandra.cqltypes.UTF8Type)
- else:
- raise ColumnFamilyNotFound("Column family {} not found".format(tablename))
-
- def get_index_meta(self, ksname, idxname):
- if ksname is None:
- ksname = self.current_keyspace
- ksmeta = self.get_keyspace_meta(ksname)
-
- if idxname not in ksmeta.indexes:
- raise IndexNotFound("Index {} not found".format(idxname))
-
- return ksmeta.indexes[idxname]
-
- def get_view_meta(self, ksname, viewname):
- if ksname is None:
- ksname = self.current_keyspace
- ksmeta = self.get_keyspace_meta(ksname)
-
- if viewname not in ksmeta.views:
- raise MaterializedViewNotFound("Materialized view '{}' not found".format(viewname))
- return ksmeta.views[viewname]
-
- def get_object_meta(self, ks, name):
- if name is None:
- if ks and ks in self.conn.metadata.keyspaces:
- return self.conn.metadata.keyspaces[ks]
- elif self.current_keyspace is None:
- raise ObjectNotFound("'{}' not found in keyspaces".format(ks))
- else:
- name = ks
- ks = self.current_keyspace
-
- if ks is None:
- ks = self.current_keyspace
-
- ksmeta = self.get_keyspace_meta(ks)
-
- if name in ksmeta.tables:
- return ksmeta.tables[name]
- elif name in ksmeta.indexes:
- return ksmeta.indexes[name]
- elif name in ksmeta.views:
- return ksmeta.views[name]
-
- raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks))
-
- def get_trigger_names(self, ksname=None):
- if ksname is None:
- ksname = self.current_keyspace
-
- return [trigger.name
- for table in list(self.get_keyspace_meta(ksname).tables.values())
- for trigger in list(table.triggers.values())]
-
- def reset_statement(self):
- self.reset_prompt()
- self.statement.truncate(0)
- self.statement.seek(0)
- self.empty_lines = 0
-
- def reset_prompt(self):
- if self.current_keyspace is None:
- self.set_prompt(self.default_prompt, True)
- else:
- self.set_prompt(self.keyspace_prompt.format(self.current_keyspace), True)
-
- def set_continue_prompt(self):
- if self.empty_lines >= 3:
- self.set_prompt("Statements are terminated with a ';'. You can press CTRL-C to cancel an incomplete statement.")
- self.empty_lines = 0
- return
- if self.current_keyspace is None:
- self.set_prompt(self.continue_prompt)
- else:
- spaces = ' ' * len(str(self.current_keyspace))
- self.set_prompt(self.keyspace_continue_prompt.format(spaces))
- self.empty_lines = self.empty_lines + 1 if not self.lastcmd else 0
-
- @contextmanager
- def prepare_loop(self):
- readline = None
- if self.tty and self.completekey:
- try:
- import readline
- except ImportError:
- pass
- else:
- old_completer = readline.get_completer()
- readline.set_completer(self.complete)
- if readline.__doc__ is not None and 'libedit' in readline.__doc__:
- readline.parse_and_bind("bind -e")
- readline.parse_and_bind("bind '" + self.completekey + "' rl_complete")
- readline.parse_and_bind("bind ^R em-inc-search-prev")
- else:
- readline.parse_and_bind(self.completekey + ": complete")
- # start coverage collection if requested, unless in subshell
- if self.coverage and not self.is_subshell:
- # check for coveragerc file, write it if missing
- if os.path.exists(CQL_DIR):
- self.coveragerc_path = os.path.join(CQL_DIR, '.coveragerc')
- covdata_path = os.path.join(CQL_DIR, '.coverage')
- if not os.path.isfile(self.coveragerc_path):
- with open(self.coveragerc_path, 'w') as f:
- f.writelines(["[run]\n",
- "concurrency = multiprocessing\n",
- "data_file = {}\n".format(covdata_path),
- "parallel = true\n"]
- )
- # start coverage
- import coverage
- self.cov = coverage.Coverage(config_file=self.coveragerc_path)
- self.cov.start()
- try:
- yield
- finally:
- if readline is not None:
- readline.set_completer(old_completer)
- if self.coverage and not self.is_subshell:
- self.stop_coverage()
-
- def get_input_line(self, prompt=''):
- if self.tty:
- self.lastcmd = input(str(prompt))
- line = self.lastcmd + '\n'
- else:
- self.lastcmd = self.stdin.readline()
- line = self.lastcmd
- if not len(line):
- raise EOFError
- self.lineno += 1
- return line
-
- def use_stdin_reader(self, until='', prompt=''):
- until += '\n'
- while True:
- try:
- newline = self.get_input_line(prompt=prompt)
- except EOFError:
- return
- if newline == until:
- return
- yield newline
-
- def cmdloop(self, intro=None):
- """
- Adapted from cmd.Cmd's version, because there is literally no way with
- cmd.Cmd.cmdloop() to tell the difference between "EOF" showing up in
- input and an actual EOF.
- """
- with self.prepare_loop():
- while not self.stop:
- try:
- if self.single_statement:
- line = self.single_statement
- self.stop = True
- else:
- line = self.get_input_line(self.prompt)
- self.statement.write(line)
- if self.onecmd(self.statement.getvalue()):
- self.reset_statement()
- except EOFError:
- self.handle_eof()
- except CQL_ERRORS as cqlerr:
- self.printerr(cqlerr.message)
- except KeyboardInterrupt:
- self.reset_statement()
- print('')
-
- def strip_comment_blocks(self, statementtext):
- comment_block_in_literal_string = re.search('["].*[/][*].*[*][/].*["]', statementtext)
- if not comment_block_in_literal_string:
- result = re.sub('[/][*].*[*][/]', "", statementtext)
- if '*/' in result and '/*' not in result and not self.in_comment:
- raise SyntaxError("Encountered comment block terminator without being in comment block")
- if '/*' in result:
- result = re.sub('[/][*].*', "", result)
- self.in_comment = True
- if '*/' in result:
- result = re.sub('.*[*][/]', "", result)
- self.in_comment = False
- if self.in_comment and not re.findall('[/][*]|[*][/]', statementtext):
- result = ''
- return result
- return statementtext
-
- def onecmd(self, statementtext):
- """
- Returns true if the statement is complete and was handled (meaning it
- can be reset).
- """
- statementtext = self.strip_comment_blocks(statementtext)
- try:
- statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext)
- except pylexotron.LexingError as e:
- if self.show_line_nums:
- self.printerr('Invalid syntax at line {0}, char {1}'
- .format(e.linenum, e.charnum))
- else:
- self.printerr('Invalid syntax at char {0}'.format(e.charnum))
- statementline = statementtext.split('\n')[e.linenum - 1]
- self.printerr(' {0}'.format(statementline))
- self.printerr(' {0}^'.format(' ' * e.charnum))
- return True
-
- while statements and not statements[-1]:
- statements = statements[:-1]
- if not statements:
- return True
- if endtoken_escaped or statements[-1][-1][0] != 'endtoken':
- self.set_continue_prompt()
- return
- for st in statements:
- try:
- self.handle_statement(st, statementtext)
- except Exception as e:
- if self.debug:
- traceback.print_exc()
- else:
- self.printerr(e)
- return True
-
- def handle_eof(self):
- if self.tty:
- print('')
- statement = self.statement.getvalue()
- if statement.strip():
- if not self.onecmd(statement):
- self.printerr('Incomplete statement at end of file')
- self.do_exit()
-
- def handle_statement(self, tokens, srcstr):
- # Concat multi-line statements and insert into history
- if readline is not None:
- nl_count = srcstr.count("\n")
-
- new_hist = srcstr.replace("\n", " ").rstrip()
-
- if nl_count > 1 and self.last_hist != new_hist:
- readline.add_history(new_hist)
-
- self.last_hist = new_hist
- cmdword = tokens[0][1]
- if cmdword == '?':
- cmdword = 'help'
- custom_handler = getattr(self, 'do_' + cmdword.lower(), None)
- if custom_handler:
- parsed = cqlruleset.cql_whole_parse_tokens(tokens, srcstr=srcstr,
- startsymbol='cqlshCommand')
- if parsed and not parsed.remainder:
- # successful complete parse
- return custom_handler(parsed)
- else:
- return self.handle_parse_error(cmdword, tokens, parsed, srcstr)
- return self.perform_statement(cqlruleset.cql_extract_orig(tokens, srcstr))
-
- def handle_parse_error(self, cmdword, tokens, parsed, srcstr):
- if cmdword.lower() in ('select', 'insert', 'update', 'delete', 'truncate',
- 'create', 'drop', 'alter', 'grant', 'revoke',
- 'batch', 'list'):
- # hey, maybe they know about some new syntax we don't. type
- # assumptions won't work, but maybe the query will.
- return self.perform_statement(cqlruleset.cql_extract_orig(tokens, srcstr))
- if parsed:
- self.printerr('Improper %s command (problem at %r).' % (cmdword, parsed.remainder[0]))
- else:
- self.printerr('Improper %s command.' % cmdword)
-
- def do_use(self, parsed):
- ksname = parsed.get_binding('ksname')
- success, _ = self.perform_simple_statement(SimpleStatement(parsed.extract_orig()))
- if success:
- if ksname[0] == '"' and ksname[-1] == '"':
- self.current_keyspace = self.cql_unprotect_name(ksname)
- else:
- self.current_keyspace = ksname.lower()
-
- def do_select(self, parsed):
- tracing_was_enabled = self.tracing_enabled
- ksname = parsed.get_binding('ksname')
- stop_tracing = ksname == 'system_traces' or (ksname is None and self.current_keyspace == 'system_traces')
- self.tracing_enabled = self.tracing_enabled and not stop_tracing
- statement = parsed.extract_orig()
- self.perform_statement(statement)
- self.tracing_enabled = tracing_was_enabled
-
- def perform_statement(self, statement):
-
- stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None)
- success, future = self.perform_simple_statement(stmt)
-
- if future:
- if future.warnings:
- self.print_warnings(future.warnings)
-
- if self.tracing_enabled:
- try:
- for trace in future.get_all_query_traces(max_wait_per=self.max_trace_wait, query_cl=self.consistency_level):
- print_trace(self, trace)
- except TraceUnavailable:
- msg = "Statement trace did not complete within %d seconds; trace data may be incomplete." % (self.session.max_trace_wait,)
- self.writeresult(msg, color=RED)
- for trace_id in future.get_query_trace_ids():
- self.show_session(trace_id, partial_session=True)
- except Exception as err:
- self.printerr("Unable to fetch query trace: %s" % (str(err),))
-
- return success
-
- def parse_for_select_meta(self, query_string):
- try:
- parsed = cqlruleset.cql_parse(query_string)[1]
- except IndexError:
- return None
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- name = self.cql_unprotect_name(parsed.get_binding('cfname', None))
- try:
- return self.get_table_meta(ks, name)
- except ColumnFamilyNotFound:
- try:
- return self.get_view_meta(ks, name)
- except MaterializedViewNotFound:
- raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks))
-
- def parse_for_update_meta(self, query_string):
- try:
- parsed = cqlruleset.cql_parse(query_string)[1]
- except IndexError:
- return None
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- cf = self.cql_unprotect_name(parsed.get_binding('cfname'))
- return self.get_table_meta(ks, cf)
-
- def perform_simple_statement(self, statement):
- if not statement:
- return False, None
-
- future = self.session.execute_async(statement, trace=self.tracing_enabled)
- result = None
- try:
- result = future.result()
- except CQL_ERRORS as err:
- err_msg = err.message if hasattr(err, 'message') else str(err)
- self.printerr(str(err.__class__.__name__) + ": " + err_msg)
- except Exception:
- import traceback
- self.printerr(traceback.format_exc())
-
- # Even if statement failed we try to refresh schema if not agreed (see CASSANDRA-9689)
- if not future.is_schema_agreed:
- try:
- self.conn.refresh_schema_metadata(5) # will throw exception if there is a schema mismatch
- except Exception:
- self.printerr("Warning: schema version mismatch detected; check the schema versions of your "
- "nodes in system.local and system.peers.")
- self.conn.refresh_schema_metadata(-1)
-
- if result is None:
- return False, None
-
- if statement.query_string[:6].lower() == 'select':
- self.print_result(result, self.parse_for_select_meta(statement.query_string))
- elif statement.query_string.lower().startswith("list users") or statement.query_string.lower().startswith("list roles"):
- self.print_result(result, self.get_table_meta('system_auth', 'roles'))
- elif statement.query_string.lower().startswith("list"):
- self.print_result(result, self.get_table_meta('system_auth', 'role_permissions'))
- elif result:
- # CAS INSERT/UPDATE
- self.writeresult("")
- self.print_static_result(result, self.parse_for_update_meta(statement.query_string), with_header=True, tty=self.tty)
- self.flush_output()
- return True, future
-
- def print_result(self, result, table_meta):
- self.decoding_errors = []
-
- self.writeresult("")
-
- def print_all(result, table_meta, tty):
- # Return the number of rows in total
- num_rows = 0
- is_first = True
- while True:
- # Always print for the first page even it is empty
- if result.current_rows or is_first:
- with_header = is_first or tty
- self.print_static_result(result, table_meta, with_header, tty, num_rows)
- num_rows += len(result.current_rows)
- if result.has_more_pages:
- if self.shunted_query_out is None and tty:
- # Only pause when not capturing.
- input("---MORE---")
- result.fetch_next_page()
- else:
- if not tty:
- self.writeresult("")
- break
- is_first = False
- return num_rows
-
- num_rows = print_all(result, table_meta, self.tty)
- self.writeresult("(%d rows)" % num_rows)
-
- if self.decoding_errors:
- for err in self.decoding_errors[:2]:
- self.writeresult(err.message(), color=RED)
- if len(self.decoding_errors) > 2:
- self.writeresult('%d more decoding errors suppressed.'
- % (len(self.decoding_errors) - 2), color=RED)
-
- def print_static_result(self, result, table_meta, with_header, tty, row_count_offset=0):
- if not result.column_names and not table_meta:
- return
-
- column_names = result.column_names or list(table_meta.columns.keys())
- formatted_names = [self.myformat_colname(name, table_meta) for name in column_names]
- if not result.current_rows:
- # print header only
- self.print_formatted_result(formatted_names, None, with_header=True, tty=tty)
- return
-
- cql_types = []
- if result.column_types:
- ks_name = table_meta.keyspace_name if table_meta else self.current_keyspace
- ks_meta = self.conn.metadata.keyspaces.get(ks_name, None)
- cql_types = [CqlType(cql_typename(t), ks_meta) for t in result.column_types]
-
- formatted_values = [list(map(self.myformat_value, [row[c] for c in column_names], cql_types)) for row in result.current_rows]
-
- if self.expand_enabled:
- self.print_formatted_result_vertically(formatted_names, formatted_values, row_count_offset)
- else:
- self.print_formatted_result(formatted_names, formatted_values, with_header, tty)
-
- def print_formatted_result(self, formatted_names, formatted_values, with_header, tty):
- # determine column widths
- widths = [n.displaywidth for n in formatted_names]
- if formatted_values is not None:
- for fmtrow in formatted_values:
- for num, col in enumerate(fmtrow):
- widths[num] = max(widths[num], col.displaywidth)
-
- # print header
- if with_header:
- header = ' | '.join(hdr.ljust(w, color=self.color) for (hdr, w) in zip(formatted_names, widths))
- self.writeresult(' ' + header.rstrip())
- self.writeresult('-%s-' % '-+-'.join('-' * w for w in widths))
-
- # stop if there are no rows
- if formatted_values is None:
- self.writeresult("")
- return
-
- # print row data
- for row in formatted_values:
- line = ' | '.join(col.rjust(w, color=self.color) for (col, w) in zip(row, widths))
- self.writeresult(' ' + line)
-
- if tty:
- self.writeresult("")
-
- def print_formatted_result_vertically(self, formatted_names, formatted_values, row_count_offset):
- max_col_width = max([n.displaywidth for n in formatted_names])
- max_val_width = max([n.displaywidth for row in formatted_values for n in row])
-
- # for each row returned, list all the column-value pairs
- for i, row in enumerate(formatted_values):
- self.writeresult("@ Row %d" % (row_count_offset + i + 1))
- self.writeresult('-%s-' % '-+-'.join(['-' * max_col_width, '-' * max_val_width]))
- for field_id, field in enumerate(row):
- column = formatted_names[field_id].ljust(max_col_width, color=self.color)
- value = field.ljust(field.displaywidth, color=self.color)
- self.writeresult(' ' + " | ".join([column, value]))
- self.writeresult('')
-
- def print_warnings(self, warnings):
- if warnings is None or len(warnings) == 0:
- return
-
- self.writeresult('')
- self.writeresult('Warnings :')
- for warning in warnings:
- self.writeresult(warning)
- self.writeresult('')
-
- def emptyline(self):
- pass
-
- def parseline(self, line):
- # this shouldn't be needed
- raise NotImplementedError
-
- def complete(self, text, state):
- if readline is None:
- return
- if state == 0:
- try:
- self.completion_matches = self.find_completions(text)
- except Exception:
- if debug_completion:
- import traceback
- traceback.print_exc()
- else:
- raise
- try:
- return self.completion_matches[state]
- except IndexError:
- return None
-
- def find_completions(self, text):
- curline = readline.get_line_buffer()
- prevlines = self.statement.getvalue()
- wholestmt = prevlines + curline
- begidx = readline.get_begidx() + len(prevlines)
- stuff_to_complete = wholestmt[:begidx]
- return cqlruleset.cql_complete(stuff_to_complete, text, cassandra_conn=self,
- debug=debug_completion, startsymbol='cqlshCommand')
-
- def set_prompt(self, prompt, prepend_user=False):
- if prepend_user and self.username:
- self.prompt = "{0}@{1}".format(self.username, prompt)
- return
- self.prompt = prompt
-
- def cql_unprotect_name(self, namestr):
- if namestr is None:
- return
- return cqlruleset.dequote_name(namestr)
-
- def cql_unprotect_value(self, valstr):
- if valstr is not None:
- return cqlruleset.dequote_value(valstr)
-
- def _columnize_unicode(self, name_list):
- """
- Used when columnizing identifiers that may contain unicode
- """
- names = [n for n in name_list]
- cmd.Cmd.columnize(self, names)
- print('')
-
- def do_describe(self, parsed):
-
- """
- DESCRIBE [cqlsh only]
-
- (DESC may be used as a shorthand.)
-
- Outputs information about the connected Cassandra cluster, or about
- the data objects stored in the cluster. Use in one of the following ways:
-
- DESCRIBE KEYSPACES
-
- Output the names of all keyspaces.
-
- DESCRIBE KEYSPACE [<keyspacename>]
-
- Output CQL commands that could be used to recreate the given keyspace,
- and the objects in it (such as tables, types, functions, etc.).
- In some cases, as the CQL interface matures, there will be some metadata
- about a keyspace that is not representable with CQL. That metadata will not be shown.
- The '<keyspacename>' argument may be omitted, in which case the current
- keyspace will be described.
-
- DESCRIBE TABLES
-
- Output the names of all tables in the current keyspace, or in all
- keyspaces if there is no current keyspace.
-
- DESCRIBE TABLE [<keyspace>.]<tablename>
-
- Output CQL commands that could be used to recreate the given table.
- In some cases, as above, there may be table metadata which is not
- representable and which will not be shown.
-
- DESCRIBE INDEX <indexname>
-
- Output the CQL command that could be used to recreate the given index.
- In some cases, there may be index metadata which is not representable
- and which will not be shown.
-
- DESCRIBE MATERIALIZED VIEW <viewname>
-
- Output the CQL command that could be used to recreate the given materialized view.
- In some cases, there may be materialized view metadata which is not representable
- and which will not be shown.
-
- DESCRIBE CLUSTER
-
- Output information about the connected Cassandra cluster, such as the
- cluster name, and the partitioner and snitch in use. When you are
- connected to a non-system keyspace, also shows endpoint-range
- ownership information for the Cassandra ring.
-
- DESCRIBE [FULL] SCHEMA
-
- Output CQL commands that could be used to recreate the entire (non-system) schema.
- Works as though "DESCRIBE KEYSPACE k" was invoked for each non-system keyspace
- k. Use DESCRIBE FULL SCHEMA to include the system keyspaces.
-
- DESCRIBE TYPES
-
- Output the names of all user-defined-types in the current keyspace, or in all
- keyspaces if there is no current keyspace.
-
- DESCRIBE TYPE [<keyspace>.]<type>
-
- Output the CQL command that could be used to recreate the given user-defined-type.
-
- DESCRIBE FUNCTIONS
-
- Output the names of all user-defined-functions in the current keyspace, or in all
- keyspaces if there is no current keyspace.
-
- DESCRIBE FUNCTION [<keyspace>.]<function>
-
- Output the CQL command that could be used to recreate the given user-defined-function.
-
- DESCRIBE AGGREGATES
-
- Output the names of all user-defined-aggregates in the current keyspace, or in all
- keyspaces if there is no current keyspace.
-
- DESCRIBE AGGREGATE [<keyspace>.]<aggregate>
-
- Output the CQL command that could be used to recreate the given user-defined-aggregate.
-
- DESCRIBE <objname>
-
- Output CQL commands that could be used to recreate the entire object schema,
- where object can be either a keyspace or a table or an index or a materialized
- view (in this order).
- """
- stmt = SimpleStatement(parsed.extract_orig(), consistency_level=cassandra.ConsistencyLevel.LOCAL_ONE, fetch_size=self.page_size if self.use_paging else None)
- future = self.session.execute_async(stmt)
-
- if self.connection_versions['build'][0] < '4':
- print('\nWARN: DESCRIBE|DESC was moved to server side in Cassandra 4.0. As a consequence DESRIBE|DESC '
- 'will not work in cqlsh %r connected to Cassandra %r, the version that you are connected to. '
- 'DESCRIBE does not exist server side prior Cassandra 4.0.'
- % (version, self.connection_versions['build']))
- else:
- try:
- result = future.result()
-
- what = parsed.matched[1][1].lower()
-
- if what in ('columnfamilies', 'tables', 'types', 'functions', 'aggregates'):
- self.describe_list(result)
- elif what == 'keyspaces':
- self.describe_keyspaces(result)
- elif what == 'cluster':
- self.describe_cluster(result)
- elif what:
- self.describe_element(result)
-
- except CQL_ERRORS as err:
- err_msg = err.message if hasattr(err, 'message') else str(err)
- self.printerr(err_msg.partition("message=")[2].strip('"'))
- except Exception:
- import traceback
- self.printerr(traceback.format_exc())
-
- if future:
- if future.warnings:
- self.print_warnings(future.warnings)
-
- do_desc = do_describe
-
- def describe_keyspaces(self, rows):
- """
- Print the output for a DESCRIBE KEYSPACES query
- """
- names = [r['name'] for r in rows]
-
- print('')
- cmd.Cmd.columnize(self, names)
- print('')
-
- def describe_list(self, rows):
- """
- Print the output for all the DESCRIBE queries for element names (e.g DESCRIBE TABLES, DESCRIBE FUNCTIONS ...)
- """
- keyspace = None
- names = list()
- for row in rows:
- if row['keyspace_name'] != keyspace:
- if keyspace is not None:
- self.print_keyspace_element_names(keyspace, names)
-
- keyspace = row['keyspace_name']
- names = list()
-
- names.append(str(row['name']))
-
- if keyspace is not None:
- self.print_keyspace_element_names(keyspace, names)
- print('')
-
- def print_keyspace_element_names(self, keyspace, names):
- print('')
- if self.current_keyspace is None:
- print('Keyspace %s' % (keyspace))
- print('---------%s' % ('-' * len(keyspace)))
- cmd.Cmd.columnize(self, names)
-
- def describe_element(self, rows):
- """
- Print the output for all the DESCRIBE queries where an element name as been specified (e.g DESCRIBE TABLE, DESCRIBE INDEX ...)
- """
- for row in rows:
- print('')
- self.query_out.write(row['create_statement'])
- print('')
-
- def describe_cluster(self, rows):
- """
- Print the output for a DESCRIBE CLUSTER query.
-
- If a specified keyspace was in use the returned ResultSet will contains a 'range_ownership' column,
- otherwise not.
- """
- for row in rows:
- print('\nCluster: %s' % row['cluster'])
- print('Partitioner: %s' % row['partitioner'])
- print('Snitch: %s\n' % row['snitch'])
- if 'range_ownership' in row:
- print("Range ownership:")
- for entry in list(row['range_ownership'].items()):
- print(' %39s [%s]' % (entry[0], ', '.join([host for host in entry[1]])))
- print('')
-
- def do_copy(self, parsed):
- r"""
- COPY [cqlsh only]
-
- COPY x FROM: Imports CSV data into a Cassandra table
- COPY x TO: Exports data from a Cassandra table in CSV format.
-
- COPY <table_name> [ ( column [, ...] ) ]
- FROM ( '<file_pattern_1, file_pattern_2, ... file_pattern_n>' | STDIN )
- [ WITH <option>='value' [AND ...] ];
-
- File patterns are either file names or valid python glob expressions, e.g. *.csv or folder/*.csv.
-
- COPY <table_name> [ ( column [, ...] ) ]
- TO ( '<filename>' | STDOUT )
- [ WITH <option>='value' [AND ...] ];
-
- Available common COPY options and defaults:
-
- DELIMITER=',' - character that appears between records
- QUOTE='"' - quoting character to be used to quote fields
- ESCAPE='\' - character to appear before the QUOTE char when quoted
- HEADER=false - whether to ignore the first line
- NULL='' - string that represents a null value
- DATETIMEFORMAT= - timestamp strftime format
- '%Y-%m-%d %H:%M:%S%z' defaults to time_format value in cqlshrc
- MAXATTEMPTS=5 - the maximum number of attempts per batch or range
- REPORTFREQUENCY=0.25 - the frequency with which we display status updates in seconds
- DECIMALSEP='.' - the separator for decimal values
- THOUSANDSSEP='' - the separator for thousands digit groups
- BOOLSTYLE='True,False' - the representation for booleans, case insensitive, specify true followed by false,
- for example yes,no or 1,0
- NUMPROCESSES=n - the number of worker processes, by default the number of cores minus one
- capped at 16
- CONFIGFILE='' - a configuration file with the same format as .cqlshrc (see the Python ConfigParser
- documentation) where you can specify WITH options under the following optional
- sections: [copy], [copy-to], [copy-from], [copy:ks.table], [copy-to:ks.table],
- [copy-from:ks.table], where <ks> is your keyspace name and <table> is your table
- name. Options are read from these sections, in the order specified
- above, and command line options always override options in configuration files.
- Depending on the COPY direction, only the relevant copy-from or copy-to sections
- are used. If no configfile is specified then .cqlshrc is searched instead.
- RATEFILE='' - an optional file where to print the output statistics
-
- Available COPY FROM options and defaults:
-
- CHUNKSIZE=5000 - the size of chunks passed to worker processes
- INGESTRATE=100000 - an approximate ingest rate in rows per second
- MINBATCHSIZE=10 - the minimum size of an import batch
- MAXBATCHSIZE=20 - the maximum size of an import batch
- MAXROWS=-1 - the maximum number of rows, -1 means no maximum
- SKIPROWS=0 - the number of rows to skip
- SKIPCOLS='' - a comma separated list of column names to skip
- MAXPARSEERRORS=-1 - the maximum global number of parsing errors, -1 means no maximum
- MAXINSERTERRORS=1000 - the maximum global number of insert errors, -1 means no maximum
- ERRFILE='' - a file where to store all rows that could not be imported, by default this is
- import_ks_table.err where <ks> is your keyspace and <table> is your table name.
- PREPAREDSTATEMENTS=True - whether to use prepared statements when importing, by default True. Set this to
- False if you don't mind shifting data parsing to the cluster. The cluster will also
- have to compile every batch statement. For large and oversized clusters
- this will result in a faster import but for smaller clusters it may generate
- timeouts.
- TTL=3600 - the time to live in seconds, by default data will not expire
-
- Available COPY TO options and defaults:
-
- ENCODING='utf8' - encoding for CSV output
- PAGESIZE='1000' - the page size for fetching results
- PAGETIMEOUT=10 - the page timeout in seconds for fetching results
- BEGINTOKEN='' - the minimum token string to consider when exporting data
- ENDTOKEN='' - the maximum token string to consider when exporting data
- MAXREQUESTS=6 - the maximum number of requests each worker process can work on in parallel
- MAXOUTPUTSIZE='-1' - the maximum size of the output file measured in number of lines,
- beyond this maximum the output file will be split into segments,
- -1 means unlimited.
- FLOATPRECISION=5 - the number of digits displayed after the decimal point for cql float values
- DOUBLEPRECISION=12 - the number of digits displayed after the decimal point for cql double values
-
- When entering CSV data on STDIN, you can use the sequence "\."
- on a line by itself to end the data input.
- """
-
- ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
- if ks is None:
- ks = self.current_keyspace
- if ks is None:
- raise NoKeyspaceError("Not in any keyspace.")
- table = self.cql_unprotect_name(parsed.get_binding('cfname'))
- columns = parsed.get_binding('colnames', None)
- if columns is not None:
- columns = list(map(self.cql_unprotect_name, columns))
- else:
- # default to all known columns
- columns = self.get_column_names(ks, table)
-
- fname = parsed.get_binding('fname', None)
- if fname is not None:
- fname = self.cql_unprotect_value(fname)
-
- copyoptnames = list(map(str.lower, parsed.get_binding('optnames', ())))
- copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ())))
- opts = dict(list(zip(copyoptnames, copyoptvals)))
-
- direction = parsed.get_binding('dir').upper()
- if direction == 'FROM':
- task = ImportTask(self, ks, table, columns, fname, opts, self.conn.protocol_version, CONFIG_FILE)
- elif direction == 'TO':
- task = ExportTask(self, ks, table, columns, fname, opts, self.conn.protocol_version, CONFIG_FILE)
- else:
- raise SyntaxError("Unknown direction %s" % direction)
-
- task.run()
-
- def do_show(self, parsed):
- """
- SHOW [cqlsh only]
-
- Displays information about the current cqlsh session. Can be called in
- the following ways:
-
- SHOW VERSION
-
- Shows the version and build of the connected Cassandra instance, as
- well as the version of the CQL spec that the connected Cassandra
- instance understands.
-
- SHOW HOST
-
- Shows where cqlsh is currently connected.
-
- SHOW SESSION <sessionid>
-
- Pretty-prints the requested tracing session.
- """
- showwhat = parsed.get_binding('what').lower()
- if showwhat == 'version':
- self.get_connection_versions()
- self.show_version()
- elif showwhat == 'host':
- self.show_host()
- elif showwhat.startswith('session'):
- session_id = parsed.get_binding('sessionid').lower()
- self.show_session(UUID(session_id))
- else:
- self.printerr('Wait, how do I show %r?' % (showwhat,))
-
- def do_source(self, parsed):
- """
- SOURCE [cqlsh only]
-
- Executes a file containing CQL statements. Gives the output for each
- statement in turn, if any, or any errors that occur along the way.
-
- Errors do NOT abort execution of the CQL source file.
-
- Usage:
-
- SOURCE '<file>';
-
- That is, the path to the file to be executed must be given inside a
- string literal. The path is interpreted relative to the current working
- directory. The tilde shorthand notation ('~/mydir') is supported for
- referring to $HOME.
-
- See also the --file option to cqlsh.
- """
- fname = parsed.get_binding('fname')
- fname = os.path.expanduser(self.cql_unprotect_value(fname))
- try:
- encoding, bom_size = get_file_encoding_bomsize(fname)
- f = codecs.open(fname, 'r', encoding)
- f.seek(bom_size)
- except IOError as e:
- self.printerr('Could not open %r: %s' % (fname, e))
- return
- subshell = Shell(self.hostname, self.port, color=self.color,
- username=self.username,
- encoding=self.encoding, stdin=f, tty=False, use_conn=self.conn,
- cqlver=self.cql_version, keyspace=self.current_keyspace,
- tracing_enabled=self.tracing_enabled,
- display_nanotime_format=self.display_nanotime_format,
- display_timestamp_format=self.display_timestamp_format,
- display_date_format=self.display_date_format,
- display_float_precision=self.display_float_precision,
- display_double_precision=self.display_double_precision,
- display_timezone=self.display_timezone,
- max_trace_wait=self.max_trace_wait, ssl=self.ssl,
- request_timeout=self.session.default_timeout,
- connect_timeout=self.conn.connect_timeout,
- is_subshell=True,
- auth_provider=self.auth_provider)
- # duplicate coverage related settings in subshell
- if self.coverage:
- subshell.coverage = True
- subshell.coveragerc_path = self.coveragerc_path
- subshell.cmdloop()
- f.close()
-
- def do_capture(self, parsed):
- """
- CAPTURE [cqlsh only]
-
- Begins capturing command output and appending it to a specified file.
- Output will not be shown at the console while it is captured.
-
- Usage:
-
- CAPTURE '<file>';
- CAPTURE OFF;
- CAPTURE;
-
- That is, the path to the file to be appended to must be given inside a
- string literal. The path is interpreted relative to the current working
- directory. The tilde shorthand notation ('~/mydir') is supported for
- referring to $HOME.
-
- Only query result output is captured. Errors and output from cqlsh-only
- commands will still be shown in the cqlsh session.
-
- To stop capturing output and show it in the cqlsh session again, use
- CAPTURE OFF.
-
- To inspect the current capture configuration, use CAPTURE with no
- arguments.
- """
- fname = parsed.get_binding('fname')
- if fname is None:
- if self.shunted_query_out is not None:
- print("Currently capturing query output to %r." % (self.query_out.name,))
- else:
- print("Currently not capturing query output.")
- return
-
- if fname.upper() == 'OFF':
- if self.shunted_query_out is None:
- self.printerr('Not currently capturing output.')
- return
- self.query_out.close()
- self.query_out = self.shunted_query_out
- self.color = self.shunted_color
- self.shunted_query_out = None
- del self.shunted_color
- return
-
- if self.shunted_query_out is not None:
- self.printerr('Already capturing output to %s. Use CAPTURE OFF'
- ' to disable.' % (self.query_out.name,))
- return
-
- fname = os.path.expanduser(self.cql_unprotect_value(fname))
- try:
- f = open(fname, 'a')
- except IOError as e:
- self.printerr('Could not open %r for append: %s' % (fname, e))
- return
- self.shunted_query_out = self.query_out
- self.shunted_color = self.color
- self.query_out = f
- self.color = False
- print('Now capturing query output to %r.' % (fname,))
-
- def do_tracing(self, parsed):
- """
- TRACING [cqlsh]
-
- Enables or disables request tracing.
-
- TRACING ON
-
- Enables tracing for all further requests.
-
- TRACING OFF
-
- Disables tracing.
-
- TRACING
-
- TRACING with no arguments shows the current tracing status.
- """
- self.tracing_enabled = SwitchCommand("TRACING", "Tracing").execute(self.tracing_enabled, parsed, self.printerr)
-
- def do_expand(self, parsed):
- """
- EXPAND [cqlsh]
-
- Enables or disables expanded (vertical) output.
-
- EXPAND ON
-
- Enables expanded (vertical) output.
-
- EXPAND OFF
-
- Disables expanded (vertical) output.
-
- EXPAND
-
- EXPAND with no arguments shows the current value of expand setting.
- """
- self.expand_enabled = SwitchCommand("EXPAND", "Expanded output").execute(self.expand_enabled, parsed, self.printerr)
-
- def do_consistency(self, parsed):
- """
- CONSISTENCY [cqlsh only]
-
- Overrides default consistency level (default level is ONE).
-
- CONSISTENCY <level>
-
- Sets consistency level for future requests.
-
- Valid consistency levels:
-
- ANY, ONE, TWO, THREE, QUORUM, ALL, LOCAL_ONE, LOCAL_QUORUM, EACH_QUORUM, SERIAL and LOCAL_SERIAL.
-
- SERIAL and LOCAL_SERIAL may be used only for SELECTs; will be rejected with updates.
-
- CONSISTENCY
-
- CONSISTENCY with no arguments shows the current consistency level.
- """
- level = parsed.get_binding('level')
- if level is None:
- print('Current consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.consistency_level]))
- return
-
- self.consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()]
- print('Consistency level set to %s.' % (level.upper(),))
-
- def do_serial(self, parsed):
- """
- SERIAL CONSISTENCY [cqlsh only]
-
- Overrides serial consistency level (default level is SERIAL).
-
- SERIAL CONSISTENCY <level>
-
- Sets consistency level for future conditional updates.
-
- Valid consistency levels:
-
- SERIAL, LOCAL_SERIAL.
-
- SERIAL CONSISTENCY
-
- SERIAL CONSISTENCY with no arguments shows the current consistency level.
- """
- level = parsed.get_binding('level')
- if level is None:
- print('Current serial consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.serial_consistency_level]))
- return
-
- self.serial_consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()]
- print('Serial consistency level set to %s.' % (level.upper(),))
-
- def do_login(self, parsed):
- """
- LOGIN [cqlsh only]
-
- Changes login information without requiring restart.
-
- LOGIN <username> (<password>)
-
- Login using the specified username. If password is specified, it will be used
- otherwise, you will be prompted to enter.
- """
- username = parsed.get_binding('username')
- password = parsed.get_binding('password')
- if password is None:
- password = getpass.getpass()
- else:
- password = password[1:-1]
-
- auth_provider = PlainTextAuthProvider(username=username, password=password)
-
- conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=self.conn.cql_version,
- protocol_version=self.conn.protocol_version,
- auth_provider=auth_provider,
- ssl_options=self.conn.ssl_options,
- load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
- control_connection_timeout=self.conn.connect_timeout,
- connect_timeout=self.conn.connect_timeout)
-
- if self.current_keyspace:
- session = conn.connect(self.current_keyspace)
- else:
- session = conn.connect()
-
- # Copy session properties
- session.default_timeout = self.session.default_timeout
- session.row_factory = self.session.row_factory
- session.default_consistency_level = self.session.default_consistency_level
- session.max_trace_wait = self.session.max_trace_wait
-
- # Update after we've connected in case we fail to authenticate
- self.conn = conn
- self.auth_provider = auth_provider
- self.username = username
- self.session = session
-
- def do_exit(self, parsed=None):
- """
- EXIT/QUIT [cqlsh only]
-
- Exits cqlsh.
- """
- self.stop = True
- if self.owns_connection:
- self.conn.shutdown()
- do_quit = do_exit
-
- def do_clear(self, parsed):
- """
- CLEAR/CLS [cqlsh only]
-
- Clears the console.
- """
- subprocess.call('clear', shell=True)
- do_cls = do_clear
-
- def do_debug(self, parsed):
- import pdb
- pdb.set_trace()
-
- def get_help_topics(self):
- topics = [t[3:] for t in dir(self) if t.startswith('do_') and getattr(self, t, None).__doc__]
- for hide_from_help in ('quit',):
- topics.remove(hide_from_help)
- return topics
-
- def columnize(self, slist, *a, **kw):
- return cmd.Cmd.columnize(self, sorted([u.upper() for u in slist]), *a, **kw)
-
- def do_help(self, parsed):
- """
- HELP [cqlsh only]
-
- Gives information about cqlsh commands. To see available topics,
- enter "HELP" without any arguments. To see help on a topic,
- use "HELP <topic>".
- """
- topics = parsed.get_binding('topic', ())
- if not topics:
- shell_topics = [t.upper() for t in self.get_help_topics()]
- self.print_topics("\nDocumented shell commands:", shell_topics, 15, 80)
- cql_topics = [t.upper() for t in cqldocs.get_help_topics()]
- self.print_topics("CQL help topics:", cql_topics, 15, 80)
- return
- for t in topics:
- if t.lower() in self.get_help_topics():
- doc = getattr(self, 'do_' + t.lower()).__doc__
- self.stdout.write(doc + "\n")
- elif t.lower() in cqldocs.get_help_topics():
- urlpart = cqldocs.get_help_topic(t)
- if urlpart is not None:
- url = "%s#%s" % (CASSANDRA_CQL_HTML, urlpart)
- if self.browser is not None:
- opened = webbrowser.get(self.browser).open_new_tab(url)
- else:
- opened = webbrowser.open_new_tab(url)
- if not opened:
- self.printerr("*** No browser to display CQL help. URL for help topic %s : %s" % (t, url))
- else:
- self.printerr("*** No help on %s" % (t,))
-
- def do_unicode(self, parsed):
- """
- Textual input/output
-
- When control characters, or other characters which can't be encoded
- in your current locale, are found in values of 'text' or 'ascii'
- types, it will be shown as a backslash escape. If color is enabled,
- any such backslash escapes will be shown in a different color from
- the surrounding text.
-
- Unicode code points in your data will be output intact, if the
- encoding for your locale is capable of decoding them. If you prefer
- that non-ascii characters be shown with Python-style "\\uABCD"
- escape sequences, invoke cqlsh with an ASCII locale (for example,
- by setting the $LANG environment variable to "C").
- """
-
- def do_paging(self, parsed):
- """
- PAGING [cqlsh]
-
- Enables or disables query paging.
-
- PAGING ON
-
- Enables query paging for all further queries.
-
- PAGING OFF
-
- Disables paging.
-
- PAGING
-
- PAGING with no arguments shows the current query paging status.
- """
- (self.use_paging, requested_page_size) = SwitchCommandWithValue(
- "PAGING", "Query paging", value_type=int).execute(self.use_paging, parsed, self.printerr)
- if self.use_paging and requested_page_size is not None:
- self.page_size = requested_page_size
- if self.use_paging:
- print(("Page size: {}".format(self.page_size)))
- else:
- self.page_size = self.default_page_size
-
- def applycolor(self, text, color=None):
- if not color or not self.color:
- return text
- return color + text + ANSI_RESET
-
- def writeresult(self, text, color=None, newline=True, out=None):
- if out is None:
- out = self.query_out
-
- # convert Exceptions, etc to text
- if not isinstance(text, str):
- text = str(text)
-
- to_write = self.applycolor(text, color) + ('\n' if newline else '')
- out.write(to_write)
-
- def flush_output(self):
- self.query_out.flush()
-
- def printerr(self, text, color=RED, newline=True, shownum=None):
- self.statement_error = True
- if shownum is None:
- shownum = self.show_line_nums
- if shownum:
- text = '%s:%d:%s' % (self.stdin.name, self.lineno, text)
- self.writeresult(text, color, newline=newline, out=sys.stderr)
-
- def stop_coverage(self):
- if self.coverage and self.cov is not None:
- self.cov.stop()
- self.cov.save()
- self.cov = None
-
-
-class SwitchCommand(object):
- command = None
- description = None
-
- def __init__(self, command, desc):
- self.command = command
- self.description = desc
-
- def execute(self, state, parsed, printerr):
- switch = parsed.get_binding('switch')
- if switch is None:
- if state:
- print("%s is currently enabled. Use %s OFF to disable"
- % (self.description, self.command))
- else:
- print("%s is currently disabled. Use %s ON to enable."
- % (self.description, self.command))
- return state
-
- if switch.upper() == 'ON':
- if state:
- printerr('%s is already enabled. Use %s OFF to disable.'
- % (self.description, self.command))
- return state
- print('Now %s is enabled' % (self.description,))
- return True
-
- if switch.upper() == 'OFF':
- if not state:
- printerr('%s is not enabled.' % (self.description,))
- return state
- print('Disabled %s.' % (self.description,))
- return False
-
-
-class SwitchCommandWithValue(SwitchCommand):
- """The same as SwitchCommand except it also accepts a value in place of ON.
-
- This returns a tuple of the form: (SWITCH_VALUE, PASSED_VALUE)
- eg: PAGING 50 returns (True, 50)
- PAGING OFF returns (False, None)
- PAGING ON returns (True, None)
-
- The value_type must match for the PASSED_VALUE, otherwise it will return None.
- """
- def __init__(self, command, desc, value_type=int):
- SwitchCommand.__init__(self, command, desc)
- self.value_type = value_type
-
- def execute(self, state, parsed, printerr):
- binary_switch_value = SwitchCommand.execute(self, state, parsed, printerr)
- switch = parsed.get_binding('switch')
- try:
- value = self.value_type(switch)
- binary_switch_value = True
- except (ValueError, TypeError):
- value = None
- return binary_switch_value, value
-
-
-def option_with_default(cparser_getter, section, option, default=None):
- try:
- return cparser_getter(section, option)
- except configparser.Error:
- return default
-
-
-def raw_option_with_default(configs, section, option, default=None):
- """
- Same (almost) as option_with_default() but won't do any string interpolation.
- Useful for config values that include '%' symbol, e.g. time format string.
- """
- try:
- return configs.get(section, option, raw=True)
- except configparser.Error:
- return default
-
-
-def should_use_color():
- if not sys.stdout.isatty():
- return False
- if os.environ.get('TERM', '') in ('dumb', ''):
- return False
- try:
- p = subprocess.Popen(['tput', 'colors'], stdout=subprocess.PIPE)
- stdout, _ = p.communicate()
- if int(stdout.strip()) < 8:
- return False
- except (OSError, ImportError, ValueError):
- # oh well, we tried. at least we know there's a $TERM and it's
- # not "dumb".
- pass
- return True
-
-
-def read_options(cmdlineargs, environment):
- configs = configparser.ConfigParser()
- configs.read(CONFIG_FILE)
-
- rawconfigs = configparser.RawConfigParser()
- rawconfigs.read(CONFIG_FILE)
-
- username_from_cqlshrc = option_with_default(configs.get, 'authentication', 'username')
- password_from_cqlshrc = option_with_default(rawconfigs.get, 'authentication', 'password')
- if username_from_cqlshrc or password_from_cqlshrc:
- if password_from_cqlshrc and not is_file_secure(os.path.expanduser(CONFIG_FILE)):
- print("\nWarning: Password is found in an insecure cqlshrc file. The file is owned or readable by other users on the system.",
- end='', file=sys.stderr)
- print("\nNotice: Credentials in the cqlshrc file is deprecated and will be ignored in the future."
- "\nPlease use a credentials file to specify the username and password.\n", file=sys.stderr)
-
- optvalues = optparse.Values()
-
- optvalues.username = None
- optvalues.password = None
- optvalues.credentials = os.path.expanduser(option_with_default(configs.get, 'authentication', 'credentials',
- os.path.join(CQL_DIR, 'credentials')))
- optvalues.keyspace = option_with_default(configs.get, 'authentication', 'keyspace')
- optvalues.browser = option_with_default(configs.get, 'ui', 'browser', None)
- optvalues.completekey = option_with_default(configs.get, 'ui', 'completekey',
- DEFAULT_COMPLETEKEY)
- optvalues.color = option_with_default(configs.getboolean, 'ui', 'color')
- optvalues.time_format = raw_option_with_default(configs, 'ui', 'time_format',
- DEFAULT_TIMESTAMP_FORMAT)
- optvalues.nanotime_format = raw_option_with_default(configs, 'ui', 'nanotime_format',
- DEFAULT_NANOTIME_FORMAT)
- optvalues.date_format = raw_option_with_default(configs, 'ui', 'date_format',
- DEFAULT_DATE_FORMAT)
- optvalues.float_precision = option_with_default(configs.getint, 'ui', 'float_precision',
- DEFAULT_FLOAT_PRECISION)
- optvalues.double_precision = option_with_default(configs.getint, 'ui', 'double_precision',
- DEFAULT_DOUBLE_PRECISION)
- optvalues.field_size_limit = option_with_default(configs.getint, 'csv', 'field_size_limit', csv.field_size_limit())
- optvalues.max_trace_wait = option_with_default(configs.getfloat, 'tracing', 'max_trace_wait',
- DEFAULT_MAX_TRACE_WAIT)
- optvalues.timezone = option_with_default(configs.get, 'ui', 'timezone', None)
-
- optvalues.debug = False
-
- optvalues.coverage = False
- if 'CQLSH_COVERAGE' in environment.keys():
- optvalues.coverage = True
-
- optvalues.file = None
- optvalues.ssl = option_with_default(configs.getboolean, 'connection', 'ssl', DEFAULT_SSL)
- optvalues.encoding = option_with_default(configs.get, 'ui', 'encoding', UTF8)
-
- optvalues.tty = option_with_default(configs.getboolean, 'ui', 'tty', sys.stdin.isatty())
- optvalues.protocol_version = option_with_default(configs.getint, 'protocol', 'version', None)
- optvalues.cqlversion = option_with_default(configs.get, 'cql', 'version', None)
- optvalues.connect_timeout = option_with_default(configs.getint, 'connection', 'timeout', DEFAULT_CONNECT_TIMEOUT_SECONDS)
- optvalues.request_timeout = option_with_default(configs.getint, 'connection', 'request_timeout', DEFAULT_REQUEST_TIMEOUT_SECONDS)
- optvalues.execute = None
- optvalues.insecure_password_without_warning = False
-
- (options, arguments) = parser.parse_args(cmdlineargs, values=optvalues)
-
- # Credentials from cqlshrc will be expanded,
- # credentials from the command line are also expanded if there is a space...
- # we need the following so that these two scenarios will work
- # cqlsh --credentials=~/.cassandra/creds
- # cqlsh --credentials ~/.cassandra/creds
- options.credentials = os.path.expanduser(options.credentials)
-
- if not is_file_secure(options.credentials):
- print("\nWarning: Credentials file '{0}' exists but is not used, because:"
- "\n a. the file owner is not the current user; or"
- "\n b. the file is readable by group or other."
- "\nPlease ensure the file is owned by the current user and is not readable by group or other."
- "\nOn a Linux or UNIX-like system, you often can do this by using the `chown` and `chmod` commands:"
- "\n chown YOUR_USERNAME credentials"
- "\n chmod 600 credentials\n".format(options.credentials),
- file=sys.stderr)
- options.credentials = '' # ConfigParser.read() will ignore unreadable files
-
- if not options.username:
- credentials = configparser.ConfigParser()
- credentials.read(options.credentials)
-
- # use the username from credentials file but fallback to cqlshrc if username is absent from the command line parameters
- options.username = username_from_cqlshrc
-
- if not options.password:
- rawcredentials = configparser.RawConfigParser()
- rawcredentials.read(options.credentials)
-
- # handling password in the same way as username, priority cli > credentials > cqlshrc
- options.password = option_with_default(rawcredentials.get, 'plain_text_auth', 'password', password_from_cqlshrc)
- options.password = password_from_cqlshrc
- elif not options.insecure_password_without_warning:
- print("\nWarning: Using a password on the command line interface can be insecure."
- "\nRecommendation: use the credentials file to securely provide the password.\n", file=sys.stderr)
-
- # Make sure some user values read from the command line are in unicode
- options.execute = maybe_ensure_text(options.execute)
- options.username = maybe_ensure_text(options.username)
- options.password = maybe_ensure_text(options.password)
- options.keyspace = maybe_ensure_text(options.keyspace)
-
- hostname = option_with_default(configs.get, 'connection', 'hostname', DEFAULT_HOST)
- port = option_with_default(configs.get, 'connection', 'port', DEFAULT_PORT)
-
- try:
- options.connect_timeout = int(options.connect_timeout)
- except ValueError:
- parser.error('"%s" is not a valid connect timeout.' % (options.connect_timeout,))
- options.connect_timeout = DEFAULT_CONNECT_TIMEOUT_SECONDS
-
- try:
- options.request_timeout = int(options.request_timeout)
- except ValueError:
- parser.error('"%s" is not a valid request timeout.' % (options.request_timeout,))
- options.request_timeout = DEFAULT_REQUEST_TIMEOUT_SECONDS
-
- hostname = environment.get('CQLSH_HOST', hostname)
- port = environment.get('CQLSH_PORT', port)
-
- if len(arguments) > 0:
- hostname = arguments[0]
- if len(arguments) > 1:
- port = arguments[1]
-
- if options.file or options.execute:
- options.tty = False
-
- if options.execute and not options.execute.endswith(';'):
- options.execute += ';'
-
- if optvalues.color in (True, False):
- options.color = optvalues.color
- else:
- if options.file is not None:
- options.color = False
- else:
- options.color = should_use_color()
-
- if options.cqlversion is not None:
- options.cqlversion, cqlvertup = full_cql_version(options.cqlversion)
- if cqlvertup[0] < 3:
- parser.error('%r is not a supported CQL version.' % options.cqlversion)
- options.cqlmodule = cql3handling
-
- try:
- port = int(port)
- except ValueError:
- parser.error('%r is not a valid port number.' % port)
- return options, hostname, port
-
-
-def setup_cqlruleset(cqlmodule):
- global cqlruleset
- cqlruleset = cqlmodule.CqlRuleSet
- cqlruleset.append_rules(cqlshhandling.cqlsh_extra_syntax_rules)
- for rulename, termname, func in cqlshhandling.cqlsh_syntax_completers:
- cqlruleset.completer_for(rulename, termname)(func)
- cqlruleset.commands_end_with_newline.update(cqlshhandling.my_commands_ending_with_newline)
-
-
-def setup_cqldocs(cqlmodule):
- global cqldocs
- cqldocs = cqlmodule.cqldocs
-
-
-def init_history():
- if readline is not None:
- try:
- readline.read_history_file(HISTORY)
- except IOError:
- pass
- delims = readline.get_completer_delims()
- delims.replace("'", "")
- delims += '.'
- readline.set_completer_delims(delims)
-
-
-def save_history():
- if readline is not None:
- try:
- readline.write_history_file(HISTORY)
- except IOError:
- pass
-
-
-def main(options, hostname, port):
- setup_cqlruleset(options.cqlmodule)
- setup_cqldocs(options.cqlmodule)
- init_history()
- csv.field_size_limit(options.field_size_limit)
-
- if options.file is None:
- stdin = None
- else:
- try:
- encoding, bom_size = get_file_encoding_bomsize(options.file)
- stdin = codecs.open(options.file, 'r', encoding)
- stdin.seek(bom_size)
- except IOError as e:
- sys.exit("Can't open %r: %s" % (options.file, e))
-
- if options.debug:
- sys.stderr.write("Using CQL driver: %s\n" % (cassandra,))
- sys.stderr.write("Using connect timeout: %s seconds\n" % (options.connect_timeout,))
- sys.stderr.write("Using '%s' encoding\n" % (options.encoding,))
- sys.stderr.write("Using ssl: %s\n" % (options.ssl,))
-
- # create timezone based on settings, environment or auto-detection
- timezone = None
- if options.timezone or 'TZ' in os.environ:
- try:
- import pytz
- if options.timezone:
- try:
- timezone = pytz.timezone(options.timezone)
- except Exception:
- sys.stderr.write("Warning: could not recognize timezone '%s' specified in cqlshrc\n\n" % (options.timezone))
- if 'TZ' in os.environ:
- try:
- timezone = pytz.timezone(os.environ['TZ'])
- except Exception:
- sys.stderr.write("Warning: could not recognize timezone '%s' from environment value TZ\n\n" % (os.environ['TZ']))
- except ImportError:
- sys.stderr.write("Warning: Timezone defined and 'pytz' module for timezone conversion not installed. Timestamps will be displayed in UTC timezone.\n\n")
-
- # try auto-detect timezone if tzlocal is installed
- if not timezone:
- try:
- from tzlocal import get_localzone
- timezone = get_localzone()
- except ImportError:
- # we silently ignore and fallback to UTC unless a custom timestamp format (which likely
- # does contain a TZ part) was specified
- if options.time_format != DEFAULT_TIMESTAMP_FORMAT:
- sys.stderr.write("Warning: custom timestamp format specified in cqlshrc, "
- + "but local timezone could not be detected.\n"
- + "Either install Python 'tzlocal' module for auto-detection "
- + "or specify client timezone in your cqlshrc.\n\n")
-
- try:
- shell = Shell(hostname,
- port,
- color=options.color,
- username=options.username,
- stdin=stdin,
- tty=options.tty,
- completekey=options.completekey,
- browser=options.browser,
- protocol_version=options.protocol_version,
- cqlver=options.cqlversion,
- keyspace=options.keyspace,
- display_timestamp_format=options.time_format,
- display_nanotime_format=options.nanotime_format,
- display_date_format=options.date_format,
- display_float_precision=options.float_precision,
- display_double_precision=options.double_precision,
- display_timezone=timezone,
- max_trace_wait=options.max_trace_wait,
- ssl=options.ssl,
- single_statement=options.execute,
- request_timeout=options.request_timeout,
- connect_timeout=options.connect_timeout,
- encoding=options.encoding,
- auth_provider=authproviderhandling.load_auth_provider(
- config_file=CONFIG_FILE,
- cred_file=options.credentials,
- username=options.username,
- password=options.password))
- except KeyboardInterrupt:
- sys.exit('Connection aborted.')
- except CQL_ERRORS as e:
- sys.exit('Connection error: %s' % (e,))
- except VersionNotSupported as e:
- sys.exit('Unsupported CQL version: %s' % (e,))
- if options.debug:
- shell.debug = True
- if options.coverage:
- shell.coverage = True
- import signal
-
- def handle_sighup():
- shell.stop_coverage()
- shell.do_exit()
-
- signal.signal(signal.SIGHUP, handle_sighup)
-
- shell.cmdloop()
- save_history()
-
- if shell.batch_mode and shell.statement_error:
- sys.exit(2)
-
+from cqlshlib.cqlshmain import main
# always call this regardless of module name: when a sub-process is spawned
# on Windows then the module name is not __main__, see CASSANDRA-9304 (Windows support was dropped in CASSANDRA-16956)
-insert_driver_hooks()
if __name__ == '__main__':
- main(*read_options(sys.argv[1:], os.environ))
-
-# vim: set ft=python et ts=4 sw=4 :
+ main(sys.argv[1:], CASSANDRA_PATH)
diff --git a/build.xml b/build.xml
index e56c1f8..a3ef1fc 100644
--- a/build.xml
+++ b/build.xml
@@ -15,7 +15,6 @@
limitations under the License.
-->
<project basedir="." default="jar" name="apache-cassandra"
- xmlns:artifact="antlib:org.apache.maven.artifact.ant"
xmlns:if="ant:if"
xmlns:unless="ant:unless">
@@ -33,11 +32,20 @@
<property name="debuglevel" value="source,lines,vars"/>
<!-- default version and SCM information -->
- <property name="base.version" value="4.1.3"/>
+ <property name="base.version" value="5.0"/>
<property name="scm.connection" value="scm:https://gitbox.apache.org/repos/asf/cassandra.git"/>
<property name="scm.developerConnection" value="scm:https://gitbox.apache.org/repos/asf/cassandra.git"/>
<property name="scm.url" value="https://gitbox.apache.org/repos/asf?p=cassandra.git;a=tree"/>
+ <!-- JDKs supported.
+ All releases are built with the default JDK.
+ Builds with non-default JDKs are considered experimental and for development and testing purposes.
+ When building, javac's source and target flags are set to the jdk used, so lower JDKs are not supported at runtime.
+ The use of both CASSANDRA_USE_JDK11 and use-jdk11 is deprecated.
+ -->
+ <property name="java.default" value="1.8" />
+ <property name="java.supported" value="1.8,11,17" />
+
<!-- directory details -->
<property name="basedir" value="."/>
<property name="build.src" value="${basedir}/src"/>
@@ -94,11 +102,6 @@
<property name="local.repository" value="${user.home}/.m2/repository" />
- <!-- details of what version of Maven ANT Tasks to fetch -->
- <property name="maven-ant-tasks.version" value="2.1.3" />
- <property name="maven-ant-tasks.local" value="${local.repository}/org/apache/maven/maven-ant-tasks"/>
- <property name="maven-ant-tasks.url"
- value="https://repo.maven.apache.org/maven2/org/apache/maven/maven-ant-tasks" />
<!-- details of how and which Maven repository we publish to -->
<property name="maven.version" value="3.0.3" />
<condition property="maven-repository-url" value="https://repository.apache.org/service/local/staging/deploy/maven2">
@@ -130,37 +133,20 @@
<property name="cassandra.test.messagingService.nonGracefulShutdown" value="true"/>
<!-- https://www.eclemma.org/jacoco/ -->
+ <property name="jacoco.version" value="0.8.8"/>
<property name="jacoco.export.dir" value="${build.dir}/jacoco/" />
<property name="jacoco.partials.dir" value="${jacoco.export.dir}/partials" />
<property name="jacoco.partialexecfile" value="${jacoco.partials.dir}/partial.exec" />
<property name="jacoco.finalexecfile" value="${jacoco.export.dir}/jacoco.exec" />
- <property name="jacoco.version" value="0.8.6"/>
- <property name="byteman.version" value="4.0.6"/>
+ <property name="jflex.version" value="1.8.2"/>
<property name="jamm.version" value="0.3.2"/>
<property name="ecj.version" value="4.6.1"/>
- <property name="ohc.version" value="0.5.1"/>
- <property name="asm.version" value="9.1"/>
+ <!-- When updating ASM, please, do consider whether you might need to update also FBUtilities#ASM_BYTECODE_VERSION
+ and the simulator InterceptClasses#BYTECODE_VERSION, in particular if we are looking to provide Cassandra support
+ for newer JDKs (CASSANDRA-17873). -->
+ <property name="asm.version" value="9.4"/>
<property name="allocation-instrumenter.version" value="3.1.0"/>
- <property name="bytebuddy.version" value="1.10.10"/>
- <property name="jflex.version" value="1.8.2"/>
-
- <!-- https://mvnrepository.com/artifact/net.openhft/chronicle-bom/1.16.23 -->
- <property name="chronicle-queue.version" value="5.20.123" />
- <property name="chronicle-core.version" value="2.20.126" />
- <property name="chronicle-bytes.version" value="2.20.111" />
- <property name="chronicle-wire.version" value="2.20.117" />
- <property name="chronicle-threads.version" value="2.20.111" />
-
- <property name="dtest-api.version" value="0.0.13" />
-
- <condition property="maven-ant-tasks.jar.exists">
- <available file="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar" />
- </condition>
-
- <condition property="maven-ant-tasks.jar.local">
- <available file="${maven-ant-tasks.local}/${maven-ant-tasks.version}/maven-ant-tasks-${maven-ant-tasks.version}.jar" />
- </condition>
<condition property="is.source.artifact">
<available file="${build.src.java}" type="dir" />
@@ -187,20 +173,23 @@
</macrodef>
<set-keepbrief-property test-name="${test.name}" />
- <condition property="java.version.8">
- <equals arg1="${ant.java.version}" arg2="1.8"/>
+ <fail message="Unsupported JDK version used: ${ant.java.version}">
+ <condition><not><contains string="${java.supported}" substring="${ant.java.version}"/></not></condition>
+ </fail>
+ <condition property="is.java.default"><equals arg1="${ant.java.version}" arg2="${java.default}"/></condition>
+ <echo unless:true="${is.java.default}" message="Non default JDK version used: ${ant.java.version}"/>
+
+ <condition property="arch_x86">
+ <equals arg1="${os.arch}" arg2="x86" />
</condition>
- <condition property="java.version.11">
- <not><isset property="java.version.8"/></not>
+ <!-- On non-X86 JDK 8 (such as M1 Mac) the smallest allowed Xss is 384k; so need a larger value
+ when on these platforms. -->
+ <condition property="jvm_xss" value="-Xss256k" else="-Xss384k">
+ <isset property="arch_x86" />
</condition>
- <fail><condition><not><or>
- <isset property="java.version.8"/>
- <isset property="java.version.11"/>
- </or></not></condition></fail>
<resources id="_jvm11_arg_items">
<string>-Djdk.attach.allowAttachSelf=true</string>
-
<string>-XX:+UseConcMarkSweepGC</string>
<string>-XX:+CMSParallelRemarkEnabled</string>
<string>-XX:SurvivorRatio=8</string>
@@ -211,6 +200,7 @@
<string>-XX:+CMSParallelInitialMarkEnabled</string>
<string>-XX:+CMSEdenChunksRecordAlways</string>
+
<string>--add-exports java.base/jdk.internal.misc=ALL-UNNAMED</string>
<string>--add-exports java.base/jdk.internal.ref=ALL-UNNAMED</string>
<string>--add-exports java.base/sun.nio.ch=ALL-UNNAMED</string>
@@ -231,9 +221,52 @@
</resources>
<pathconvert property="_jvm_args_concat" refid="_jvm11_arg_items" pathsep=" "/>
<condition property="java11-jvmargs" value="${_jvm_args_concat}" else="">
- <not>
- <equals arg1="${ant.java.version}" arg2="1.8"/>
- </not>
+ <equals arg1="${ant.java.version}" arg2="11"/>
+ </condition>
+
+ <resources id="_jvm17_arg_items">
+ <string>-Djdk.attach.allowAttachSelf=true</string>
+ <string>-XX:+UseG1GC</string>
+ <string>-XX:+ParallelRefProcEnabled</string>
+ <string>-XX:MaxTenuringThreshold=1</string>
+ <string>-XX:G1HeapRegionSize=16m</string>
+
+ <string>--add-exports java.base/jdk.internal.misc=ALL-UNNAMED</string>
+ <string>--add-exports java.base/jdk.internal.ref=ALL-UNNAMED</string>
+ <string>--add-exports java.base/sun.nio.ch=ALL-UNNAMED</string>
+ <string>--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED</string>
+ <string>--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED</string>
+ <string>--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED</string>
+ <string>--add-exports java.sql/java.sql=ALL-UNNAMED</string>
+ <string>--add-exports java.base/java.lang.ref=ALL-UNNAMED</string>
+ <string>--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED</string>
+ <string>--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</string>
+
+ <string>--add-opens java.base/java.lang.module=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.net=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.loader=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.ref=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.math=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.module=ALL-UNNAMED</string>
+ <string>--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED</string>
+ <string>--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED</string>
+ <string>--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED</string>
+
+ <string>--add-opens java.base/sun.nio.ch=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.io=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.nio=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.util.concurrent=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.util=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.lang=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.math=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.lang.reflect=ALL-UNNAMED</string>
+ <string>--add-opens java.base/java.net=ALL-UNNAMED</string>
+ </resources>
+ <pathconvert property="_jvm_args_concat2" refid="_jvm17_arg_items" pathsep=" "/>
+ <condition property="java17-jvmargs" value="${_jvm_args_concat2}" else="">
+ <equals arg1="${ant.java.version}" arg2="17"/>
</condition>
<!--
@@ -248,6 +281,7 @@
In java 11 we also need to set a system property to enable netty to use Unsafe direct byte
buffer construction (see CASSANDRA-16493)
-->
+ <!-- Leaving _jvm8_test_arg_items for now as we are not dropping Java 8 yet but it will be useless after CASSANDRA-18258-->
<resources id="_jvm8_test_arg_items">
<!-- TODO see CASSANDRA-16212 - we seem to OOM non stop now after CASSANDRA-16212, so to have clean CI while this gets looked into, disabling limiting metaspace
<string>-XX:MaxMetaspaceExpansion=64M</string>
@@ -261,39 +295,35 @@
<string>-Dio.netty.tryReflectionSetAccessible=true</string>
</resources>
<pathconvert property="_jvm11_test_arg_items_concat" refid="_jvm11_test_arg_items" pathsep=" "/>
- <condition property="_std-test-jvmargs" value="${_jvm11_test_arg_items_concat}" else="${_jvm8_test_arg_items_concat}">
- <not>
- <equals arg1="${ant.java.version}" arg2="1.8"/>
- </not>
+ <resources id="_jvm17_test_arg_items">
+ <string>-Dio.netty.tryReflectionSetAccessible=true</string>
+ </resources>
+ <pathconvert property="_jvm17_test_arg_items_concat" refid="_jvm17_test_arg_items" pathsep=" "/>
+ <condition property="_std-test-jvmargs11" value="${_jvm11_test_arg_items_concat}" else=" ">
+ <equals arg1="${ant.java.version}" arg2="11"/>
+ </condition>
+ <condition property="_std-test-jvmargs17" value="${_jvm17_test_arg_items_concat}" else=" ">
+ <equals arg1="${ant.java.version}" arg2="17"/>
</condition>
<!-- needed to compile org.apache.cassandra.utils.JMXServerUtils -->
- <condition property="jdk11-javac-exports" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED" else="">
+ <condition property="jdk11plus-javac-exports" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED" else="">
<not>
<equals arg1="${ant.java.version}" arg2="1.8"/>
</not>
</condition>
- <condition property="jdk11-javadoc-exports" value="${jdk11-javac-exports} --frames" else="">
+ <condition property="jdk11plus-javadoc-exports" value="${jdk11plus-javac-exports} --frames" else="">
<not>
<equals arg1="${ant.java.version}" arg2="1.8"/>
</not>
</condition>
-
- <condition property="build.java.11">
- <istrue value="${use.jdk11}"/>
- </condition>
-
- <condition property="source.version" value="8" else="11">
- <equals arg1="${java.version.8}" arg2="true"/>
- </condition>
- <condition property="target.version" value="8" else="11">
- <equals arg1="${java.version.8}" arg2="true"/>
+ <condition property="checkstyle.version" value="8.45.1" else="10.8.1">
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
</condition>
<!--
Add all the dependencies.
-->
- <path id="maven-ant-tasks.classpath" path="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar" />
<path id="cassandra.classpath">
<pathelement location="${build.classes.main}" />
<fileset dir="${build.dir.lib}">
@@ -319,7 +349,7 @@
windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
bottom="Copyright &copy; 2009- The Apache Software Foundation"
useexternalfile="yes" encoding="UTF-8" failonerror="false"
- maxmemory="256m" additionalparam="${jdk11-javadoc-exports}">
+ maxmemory="256m" additionalparam="${jdk11plus-javadoc-exports}">
<filesets/>
</javadoc>
<fail message="javadoc failed">
@@ -332,57 +362,10 @@
</sequential>
</macrodef>
- <target name="validate-build-conf">
- <condition property="use-jdk11">
- <or>
- <isset property="build.java.11"/>
- <istrue value="${env.CASSANDRA_USE_JDK11}"/>
- </or>
- </condition>
- <fail message="Inconsistent JDK11 options set">
- <condition>
- <and>
- <istrue value="${env.CASSANDRA_USE_JDK11}"/>
- <isset property="use.jdk11"/>
- <not>
- <istrue value="${use.jdk11}"/>
- </not>
- </and>
- </condition>
- </fail>
- <fail message="Inconsistent JDK11 options set">
- <condition>
- <and>
- <isset property="env.CASSANDRA_USE_JDK11"/>
- <not>
- <istrue value="${env.CASSANDRA_USE_JDK11}"/>
- </not>
- <istrue value="${use.jdk11}"/>
- </and>
- </condition>
- </fail>
- <fail message="-Duse.jdk11=true or $CASSANDRA_USE_JDK11=true cannot be set when building from java 8">
- <condition>
- <not><or>
- <not><isset property="java.version.8"/></not>
- <not><isset property="use-jdk11"/></not>
- </or></not>
- </condition>
- </fail>
- <fail message="-Duse.jdk11=true or $CASSANDRA_USE_JDK11=true must be set when building from java 11">
- <condition>
- <not><or>
- <isset property="java.version.8"/>
- <isset property="use-jdk11"/>
- </or></not>
- </condition>
- </fail>
- </target>
-
<!--
Setup the output directories.
-->
- <target name="init" depends="validate-build-conf">
+ <target name="init" >
<fail unless="is.source.artifact"
message="Not a source artifact, stopping here." />
<mkdir dir="${build.classes.main}"/>
@@ -394,6 +377,13 @@
<mkdir dir="${build.dir.lib}"/>
<mkdir dir="${jacoco.export.dir}"/>
<mkdir dir="${jacoco.partials.dir}"/>
+
+ <!-- Set up jdk specific properties -->
+ <javac includes="**/JdkProperties.java" srcdir="test/anttasks" destdir="${test.classes}" includeantruntime="true" source="${java.default}" target="${java.default}">
+ <compilerarg value="-Xlint:-options"/>
+ </javac>
+ <taskdef name="JdkProperties" classname="org.apache.cassandra.anttasks.JdkProperties" classpath="${test.classes}"/>
+ <JdkProperties/>
</target>
<target name="clean" description="Remove all locally created artifacts">
@@ -472,436 +462,13 @@
<jflex file="${build.src.java}/org/apache/cassandra/index/sasi/analyzer/StandardTokenizerImpl.jflex" destdir="${build.src.gen-java}/" />
</target>
- <!--
- Fetch Maven Ant Tasks and Cassandra's dependencies
- These targets are intentionally free of dependencies so that they
- can be run stand-alone from a binary release artifact.
- -->
- <target name="maven-ant-tasks-localrepo" unless="maven-ant-tasks.jar.exists" if="maven-ant-tasks.jar.local"
- depends="init" description="Fetch Maven ANT Tasks from Maven Local Repository">
- <copy file="${maven-ant-tasks.local}/${maven-ant-tasks.version}/maven-ant-tasks-${maven-ant-tasks.version}.jar"
- tofile="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar"/>
- <property name="maven-ant-tasks.jar.exists" value="true"/>
- </target>
-
- <target name="maven-ant-tasks-download" depends="init,maven-ant-tasks-localrepo" unless="maven-ant-tasks.jar.exists"
- description="Fetch Maven ANT Tasks from Maven Central Repositroy">
- <echo>Downloading Maven ANT Tasks...</echo>
- <get src="${maven-ant-tasks.url}/${maven-ant-tasks.version}/maven-ant-tasks-${maven-ant-tasks.version}.jar"
- dest="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar" usetimestamp="true" />
- <copy file="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar"
- tofile="${maven-ant-tasks.local}/${maven-ant-tasks.version}/maven-ant-tasks-${maven-ant-tasks.version}.jar"/>
- </target>
-
- <target name="maven-ant-tasks-init" depends="maven-ant-tasks-download,resolver-init" unless="maven-ant-tasks.initialized"
- description="Initialize Maven ANT Tasks">
- <typedef uri="antlib:org.apache.maven.artifact.ant" classpathref="maven-ant-tasks.classpath" />
- <property name="maven-ant-tasks.initialized" value="true"/>
- </target>
-
- <!-- this task defines the dependencies that will be fetched by Maven ANT Tasks
- the dependencies are re-used for publishing artifacts to Maven Central
- in order to keep everything consistent -->
- <target name="maven-declare-dependencies" depends="maven-ant-tasks-init"
- description="Define dependencies and dependency versions">
- <!-- The parent pom defines the versions of all dependencies -->
- <artifact:pom id="parent-pom"
- groupId="org.apache.cassandra"
- artifactId="cassandra-parent"
- packaging="pom"
- version="${version}"
- url="https://cassandra.apache.org"
- name="Apache Cassandra"
- inceptionYear="2009"
- description="The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.">
-
- <!-- Inherit from the ASF template pom file, ref http://maven.apache.org/pom/asf/ -->
- <parent groupId="org.apache" artifactId="apache" version="22"/>
- <license name="The Apache Software License, Version 2.0" url="https://www.apache.org/licenses/LICENSE-2.0.txt"/>
- <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
- <dependencyManagement>
- <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.1.8.4"/>
- <dependency groupId="org.lz4" artifactId="lz4-java" version="1.8.0"/>
- <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4" scope="provided"/>
- <dependency groupId="com.github.luben" artifactId="zstd-jni" version="1.5.5-1"/>
- <dependency groupId="com.google.guava" artifactId="guava" version="27.0-jre">
- <exclusion groupId="com.google.code.findbugs" artifactId="jsr305" />
- <exclusion groupId="org.codehaus.mojo" artifactId="animal-sniffer-annotations" />
- <exclusion groupId="com.google.guava" artifactId="listenablefuture" />
- <exclusion groupId="com.google.guava" artifactId="failureaccess" />
- <exclusion groupId="org.checkerframework" artifactId="checker-qual" />
- <exclusion groupId="com.google.errorprone" artifactId="error_prone_annotations" />
- </dependency>
- <dependency groupId="com.google.jimfs" artifactId="jimfs" version="1.1"/>
- <dependency groupId="org.hdrhistogram" artifactId="HdrHistogram" version="2.1.9"/>
- <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
- <dependency groupId="commons-codec" artifactId="commons-codec" version="1.9"/>
- <dependency groupId="commons-io" artifactId="commons-io" version="2.6"/>
- <dependency groupId="org.apache.commons" artifactId="commons-lang3" version="3.11"/>
- <dependency groupId="org.apache.commons" artifactId="commons-math3" version="3.2"/>
- <dependency groupId="org.antlr" artifactId="antlr" version="3.5.2" scope="provided">
- <exclusion groupId="org.antlr" artifactId="stringtemplate"/>
- </dependency>
- <dependency groupId="org.antlr" artifactId="ST4" version="4.0.8"/>
- <dependency groupId="org.antlr" artifactId="antlr-runtime" version="3.5.2">
- <exclusion groupId="org.antlr" artifactId="stringtemplate"/>
- </dependency>
- <dependency groupId="org.slf4j" artifactId="slf4j-api" version="1.7.25"/>
- <dependency groupId="org.slf4j" artifactId="log4j-over-slf4j" version="1.7.25"/>
- <dependency groupId="org.slf4j" artifactId="jcl-over-slf4j" version="1.7.25" />
- <dependency groupId="ch.qos.logback" artifactId="logback-core" version="1.2.9"/>
- <dependency groupId="ch.qos.logback" artifactId="logback-classic" version="1.2.9"/>
- <dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-core" version="2.13.2"/>
- <dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-databind" version="2.13.2.2"/>
- <dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-annotations" version="2.13.2"/>
- <dependency groupId="com.fasterxml.jackson.datatype" artifactId="jackson-datatype-jsr310" version="2.13.2"/>
- <dependency groupId="com.fasterxml.jackson.dataformat" artifactId="jackson-dataformat-yaml" version="2.13.2" scope="test">
- <exclusion groupId="org.yaml" artifactId="snakeyaml"/>
- </dependency>
- <dependency groupId="com.googlecode.json-simple" artifactId="json-simple" version="1.1"/>
- <dependency groupId="com.boundary" artifactId="high-scale-lib" version="1.0.6"/>
- <dependency groupId="com.github.jbellis" artifactId="jamm" version="${jamm.version}"/>
- <dependency groupId="org.yaml" artifactId="snakeyaml" version="1.26"/>
- <dependency groupId="junit" artifactId="junit" version="4.12" scope="test">
- <exclusion groupId="org.hamcrest" artifactId="hamcrest-core"/>
- </dependency>
- <dependency groupId="org.mockito" artifactId="mockito-core" version="3.2.4" scope="test"/>
- <dependency groupId="org.quicktheories" artifactId="quicktheories" version="0.26" scope="test"/>
- <dependency groupId="com.google.code.java-allocation-instrumenter" artifactId="java-allocation-instrumenter" version="${allocation-instrumenter.version}" scope="test">
- <exclusion groupId="com.google.guava" artifactId="guava"/>
- </dependency>
- <dependency groupId="org.apache.cassandra" artifactId="harry-core" version="0.0.1" scope="test"/>
- <dependency groupId="org.reflections" artifactId="reflections" version="0.10.2" scope="test"/>
- <dependency groupId="org.apache.cassandra" artifactId="dtest-api" version="${dtest-api.version}" scope="test"/>
- <dependency groupId="com.puppycrawl.tools" artifactId="checkstyle" version="8.40" scope="test"/>
- <dependency groupId="org.apache.hadoop" artifactId="hadoop-core" version="1.0.3" scope="provided">
- <exclusion groupId="org.mortbay.jetty" artifactId="servlet-api"/>
- <exclusion groupId="commons-logging" artifactId="commons-logging"/>
- <exclusion groupId="commons-lang" artifactId="commons-lang"/>
- <exclusion groupId="org.eclipse.jdt" artifactId="core"/>
- <exclusion groupId="ant" artifactId="ant"/>
- <exclusion groupId="junit" artifactId="junit"/>
- <exclusion groupId="org.codehaus.jackson" artifactId="jackson-mapper-asl"/>
- <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
- </dependency>
- <dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster" version="1.0.3" scope="provided">
- <exclusion groupId="asm" artifactId="asm"/> <!-- this is the outdated version 3.1 -->
- <exclusion groupId="org.codehaus.jackson" artifactId="jackson-mapper-asl"/>
- <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
- </dependency>
- <dependency groupId="net.java.dev.jna" artifactId="jna" version="5.9.0"/>
-
- <dependency groupId="org.jacoco" artifactId="org.jacoco.agent" version="${jacoco.version}" scope="test"/>
- <dependency groupId="org.jacoco" artifactId="org.jacoco.ant" version="${jacoco.version}" scope="test">
- <exclusion groupId="org.ow2.asm" artifactId="asm"/>
- </dependency>
-
- <dependency groupId="org.jboss.byteman" artifactId="byteman-install" version="${byteman.version}" scope="provided"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman" version="${byteman.version}" scope="provided"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman-submit" version="${byteman.version}" scope="provided"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman-bmunit" version="${byteman.version}" scope="provided"/>
-
- <dependency groupId="net.bytebuddy" artifactId="byte-buddy" version="${bytebuddy.version}" />
- <dependency groupId="net.bytebuddy" artifactId="byte-buddy-agent" version="${bytebuddy.version}" />
-
- <dependency groupId="org.openjdk.jmh" artifactId="jmh-core" version="1.21" scope="test"/>
- <dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess" version="1.21" scope="test"/>
-
- <dependency groupId="org.apache.ant" artifactId="ant-junit" version="1.10.12" scope="test"/>
-
- <dependency groupId="org.apache.cassandra" artifactId="cassandra-all" version="${version}" />
- <dependency groupId="io.dropwizard.metrics" artifactId="metrics-core" version="3.1.5" />
- <dependency groupId="io.dropwizard.metrics" artifactId="metrics-jvm" version="3.1.5" />
- <dependency groupId="io.dropwizard.metrics" artifactId="metrics-logback" version="3.1.5" />
- <dependency groupId="com.addthis.metrics" artifactId="reporter-config3" version="3.0.3">
- <exclusion groupId="org.hibernate" artifactId="hibernate-validator" />
- </dependency>
- <dependency groupId="org.mindrot" artifactId="jbcrypt" version="0.4" />
- <dependency groupId="io.airlift" artifactId="airline" version="0.8">
- <exclusion groupId="com.google.code.findbugs" artifactId="jsr305" />
- </dependency>
- <dependency groupId="io.netty" artifactId="netty-bom" version="4.1.58.Final" type="pom" scope="provided"/>
- <dependency groupId="io.netty" artifactId="netty-all" version="4.1.58.Final" />
- <dependency groupId="io.netty" artifactId="netty-tcnative-boringssl-static" version="2.0.36.Final"/>
- <dependency groupId="net.openhft" artifactId="chronicle-queue" version="${chronicle-queue.version}">
- <exclusion groupId="com.sun" artifactId="tools" />
- </dependency>
- <dependency groupId="net.openhft" artifactId="chronicle-core" version="${chronicle-core.version}">
- <exclusion groupId="net.openhft" artifactId="chronicle-analytics" />
- <exclusion groupId="org.jetbrains" artifactId="annotations" />
- </dependency>
- <dependency groupId="net.openhft" artifactId="chronicle-bytes" version="${chronicle-bytes.version}">
- <exclusion groupId="org.jetbrains" artifactId="annotations" />
- </dependency>
- <dependency groupId="net.openhft" artifactId="chronicle-wire" version="${chronicle-wire.version}">
- <exclusion groupId="net.openhft" artifactId="compiler" />
- </dependency>
- <dependency groupId="net.openhft" artifactId="chronicle-threads" version="${chronicle-threads.version}">
- <exclusion groupId="net.openhft" artifactId="affinity" />
- <!-- Exclude JNA here, as we want to avoid breaking consumers of the cassandra-all jar -->
- <exclusion groupId="net.java.dev.jna" artifactId="jna" />
- <exclusion groupId="net.java.dev.jna" artifactId="jna-platform" />
- </dependency>
- <dependency groupId="com.google.code.findbugs" artifactId="jsr305" version="2.0.2"/>
- <dependency groupId="com.clearspring.analytics" artifactId="stream" version="2.5.2">
- <exclusion groupId="it.unimi.dsi" artifactId="fastutil" />
- </dependency>
- <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="3.11.0" classifier="shaded">
- <exclusion groupId="io.netty" artifactId="netty-buffer"/>
- <exclusion groupId="io.netty" artifactId="netty-codec"/>
- <exclusion groupId="io.netty" artifactId="netty-handler"/>
- <exclusion groupId="io.netty" artifactId="netty-transport"/>
- <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
- <exclusion groupId="com.github.jnr" artifactId="jnr-ffi"/>
- <exclusion groupId="com.github.jnr" artifactId="jnr-posix"/>
- </dependency>
- <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj" version="${ecj.version}" />
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="${ohc.version}">
- <exclusion groupId="org.slf4j" artifactId="slf4j-api"/>
- </dependency>
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="${ohc.version}" />
- <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations" version="1.2.0" scope="provided"/>
- <dependency groupId="org.fusesource" artifactId="sigar" version="1.6.4">
- <exclusion groupId="log4j" artifactId="log4j"/>
- </dependency>
- <dependency groupId="com.carrotsearch" artifactId="hppc" version="0.8.1" />
- <dependency groupId="de.jflex" artifactId="jflex" version="${jflex.version}">
- <exclusion groupId="org.apache.ant" artifactId="ant"/>
- </dependency>
- <dependency groupId="com.github.rholder" artifactId="snowball-stemmer" version="1.3.0.581.1" />
- <dependency groupId="com.googlecode.concurrent-trees" artifactId="concurrent-trees" version="2.4.0" />
- <dependency groupId="com.github.ben-manes.caffeine" artifactId="caffeine" version="2.9.2" />
- <dependency groupId="org.jctools" artifactId="jctools-core" version="3.1.0"/>
- <dependency groupId="org.ow2.asm" artifactId="asm" version="${asm.version}"/>
- <dependency groupId="org.ow2.asm" artifactId="asm-tree" version="${asm.version}" scope="test"/>
- <dependency groupId="org.ow2.asm" artifactId="asm-commons" version="${asm.version}" scope="test"/>
- <dependency groupId="org.ow2.asm" artifactId="asm-util" version="${asm.version}" scope="test"/>
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-cli" version="0.14"/>
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-core" version="0.14">
- <exclusion groupId="org.gridkit.jvmtool" artifactId="sjk-hflame"/>
- <exclusion groupId="org.perfkit.sjk.parsers" artifactId="sjk-hflame"/>
- <exclusion groupId="org.perfkit.sjk.parsers" artifactId="sjk-jfr-standalone"/>
- <exclusion groupId="org.perfkit.sjk.parsers" artifactId="sjk-nps"/>
- <exclusion groupId="org.perfkit.sjk.parsers" artifactId="sjk-jfr5"/>
- <exclusion groupId="org.perfkit.sjk.parsers" artifactId="sjk-jfr6"/>
- </dependency>
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-stacktrace" version="0.14"/>
- <dependency groupId="org.gridkit.jvmtool" artifactId="mxdump" version="0.14"/>
- <dependency groupId="org.gridkit.lab" artifactId="jvm-attach-api" version="1.5"/>
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-json" version="0.14"/>
- <dependency groupId="com.beust" artifactId="jcommander" version="1.30"/>
- <dependency groupId="org.psjava" artifactId="psjava" version="0.1.19"/>
- <dependency groupId="javax.inject" artifactId="javax.inject" version="1"/>
- <dependency groupId="com.google.j2objc" artifactId="j2objc-annotations" version="1.3"/>
- <!-- adding this dependency is necessary for assertj. When updating assertj, need to also update the version of
- this that the new assertj's `assertj-parent-pom` depends on. -->
- <dependency groupId="org.junit" artifactId="junit-bom" version="5.6.0" type="pom" scope="test"/>
- <!-- when updating assertj, make sure to also update the corresponding junit-bom dependency -->
- <dependency groupId="org.assertj" artifactId="assertj-core" version="3.15.0" scope="provided"/>
- <dependency groupId="org.awaitility" artifactId="awaitility" version="4.0.3" scope="test">
- <exclusion groupId="org.hamcrest" artifactId="hamcrest"/>
- </dependency>
- <dependency groupId="org.hamcrest" artifactId="hamcrest" version="2.2" scope="test"/>
- <dependency groupId="com.github.seancfoley" artifactId="ipaddress" version="5.3.3" />
- </dependencyManagement>
- <developer id="adelapena" name="Andres de la Peña"/>
- <developer id="alakshman" name="Avinash Lakshman"/>
- <developer id="aleksey" name="Aleksey Yeschenko"/>
- <developer id="amorton" name="Aaron Morton"/>
- <developer id="aweisberg" name="Ariel Weisberg"/>
- <developer id="bdeggleston" name="Blake Eggleston"/>
- <developer id="benedict" name="Benedict Elliott Smith"/>
- <developer id="benjamin" name="Benjamin Lerer"/>
- <developer id="blambov" name="Branimir Lambov"/>
- <developer id="brandonwilliams" name="Brandon Williams"/>
- <developer id="carl" name="Carl Yeksigian"/>
- <developer id="dbrosius" name="David Brosiusd"/>
- <developer id="dikang" name="Dikang Gu"/>
- <developer id="eevans" name="Eric Evans"/>
- <developer id="edimitrova" name="Ekaterina Dimitrova"/>
- <developer id="gdusbabek" name="Gary Dusbabek"/>
- <developer id="goffinet" name="Chris Goffinet"/>
- <developer id="ifesdjeen" name="Alex Petrov"/>
- <developer id="jaakko" name="Laine Jaakko Olavi"/>
- <developer id="jake" name="T Jake Luciani"/>
- <developer id="jasonbrown" name="Jason Brown"/>
- <developer id="jbellis" name="Jonathan Ellis"/>
- <developer id="jfarrell" name="Jake Farrell"/>
- <developer id="jjirsa" name="Jeff Jirsa"/>
- <developer id="jkni" name="Joel Knighton"/>
- <developer id="jmckenzie" name="Josh McKenzie"/>
- <developer id="johan" name="Johan Oskarsson"/>
- <developer id="junrao" name="Jun Rao"/>
- <developer id="jzhuang" name="Jay Zhuang"/>
- <developer id="kohlisankalp" name="Sankalp Kohli"/>
- <developer id="marcuse" name="Marcus Eriksson"/>
- <developer id="mck" name="Michael Semb Wever"/>
- <developer id="mishail" name="Mikhail Stepura"/>
- <developer id="mshuler" name="Michael Shuler"/>
- <developer id="paulo" name="Paulo Motta"/>
- <developer id="pmalik" name="Prashant Malik"/>
- <developer id="rstupp" name="Robert Stupp"/>
- <developer id="scode" name="Peter Schuller"/>
- <developer id="beobal" name="Sam Tunnicliffe"/>
- <developer id="slebresne" name="Sylvain Lebresne"/>
- <developer id="stefania" name="Stefania Alborghetti"/>
- <developer id="tylerhobbs" name="Tyler Hobbs"/>
- <developer id="vijay" name="Vijay Parthasarathy"/>
- <developer id="xedin" name="Pavel Yaskevich"/>
- <developer id="yukim" name="Yuki Morishita"/>
- <developer id="zznate" name="Nate McCall"/>
- </artifact:pom>
-
- <!-- each dependency set then defines the subset of the dependencies for that dependency set -->
- <artifact:pom id="build-deps-pom"
- artifactId="cassandra-build-deps">
- <parent groupId="org.apache.cassandra"
- artifactId="cassandra-parent"
- version="${version}"
- relativePath="${final.name}-parent.pom"/>
- <dependency groupId="junit" artifactId="junit" scope="test"/>
- <dependency groupId="commons-io" artifactId="commons-io" scope="test"/>
- <dependency groupId="org.mockito" artifactId="mockito-core" scope="test"/>
- <dependency groupId="org.ow2.asm" artifactId="asm" version="${asm.version}"/>
- <dependency groupId="org.ow2.asm" artifactId="asm-tree" version="${asm.version}" scope="test"/>
- <dependency groupId="org.ow2.asm" artifactId="asm-commons" version="${asm.version}" scope="test"/>
- <dependency groupId="org.ow2.asm" artifactId="asm-util" version="${asm.version}" scope="test"/>
- <dependency groupId="com.google.jimfs" artifactId="jimfs" version="1.1" scope="test"/>
- <dependency groupId="com.puppycrawl.tools" artifactId="checkstyle" scope="test"/>
- <dependency groupId="org.quicktheories" artifactId="quicktheories" scope="test"/>
- <dependency groupId="org.reflections" artifactId="reflections" scope="test"/>
- <dependency groupId="com.google.code.java-allocation-instrumenter" artifactId="java-allocation-instrumenter" version="${allocation-instrumenter.version}" scope="test"/>
- <dependency groupId="org.apache.cassandra" artifactId="dtest-api" scope="test"/>
- <dependency groupId="org.openjdk.jmh" artifactId="jmh-core" scope="test"/>
- <dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess" scope="test"/>
- <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations" scope="test"/>
- <dependency groupId="org.apache.ant" artifactId="ant-junit" scope="test"/>
- <dependency groupId="org.apache.cassandra" artifactId="harry-core"/>
- <!-- adding this dependency is necessary for assertj. When updating assertj, need to also update the version of
- this that the new assertj's `assertj-parent-pom` depends on. -->
- <dependency groupId="org.junit" artifactId="junit-bom" type="pom"/>
- <dependency groupId="org.awaitility" artifactId="awaitility"/>
- <dependency groupId="org.hamcrest" artifactId="hamcrest"/>
- <!-- coverage debs -->
- <dependency groupId="org.jacoco" artifactId="org.jacoco.agent"/>
- <dependency groupId="org.jacoco" artifactId="org.jacoco.ant"/>
-
- <dependency groupId="com.fasterxml.jackson.dataformat" artifactId="jackson-dataformat-yaml"/>
- </artifact:pom>
-
- <!-- now the pom's for artifacts being deployed to Maven Central -->
- <artifact:pom id="all-pom"
- artifactId="cassandra-all"
- url="https://cassandra.apache.org"
- name="Apache Cassandra">
- <parent groupId="org.apache.cassandra"
- artifactId="cassandra-parent"
- version="${version}"
- relativePath="${final.name}-parent.pom"/>
- <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
- <dependency groupId="org.xerial.snappy" artifactId="snappy-java"/>
- <dependency groupId="org.lz4" artifactId="lz4-java"/>
- <dependency groupId="com.ning" artifactId="compress-lzf"/>
- <dependency groupId="com.google.guava" artifactId="guava"/>
- <dependency groupId="commons-cli" artifactId="commons-cli"/>
- <dependency groupId="commons-codec" artifactId="commons-codec"/>
- <dependency groupId="org.apache.commons" artifactId="commons-lang3"/>
- <dependency groupId="org.apache.commons" artifactId="commons-math3"/>
- <dependency groupId="org.antlr" artifactId="antlr" scope="provided"/>
- <dependency groupId="org.antlr" artifactId="ST4"/>
- <dependency groupId="org.antlr" artifactId="antlr-runtime"/>
- <dependency groupId="org.slf4j" artifactId="slf4j-api"/>
- <dependency groupId="org.slf4j" artifactId="log4j-over-slf4j"/>
- <dependency groupId="org.slf4j" artifactId="jcl-over-slf4j"/>
- <dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-core"/>
- <dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-databind"/>
- <dependency groupId="com.fasterxml.jackson.core" artifactId="jackson-annotations"/>
- <dependency groupId="com.fasterxml.jackson.datatype" artifactId="jackson-datatype-jsr310"/>
- <dependency groupId="com.googlecode.json-simple" artifactId="json-simple"/>
- <dependency groupId="com.boundary" artifactId="high-scale-lib"/>
- <dependency groupId="org.yaml" artifactId="snakeyaml"/>
- <dependency groupId="org.mindrot" artifactId="jbcrypt"/>
- <dependency groupId="io.airlift" artifactId="airline"/>
- <dependency groupId="io.dropwizard.metrics" artifactId="metrics-core"/>
- <dependency groupId="io.dropwizard.metrics" artifactId="metrics-jvm"/>
- <dependency groupId="io.dropwizard.metrics" artifactId="metrics-logback"/>
- <dependency groupId="com.addthis.metrics" artifactId="reporter-config3"/>
- <dependency groupId="com.clearspring.analytics" artifactId="stream"/>
-
- <dependency groupId="ch.qos.logback" artifactId="logback-core"/>
- <dependency groupId="ch.qos.logback" artifactId="logback-classic"/>
-
- <!-- don't need hadoop classes to run, but if you use the hadoop stuff -->
- <dependency groupId="org.apache.hadoop" artifactId="hadoop-core" optional="true"/>
- <dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster" optional="true"/>
-
- <!-- don't need the Java Driver to run, but if you use the hadoop stuff or UDFs -->
- <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded" optional="true"/>
- <!-- don't need jna to run, but nice to have -->
- <dependency groupId="net.java.dev.jna" artifactId="jna"/>
-
- <!-- don't need jamm unless running a server in which case it needs to be a -javagent to be used anyway -->
- <dependency groupId="com.github.jbellis" artifactId="jamm"/>
-
- <dependency groupId="io.netty" artifactId="netty-bom" type="pom" />
- <dependency groupId="io.netty" artifactId="netty-all"/>
- <dependency groupId="net.openhft" artifactId="chronicle-queue"/>
- <dependency groupId="net.openhft" artifactId="chronicle-core"/>
- <dependency groupId="net.openhft" artifactId="chronicle-bytes"/>
- <dependency groupId="net.openhft" artifactId="chronicle-wire"/>
- <dependency groupId="net.openhft" artifactId="chronicle-threads"/>
- <dependency groupId="org.fusesource" artifactId="sigar"/>
- <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core"/>
- <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8"/>
- <dependency groupId="com.github.ben-manes.caffeine" artifactId="caffeine" />
- <dependency groupId="org.jctools" artifactId="jctools-core"/>
- <dependency groupId="org.ow2.asm" artifactId="asm" />
- <dependency groupId="com.carrotsearch" artifactId="hppc" />
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-cli" />
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-core" />
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-stacktrace" />
- <dependency groupId="org.gridkit.jvmtool" artifactId="mxdump" />
- <dependency groupId="org.gridkit.lab" artifactId="jvm-attach-api" />
- <dependency groupId="com.beust" artifactId="jcommander" />
- <dependency groupId="org.gridkit.jvmtool" artifactId="sjk-json"/>
- <dependency groupId="com.github.luben" artifactId="zstd-jni"/>
- <dependency groupId="org.psjava" artifactId="psjava"/>
- <dependency groupId="io.netty" artifactId="netty-tcnative-boringssl-static"/>
- <dependency groupId="javax.inject" artifactId="javax.inject"/>
- <dependency groupId="com.google.j2objc" artifactId="j2objc-annotations"/>
- <dependency groupId="org.hdrhistogram" artifactId="HdrHistogram"/>
-
- <!-- sasi deps -->
- <dependency groupId="de.jflex" artifactId="jflex" />
- <dependency groupId="com.github.rholder" artifactId="snowball-stemmer" />
- <dependency groupId="com.googlecode.concurrent-trees" artifactId="concurrent-trees" />
-
- <!-- compile tools -->
- <dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
- <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations"/>
- <dependency groupId="org.assertj" artifactId="assertj-core"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman-install"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman-submit"/>
- <dependency groupId="org.jboss.byteman" artifactId="byteman-bmunit"/>
- <dependency groupId="com.github.seancfoley" artifactId="ipaddress" />
- </artifact:pom>
- </target>
-
- <!-- deprecated: legacy compatibility for build scripts in other repositories -->
- <target name="maven-ant-tasks-retrieve-build" depends="resolver-retrieve-build"/>
-
- <target name="echo-base-version">
- <echo message="${base.version}" />
- </target>
-
<!-- create properties file with C version -->
- <target name="createVersionPropFile">
+ <target name="createVersionPropFile" depends="get-git-sha">
<taskdef name="propertyfile" classname="org.apache.tools.ant.taskdefs.optional.PropertyFile"/>
<mkdir dir="${version.properties.dir}"/>
<propertyfile file="${version.properties.dir}/version.properties">
<entry key="CassandraVersion" value="${version}"/>
+ <entry key="GitSHA" value="${git.sha}"/>
</propertyfile>
</target>
@@ -917,6 +484,7 @@
<jvmarg value="-javaagent:${build.lib}/jamm-${jamm.version}.jar" />
<jvmarg value="-ea"/>
<jvmarg line="${java11-jvmargs}"/>
+ <jvmarg line="${java17-jvmargs}"/>
</java>
</target>
@@ -929,22 +497,22 @@
<target name="_build_java">
<!-- Note: we cannot use javac's 'release' option, as that does not allow accessing sun.misc.Unsafe nor
- Nashorn's ClassFilter class as any javac modules option is invalid for relase 8. -->
- <echo message="Compiling for Java ${target.version}..."/>
+ Nashorn's ClassFilter class as any javac modules option is invalid for release 8. -->
+ <echo message="Compiling for Java ${ant.java.version}..."/>
<javac
debug="true" debuglevel="${debuglevel}" encoding="utf-8"
- destdir="${build.classes.main}" includeantruntime="false" source="${source.version}" target="${target.version}">
+ destdir="${build.classes.main}" includeantruntime="false" source="${ant.java.version}" target="${ant.java.version}">
<src path="${build.src.java}"/>
<src path="${build.src.gen-java}"/>
<compilerarg value="-XDignore.symbol.file"/>
- <compilerarg line="${jdk11-javac-exports}"/>
+ <compilerarg line="${jdk11plus-javac-exports}"/>
<classpath>
<path refid="cassandra.classpath"/>
</classpath>
</javac>
</target>
- <target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java,rat-check"
+ <target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java,rat-check,get-git-sha"
name="build-project">
<echo message="${ant.project.name}: ${ant.file}"/>
<!-- Order matters! -->
@@ -965,7 +533,7 @@
<target name="stress-build-test" depends="stress-build" description="Compile stress tests">
<javac debug="true" debuglevel="${debuglevel}" destdir="${stress.test.classes}"
- source="${source.version}" target="${target.version}"
+ source="${ant.java.version}" target="${ant.java.version}"
includeantruntime="false" encoding="utf-8">
<classpath>
<path refid="cassandra.classpath.test"/>
@@ -982,7 +550,7 @@
<target name="_stress_build">
<mkdir dir="${stress.build.classes}" />
<javac compiler="modern" debug="true" debuglevel="${debuglevel}"
- source="${source.version}" target="${target.version}"
+ source="${ant.java.version}" target="${ant.java.version}"
encoding="utf-8" destdir="${stress.build.classes}" includeantruntime="true">
<src path="${stress.build.src}" />
<classpath>
@@ -1023,7 +591,7 @@
<target name="fqltool-build-test" depends="fqltool-build" description="Compile fqltool tests">
<javac debug="true" debuglevel="${debuglevel}" destdir="${fqltool.test.classes}"
- source="${source.version}" target="${target.version}"
+ source="${ant.java.version}" target="${ant.java.version}"
includeantruntime="false" encoding="utf-8">
<classpath>
<path refid="cassandra.classpath.test"/>
@@ -1040,7 +608,7 @@
<target name="_fqltool_build">
<mkdir dir="${fqltool.build.classes}" />
<javac compiler="modern" debug="true" debuglevel="${debuglevel}"
- source="${source.version}" target="${target.version}"
+ source="${ant.java.version}" target="${ant.java.version}"
encoding="utf-8" destdir="${fqltool.build.classes}" includeantruntime="true">
<src path="${fqltool.build.src}" />
<classpath>
@@ -1049,7 +617,7 @@
</javac>
</target>
- <target name="fqltool-test" depends="fqltool-build-test, maybe-build-test" description="Runs fqltool tests">
+ <target name="fqltool-test" depends="fqltool-build-test, build-test" description="Runs fqltool tests">
<testmacro inputdir="${fqltool.test.src}"
timeout="${test.timeout}">
</testmacro>
@@ -1077,7 +645,7 @@
<target name="_simulator-asm_build">
<mkdir dir="${simulator-asm.build.classes}" />
<javac compiler="modern" debug="true" debuglevel="${debuglevel}"
- source="${source.version}" target="${target.version}"
+ source="${ant.java.version}" target="${ant.java.version}"
encoding="utf-8" destdir="${simulator-asm.build.classes}" includeantruntime="true">
<src path="${simulator-asm.build.src}" />
<classpath>
@@ -1094,7 +662,7 @@
<target name="_simulator-bootstrap_build">
<mkdir dir="${simulator-bootstrap.build.classes}" />
<javac compiler="modern" debug="true" debuglevel="${debuglevel}"
- source="${source.version}" target="${target.version}"
+ source="${ant.java.version}" target="${ant.java.version}"
encoding="utf-8" destdir="${simulator-bootstrap.build.classes}" includeantruntime="true">
<src path="${simulator-bootstrap.build.src}" />
<classpath>
@@ -1109,15 +677,28 @@
</javac>
</target>
- <target name="_write-poms" depends="maven-declare-dependencies">
- <artifact:writepom pomRefId="parent-pom" file="${build.dir}/${final.name}-parent.pom"/>
- <artifact:writepom pomRefId="all-pom" file="${build.dir}/${final.name}.pom"/>
- <artifact:writepom pomRefId="build-deps-pom" file="${build.dir}/tmp-${final.name}-deps.pom"/>
- </target>
+ <target name="write-poms" unless="without.maven">
+ <filterset id="pom-template">
+ <filter token="version" value="${version}"/>
+ <filter token="final.name" value="${final.name}"/>
+ <filter token="jamm.version" value="${jamm.version}"/>
+ <filter token="allocation-instrumenter.version" value="${allocation-instrumenter.version}"/>
+ <filter token="ecj.version" value="${ecj.version}"/>
+ <filter token="asm.version" value="${asm.version}"/>
+ <filter token="jacoco.version" value="${jacoco.version}"/>
+ <filter token="jflex.version" value="${jflex.version}"/>
+ </filterset>
- <target name="write-poms" unless="without.maven">
- <antcall target="_write-poms" />
- </target>
+ <copy file=".build/cassandra-deps-template.xml" tofile="${build.dir}/${final.name}.pom">
+ <filterset refid="pom-template"/>
+ </copy>
+ <copy file=".build/parent-pom-template.xml" tofile="${build.dir}/${final.name}-parent.pom">
+ <filterset refid="pom-template"/>
+ </copy>
+ <copy file=".build/cassandra-build-deps-template.xml" tofile="${build.dir}/tmp-${final.name}-deps.pom">
+ <filterset refid="pom-template"/>
+ </copy>
+ </target>
<!--
The jar target makes cassandra.jar output.
@@ -1141,6 +722,7 @@
<attribute name="Implementation-Title" value="Cassandra"/>
<attribute name="Implementation-Version" value="${version}"/>
<attribute name="Implementation-Vendor" value="Apache"/>
+ <attribute name="Implementation-Git-SHA" value="${git.sha}"/>
<!-- </section> -->
</manifest>
</jar>
@@ -1424,6 +1006,7 @@
<target name="build-test" depends="_main-jar,stress-build-test,fqltool-build,resolver-dist-lib,simulator-jars,checkstyle-test"
description="Compile test classes">
<antcall target="_build-test"/>
+ <checktestnameshelper/>
</target>
<target name="_build-test">
@@ -1433,8 +1016,8 @@
debuglevel="${debuglevel}"
destdir="${test.classes}"
includeantruntime="true"
- source="${source.version}"
- target="${target.version}"
+ source="${ant.java.version}"
+ target="${ant.java.version}"
encoding="utf-8">
<classpath>
<path refid="cassandra.classpath.test"/>
@@ -1454,8 +1037,6 @@
<src path="${test.simulator-test.src}"/>
</javac>
- <checktestnameshelper/>
-
<!-- Non-java resources needed by the test suite -->
<copy todir="${test.classes}">
<fileset dir="${test.resources}"/>
@@ -1537,7 +1118,7 @@
<jvmarg value="-ea"/>
<jvmarg value="-Djava.io.tmpdir=${tmp.dir}"/>
<jvmarg value="-Dcassandra.debugrefcount=true"/>
- <jvmarg value="-Xss256k"/>
+ <jvmarg value="${jvm_xss}"/>
<!-- When we do classloader manipulation SoftReferences can cause memory leaks
that can OOM our test runs. The next two settings informs our GC
algorithm to limit the metaspace size and clean up SoftReferences
@@ -1545,6 +1126,7 @@
-->
<jvmarg value="-XX:SoftRefLRUPolicyMSPerMB=0" />
<jvmarg value="-XX:ActiveProcessorCount=${cassandra.test.processorCount}" />
+ <jvmarg value="-XX:HeapDumpPath=build/test" />
<jvmarg value="-Dcassandra.test.driver.connection_timeout_ms=${test.driver.connection_timeout_ms}"/>
<jvmarg value="-Dcassandra.test.driver.read_timeout_ms=${test.driver.read_timeout_ms}"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
@@ -1552,7 +1134,7 @@
<jvmarg value="-Dcassandra.test.sstableformatdevelopment=true"/>
<!-- The first time SecureRandom initializes can be slow if it blocks on /dev/random -->
<jvmarg value="-Djava.security.egd=file:/dev/urandom" />
- <jvmarg value="-Dcassandra.testtag=@{testtag}"/>
+ <jvmarg value="-Dcassandra.testtag=@{testtag}.jdk${ant.java.version}"/>
<jvmarg value="-Dcassandra.keepBriefBrief=${cassandra.keepBriefBrief}" />
<jvmarg value="-Dcassandra.strict.runtime.checks=true" />
<jvmarg value="-Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" /> <!-- enable defensive checks -->
@@ -1560,9 +1142,11 @@
<jvmarg value="-Dcassandra.test.messagingService.nonGracefulShutdown=${cassandra.test.messagingService.nonGracefulShutdown}"/>
<jvmarg value="-Dcassandra.use_nix_recursive_delete=${cassandra.use_nix_recursive_delete}"/>
<jvmarg line="${java11-jvmargs}"/>
+ <jvmarg line="${java17-jvmargs}"/>
<!-- disable shrinks in quicktheories CASSANDRA-15554 -->
<jvmarg value="-DQT_SHRINKS=0"/>
- <jvmarg line="${_std-test-jvmargs}" />
+ <jvmarg line="${_std-test-jvmargs11}" />
+ <jvmarg line="${_std-test-jvmargs17}" />
<jvmarg line="${test.jvm.args}" />
<optjvmargs/>
<!-- Uncomment to debug unittest, attach debugger to port 1416 -->
@@ -1669,7 +1253,27 @@
</sequential>
</macrodef>
- <macrodef name="testlist-system-keyspace-directory">
+ <macrodef name="testlist-trie">
+ <attribute name="test.file.list" />
+ <sequential>
+ <property name="trie_yaml" value="${build.test.dir}/cassandra.trie.yaml"/>
+ <concat destfile="${trie_yaml}">
+ <fileset file="${test.conf}/cassandra.yaml"/>
+ <fileset file="${test.conf}/trie_memtable.yaml"/>
+ </concat>
+ <testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}"
+ exclude="**/*.java" timeout="${test.timeout}" testtag="trie">
+ <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
+ <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
+ <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
+ <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+ <jvmarg value="-Dcassandra.config=file:///${trie_yaml}"/>
+ <jvmarg value="-Dcassandra.skip_sync=true" />
+ </testmacrohelper>
+ </sequential>
+ </macrodef>
+
+ <macrodef name="testlist-system-keyspace-directory">
<attribute name="test.file.list" />
<sequential>
<property name="system_keyspaces_directory_yaml" value="${build.test.dir}/cassandra.system.yaml"/>
@@ -1706,9 +1310,27 @@
ant testsome -Dtest.name=org.apache.cassandra.service.StorageServiceServerTest -Dtest.methods=testRegularMode,testGetAllRangesEmpty
-->
<target name="testsome" depends="maybe-build-test" description="Execute specific unit tests" >
+ <condition property="withoutMethods">
+ <and>
+ <equals arg1="${test.methods}" arg2=""/>
+ <not>
+ <contains string="${test.name}" substring="*"/>
+ </not>
+ </and>
+ </condition>
+ <condition property="withMethods">
+ <and>
+ <not>
+ <equals arg1="${test.methods}" arg2=""/>
+ </not>
+ <not>
+ <contains string="${test.name}" substring="*"/>
+ </not>
+ </and>
+ </condition>
<testmacro inputdir="${test.unit.src}" timeout="${test.timeout}">
- <test unless:blank="${test.methods}" name="${test.name}" methods="${test.methods}" outfile="build/test/output/TEST-${test.name}-${test.methods}"/>
- <test if:blank="${test.methods}" name="${test.name}" outfile="build/test/output/TEST-${test.name}"/>
+ <test if="withMethods" name="${test.name}" methods="${test.methods}" outfile="build/test/output/TEST-${test.name}-${test.methods}"/>
+ <test if="withoutMethods" name="${test.name}" outfile="build/test/output/TEST-${test.name}"/>
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
@@ -1759,6 +1381,14 @@
<testhelper testdelegate="testlist-cdc" />
</target>
+ <target name="test-trie" depends="maybe-build-test" description="Execute unit tests with trie memtables">
+ <path id="all-test-classes-path">
+ <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
+ </path>
+ <property name="all-test-classes" refid="all-test-classes-path"/>
+ <testhelper testdelegate="testlist-trie" />
+ </target>
+
<target name="test-system-keyspace-directory" depends="maybe-build-test" description="Execute unit tests with a system keyspaces directory configured">
<path id="all-test-classes-path">
<fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
@@ -1825,7 +1455,7 @@
<jvmarg value="-Djava.awt.headless=true"/>
<jvmarg value="-javaagent:${build.lib}/jamm-${jamm.version}.jar" />
<jvmarg value="-ea"/>
- <jvmarg value="-Xss256k"/>
+ <jvmarg value="${jvm_xss}"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
@@ -1871,7 +1501,7 @@
<jvmarg value="-Djava.awt.headless=true"/>
<jvmarg value="-javaagent:${build.lib}/jamm-${jamm.version}.jar" />
<jvmarg value="-ea"/>
- <jvmarg value="-Xss256k"/>
+ <jvmarg value="${jvm_xss}"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
@@ -2006,6 +1636,13 @@
<property name="all-test-classes" refid="all-test-classes-path"/>
<testhelper testdelegate="testlist-compression"/>
</target>
+ <target name="testclasslist-trie" depends="maybe-build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
+ <path id="all-test-classes-path">
+ <fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
+ </path>
+ <property name="all-test-classes" refid="all-test-classes-path"/>
+ <testhelper testdelegate="testlist-trie"/>
+ </target>
<target name="testclasslist-cdc" depends="maybe-build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
<path id="all-test-classes-path">
<fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
@@ -2055,6 +1692,7 @@
</testmacro>
</target>
+ <property name="simulator.asm.print" value="none"/> <!-- Supports: NONE, CLASS_SUMMARY, CLASS_DETAIL, METHOD_SUMMARY, METHOD_DETAIL, ASM; see org.apache.cassandra.simulator.asm.MethodLogger.Level -->
<target name="test-simulator-dtest" depends="maybe-build-test" description="Execute simulator dtests">
<testmacro inputdir="${test.simulator-test.src}" timeout="${test.simulation.timeout}" forkmode="perTest" showoutput="true" filter="**/test/${test.name}.java">
<jvmarg value="-Dlogback.configurationFile=test/conf/logback-simulator.xml"/>
@@ -2063,6 +1701,7 @@
<jvmarg value="-Dcassandra.skip_sync=true" />
<jvmarg value="-Dcassandra.debugrefcount=false"/>
<jvmarg value="-Dcassandra.test.simulator.determinismcheck=strict"/>
+ <jvmarg value="-Dcassandra.test.simulator.print_asm=${simulator.asm.print}" />
<!-- Support Simulator Tests -->
<jvmarg line="-javaagent:${test.lib}/jars/simulator-asm.jar"/>
<jvmarg line="-Xbootclasspath/a:${test.lib}/jars/simulator-bootstrap.jar"/>
@@ -2154,7 +1793,7 @@
</java>
</target>
- <target name="_maybe_update_idea_to_java11" if="java.version.11">
+ <target name="_maybe_update_idea_to_java11" depends="init" if="java.version.11">
<replace file="${eclipse.project.name}.iml" token="JDK_1_8" value="JDK_11"/>
<replace file=".idea/misc.xml" token="JDK_1_8" value="JDK_11"/>
<replace file=".idea/misc.xml" token="1.8" value="11"/>
@@ -2169,10 +1808,11 @@
<option name="ADDITIONAL_OPTIONS_STRING" value="--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED" />
</component>
</project>]]></echo>
+ <echo>"IDE configuration updated for use with JDK11"</echo>
</target>
<!-- Generate IDEA project description files -->
- <target name="generate-idea-files" depends="init,maven-ant-tasks-init,resolver-dist-lib,gen-cql3-grammar,generate-jflex-java,createVersionPropFile" description="Generate IDEA files">
+ <target name="generate-idea-files" depends="init,resolver-dist-lib,gen-cql3-grammar,generate-jflex-java,createVersionPropFile" description="Generate IDEA files">
<mkdir dir=".idea"/>
<mkdir dir=".idea/libraries"/>
<copy todir=".idea" overwrite="true">
@@ -2291,9 +1931,9 @@
failonerror="true"
maxmemory="512m">
<arg value="-source"/>
- <arg value="${source.version}" />
+ <arg value="${java.default}" />
<arg value="-target"/>
- <arg value="${target.version}" />
+ <arg value="${ant.java.version}" />
<arg value="-d" />
<arg value="none" />
<arg value="-proc:none" />
@@ -2307,17 +1947,15 @@
</java>
</target>
- <target name="init-checkstyle" depends="maven-ant-tasks-retrieve-build,build-project" unless="no-checkstyle">
- <path id="checkstyle.lib.path">
- <fileset dir="${test.lib}/jars" includes="*.jar"/>
- </path>
+ <target name="init-checkstyle" depends="resolver-retrieve-build,build-project" unless="no-checkstyle">
+ <echo message="checkstyle.version=${checkstyle.version}"/>
<!-- Sevntu custom checks are retrieved by Ivy into lib folder
and will be accessible to checkstyle-->
<taskdef resource="com/puppycrawl/tools/checkstyle/ant/checkstyle-ant-task.properties"
- classpathref="checkstyle.lib.path"/>
+ classpathref="checkstyle.classpath"/>
</target>
- <target name="checkstyle" depends="init-checkstyle,maven-ant-tasks-retrieve-build,build-project" description="Run custom checkstyle code analysis" if="java.version.8" unless="no-checkstyle">
+ <target name="checkstyle" depends="init-checkstyle,build-project" description="Run custom checkstyle code analysis" unless="no-checkstyle">
<property name="checkstyle.log.dir" value="${build.dir}/checkstyle" />
<property name="checkstyle.report.file" value="${checkstyle.log.dir}/checkstyle_report.xml"/>
<mkdir dir="${checkstyle.log.dir}" />
@@ -2333,7 +1971,7 @@
</checkstyle>
</target>
- <target name="checkstyle-test" depends="init-checkstyle,maven-ant-tasks-retrieve-build,build-project" description="Run custom checkstyle code analysis on tests" if="java.version.8" unless="no-checkstyle">
+ <target name="checkstyle-test" depends="init-checkstyle,resolver-retrieve-build,build-project" description="Run custom checkstyle code analysis on tests" unless="no-checkstyle">
<property name="checkstyle.log.dir" value="${build.dir}/checkstyle" />
<property name="checkstyle_test.report.file" value="${checkstyle.log.dir}/checkstyle_report_test.xml"/>
<mkdir dir="${checkstyle.log.dir}" />
@@ -2351,7 +1989,7 @@
<!-- Installs artifacts to local Maven repository -->
<target name="mvn-install"
- depends="maven-declare-dependencies,jar,sources-jar,javadoc-jar"
+ depends="jar,sources-jar,javadoc-jar"
description="Installs the artifacts in the Maven Local Repository">
<!-- the parent -->
@@ -2399,4 +2037,5 @@
<import file="${basedir}/.build/build-resolver.xml"/>
<import file="${basedir}/.build/build-rat.xml"/>
<import file="${basedir}/.build/build-owasp.xml"/>
+ <import file="${basedir}/.build/build-git.xml"/>
</project>
diff --git a/checkstyle.xml b/checkstyle.xml
index 053cc73..fc77fdd 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -96,7 +96,7 @@
<property name="illegalClasses" value="java.io.File,java.io.FileInputStream,java.io.FileOutputStream,java.io.FileReader,java.io.FileWriter,java.io.RandomAccessFile,java.util.concurrent.Semaphore,java.util.concurrent.CountDownLatch,java.util.concurrent.Executors,java.util.concurrent.LinkedBlockingQueue,java.util.concurrent.SynchronousQueue,java.util.concurrent.ArrayBlockingQueue,com.google.common.util.concurrent.Futures,java.util.concurrent.CompletableFuture,io.netty.util.concurrent.Future,io.netty.util.concurrent.Promise,io.netty.util.concurrent.AbstractFuture,com.google.common.util.concurrent.ListenableFutureTask,com.google.common.util.concurrent.ListenableFuture,com.google.common.util.concurrent.AbstractFuture,java.nio.file.Paths"/>
</module>
<module name="IllegalInstantiation">
- <property name="classes" value="java.io.File,java.lang.Thread,java.util.concurrent.FutureTask,java.util.concurrent.Semaphore,java.util.concurrent.CountDownLatch,java.util.concurrent.ScheduledThreadPoolExecutor,java.util.concurrent.ThreadPoolExecutor,java.util.concurrent.ForkJoinPool,java.lang.OutOfMemoryError"/>
+ <property name="classes" value="com.fasterxml.jackson.databind.ObjectMapper,java.io.File,java.lang.Thread,java.util.concurrent.FutureTask,java.util.concurrent.Semaphore,java.util.concurrent.CountDownLatch,java.util.concurrent.ScheduledThreadPoolExecutor,java.util.concurrent.ThreadPoolExecutor,java.util.concurrent.ForkJoinPool,java.lang.OutOfMemoryError"/>
</module>
<module name="RegexpSinglelineJava">
@@ -106,6 +106,67 @@
<property name="message" value="Avoid Path#toFile(), as some implementations may not support it." />
</module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Integer() -->
+ <property name="id" value="blockIntegerInstantiation"/>
+ <property name="format" value="new Integer\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Integer() and use Integer.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Long() -->
+ <property name="id" value="blockLongInstantiation"/>
+ <property name="format" value="new Long\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Long() and use Long.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Float() -->
+ <property name="id" value="blockFloatInstantiation"/>
+ <property name="format" value="new Float\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Float() and use Float.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Byte() -->
+ <property name="id" value="blockByteInstantiation"/>
+ <property name="format" value="new Byte\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Byte() and use Byte.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Double() -->
+ <property name="id" value="blockDoubleInstantiation"/>
+ <property name="format" value="new Double\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Double() and use Double.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Short() -->
+ <property name="id" value="blockShortInstantiation"/>
+ <property name="format" value="new Short\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Short() and use Short.valueOf()" />
+ </module>
+
+ <module name="SuppressionCommentFilter">
+ <property name="offCommentFormat" value="checkstyle: suppress below '([\w\|]+)'"/>
+ <property name="idFormat" value="$1"/>
+ </module>
+
+ <module name="SuppressWithNearbyCommentFilter">
+ <property name="commentFormat" value="checkstyle: suppress nearby '([\w\|]+)'"/>
+ <property name="idFormat" value="$1"/>
+ <property name="influenceFormat" value="0"/>
+ </module>
+
+ <module name="RegexpSinglelineJava">
+ <property name="id" value="blockSystemPropertyUsage"/>
+ <property name="format" value="(System\.getenv)|(System\.(getProperty|setProperty))|(Integer\.getInteger)|(Long\.getLong)|(Boolean\.getBoolean)"/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Use the CassandraRelevantProperties or CassandraRelevantEnv instead." />
+ </module>
+
<module name="RedundantImport"/>
<module name="UnusedImports"/>
</module>
diff --git a/checkstyle_test.xml b/checkstyle_test.xml
index d237827..720bc81 100644
--- a/checkstyle_test.xml
+++ b/checkstyle_test.xml
@@ -54,7 +54,75 @@
<property name="illegalClasses" value=""/>
</module>
<module name="IllegalInstantiation">
- <property name="classes" value=""/>
+ <property name="classes" value="com.fasterxml.jackson.databind.ObjectMapper"/>
+ </module>
+
+ <module name="RegexpSinglelineJava">
+ <!-- block Integer() -->
+ <property name="id" value="blockIntegerInstantiation"/>
+ <property name="format" value="new Integer\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Integer() and use Integer.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Long() -->
+ <property name="id" value="blockLongInstantiation"/>
+ <property name="format" value="new Long\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Long() and use Long.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Float() -->
+ <property name="id" value="blockFloatInstantiation"/>
+ <property name="format" value="new Float\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Float() and use Float.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Byte() -->
+ <property name="id" value="blockByteInstantiation"/>
+ <property name="format" value="new Byte\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Byte() and use Byte.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Double() -->
+ <property name="id" value="blockDoubleInstantiation"/>
+ <property name="format" value="new Double\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Double() and use Double.valueOf()" />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <!-- block Short() -->
+ <property name="id" value="blockShortInstantiation"/>
+ <property name="format" value="new Short\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Avoid Short() and use Short.valueOf()" />
+ </module>
+
+ <module name="SuppressionCommentFilter">
+ <property name="offCommentFormat" value="checkstyle: suppress below '([\w\|]+)'"/>
+ <property name="idFormat" value="$1"/>
+ </module>
+
+ <module name="SuppressWithNearbyCommentFilter">
+ <property name="commentFormat" value="checkstyle: suppress nearby '([\w\|]+)'"/>
+ <property name="idFormat" value="$1"/>
+ <property name="influenceFormat" value="0"/>
+ </module>
+
+ <module name="RegexpSinglelineJava">
+ <property name="id" value="blockSystemPropertyUsage"/>
+ <property name="format" value="(System\.getenv)|(System\.(getProperty|setProperty))|(Integer\.getInteger)|(Long\.getLong)|(Boolean\.getBoolean)"/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Use the CassandraRelevantProperties or CassandraRelevantEnv instead." />
+ </module>
+
+ <module name="RegexpSinglelineJava">
+ <property name="id" value="clearValueSystemPropertyUsage"/>
+ <property name="format" value="\.clearValue\("/>
+ <property name="ignoreComments" value="true"/>
+ <property name="message" value="Please use WithProperties in try-with-resources instead. See CASSANDRA-18453." />
</module>
<module name="RedundantImport"/>
diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh
index 2e3c8c9..25ba505 100644
--- a/conf/cassandra-env.sh
+++ b/conf/cassandra-env.sh
@@ -92,7 +92,7 @@
fi
#GC log path has to be defined here because it needs to access CASSANDRA_HOME
-if [ $JAVA_VERSION -ge 11 ] ; then
+if [ $JAVA_VERSION -ge 11 ] || [ $JAVA_VERSION -ge 17 ] ; then
# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
echo "$JVM_OPTS" | grep -qe "-[X]log:gc"
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 4b2711c..ca865b1 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -21,7 +21,7 @@
# Specifying initial_token will override this setting on the node's initial start,
# on subsequent starts, this setting will apply even if initial token is set.
#
-# See https://cassandra.apache.org/doc/latest/getting_started/production.html#tokens for
+# See https://cassandra.apache.org/doc/latest/getting-started/production.html#tokens for
# best practice information about num_tokens.
#
num_tokens: 16
@@ -44,8 +44,8 @@
allocate_tokens_for_local_replication_factor: 3
# initial_token allows you to specify tokens manually. While you can use it with
-# vnodes (num_tokens > 1, above) -- in which case you should provide a
-# comma-separated list -- it's primarily used when adding nodes to legacy clusters
+# vnodes (num_tokens > 1, above) -- in which case you should provide a
+# comma-separated list -- it's primarily used when adding nodes to legacy clusters
# that do not have vnodes enabled.
# initial_token:
@@ -99,6 +99,11 @@
# Disable the option in order to preserve those hints on the disk.
auto_hints_cleanup_enabled: false
+# Enable/disable transfering hints to a peer during decommission. Even when enabled, this does not guarantee
+# consistency for logged batches, and it may delay decommission when coupled with a strict hinted_handoff_throttle.
+# Default: true
+# transfer_hints_on_decommission: true
+
# Compression to apply to the hint files. If omitted, hints files
# will be written uncompressed. LZ4, Snappy, and Deflate compressors
# are supported.
@@ -107,6 +112,16 @@
# parameters:
# -
+# Directory where Cassandra should store results of a One-Shot troubleshooting heapdump for uncaught exceptions.
+# Note: this value can be overridden by the -XX:HeapDumpPath JVM env param with a relative local path for testing if
+# so desired.
+# If not set, the default directory is $CASSANDRA_HOME/heapdump
+# heap_dump_path: /var/lib/cassandra/heapdump
+
+# Enable / disable automatic dump of heap on first uncaught exception
+# If not set, the default value is false
+# dump_heap_on_uncaught_exception: true
+
# Enable / disable persistent hint windows.
#
# If set to false, a hint will be stored only in case a respective node
@@ -275,7 +290,7 @@
partitioner: org.apache.cassandra.dht.Murmur3Partitioner
# Directories where Cassandra should store data on disk. If multiple
-# directories are specified, Cassandra will spread data evenly across
+# directories are specified, Cassandra will spread data evenly across
# them by partitioning the token ranges.
# If not set, the default directory is $CASSANDRA_HOME/data/data.
# data_file_directories:
@@ -298,6 +313,18 @@
# containing a CDC-enabled table if at space limit in cdc_raw_directory).
cdc_enabled: false
+# Specify whether writes to the CDC-enabled tables should be blocked when CDC data on disk has reached to the limit.
+# When setting to false, the writes will not be blocked and the oldest CDC data on disk will be deleted to
+# ensure the size constraint. The default is true.
+# cdc_block_writes: true
+
+# Specify whether CDC mutations are replayed through the write path on streaming, e.g. repair.
+# When enabled, CDC data streamed to the destination node will be written into commit log first. When setting to false,
+# the streamed CDC data is written into SSTables just the same as normal streaming. The default is true.
+# If this is set to false, streaming will be considerably faster however it's possible that, in extreme situations
+# (losing > quorum # nodes in a replica set), you may have data in your SSTables that never makes it to the CDC log.
+# cdc_on_repair_enabled: true
+
# CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the
# segment contains mutations for a CDC-enabled table. This should be placed on a
# separate spindle than the data directories. If not set, the default directory is
@@ -376,6 +403,8 @@
# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
#
# Default value is empty to make it "auto" (min(5% of Heap (in MiB), 100MiB)). Set to 0 to disable key cache.
+#
+# This is only relevant to SSTable formats that use key cache, e.g. BIG.
# Min unit: MiB
key_cache_size:
@@ -387,12 +416,14 @@
# terms of I/O for the key cache. Row cache saving is much more expensive and
# has limited use.
#
+# This is only relevant to SSTable formats that use key cache, e.g. BIG.
# Default is 14400 or 4 hours.
# Min unit: s
key_cache_save_period: 4h
# Number of keys from the key cache to save
# Disabled by default, meaning all keys are going to be saved
+# This is only relevant to SSTable formats that use key cache, e.g. BIG.
# key_cache_keys_to_save: 100
# Row cache implementation class name. Available implementations:
@@ -401,7 +432,7 @@
# Fully off-heap row cache implementation (default).
#
# org.apache.cassandra.cache.SerializingCacheProvider
-# This is the row cache implementation availabile
+# This is the row cache implementation available
# in previous releases of Cassandra.
# row_cache_class_name: org.apache.cassandra.cache.OHCProvider
@@ -469,8 +500,8 @@
# Min unit: s
# cache_load_timeout: 30s
-# commitlog_sync may be either "periodic", "group", or "batch."
-#
+# commitlog_sync may be either "periodic", "group", or "batch."
+#
# When in batch mode, Cassandra won't ack writes until the commit log
# has been flushed to disk. Each incoming write will trigger the flush task.
# commitlog_sync_batch_window_in_ms is a deprecated value. Previously it had
@@ -551,6 +582,9 @@
# seeds is actually a comma-delimited list of addresses.
# Ex: "<ip1>,<ip2>,<ip3>"
- seeds: "127.0.0.1:7000"
+ # If set to "true", SimpleSeedProvider will return all IP addresses for a DNS name,
+ # based on the configured name service on the system. Defaults to "false".
+ # resolve_multiple_ip_addresses_per_dns_record: "false"
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
@@ -896,9 +930,11 @@
# internode_socket_receive_buffer_size:
# Set to true to have Cassandra create a hard link to each sstable
-# flushed or streamed locally in a backups/ subdirectory of the
-# keyspace data. Removing these links is the operator's
-# responsibility.
+# flushed or streamed locally in a backups/ subdirectory of all the
+# keyspace data in this node. Removing these links is the operator's
+# responsibility. The operator can also turn off incremental backups
+# for specified table by setting table parameter incremental_backups to
+# false, which is set to true by default. See CASSANDRA-15402
incremental_backups: false
# Whether or not to take a snapshot before each compaction. Be
@@ -908,7 +944,7 @@
snapshot_before_compaction: false
# Whether or not a snapshot is taken of the data before keyspace truncation
-# or dropping of column families. The STRONGLY advised default of true
+# or dropping of column families. The STRONGLY advised default of true
# should be used to provide data safety. If you set this flag to false, you will
# lose data on truncation or drop.
auto_snapshot: true
@@ -928,18 +964,31 @@
# taking and clearing snapshots
snapshot_links_per_second: 0
+# The sstable formats configuration. SSTable formats implementations are
+# loaded using the service loader mechanism. In this section, one can select
+# the format for created sstables and pass additional parameters for the formats
+# available on the classpath.
+# The default format is "big", the legacy SSTable format in use since Cassandra 3.0.
+# Cassandra versions 5.0 and later also support the trie-indexed "bti" format,
+# which offers better performance.
+#sstable:
+# selected_format: big
+
# Granularity of the collation index of rows within a partition.
-# Increase if your rows are large, or if you have a very large
-# number of rows per partition. The competing goals are these:
+# Applies to both BIG and BTI SSTable formats. In both formats,
+# a smaller granularity results in faster lookup of rows within
+# a partition, but a bigger index file size.
+# Using smaller granularities with the BIG format is not recommended
+# because bigger collation indexes cannot be cached efficiently
+# or at all if they become sufficiently large. Further, if
+# large rows, or a very large number of rows per partition are
+# present, it is recommended to increase the index granularity
+# or switch to the BTI SSTable format.
#
-# - a smaller granularity means more index entries are generated
-# and looking up rows withing the partition by collation column
-# is faster
-# - but, Cassandra will keep the collation index in memory for hot
-# rows (as part of the key cache), so a larger granularity means
-# you can cache more hot rows
+# Leave undefined to use a default suitable for the SSTable format
+# in use (64 KiB for BIG, 16KiB for BTI).
# Min unit: KiB
-column_index_size: 64KiB
+# column_index_size: 4KiB
# Per sstable indexed key cache entries (the collation index in memory
# mentioned above) exceeding this size will not be held on heap.
@@ -948,6 +997,8 @@
#
# Note that this size refers to the size of the
# serialized index information and not the size of the partition.
+#
+# This is only relevant to SSTable formats that use key cache, e.g. BIG.
# Min unit: KiB
column_index_cache_size: 2KiB
@@ -962,7 +1013,7 @@
#
# concurrent_compactors defaults to the smaller of (number of disks,
# number of cores), with a minimum of 2 and a maximum of 8.
-#
+#
# If your data directories are backed by SSD, you should increase this
# to the number of cores.
# concurrent_compactors: 1
@@ -990,7 +1041,7 @@
# When compacting, the replacement sstable(s) can be opened before they
# are completely written, and used in place of the prior sstables for
-# any range that has been written. This helps to smoothly transfer reads
+# any range that has been written. This helps to smoothly transfer reads
# between the sstables, reducing page cache churn and keeping hot rows hot
# Set sstable_preemptive_open_interval to null for disabled which is equivalent to
# sstable_preemptive_open_interval_in_mb being negative
@@ -1138,10 +1189,10 @@
# Enable operation timeout information exchange between nodes to accurately
# measure request timeouts. If disabled, replicas will assume that requests
# were forwarded to them instantly by the coordinator, which means that
-# under overload conditions we will waste that much extra time processing
+# under overload conditions we will waste that much extra time processing
# already-timed-out requests.
#
-# Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync,
+# Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync,
# since this is a requirement for general correctness of last write wins.
# internode_timeout: true
@@ -1329,17 +1380,23 @@
legacy_ssl_storage_port_enabled: false
# Set to a valid keystore if internode_encryption is dc, rack or all
keystore: conf/.keystore
- keystore_password: cassandra
+ #keystore_password: cassandra
# Configure the way Cassandra creates SSL contexts.
# To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory
# ssl_context_factory:
# # Must be an instance of org.apache.cassandra.security.ISslContextFactory
# class_name: org.apache.cassandra.security.DefaultSslContextFactory
+ # During internode mTLS authentication, inbound connections (acting as servers) use keystore, keystore_password
+ # containing server certificate to create SSLContext and
+ # outbound connections (acting as clients) use outbound_keystore & outbound_keystore_password with client certificates
+ # to create SSLContext. By default, outbound_keystore is the same as keystore indicating mTLS is not enabled.
+# outbound_keystore: conf/.keystore
+# outbound_keystore_password: cassandra
# Verify peer server certificates
require_client_auth: false
# Set to a valid trustore if require_client_auth is true
truststore: conf/.truststore
- truststore_password: cassandra
+ #truststore_password: cassandra
# Verify that the host name in the certificate matches the connected host
require_endpoint_verification: false
# More advanced defaults:
@@ -1373,7 +1430,7 @@
# optional: true
# Set keystore and keystore_password to valid keystores if enabled is true
keystore: conf/.keystore
- keystore_password: cassandra
+ #keystore_password: cassandra
# Configure the way Cassandra creates SSL contexts.
# To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory
# ssl_context_factory:
@@ -1381,6 +1438,7 @@
# class_name: org.apache.cassandra.security.DefaultSslContextFactory
# Verify client certificates
require_client_auth: false
+ # require_endpoint_verification: false
# Set trustore and truststore_password if require_client_auth is true
# truststore: conf/.truststore
# truststore_password: cassandra
@@ -1426,12 +1484,6 @@
# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code.
user_defined_functions_enabled: false
-# Enables scripted UDFs (JavaScript UDFs).
-# Java UDFs are always enabled, if user_defined_functions_enabled is true.
-# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider.
-# This option has no effect, if user_defined_functions_enabled is false.
-scripted_user_defined_functions_enabled: false
-
# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from
# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by
# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys
@@ -1508,7 +1560,8 @@
# Log WARN on any batches not of type LOGGED than span across more partitions than this limit
unlogged_batch_across_partitions_warn_threshold: 10
-# Log a warning when compacting partitions larger than this value
+# Log a warning when compacting partitions larger than this value.
+# As of Cassandra 5.0, this property is deprecated in favour of partition_size_warn_threshold.
compaction_large_partition_warning_threshold: 100MiB
# Log a warning when writing more tombstones than this value to a partition
@@ -1557,7 +1610,7 @@
# max_concurrent_automatic_sstable_upgrades: 1
# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
-# on audit_logging for full details about the various configuration options.
+# on audit_logging for full details about the various configuration options and production tips.
audit_logging_options:
enabled: false
logger:
@@ -1573,11 +1626,13 @@
# block: true
# max_queue_weight: 268435456 # 256 MiB
# max_log_size: 17179869184 # 16 GiB
- ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+ #
+ ## If archive_command is empty or unset, Cassandra uses a built-in DeletingArchiver that deletes the oldest files if ``max_log_size`` is reached.
+ ## If archive_command is set, Cassandra does not use DeletingArchiver, so it is the responsibility of the script to make any required cleanup.
+ ## Example: "/path/to/script.sh %path" where %path is replaced with the file being rolled.
# archive_command:
# max_archive_retries: 10
-
# default options for full query logging - these can be overridden from command line when executing
# nodetool enablefullquerylog
# full_query_logging_options:
@@ -1637,6 +1692,14 @@
# warming of auth caches prior to node completing startup. See CASSANDRA-16958
# auth_cache_warming_enabled: false
+# If enabled, dynamic data masking allows to attach CQL masking functions to the columns of a table.
+# Users without the UNMASK permission will see an obscured version of the values of the columns with an attached mask.
+# If dynamic data masking is disabled it won't be allowed to create new column masks, although it will still be possible
+# to drop any previously existing masks. Also, any existing mask will be ignored at query time, so all users will see
+# the clear values of the masked columns.
+# Defaults to false to disable dynamic data masking.
+# dynamic_data_masking_enabled: false
+
#########################
# EXPERIMENTAL FEATURES #
#########################
@@ -1691,58 +1754,112 @@
# The two thresholds default to -1 to disable.
# keyspaces_warn_threshold: -1
# keyspaces_fail_threshold: -1
+#
# Guardrail to warn or fail when creating more user tables than threshold.
# The two thresholds default to -1 to disable.
# tables_warn_threshold: -1
# tables_fail_threshold: -1
+#
# Guardrail to enable or disable the ability to create uncompressed tables
# uncompressed_tables_enabled: true
+#
# Guardrail to warn or fail when creating/altering a table with more columns per table than threshold.
# The two thresholds default to -1 to disable.
# columns_per_table_warn_threshold: -1
# columns_per_table_fail_threshold: -1
+#
# Guardrail to warn or fail when creating more secondary indexes per table than threshold.
# The two thresholds default to -1 to disable.
# secondary_indexes_per_table_warn_threshold: -1
# secondary_indexes_per_table_fail_threshold: -1
+#
# Guardrail to enable or disable the creation of secondary indexes
# secondary_indexes_enabled: true
+#
# Guardrail to warn or fail when creating more materialized views per table than threshold.
# The two thresholds default to -1 to disable.
# materialized_views_per_table_warn_threshold: -1
# materialized_views_per_table_fail_threshold: -1
+#
# Guardrail to warn about, ignore or reject properties when creating tables. By default all properties are allowed.
# table_properties_warned: []
# table_properties_ignored: []
# table_properties_disallowed: []
+#
# Guardrail to allow/disallow user-provided timestamps. Defaults to true.
# user_timestamps_enabled: true
+#
+# Guardrail to bound user-provided timestamps within a given range. Default is infinite (denoted by null).
+# Accepted values are durations of the form 12h, 24h, etc.
+# maximum_timestamp_warn_threshold:
+# maximum_timestamp_fail_threshold:
+# minimum_timestamp_warn_threshold:
+# minimum_timestamp_fail_threshold:
+#
# Guardrail to allow/disallow GROUP BY functionality.
# group_by_enabled: true
+#
# Guardrail to allow/disallow TRUNCATE and DROP TABLE statements
# drop_truncate_table_enabled: true
+#
+# Guardrail to allow/disallow DROP KEYSPACE statements
+# drop_keyspace_enabled: true
+#
# Guardrail to warn or fail when using a page size greater than threshold.
# The two thresholds default to -1 to disable.
# page_size_warn_threshold: -1
# page_size_fail_threshold: -1
+#
# Guardrail to allow/disallow list operations that require read before write, i.e. setting list element by index and
# removing list elements by either index or value. Defaults to true.
# read_before_write_list_operations_enabled: true
+#
# Guardrail to warn or fail when querying with an IN restriction selecting more partition keys than threshold.
# The two thresholds default to -1 to disable.
# partition_keys_in_select_warn_threshold: -1
# partition_keys_in_select_fail_threshold: -1
+#
# Guardrail to warn or fail when an IN query creates a cartesian product with a size exceeding threshold,
# eg. "a in (1,2,...10) and b in (1,2...10)" results in cartesian product of 100.
# The two thresholds default to -1 to disable.
# in_select_cartesian_product_warn_threshold: -1
# in_select_cartesian_product_fail_threshold: -1
+#
# Guardrail to warn about or reject read consistency levels. By default, all consistency levels are allowed.
# read_consistency_levels_warned: []
# read_consistency_levels_disallowed: []
+#
# Guardrail to warn about or reject write consistency levels. By default, all consistency levels are allowed.
# write_consistency_levels_warned: []
# write_consistency_levels_disallowed: []
+#
+# Guardrail to warn or fail when writing partitions larger than threshold, expressed as 100MiB, 1GiB, etc.
+# The guardrail is only checked when writing sstables (flush and compaction), and exceeding the fail threshold on that
+# moment will only log an error message, without interrupting the operation.
+# This operates on a per-sstable basis, so it won't detect a large partition if it is spread across multiple sstables.
+# The warning threshold replaces the deprecated config property compaction_large_partition_warning_threshold.
+# The two thresholds default to null to disable.
+# partition_size_warn_threshold:
+# partition_size_fail_threshold:
+#
+# Guardrail to warn or fail when writing column values larger than threshold.
+# This guardrail is only applied to the values of regular columns because both the serialized partitions keys and the
+# values of the components of the clustering key already have a fixed, relatively small size limit of 65535 bytes, which
+# is probably lesser than the thresholds defined here.
+# Deleting individual elements of non-frozen sets and maps involves creating tombstones that contain the value of the
+# deleted element, independently on whether the element existed or not. That tombstone value is also guarded by this
+# guardrail, to prevent the insertion of tombstones over the threshold. The downside is that enabling or raising this
+# threshold can prevent users from deleting set/map elements that were written when the guardrail was disabled or with a
+# lower value. Deleting the entire column, row or partition is always allowed, since the tombstones created for those
+# operations don't contain the CQL column values.
+# This guardrail is different to max_value_size. max_value_size is checked when deserializing any value to detect
+# sstable corruption, whereas this guardrail is checked on the CQL layer at write time to reject regular user queries
+# inserting too large columns.
+# The two thresholds default to null to disable.
+# Min unit: B
+# column_value_size_warn_threshold:
+# column_value_size_fail_threshold:
+#
# Guardrail to warn or fail when encountering larger size of collection data than threshold.
# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case
# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to
@@ -1753,6 +1870,7 @@
# collection_size_warn_threshold:
# Min unit: B
# collection_size_fail_threshold:
+#
# Guardrail to warn or fail when encountering more elements in collection than threshold.
# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case
# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to
@@ -1761,12 +1879,22 @@
# The two thresholds default to -1 to disable.
# items_per_collection_warn_threshold: -1
# items_per_collection_fail_threshold: -1
+#
# Guardrail to allow/disallow querying with ALLOW FILTERING. Defaults to true.
+# ALLOW FILTERING can potentially visit all the data in the table and have unpredictable performance.
# allow_filtering_enabled: true
+#
+# Guardrail to allow/disallow setting SimpleStrategy via keyspace creation or alteration. Defaults to true.
+# simplestrategy_enabled: true
+#
# Guardrail to warn or fail when creating a user-defined-type with more fields in than threshold.
# Default -1 to disable.
# fields_per_udt_warn_threshold: -1
# fields_per_udt_fail_threshold: -1
+#
+# Guardrail to indicate whether or not users are allowed to use ALTER TABLE commands to make column changes to tables
+# alter_table_enabled: true
+#
# Guardrail to warn or fail when local data disk usage percentage exceeds threshold. Valid values are in [1, 100].
# This is only used for the disks storing data directories, so it won't count any separate disks used for storing
# the commitlog, hints nor saved caches. The disk usage is the ratio between the amount of space used by the data
@@ -1778,7 +1906,8 @@
# The two thresholds default to -1 to disable.
# data_disk_usage_percentage_warn_threshold: -1
# data_disk_usage_percentage_fail_threshold: -1
-# Allows defining the max disk size of the data directories when calculating thresholds for
+#
+# Guardrail that allows users to define the max disk size of the data directories when calculating thresholds for
# disk_usage_percentage_warn_threshold and disk_usage_percentage_fail_threshold, so if this is greater than zero they
# become percentages of a fixed size on disk instead of percentages of the physically available disk size. This should
# be useful when we have a large disk and we only want to use a part of it for Cassandra's data directories.
@@ -1786,11 +1915,30 @@
# Defaults to null to disable and use the physically available disk size of data directories during calculations.
# Min unit: B
# data_disk_usage_max_disk_size:
+#
# Guardrail to warn or fail when the minimum replication factor is lesser than threshold.
# This would also apply to system keyspaces.
# Suggested value for use in production: 2 or higher
# minimum_replication_factor_warn_threshold: -1
# minimum_replication_factor_fail_threshold: -1
+#
+# Guardrail to warn or fail when the maximum replication factor is greater than threshold.
+# This would also apply to system keyspaces.
+# maximum_replication_factor_warn_threshold: -1
+# maximum_replication_factor_fail_threshold: -1
+
+# Guardrail to enable a CREATE or ALTER TABLE statement when default_time_to_live is set to 0
+# and the table is using TimeWindowCompactionStrategy compaction or a subclass of it.
+# It is suspicious to use default_time_to_live set to 0 with such compaction strategy.
+# Please keep in mind that data will not start to automatically expire after they are older than
+# a respective compaction window unit of a certain size. Please set TTL for your INSERT or UPDATE
+# statements if you expect data to be expired as table settings will not do it.
+# Defaults to true. If set to false, such statements fail and zero_ttl_on_twcs_warned flag is irrelevant.
+#zero_ttl_on_twcs_enabled: true
+# Guardrail to warn a user upon executing CREATE or ALTER TABLE statement when default_time_to_live is set to 0
+# and the table is using TimeWindowCompactionStrategy compaction or a subclass of it. Defaults to true.
+# if zero_ttl_on_twcs_enabled is set to false, this property is irrelevant as such statements will fail.
+#zero_ttl_on_twcs_warned: true
# Startup Checks are executed as part of Cassandra startup process, not all of them
# are configurable (so you can disable them) but these which are enumerated bellow.
diff --git a/conf/cqlshrc.sample b/conf/cqlshrc.sample
index 4878b58..56011f4 100644
--- a/conf/cqlshrc.sample
+++ b/conf/cqlshrc.sample
@@ -15,7 +15,7 @@
; specific language governing permissions and limitations
; under the License.
;
-; Sample ~/.cqlshrc file.
+; Sample ~/.cassandra/cqlshrc file.
[authentication]
;; If Cassandra has auth enabled, fill out these options
@@ -23,7 +23,6 @@
; credentials = ~/.cassandra/credentials
; keyspace = ks1
-
[auth_provider]
;; you can specify any auth provider found in your python environment
;; module and class will be used to dynamically load the class
@@ -33,6 +32,10 @@
; classname = PlainTextAuthProvider
; username = user1
+[protocol]
+;; Specify a specific protcol version otherwise the client will default and downgrade as necessary
+; version = None
+
[ui]
;; Whether or not to display query results with colors
; color = on
@@ -153,9 +156,8 @@
; boolstyle = True,False
;; The number of child worker processes to create for
-;; COPY tasks. Defaults to a max of 4 for COPY FROM and 16
-;; for COPY TO. However, at most (num_cores - 1) processes
-;; will be created.
+;; COPY tasks. Defaults to 16 for `COPY` tasks.
+;; However, at most (num_cores - 1) processes will be created.
; numprocesses =
;; The maximum number of failed attempts to fetch a range of data (when using
diff --git a/conf/jvm-server.options b/conf/jvm-server.options
index 46967f4..a639ee5 100644
--- a/conf/jvm-server.options
+++ b/conf/jvm-server.options
@@ -108,9 +108,6 @@
# transparent hugepage allocation more effective.
-XX:+AlwaysPreTouch
-# Disable biased locking as it does not benefit Cassandra.
--XX:-UseBiasedLocking
-
# Enable thread-local allocation blocks and allow the JVM to automatically
# resize them at runtime.
-XX:+UseTLAB
diff --git a/conf/jvm11-server.options b/conf/jvm11-server.options
index 1fc3503..5fd1f26 100644
--- a/conf/jvm11-server.options
+++ b/conf/jvm11-server.options
@@ -4,6 +4,15 @@
# See jvm-server.options. This file is specific for Java 11 and newer. #
###########################################################################
+
+########################
+# GENERAL JVM SETTINGS #
+########################
+
+# Disable biased locking as it does not benefit Cassandra.
+-XX:-UseBiasedLocking
+
+
#################
# GC SETTINGS #
#################
@@ -11,41 +20,41 @@
### CMS Settings
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:SurvivorRatio=8
--XX:MaxTenuringThreshold=1
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
--XX:CMSWaitDuration=10000
--XX:+CMSParallelInitialMarkEnabled
--XX:+CMSEdenChunksRecordAlways
+##-XX:+UseConcMarkSweepGC
+##-XX:+CMSParallelRemarkEnabled
+##-XX:SurvivorRatio=8
+##-XX:MaxTenuringThreshold=1
+##-XX:CMSInitiatingOccupancyFraction=75
+##-XX:+UseCMSInitiatingOccupancyOnly
+##-XX:CMSWaitDuration=10000
+##-XX:+CMSParallelInitialMarkEnabled
+##-XX:+CMSEdenChunksRecordAlways
## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
--XX:+CMSClassUnloadingEnabled
+##-XX:+CMSClassUnloadingEnabled
### G1 Settings
## Use the Hotspot garbage-first collector.
-#-XX:+UseG1GC
-#-XX:+ParallelRefProcEnabled
-#-XX:MaxTenuringThreshold=1
-#-XX:G1HeapRegionSize=16m
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
+-XX:MaxTenuringThreshold=1
+-XX:G1HeapRegionSize=16m
#
## Have the JVM do less remembered set work during STW, instead
## preferring concurrent GC. Reduces p99.9 latency.
-#-XX:G1RSetUpdatingPauseTimePercent=5
+-XX:G1RSetUpdatingPauseTimePercent=5
#
## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
## 200ms is the JVM default and lowest viable setting
## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
-#-XX:MaxGCPauseMillis=300
+-XX:MaxGCPauseMillis=300
## Optional G1 Settings
# Save CPU time on large (>= 16GB) heaps by delaying region scanning
# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
-#-XX:InitiatingHeapOccupancyPercent=70
+-XX:InitiatingHeapOccupancyPercent=70
# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
# Otherwise equal to the number of cores when 8 or less.
@@ -61,6 +70,7 @@
-Djdk.attach.allowAttachSelf=true
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/jdk.internal.util=ALL-UNNAMED
--add-exports java.base/sun.nio.ch=ALL-UNNAMED
--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
diff --git a/conf/jvm17-clients.options b/conf/jvm17-clients.options
new file mode 100644
index 0000000..e006fee
--- /dev/null
+++ b/conf/jvm17-clients.options
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###########################################################################
+# jvm17-clients.options #
+# #
+# See jvm-clients.options. This file is specific for Java 17 and newer. #
+###########################################################################
+
+###################
+# JPMS SETTINGS #
+###################
+
+-Djdk.attach.allowAttachSelf=true
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.attach/sun.tools.attach=ALL-UNNAMED
+
+
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+# to be addressed in CASSANDRA-17850
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.nio=ALL-UNNAMED
+# to be addressed during jamm maintenance
+--add-opens java.base/java.util.concurrent=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
+--add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
+# jamm and the in-jvm tests...
+--add-opens java.base/java.lang=ALL-UNNAMED
+# jamm
+--add-opens java.base/java.math=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+#jamm post CASSANDRA-17199
+--add-opens java.base/java.net=ALL-UNNAMED
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm17-server.options b/conf/jvm17-server.options
new file mode 100644
index 0000000..98a70a2
--- /dev/null
+++ b/conf/jvm17-server.options
@@ -0,0 +1,134 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###########################################################################
+# jvm17-server.options #
+# #
+# See jvm-server.options. This file is specific for Java 17 and newer. #
+###########################################################################
+
+#################
+# GC SETTINGS #
+#################
+
+
+
+### G1 Settings
+## Use the Hotspot garbage-first collector.
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
+-XX:MaxTenuringThreshold=1
+-XX:G1HeapRegionSize=16m
+
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+-XX:MaxGCPauseMillis=300
+
+## Optional G1 Settings
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+
+### JPMS
+
+-Djdk.attach.allowAttachSelf=true
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+# https://chronicle.software/chronicle-support-java-17/
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports java.sql/java.sql=ALL-UNNAMED
+
+#chronicle, AuditLog https://chronicle.software/chronicle-support-java-17/
+--add-exports java.base/java.lang.ref=ALL-UNNAMED
+--add-exports java.base/jdk.internal.util=ALL-UNNAMED
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+
+#to be addressed in CASSANDRA-17850
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+# https://chronicle.software/chronicle-support-java-17/
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.nio=ALL-UNNAMED
+#to be addressed during jamm maintenance
+--add-opens java.base/java.util.concurrent=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
+--add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
+# https://chronicle.software/chronicle-support-java-17/ explains also --add-opens java.base/java.util=ALL-UNNAMED, further to jamm
+# many cqlsh tests fail if we do not open the below one - jamm and at org.apache.cassandra.net.Verb.getModifiersField(Verb.java:388)
+# in-jvm tests
+--add-opens java.base/java.lang=ALL-UNNAMED
+#jamm
+--add-opens java.base/java.math=ALL-UNNAMED
+#in-jvm tests? plus # https://chronicle.software/chronicle-support-java-17/
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+#jamm post CASSANDRA-17199
+--add-opens java.base/java.net=ALL-UNNAMED
+
+### GC logging options -- uncomment to enable
+
+# Java 11 (and newer) GC logging options:
+# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
+# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+
+# Notes for Java 8 migration:
+#
+# -XX:+PrintGCDetails maps to -Xlog:gc*:... - i.e. add a '*' after "gc"
+# -XX:+PrintGCDateStamps maps to decorator 'time'
+#
+# -XX:+PrintHeapAtGC maps to 'heap' with level 'trace'
+# -XX:+PrintTenuringDistribution maps to 'age' with level 'debug'
+# -XX:+PrintGCApplicationStoppedTime maps to 'safepoint' with level 'info'
+# -XX:+PrintPromotionFailure maps to 'promotion' with level 'trace'
+# -XX:PrintFLSStatistics=1 maps to 'freelist' with level 'trace'
+
+### Netty Options
+
+# On Java >= 9 Netty requires the io.netty.tryReflectionSetAccessible system property to be set to true to enable
+# creation of direct buffers using Unsafe. Without it, this falls back to ByteBuffer.allocateDirect which has
+# inferior performance and risks exceeding MaxDirectMemory
+-Dio.netty.tryReflectionSetAccessible=true
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm8-server.options b/conf/jvm8-server.options
index ba800db..1436a7e 100644
--- a/conf/jvm8-server.options
+++ b/conf/jvm8-server.options
@@ -13,45 +13,48 @@
# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workaround.html
-XX:ThreadPriorityPolicy=42
+# Disable biased locking as it does not benefit Cassandra.
+-XX:-UseBiasedLocking
+
#################
# GC SETTINGS #
#################
### CMS Settings
--XX:+UseParNewGC
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:SurvivorRatio=8
--XX:MaxTenuringThreshold=1
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
--XX:CMSWaitDuration=10000
--XX:+CMSParallelInitialMarkEnabled
--XX:+CMSEdenChunksRecordAlways
+##-XX:+UseParNewGC
+##-XX:+UseConcMarkSweepGC
+##-XX:+CMSParallelRemarkEnabled
+##-XX:SurvivorRatio=8
+##-XX:MaxTenuringThreshold=1
+##-XX:CMSInitiatingOccupancyFraction=75
+##-XX:+UseCMSInitiatingOccupancyOnly
+##-XX:CMSWaitDuration=10000
+##-XX:+CMSParallelInitialMarkEnabled
+##-XX:+CMSEdenChunksRecordAlways
## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
--XX:+CMSClassUnloadingEnabled
+##-XX:+CMSClassUnloadingEnabled
### G1 Settings
## Use the Hotspot garbage-first collector.
-#-XX:+UseG1GC
-#-XX:+ParallelRefProcEnabled
-#-XX:MaxTenuringThreshold=1
-#-XX:G1HeapRegionSize=16m
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
+-XX:MaxTenuringThreshold=1
+-XX:G1HeapRegionSize=16m
-#
+
## Have the JVM do less remembered set work during STW, instead
## preferring concurrent GC. Reduces p99.9 latency.
-#-XX:G1RSetUpdatingPauseTimePercent=5
-#
+-XX:G1RSetUpdatingPauseTimePercent=5
+
## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
## 200ms is the JVM default and lowest viable setting
## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
-#-XX:MaxGCPauseMillis=300
+-XX:MaxGCPauseMillis=300
## Optional G1 Settings
# Save CPU time on large (>= 16GB) heaps by delaying region scanning
# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
-#-XX:InitiatingHeapOccupancyPercent=70
+-XX:InitiatingHeapOccupancyPercent=70
# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
# Otherwise equal to the number of cores when 8 or less.
diff --git a/conf/logback.xml b/conf/logback.xml
index e98fea4..102cf06 100644
--- a/conf/logback.xml
+++ b/conf/logback.xml
@@ -111,6 +111,14 @@
<appender name="LogbackMetrics" class="com.codahale.metrics.logback.InstrumentedAppender" />
-->
+ <!-- Uncomment below configuration and corresponding appender-ref to activate
+ logging into system_views.system_logs virtual table. -->
+ <!-- <appender name="CQLLOG" class="org.apache.cassandra.utils.logging.VirtualTableAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>WARN</level>
+ </filter>
+ </appender> -->
+
<root level="INFO">
<appender-ref ref="SYSTEMLOG" />
<appender-ref ref="STDOUT" />
@@ -118,6 +126,9 @@
<!--
<appender-ref ref="LogbackMetrics" />
-->
+ <!--
+ <appender-ref ref="CQLLOG"/>
+ -->
</root>
<logger name="org.apache.cassandra" level="DEBUG"/>
diff --git a/debian/cassandra.install b/debian/cassandra.install
index f54d1ad..dced5a2 100644
--- a/debian/cassandra.install
+++ b/debian/cassandra.install
@@ -25,6 +25,7 @@
tools/bin/auditlogviewer usr/bin
tools/bin/jmxtool usr/bin
tools/bin/hash_password usr/bin
+tools/bin/sstablepartitions usr/bin
lib/*.jar usr/share/cassandra/lib
lib/*.zip usr/share/cassandra/lib
lib/sigar-bin/* usr/share/cassandra/lib/sigar-bin
diff --git a/debian/cassandra.postinst b/debian/cassandra.postinst
index 752ff1f..95882e3 100644
--- a/debian/cassandra.postinst
+++ b/debian/cassandra.postinst
@@ -37,6 +37,8 @@
if [ -z "$2" ]; then
chown -R cassandra: /var/lib/cassandra
chown -R cassandra: /var/log/cassandra
+ chmod 750 /var/lib/cassandra/
+ chmod 750 /var/log/cassandra/
fi
if ! sysctl -p /etc/sysctl.d/cassandra.conf; then
echo >&2
diff --git a/debian/changelog b/debian/changelog
index 1bb1d0d..cdb1af9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,44 +1,8 @@
-cassandra (4.1.3) UNRELEASED; urgency=medium
+cassandra (5.0) UNRELEASED; urgency=medium
* New release
- -- Mick Semb Wever <mck@apache.org> Thu, 25 May 2023 16:11:28 +0200
-
-cassandra (4.1.2) unstable; urgency=medium
-
- * New release
-
- -- Mick Semb Wever <mck@apache.org> Thu, 25 May 2023 16:11:28 +0200
-
-cassandra (4.1.1) unstable; urgency=medium
-
- * New release
-
- -- Stefan Miklosovic <smiklosovic@apache.org> Wed, 15 Mar 2023 09:08:40 +0100
-
-cassandra (4.1.0) unstable; urgency=medium
-
- * New release
-
- -- Mick Semb Wever <mck@apache.org> Wed, 07 Dec 2022 21:53:33 +0100
-
-cassandra (4.1~rc1) unstable; urgency=medium
-
- * New release
-
- -- Mick Semb Wever <mck@apache.org> Thu, 17 Nov 2022 11:33:25 +0100
-
-cassandra (4.1~beta1) unstable; urgency=medium
-
- * New release
-
- -- Mick Semb Wever <mck@apache.org> Tue, 27 Sep 2022 00:08:07 +0200
-
-cassandra (4.1~alpha1) unstable; urgency=medium
-
- * New release
-
- -- Mick Semb Wever <mck@apache.org> Fri, 20 May 2022 22:02:50 +0200
+ -- Mick Semb Wever <mck@apache.org> Wed, 21 Apr 2021 19:24:28 +0200
cassandra (4.0~rc1) unstable; urgency=medium
diff --git a/doc/antora.yml b/doc/antora.yml
index 08ed5e2..401cbd4 100644
--- a/doc/antora.yml
+++ b/doc/antora.yml
@@ -1,6 +1,7 @@
name: Cassandra
-version: '4.1'
-display_version: '4.1'
+version: 'trunk'
+display_version: 'trunk'
+prerelease: true
asciidoc:
attributes:
cass_url: 'http://cassandra.apache.org/'
diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile
index d2a4b72..c1ce85f 100644
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@ -18,7 +18,7 @@
#
-->
-h1. Cassandra Query Language (CQL) v3.4.6
+h1. Cassandra Query Language (CQL) v3.4.7
@@ -252,9 +252,11 @@
'(' <column-definition> ( ',' <column-definition> )* ')'
( WITH <option> ( AND <option>)* )?
-<column-definition> ::= <identifier> <type> ( STATIC )? ( PRIMARY KEY )?
+<column-definition> ::= <identifier> <type> ( STATIC )? ( <column_mask> )? ( PRIMARY KEY )?
| PRIMARY KEY '(' <partition-key> ( ',' <identifier> )* ')'
+<column-mask> ::= MASKED WITH ( DEFAULT | <function> '(' ( <term> (',' <term>)* )? ')' )
+
<partition-key> ::= <identifier>
| '(' <identifier> (',' <identifier> )* ')'
@@ -362,7 +364,7 @@
h4(#compactionOptions). Compaction options
-The @compaction@ property must at least define the @'class'@ sub-option, that defines the compaction strategy class to use. The default supported class are @'SizeTieredCompactionStrategy'@, @'LeveledCompactionStrategy'@, @'DateTieredCompactionStrategy'@ and @'TimeWindowCompactionStrategy'@. Custom strategy can be provided by specifying the full class name as a "string constant":#constants. The rest of the sub-options depends on the chosen class. The sub-options supported by the default classes are:
+The @compaction@ property must at least define the @'class'@ sub-option, that defines the compaction strategy class to use. The default supported class are @'SizeTieredCompactionStrategy'@, @'LeveledCompactionStrategy'@ and @'TimeWindowCompactionStrategy'@. Custom strategy can be provided by specifying the full class name as a "string constant":#constants. The rest of the sub-options depends on the chosen class. The sub-options supported by the default classes are:
|_. option |_. supported compaction strategy |_. default |_. description |
| @enabled@ | _all_ | true | A boolean denoting whether compaction should be enabled or not.|
@@ -375,9 +377,6 @@
| @bucket_low@ | SizeTieredCompactionStrategy | 0.5 | Size tiered consider sstables to be within the same bucket if their size is within [average_size * @bucket_low@, average_size * @bucket_high@ ] (i.e the default groups sstable whose sizes diverges by at most 50%)|
| @bucket_high@ | SizeTieredCompactionStrategy | 1.5 | Size tiered consider sstables to be within the same bucket if their size is within [average_size * @bucket_low@, average_size * @bucket_high@ ] (i.e the default groups sstable whose sizes diverges by at most 50%).|
| @sstable_size_in_mb@ | LeveledCompactionStrategy | 5MB | The target size (in MB) for sstables in the leveled strategy. Note that while sstable sizes should stay less or equal to @sstable_size_in_mb@, it is possible to exceptionally have a larger sstable as during compaction, data for a given partition key are never split into 2 sstables|
-| @timestamp_resolution@ | DateTieredCompactionStrategy | MICROSECONDS | The timestamp resolution used when inserting data, could be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit) - don't change this unless you do mutations with USING TIMESTAMP <non_microsecond_timestamps> (or equivalent directly in the client)|
-| @base_time_seconds@ | DateTieredCompactionStrategy | 60 | The base size of the time windows. |
-| @max_sstable_age_days@ | DateTieredCompactionStrategy | 365 | SSTables only containing data that is older than this will never be compacted. |
| @timestamp_resolution@ | TimeWindowCompactionStrategy | MICROSECONDS | The timestamp resolution used when inserting data, could be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit) - don't change this unless you do mutations with USING TIMESTAMP <non_microsecond_timestamps> (or equivalent directly in the client)|
| @compaction_window_unit@ | TimeWindowCompactionStrategy | DAYS | The Java TimeUnit used for the window size, set in conjunction with @compaction_window_size@. Must be one of DAYS, HOURS, MINUTES |
| @compaction_window_size@ | TimeWindowCompactionStrategy | 1 | The number of @compaction_window_unit@ units that make up a time window. |
@@ -413,11 +412,15 @@
bc(syntax)..
<alter-table-stmt> ::= ALTER (TABLE | COLUMNFAMILY) (IF NOT EXISTS)? <tablename> <instruction>
-<instruction> ::= ADD (IF NOT EXISTS)? ( <identifier> <type> ( , <identifier> <type> )* )
+<instruction> ::= ADD (IF NOT EXISTS)? ( <identifier> <type> ( <column-mask> )?
+ ( , <identifier> <type> ( <column-mask> )? )* )
| DROP (IF EXISTS)? ( <identifier> ( , <identifier> )* )
+ | ALTER <identifier> ( <column-mask> | DROP MASKED )
| RENAME (IF EXISTS)? <identifier> to <identifier> (AND <identifier> to <identifier>)*
| DROP COMPACT STORAGE
| WITH <option> ( AND <option> )*
+
+<column-mask> ::= MASKED WITH ( DEFAULT | <function> '(' ( <term> (',' <term>)* )? ')' )
p.
__Sample:__
@@ -437,6 +440,7 @@
The @<tablename>@ is the table name optionally preceded by the keyspace name. The @<instruction>@ defines the alteration to perform:
* @ADD@: Adds a new column to the table. The @<identifier>@ for the new column must not conflict with an existing column. Moreover, columns cannot be added to tables defined with the @COMPACT STORAGE@ option. If the new column already exists, the statement will return an error, unless @IF NOT EXISTS@ is used in which case the operation is a no-op.
* @DROP@: Removes a column from the table. Dropped columns will immediately become unavailable in the queries and will not be included in compacted sstables in the future. If a column is readded, queries won't return values written before the column was last dropped. It is assumed that timestamps represent actual time, so if this is not your case, you should NOT readd previously dropped columns. Columns can't be dropped from tables defined with the @COMPACT STORAGE@ option. If the dropped column does not already exist, the statement will return an error, unless @IF EXISTS@ is used in which case the operation is a no-op.
+* @ALTER@: Alters an existing column. It can be used to set its data mask, or to set it as unmasked. The data mask is any function meant to obscure the real values of the column.
* @RENAME@ a primary key column of a table. Non primary key columns cannot be renamed. Furthermore, renaming a column to another name which already exists isn't allowed. It's important to keep in mind that renamed columns shouldn't have dependent secondary indexes. If the renamed column does not already exist, the statement will return an error, unless @IF EXISTS@ is used in which case the operation is a no-op.
* @DROP COMPACT STORAGE@: Removes Thrift compatibility mode from the table.
* @WITH@: Allows to update the options of the table. The "supported @<option>@":#createTableOptions (and syntax) are the same as for the @CREATE TABLE@ statement except that @COMPACT STORAGE@ is not supported. Note that setting any @compaction@ sub-options has the effect of erasing all previous @compaction@ options, so you need to re-specify all the sub-options if you want to keep them. The same note applies to the set of @compression@ sub-options.
@@ -1086,9 +1090,10 @@
<selector> ::= <identifier>
| <term>
- | WRITETIME '(' <identifier> ')'
+ | WRITETIME '(' <selector> ')'
+ | MAXWRITETIME '(' <selector> ')'
| COUNT '(' '*' ')'
- | TTL '(' <identifier> ')'
+ | TTL '(' <selector> ')'
| CAST '(' <selector> AS <type> ')'
| <function> '(' (<selector> (',' <selector>)*)? ')'
| <selector> '.' <identifier>
@@ -1135,7 +1140,7 @@
The @<select-clause>@ determines which columns needs to be queried and returned in the result-set. It consists of either the comma-separated list of <selector> or the wildcard character (@*@) to select all the columns defined for the table. Please note that for wildcard @SELECT@ queries the order of columns returned is not specified and is not guaranteed to be stable between Cassandra versions.
-A @<selector>@ is either a column name to retrieve or a @<function>@ of one or more @<term>@s. The function allowed are the same as for @<term>@ and are described in the "function section":#functions. In addition to these generic functions, the @WRITETIME@ (resp. @TTL@) function allows to select the timestamp of when the column was inserted (resp. the time to live (in seconds) for the column (or null if the column has no expiration set)) and the "@CAST@":#castFun function can be used to convert one data type to another.
+A @<selector>@ is either a column name to retrieve or a @<function>@ of one or more @<term>@s. The function allowed are the same as for @<term>@ and are described in the "function section":#functions. In addition to these generic functions, the @WRITETIME@ and @MAXWRITETIME@ (resp. @TTL@) function allows to select the timestamp of when the column was inserted (resp. the time to live (in seconds) for the column (or null if the column has no expiration set)) and the "@CAST@":#castFun function can be used to convert one data type to another.
Additionally, individual values of maps and sets can be selected using @[ <term> ]@. For maps, this will return the value corresponding to the key, if such entry exists. For sets, this will return the key that is selected if it exists and is thus mainly a way to check element existence. It is also possible to select a slice of a set or map with @[ <term> ... <term> @], where both bound can be omitted.
@@ -1520,6 +1525,8 @@
* @AUTHORIZE@
* @DESCRIBE@
* @EXECUTE@
+* @UNMASK@
+* @SELECT_MASKED@
Not all permissions are applicable to every type of resource. For instance, @EXECUTE@ is only relevant in the context of functions or mbeans; granting @EXECUTE@ on a resource representing a table is nonsensical. Attempting to @GRANT@ a permission on resource to which it cannot be applied results in an error response. The following illustrates which permissions can be granted on which types of resource, and which statements are enabled by that permission.
@@ -1579,6 +1586,12 @@
| @EXECUTE@ | @ALL MBEANS@ |Execute operations on any mbean|
| @EXECUTE@ | @MBEANS@ |Execute operations on any mbean matching a wildcard pattern|
| @EXECUTE@ | @MBEAN@ |Execute operations on named mbean|
+| @UNMASK@ | @ALL KEYSPACES@ |See the clear contents of masked columns on any table|
+| @UNMASK@ | @KEYSPACE@ |See the clear contents of masked columns on any table in keyspace|
+| @UNMASK@ | @TABLE@ |See the clear contents of masked columns on the specified table|
+| @SELECT_MASKED@ | @ALL KEYSPACES@ |Select restricting masked columns on any table|
+| @SELECT_MASKED@ | @KEYSPACE@ |Select restricting masked columns on any table in keyspace|
+| @SELECT_MASKED@ | @TABLE@ |Select restricting masked columns on the specified table|
h3(#grantPermissionsStmt). GRANT PERMISSION
@@ -1588,7 +1601,7 @@
bc(syntax)..
<grant-permission-stmt> ::= GRANT ( ALL ( PERMISSIONS )? | <permission> ( PERMISSION )? (, PERMISSION)* ) ON <resource> TO <identifier>
-<permission> ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | EXECUTE
+<permission> ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | EXECUTE | UNMASK | SELECT_MASKED
<resource> ::= ALL KEYSPACES
| KEYSPACE <identifier>
@@ -1645,7 +1658,7 @@
bc(syntax)..
<revoke-permission-stmt> ::= REVOKE ( ALL ( PERMISSIONS )? | <permission> ( PERMISSION )? (, PERMISSION)* ) ON <resource> FROM <identifier>
-<permission> ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | EXECUTE
+<permission> ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | EXECUTE | UNMASK | SELECT_MASKED
<resource> ::= ALL KEYSPACES
| KEYSPACE <identifier>
@@ -2105,21 +2118,21 @@
will never return any result by design, since the value returned by @now()@ is guaranteed to be unique.
-@currentTimeUUID@ is an alias of @now@.
+@current_timeuuid@ is an alias of @now@.
-h4. @minTimeuuid@ and @maxTimeuuid@
+h4. @min_timeuuid@ and @max_timeuuid@
-The @minTimeuuid@ (resp. @maxTimeuuid@) function takes a @timestamp@ value @t@ (which can be "either a timestamp or a date string":#usingtimestamps ) and return a _fake_ @timeuuid@ corresponding to the _smallest_ (resp. _biggest_) possible @timeuuid@ having for timestamp @t@. So for instance:
+The @min_timeuuid@ (resp. @max_timeuuid@) function takes a @timestamp@ value @t@ (which can be "either a timestamp or a date string":#usingtimestamps ) and return a _fake_ @timeuuid@ corresponding to the _smallest_ (resp. _biggest_) possible @timeuuid@ having for timestamp @t@. So for instance:
-bc(sample).
-SELECT * FROM myTable WHERE t > maxTimeuuid('2013-01-01 00:05+0000') AND t < minTimeuuid('2013-02-02 10:00+0000')
+bc(sample).
+SELECT * FROM myTable WHERE t > max_timeuuid('2013-01-01 00:05+0000') AND t < min_timeuuid('2013-02-02 10:00+0000')
-will select all rows where the @timeuuid@ column @t@ is strictly older than '2013-01-01 00:05+0000' but strictly younger than '2013-02-02 10:00+0000'. Please note that @t >= maxTimeuuid('2013-01-01 00:05+0000')@ would still _not_ select a @timeuuid@ generated exactly at '2013-01-01 00:05+0000' and is essentially equivalent to @t > maxTimeuuid('2013-01-01 00:05+0000')@.
+will select all rows where the @timeuuid@ column @t@ is strictly older than '2013-01-01 00:05+0000' but strictly younger than '2013-02-02 10:00+0000'. Please note that @t >= max_timeuuid('2013-01-01 00:05+0000')@ would still _not_ select a @timeuuid@ generated exactly at '2013-01-01 00:05+0000' and is essentially equivalent to @t > maxTimeuuid('2013-01-01 00:05+0000')@.
-_Warning_: We called the values generated by @minTimeuuid@ and @maxTimeuuid@ _fake_ UUID because they do no respect the Time-Based UUID generation process specified by the "RFC 4122":http://www.ietf.org/rfc/rfc4122.txt. In particular, the value returned by these 2 methods will not be unique. This means you should only use those methods for querying (as in the example above). Inserting the result of those methods is almost certainly _a bad idea_.
+_Warning_: We called the values generated by @min_timeuuid@ and @max_timeuuid@ _fake_ UUID because they do no respect the Time-Based UUID generation process specified by the "RFC 4122":http://www.ietf.org/rfc/rfc4122.txt. In particular, the value returned by these 2 methods will not be unique. This means you should only use those methods for querying (as in the example above). Inserting the result of those methods is almost certainly _a bad idea_.
h3(#datetimeFun). Datetime functions
@@ -2129,30 +2142,28 @@
The following functions can be used to retrieve the date/time at the time where the function is invoked:
|_. function name |_. output type |
-| @currentTimestamp@ | @timestamp@ |
-| @currentDate@ | @date@ |
-| @currentTime@ | @time@ |
-| @currentTimeUUID@ | @timeUUID@ |
+| @current_timestamp@ | @timestamp@ |
+| @current_date@ | @date@ |
+| @current_time@ | @time@ |
+| @current_timeuuid@ | @timeUUID@ |
For example the last 2 days of data can be retrieved using:
-bc(sample).
-SELECT * FROM myTable WHERE date >= currentDate() - 2d
+bc(sample).
+SELECT * FROM myTable WHERE date >= current_date() - 2d
h4(#timeFun). Time conversion functions
A number of functions are provided to "convert" a @timeuuid@, a @timestamp@ or a @date@ into another @native@ type.
|_. function name |_. input type |_. description|
-|@toDate@ |@timeuuid@ |Converts the @timeuuid@ argument into a @date@ type|
-|@toDate@ |@timestamp@ |Converts the @timestamp@ argument into a @date@ type|
-|@toTimestamp@ |@timeuuid@ |Converts the @timeuuid@ argument into a @timestamp@ type|
-|@toTimestamp@ |@date@ |Converts the @date@ argument into a @timestamp@ type|
-|@toUnixTimestamp@ |@timeuuid@ |Converts the @timeuuid@ argument into a @bigInt@ raw value|
-|@toUnixTimestamp@ |@timestamp@ |Converts the @timestamp@ argument into a @bigInt@ raw value|
-|@toUnixTimestamp@ |@date@ |Converts the @date@ argument into a @bigInt@ raw value|
-|@dateOf@ |@timeuuid@ |Similar to @toTimestamp(timeuuid)@ (DEPRECATED)|
-|@unixTimestampOf@ |@timeuuid@ |Similar to @toUnixTimestamp(timeuuid)@ (DEPRECATED)|
+|@to_date@ |@timeuuid@ |Converts the @timeuuid@ argument into a @date@ type|
+|@to_date@ |@timestamp@ |Converts the @timestamp@ argument into a @date@ type|
+|@to_timestamp@ |@timeuuid@ |Converts the @timeuuid@ argument into a @timestamp@ type|
+|@to_timestamp@ |@date@ |Converts the @date@ argument into a @timestamp@ type|
+|@to_unix_timestamp@ |@timeuuid@ |Converts the @timeuuid@ argument into a @bigInt@ raw value|
+|@to_unix_timestamp@ |@timestamp@ |Converts the @timestamp@ argument into a @bigInt@ raw value|
+|@to_unix_timestamp@ |@date@ |Converts the @date@ argument into a @bigInt@ raw value|
h4(#floorFun). Floor function
@@ -2166,12 +2177,12 @@
h3(#blobFun). Blob conversion functions
-A number of functions are provided to "convert" the native types into binary data (@blob@). For every @<native-type>@ @type@ supported by CQL3 (a notable exceptions is @blob@, for obvious reasons), the function @typeAsBlob@ takes a argument of type @type@ and return it as a @blob@. Conversely, the function @blobAsType@ takes a 64-bit @blob@ argument and convert it to a @bigint@ value. And so for instance, @bigintAsBlob(3)@ is @0x0000000000000003@ and @blobAsBigint(0x0000000000000003)@ is @3@.
+A number of functions are provided to "convert" the native types into binary data (@blob@). For every @<native-type>@ @type@ supported by CQL3 (a notable exceptions is @blob@, for obvious reasons), the function @type_as_blob@ takes a argument of type @type@ and return it as a @blob@. Conversely, the function @blob_as_type@ takes a 64-bit @blob@ argument and convert it to a @bigint@ value. And so for instance, @bigint_as_blob(3)@ is @0x0000000000000003@ and @blob_as_bigint(0x0000000000000003)@ is @3@.
h2(#aggregates). Aggregates
Aggregate functions work on a set of rows. They receive values for each row and returns one value for the whole set.
-If @normal@ columns, @scalar functions@, @UDT@ fields, @writetime@ or @ttl@ are selected together with aggregate functions, the values returned for them will be the ones of the first row matching the query.
+If @normal@ columns, @scalar functions@, @UDT@ fields, @writetime@, @maxwritetime@ or @ttl@ are selected together with aggregate functions, the values returned for them will be the ones of the first row matching the query.
CQL3 distinguishes between built-in aggregates (so called 'native aggregates') and "user-defined aggregates":#udas. CQL3 includes several native aggregates, described below:
@@ -2375,7 +2386,7 @@
Where possible, Cassandra will represent and accept data types in their native @JSON@ representation. Cassandra will also accept string representations matching the CQL literal format for all single-field types. For example, floats, ints, UUIDs, and dates can be represented by CQL literal strings. However, compound types, such as collections, tuples, and user-defined types must be represented by native @JSON@ collections (maps and lists) or a JSON-encoded string representation of the collection.
-The following table describes the encodings that Cassandra will accept in @INSERT JSON@ values (and @fromJson()@ arguments) as well as the format Cassandra will use when returning data for @SELECT JSON@ statements (and @fromJson()@):
+The following table describes the encodings that Cassandra will accept in @INSERT JSON@ values (and @from_json()@ arguments) as well as the format Cassandra will use when returning data for @SELECT JSON@ statements (and @from_json()@):
|_. type |_. formats accepted |_. return format |_. notes|
|@ascii@ |string |string |Uses JSON's @\u@ character escape|
@@ -2403,13 +2414,13 @@
|@varchar@ |string |string |Uses JSON's @\u@ character escape|
|@varint@ |integer, string |integer |Variable length; may overflow 32 or 64 bit integers in client-side decoder|
-h3(#fromJson). The fromJson() Function
+h3(#from_json). The from_json() Function
-The @fromJson()@ function may be used similarly to @INSERT JSON@, but for a single column value. It may only be used in the @VALUES@ clause of an @INSERT@ statement or as one of the column values in an @UPDATE@, @DELETE@, or @SELECT@ statement. For example, it cannot be used in the selection clause of a @SELECT@ statement.
+The @from_json()@ function may be used similarly to @INSERT JSON@, but for a single column value. It may only be used in the @VALUES@ clause of an @INSERT@ statement or as one of the column values in an @UPDATE@, @DELETE@, or @SELECT@ statement. For example, it cannot be used in the selection clause of a @SELECT@ statement.
-h3(#toJson). The toJson() Function
+h3(#to_json). The to_json() Function
-The @toJson()@ function may be used similarly to @SELECT JSON@, but for a single column value. It may only be used in the selection clause of a @SELECT@ statement.
+The @to_json()@ function may be used similarly to @SELECT JSON@, but for a single column value. It may only be used in the selection clause of a @SELECT@ statement.
h2(#appendixA). Appendix A: CQL Keywords
@@ -2488,6 +2499,7 @@
| @LIST@ | no |
| @LOGIN@ | no |
| @MAP@ | no |
+| @MASKED@ | no |
| @MATERIALIZED@ | yes |
| @MBEAN@ | yes |
| @MBEANS@ | yes |
@@ -2517,6 +2529,7 @@
| @ROLES@ | no |
| @SCHEMA@ | yes |
| @SELECT@ | yes |
+| @SELECT_MASKED@| no |
| @SET@ | yes |
| @SFUNC@ | no |
| @SMALLINT@ | no |
@@ -2538,6 +2551,7 @@
| @TUPLE@ | no |
| @TYPE@ | no |
| @UNLOGGED@ | yes |
+| @UNMASK@ | no |
| @UNSET@ | yes |
| @UPDATE@ | yes |
| @USE@ | yes |
@@ -2552,6 +2566,7 @@
| @WHERE@ | yes |
| @WITH@ | yes |
| @WRITETIME@ | no |
+| @MAXWRITETIME@ | no |
h2(#appendixB). Appendix B: CQL Reserved Types
@@ -2570,6 +2585,11 @@
The following describes the changes in each version of CQL.
+h3. 3.4.7
+
+* Remove deprecated functions @dateOf@ and @unixTimestampOf@, replaced by @to_timestamp@ and @to_unixtimestamp@ (see "CASSANDRA-18328":https://issues.apache.org/jira/browse/CASSANDRA-18328).
+* Adopt snake_case function names, deprecating all previous camelCase or alltogetherwithoutspaces function names (see "CASSANDRA-18037":https://issues.apache.org/jira/browse/CASSANDRA-18037).
+
h3. 3.4.6
* Add support for @IF EXISTS@ and @IF NOT EXISTS@ in @ALTER@ statements (see "CASSANDRA-16916":https://issues.apache.org/jira/browse/CASSANDRA-16916).
diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc
index 4c7a3fd..dd6a84a 100644
--- a/doc/modules/ROOT/pages/index.adoc
+++ b/doc/modules/ROOT/pages/index.adoc
@@ -15,27 +15,27 @@
[cols="a,a"]
|===
-| xref:cassandra:getting_started/index.adoc[Getting started] | Newbie starting point
+| xref:cassandra:getting-started/index.adoc[Getting started] | Newbie starting point
-| xref:cassandra:new/index.adoc[What's new in 4.1] | What's new in Cassandra 4.1
+| xref:cassandra:new/index.adoc[What's new in 5.0] | What's new in Cassandra 5.0
| xref:cassandra:architecture/index.adoc[Architecture] | Cassandra's big picture
-| xref:cassandra:data_modeling/index.adoc[Data modeling] | Hint: it's not relational
+| xref:cassandra:developing/data-modeling/index.adoc[Data modeling] | Hint: it's not relational
-| xref:cassandra:cql/index.adoc[Cassandra Query Language (CQL)] | CQL reference documentation
+| xref:cassandra:developing/cql/index.adoc[Cassandra Query Language (CQL)] | CQL reference documentation
-| xref:cassandra:configuration/index.adoc[Configuration] | Cassandra's handles and knobs
+| xref:cassandra:getting-started/configuration/index.adoc[Configuration] | Cassandra's handles and knobs
-| xref:cassandra:operating/index.adoc[Operation] | The operator's corner
+| xref:cassandra:managing/operating/index.adoc[Operation] | The operator's corner
-| xref:cassandra:tools/index.adoc[Tools] | cqlsh, nodetool, and others
+| xref:cassandra:managing/tools/index.adoc[Tools] | cqlsh, nodetool, and others
| xref:cassandra:troubleshooting/index.adoc[Troubleshooting] | What to look for when you have a problem
-| xref:cassandra:faq/index.adoc[FAQ] | Frequently asked questions
+| xref:cassandra:overview/faq/index.adoc[FAQ] | Frequently asked questions
-| xref:cassandra:plugins/index.adoc[Plug-ins] | Third-party plug-ins
+| xref:cassandra:integrating/plugins/index.adoc[Plug-ins] | Third-party plug-ins
| xref:master@_:ROOT:native_protocol.adoc[Native Protocols] | Native Cassandra protocol specifications
diff --git a/doc/modules/cassandra/examples/BASH/get_deb_package.sh b/doc/modules/cassandra/examples/BASH/get_deb_package.sh
index 69648e8..8c1b07b 100644
--- a/doc/modules/cassandra/examples/BASH/get_deb_package.sh
+++ b/doc/modules/cassandra/examples/BASH/get_deb_package.sh
@@ -1,2 +1,2 @@
-$ echo "deb https://debian.cassandra.apache.org 41x main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list
-deb https://debian.cassandra.apache.org 41x main
+$ echo "deb https://debian.cassandra.apache.org 42x main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list
+deb https://debian.cassandra.apache.org 42x main
diff --git a/doc/modules/cassandra/examples/BNF/alter_table.bnf b/doc/modules/cassandra/examples/BNF/alter_table.bnf
index 728a78a..633a8b8 100644
--- a/doc/modules/cassandra/examples/BNF/alter_table.bnf
+++ b/doc/modules/cassandra/examples/BNF/alter_table.bnf
@@ -1,5 +1,8 @@
alter_table_statement::= ALTER TABLE [ IF EXISTS ] table_name alter_table_instruction
-alter_table_instruction::= ADD [ IF NOT EXISTS ] column_name cql_type ( ',' column_name cql_type )*
- | DROP [ IF EXISTS ] column_name ( column_name )*
+alter_table_instruction::= ADD [ IF NOT EXISTS ] column_definition ( ',' column_definition)*
+ | DROP [ IF EXISTS ] column_name ( ',' column_name )*
| RENAME [ IF EXISTS ] column_name to column_name (AND column_name to column_name)*
+ | ALTER [ IF EXISTS ] column_name ( column_mask | DROP MASKED )
| WITH options
+column_definition::= column_name cql_type [ column_mask]
+column_mask::= MASKED WITH ( DEFAULT | function_name '(' term ( ',' term )* ')' )
diff --git a/doc/modules/cassandra/examples/BNF/create_table.bnf b/doc/modules/cassandra/examples/BNF/create_table.bnf
index 840573c..e4173c8 100644
--- a/doc/modules/cassandra/examples/BNF/create_table.bnf
+++ b/doc/modules/cassandra/examples/BNF/create_table.bnf
@@ -2,11 +2,12 @@
column_definition ( ',' column_definition )*
[ ',' PRIMARY KEY '(' primary_key ')' ]
')' [ WITH table_options ]
-column_definition::= column_name cql_type [ STATIC ] [ PRIMARY KEY]
+column_definition::= column_name cql_type [ STATIC ] [ column_mask ] [ PRIMARY KEY]
+column_mask::= MASKED WITH ( DEFAULT | function_name '(' term ( ',' term )* ')' )
primary_key::= partition_key [ ',' clustering_columns ]
partition_key::= column_name | '(' column_name ( ',' column_name )* ')'
clustering_columns::= column_name ( ',' column_name )*
-table_options:=: COMPACT STORAGE [ AND table_options ]
+table_options::= COMPACT STORAGE [ AND table_options ]
| CLUSTERING ORDER BY '(' clustering_order ')'
[ AND table_options ] | options
clustering_order::= column_name (ASC | DESC) ( ',' column_name (ASC | DESC) )*
diff --git a/doc/modules/cassandra/examples/BNF/grant_permission_statement.bnf b/doc/modules/cassandra/examples/BNF/grant_permission_statement.bnf
index 40f1df3..43a7eda 100644
--- a/doc/modules/cassandra/examples/BNF/grant_permission_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/grant_permission_statement.bnf
@@ -1,6 +1,6 @@
grant_permission_statement ::= GRANT permissions ON resource TO role_name
permissions ::= ALL [ PERMISSIONS ] | permission [ PERMISSION ]
-permission ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESCRIBE | EXECUTE
+permission ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESCRIBE | EXECUTE | UNMASK | SELECT_MASKED
resource ::= ALL KEYSPACES
| KEYSPACE keyspace_name
| [ TABLE ] table_name
diff --git a/doc/modules/cassandra/examples/CQL/as.cql b/doc/modules/cassandra/examples/CQL/as.cql
index a8b9f03..7270526 100644
--- a/doc/modules/cassandra/examples/CQL/as.cql
+++ b/doc/modules/cassandra/examples/CQL/as.cql
@@ -1,12 +1,12 @@
// Without alias
-SELECT intAsBlob(4) FROM t;
+SELECT int_as_blob(4) FROM t;
-// intAsBlob(4)
-// --------------
+// int_as_blob(4)
+// ----------------
// 0x00000004
// With alias
-SELECT intAsBlob(4) AS four FROM t;
+SELECT int_as_blob(4) AS four FROM t;
// four
// ------------
diff --git a/doc/modules/cassandra/examples/CQL/avg_with_cast.cql b/doc/modules/cassandra/examples/CQL/avg_with_cast.cql
new file mode 100644
index 0000000..95839b4
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/avg_with_cast.cql
@@ -0,0 +1 @@
+SELECT AVG (CAST (players AS FLOAT)) FROM plays;
diff --git a/doc/modules/cassandra/examples/CQL/current_date.cql b/doc/modules/cassandra/examples/CQL/current_date.cql
new file mode 100644
index 0000000..d82fb80
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/current_date.cql
@@ -0,0 +1 @@
+SELECT * FROM myTable WHERE date >= current_date() - 2d;
diff --git a/doc/modules/cassandra/examples/CQL/currentdate.cql b/doc/modules/cassandra/examples/CQL/currentdate.cql
deleted file mode 100644
index 0bed1b2..0000000
--- a/doc/modules/cassandra/examples/CQL/currentdate.cql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT * FROM myTable WHERE date >= currentDate() - 2d;
diff --git a/doc/modules/cassandra/examples/CQL/ddm_alter_mask.cql b/doc/modules/cassandra/examples/CQL/ddm_alter_mask.cql
new file mode 100644
index 0000000..6e10308
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_alter_mask.cql
@@ -0,0 +1 @@
+ALTER TABLE patients ALTER name MASKED WITH mask_default();
diff --git a/doc/modules/cassandra/examples/CQL/ddm_create_table.cql b/doc/modules/cassandra/examples/CQL/ddm_create_table.cql
new file mode 100644
index 0000000..65ba56d
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_create_table.cql
@@ -0,0 +1,5 @@
+CREATE TABLE patients (
+ id timeuuid PRIMARY KEY,
+ name text MASKED WITH mask_inner(1, null),
+ birth date MASKED WITH mask_default()
+);
diff --git a/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql b/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
new file mode 100644
index 0000000..23db48c
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
@@ -0,0 +1,11 @@
+CREATE FUNCTION redact(input text)
+ CALLED ON NULL INPUT
+ RETURNS text
+ LANGUAGE java
+ AS 'return "redacted";
+
+CREATE TABLE patients (
+ id timeuuid PRIMARY KEY,
+ name text MASKED WITH redact(),
+ birth date
+);
diff --git a/doc/modules/cassandra/examples/CQL/ddm_create_users.cql b/doc/modules/cassandra/examples/CQL/ddm_create_users.cql
new file mode 100644
index 0000000..7e66a1a
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_create_users.cql
@@ -0,0 +1,6 @@
+CREATE USER privileged WITH PASSWORD 'xyz';
+GRANT SELECT ON TABLE patients TO privileged;
+GRANT UNMASK ON TABLE patients TO privileged;
+
+CREATE USER unprivileged WITH PASSWORD 'xyz';
+GRANT SELECT ON TABLE patients TO unprivileged;
diff --git a/doc/modules/cassandra/examples/CQL/ddm_drop_mask.cql b/doc/modules/cassandra/examples/CQL/ddm_drop_mask.cql
new file mode 100644
index 0000000..06e032c
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_drop_mask.cql
@@ -0,0 +1 @@
+ALTER TABLE patients ALTER name DROP MASKED;
diff --git a/doc/modules/cassandra/examples/CQL/ddm_insert_data.cql b/doc/modules/cassandra/examples/CQL/ddm_insert_data.cql
new file mode 100644
index 0000000..59e916f
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_insert_data.cql
@@ -0,0 +1,2 @@
+INSERT INTO patients(id, name, birth) VALUES (now(), 'alice', '1984-01-02');
+INSERT INTO patients(id, name, birth) VALUES (now(), 'bob', '1982-02-03');
diff --git a/doc/modules/cassandra/examples/CQL/ddm_revoke_unmask.cql b/doc/modules/cassandra/examples/CQL/ddm_revoke_unmask.cql
new file mode 100644
index 0000000..93e2b90
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_revoke_unmask.cql
@@ -0,0 +1 @@
+REVOKE UNMASK ON TABLE patients FROM privileged;
diff --git a/doc/modules/cassandra/examples/CQL/ddm_select_with_masked_columns.cql b/doc/modules/cassandra/examples/CQL/ddm_select_with_masked_columns.cql
new file mode 100644
index 0000000..8acd066
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_select_with_masked_columns.cql
@@ -0,0 +1,6 @@
+SELECT name, birth FROM patients;
+
+// name | birth
+// -------+------------
+// a**** | 1970-01-01
+// b** | 1970-01-01
diff --git a/doc/modules/cassandra/examples/CQL/ddm_select_with_select_masked.cql b/doc/modules/cassandra/examples/CQL/ddm_select_with_select_masked.cql
new file mode 100644
index 0000000..0093855
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_select_with_select_masked.cql
@@ -0,0 +1,8 @@
+CREATE USER trusted_user WITH PASSWORD 'xyz';
+GRANT SELECT, SELECT_MASKED ON TABLE patients TO trusted_user;
+LOGIN trusted_user
+SELECT name, birth FROM patients WHERE name = 'Alice' ALLOW FILTERING;
+
+// name | birth
+// -------+------------
+// a**** | 1970-01-01
diff --git a/doc/modules/cassandra/examples/CQL/ddm_select_with_unmask_permission.cql b/doc/modules/cassandra/examples/CQL/ddm_select_with_unmask_permission.cql
new file mode 100644
index 0000000..35072c7
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_select_with_unmask_permission.cql
@@ -0,0 +1,7 @@
+LOGIN privileged
+SELECT name, birth FROM patients;
+
+// name | birth
+// -------+------------
+// alice | 1984-01-02
+// bob | 1982-02-03
diff --git a/doc/modules/cassandra/examples/CQL/ddm_select_without_select_masked.cql b/doc/modules/cassandra/examples/CQL/ddm_select_without_select_masked.cql
new file mode 100644
index 0000000..af15cf8
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_select_without_select_masked.cql
@@ -0,0 +1,6 @@
+CREATE USER untrusted_user WITH PASSWORD 'xyz';
+GRANT SELECT ON TABLE patients TO untrusted_user;
+LOGIN untrusted_user
+SELECT name, birth FROM patients WHERE name = 'Alice' ALLOW FILTERING;
+
+// Unauthorized: Error from server: code=2100 [Unauthorized] message="User untrusted_user has no UNMASK nor SELECT_UNMASK permission on table k.patients"
diff --git a/doc/modules/cassandra/examples/CQL/ddm_select_without_unmask_permission.cql b/doc/modules/cassandra/examples/CQL/ddm_select_without_unmask_permission.cql
new file mode 100644
index 0000000..4b80b83
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/ddm_select_without_unmask_permission.cql
@@ -0,0 +1,7 @@
+LOGIN unprivileged
+SELECT name, birth FROM patients;
+
+// name | birth
+// -------+------------
+// a**** | 1970-01-01
+// b** | 1970-01-01
diff --git a/doc/modules/cassandra/examples/CQL/select_with_mask_functions.cql b/doc/modules/cassandra/examples/CQL/select_with_mask_functions.cql
new file mode 100644
index 0000000..d7923fc
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/select_with_mask_functions.cql
@@ -0,0 +1,15 @@
+CREATE TABLE patients (
+ id timeuuid PRIMARY KEY,
+ name text,
+ birth date
+);
+
+INSERT INTO patients(id, name, birth) VALUES (now(), 'alice', '1982-01-02');
+INSERT INTO patients(id, name, birth) VALUES (now(), 'bob', '1982-01-02');
+
+SELECT mask_inner(name, 1, null), mask_default(birth) FROM patients;
+
+// system.mask_inner(name, 1, NULL) | system.mask_default(birth)
+// -----------------------------------+----------------------------
+// b** | 1970-01-01
+// a**** | 1970-01-01
diff --git a/doc/modules/cassandra/examples/CQL/sum_with_cast.cql b/doc/modules/cassandra/examples/CQL/sum_with_cast.cql
new file mode 100644
index 0000000..3261df1
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/sum_with_cast.cql
@@ -0,0 +1 @@
+SELECT SUM (CAST (players AS VARINT)) FROM plays;
diff --git a/doc/modules/cassandra/examples/CQL/timeuuid_min_max.cql b/doc/modules/cassandra/examples/CQL/timeuuid_min_max.cql
index 81353f5..5a1fc51 100644
--- a/doc/modules/cassandra/examples/CQL/timeuuid_min_max.cql
+++ b/doc/modules/cassandra/examples/CQL/timeuuid_min_max.cql
@@ -1,3 +1,3 @@
SELECT * FROM myTable
- WHERE t > maxTimeuuid('2013-01-01 00:05+0000')
- AND t < minTimeuuid('2013-02-02 10:00+0000');
+ WHERE t > max_timeuuid('2013-01-01 00:05+0000')
+ AND t < min_timeuuid('2013-02-02 10:00+0000');
diff --git a/doc/modules/cassandra/examples/RESULTS/add_yum_repo.result b/doc/modules/cassandra/examples/RESULTS/add_yum_repo.result
index 6fe704a..4b188e1 100644
--- a/doc/modules/cassandra/examples/RESULTS/add_yum_repo.result
+++ b/doc/modules/cassandra/examples/RESULTS/add_yum_repo.result
@@ -1,6 +1,6 @@
[cassandra]
name=Apache Cassandra
-baseurl=https://redhat.cassandra.apache.org/41x/
+baseurl=https://redhat.cassandra.apache.org/42x/
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://downloads.apache.org/cassandra/KEYS
diff --git a/doc/modules/cassandra/nav.adoc b/doc/modules/cassandra/nav.adoc
index 4849abb..2b63bcf 100644
--- a/doc/modules/cassandra/nav.adoc
+++ b/doc/modules/cassandra/nav.adoc
@@ -1,86 +1,86 @@
* Cassandra
-** xref:getting_started/index.adoc[Getting Started]
-*** xref:getting_started/installing.adoc[Installing Cassandra]
-*** xref:getting_started/configuring.adoc[Configuring Cassandra]
-*** xref:getting_started/querying.adoc[Inserting and querying]
-*** xref:getting_started/drivers.adoc[Client drivers]
-*** xref:getting_started/java11.adoc[Support for Java 11]
-*** xref:getting_started/production.adoc[Production recommendations]
+** xref:getting-started/index.adoc[Getting Started]
+*** xref:installing/installing.adoc[Installing Cassandra]
+*** xref:getting-started/configuring.adoc[Configuring Cassandra]
+*** xref:getting-started/querying.adoc[Inserting and querying]
+*** xref:getting-started/drivers.adoc[Client drivers]
+*** xref:getting-started/java11.adoc[Support for Java 11]
+*** xref:getting-started/production.adoc[Production recommendations]
** xref:new/index.adoc[What's new]
** xref:architecture/index.adoc[Architecture]
*** xref:architecture/overview.adoc[Overview]
*** xref:architecture/dynamo.adoc[Dynamo]
-*** xref:architecture/storage_engine.adoc[Storage engine]
+*** xref:architecture/storage-engine.adoc[Storage engine]
*** xref:architecture/guarantees.adoc[Guarantees]
*** xref:architecture/messaging.adoc[Improved internode messaging]
*** xref:architecture/streaming.adoc[Improved streaming]
-** xref:data_modeling/index.adoc[Data modeling]
-*** xref:data_modeling/intro.adoc[Introduction]
-*** xref:data_modeling/data_modeling_conceptual.adoc[Conceptual data modeling]
-*** xref:data_modeling/data_modeling_rdbms.adoc[RDBMS design]
-*** xref:data_modeling/data_modeling_queries.adoc[Defining application queries]
-*** xref:data_modeling/data_modeling_logical.adoc[Logical data modeling]
-*** xref:data_modeling/data_modeling_physical.adoc[Physical data modeling]
-*** xref:data_modeling/data_modeling_refining.adoc[Evaluating and refining data models]
-*** xref:data_modeling/data_modeling_schema.adoc[Defining database schema]
-*** xref:data_modeling/data_modeling_tools.adoc[Cassandra data modeling tools]
+** xref:developing:data-modeling/index.adoc[Data modeling]
+*** xref:developing:data-modeling/intro.adoc[Introduction]
+*** xref:developing:data-modeling/data-modeling_conceptual.adoc[Conceptual data modeling]
+*** xref:developing:data-modeling/data-modeling_rdbms.adoc[RDBMS design]
+*** xref:developing:data-modeling/data-modeling_queries.adoc[Defining application queries]
+*** xref:developing:data-modeling/data-modeling_logical.adoc[Logical data modeling]
+*** xref:developing:data-modeling/data-modeling_physical.adoc[Physical data modeling]
+*** xref:developing:data-modeling/data-modeling_refining.adoc[Evaluating and refining data models]
+*** xref:developing:data-modeling/data-modeling_schema.adoc[Defining database schema]
+*** xref:developing:data-modeling/data-modeling_tools.adoc[Cassandra data modeling tools]
-** xref:cql/index.adoc[Cassandra Query Language (CQL)]
-*** xref:cql/definitions.adoc[Definitions]
-*** xref:cql/types.adoc[Data types]
-*** xref:cql/ddl.adoc[Data definition (DDL)]
-*** xref:cql/dml.adoc[Data manipulation (DML)]
-*** xref:cql/operators.adoc[Operators]
-*** xref:cql/indexes.adoc[Secondary indexes]
-*** xref:cql/mvs.adoc[Materialized views]
-*** xref:cql/functions.adoc[Functions]
-*** xref:cql/json.adoc[JSON]
-*** xref:cql/security.adoc[Security]
-*** xref:cql/triggers.adoc[Triggers]
-*** xref:cql/appendices.adoc[Appendices]
-*** xref:cql/changes.adoc[Changes]
-*** xref:cql/SASI.adoc[SASI]
-*** xref:cql/cql_singlefile.adoc[Single file of CQL information]
+** xref:developing:cql/index.adoc[Cassandra Query Language (CQL)]
+*** xref:developing:cql/definitions.adoc[Definitions]
+*** xref:developing:cql/types.adoc[Data types]
+*** xref:developing:cql/ddl.adoc[Data definition (DDL)]
+*** xref:developing:cql/dml.adoc[Data manipulation (DML)]
+*** xref:developing:cql/operators.adoc[Operators]
+*** xref:developing:cql/indexes.adoc[Secondary indexes]
+*** xref:developing:cql/mvs.adoc[Materialized views]
+*** xref:developing:cql/functions.adoc[Functions]
+*** xref:developing:cql/json.adoc[JSON]
+*** xref:developing:cql/security.adoc[Security]
+*** xref:developing:cql/triggers.adoc[Triggers]
+*** xref:developing:cql/appendices.adoc[Appendices]
+*** xref:developing:cql/changes.adoc[Changes]
+*** xref:developing:cql/SASI.adoc[SASI]
+*** xref:developing:cql/cql_singlefile.adoc[Single file of CQL information]
-** xref:configuration/index.adoc[Configuration]
-*** xref:configuration/cass_yaml_file.adoc[cassandra.yaml]
-*** xref:configuration/cass_rackdc_file.adoc[cassandra-rackdc.properties]
-*** xref:configuration/cass_env_sh_file.adoc[cassandra-env.sh]
-*** xref:configuration/cass_topo_file.adoc[cassandra-topologies.properties]
-*** xref:configuration/cass_cl_archive_file.adoc[commitlog-archiving.properties]
-*** xref:configuration/cass_logback_xml_file.adoc[logback.xml]
-*** xref:configuration/cass_jvm_options_file.adoc[jvm-* files]
-*** xref:configuration/configuration.adoc[Liberating cassandra.yaml Parameters' Names from Their Units]
+** xref:managing:configuration/index.adoc[Configuration]
+*** xref:managing:configuration/cass_yaml_file.adoc[cassandra.yaml]
+*** xref:managing:configuration/cass_rackdc_file.adoc[cassandra-rackdc.properties]
+*** xref:managing:configuration/cass_env_sh_file.adoc[cassandra-env.sh]
+*** xref:managing:configuration/cass_topo_file.adoc[cassandra-topologies.properties]
+*** xref:managing:configuration/cass_cl_archive_file.adoc[commitlog-archiving.properties]
+*** xref:managing:configuration/cass_logback_xml_file.adoc[logback.xml]
+*** xref:managing:configuration/cass_jvm_options_file.adoc[jvm-* files]
+*** xref:managing:configuration/configuration.adoc[Liberating cassandra.yaml Parameters' Names from Their Units]
-** xref:operating/index.adoc[Operating]
-*** xref:operating/snitch.adoc[Snitches]
-*** xref:operating/topo_changes.adoc[Topology changes]
-*** xref:operating/repair.adoc[Repair]
-*** xref:operating/read_repair.adoc[Read repair]
-*** xref:operating/hints.adoc[Hints]
-*** xref:operating/bloom_filters.adoc[Bloom filters]
-*** xref:operating/compression.adoc[Compression]
-*** xref:operating/cdc.adoc[Change Data Capture (CDC)]
-*** xref:operating/backups.adoc[Backups]
-*** xref:operating/bulk_loading.adoc[Bulk loading]
-*** xref:operating/metrics.adoc[Metrics]
-*** xref:operating/security.adoc[Security]
-*** xref:operating/hardware.adoc[Hardware]
-*** xref:operating/compaction/index.adoc[Compaction]
-*** xref:operating/virtualtables.adoc[Virtual tables]
-*** xref:operating/auditlogging.adoc[Audit logging]
-*** xref:operating/audit_logging.adoc[Audit logging 2]
-*** xref:operating/fqllogging.adoc[Full query logging]
-*** xref:operating/transientreplication.adoc[Transient replication]
+** xref:managing:operating/index.adoc[Operating]
+*** xref:managing:operating/snitch.adoc[Snitches]
+*** xref:managing:operating/topo_changes.adoc[Topology changes]
+*** xref:managing:operating/repair.adoc[Repair]
+*** xref:managing:operating/read_repair.adoc[Read repair]
+*** xref:managing:operating/hints.adoc[Hints]
+*** xref:managing:operating/bloom_filters.adoc[Bloom filters]
+*** xref:managing:operating/compression.adoc[Compression]
+*** xref:managing:operating/cdc.adoc[Change Data Capture (CDC)]
+*** xref:managing:operating/backups.adoc[Backups]
+*** xref:managing:operating/bulk_loading.adoc[Bulk loading]
+*** xref:managing:operating/metrics.adoc[Metrics]
+*** xref:managing:operating/security.adoc[Security]
+*** xref:managing:operating/hardware.adoc[Hardware]
+*** xref:managing:operating/compaction/index.adoc[Compaction]
+*** xref:managing:operating/virtualtables.adoc[Virtual tables]
+*** xref:managing:operating/auditlogging.adoc[Audit logging]
+*** xref:managing:operating/audit_logging.adoc[Audit logging 2]
+*** xref:managing:operating/fqllogging.adoc[Full query logging]
+*** xref:managing:operating/transientreplication.adoc[Transient replication]
-** xref:tools/index.adoc[Tools]
-*** xref:tools/cqlsh.adoc[cqlsh: the CQL shell]
-*** xref:tools/nodetool/nodetool.adoc[nodetool]
-*** xref:tools/sstable/index.adoc[SSTable tools]
-*** xref:tools/cassandra_stress.adoc[cassandra-stress]
+** xref:managing:tools/index.adoc[Tools]
+*** xref:managing:tools/cqlsh.adoc[cqlsh: the CQL shell]
+*** xref:managing:tools/nodetool/nodetool.adoc[nodetool]
+*** xref:managing:tools/sstable/index.adoc[SSTable tools]
+*** xref:managing:tools/cassandra_stress.adoc[cassandra-stress]
** xref:troubleshooting/index.adoc[Troubleshooting]
*** xref:troubleshooting/finding_nodes.adoc[Finding misbehaving nodes]
@@ -101,7 +101,7 @@
*** xref:master@_:ROOT:development/dependencies.adoc[Dependency management]
*** xref:master@_:ROOT:development/release_process.adoc[Release process]
-** xref:faq/index.adoc[FAQ]
+** xref:overview:faq/index.adoc[FAQ]
-** xref:plugins/index.adoc[Plug-ins]
+** xref:integrating:plugins/index.adoc[Plug-ins]
diff --git a/doc/modules/cassandra/pages/architecture/dynamo.adoc b/doc/modules/cassandra/pages/architecture/dynamo.adoc
index e90390a..aa1cf5a 100644
--- a/doc/modules/cassandra/pages/architecture/dynamo.adoc
+++ b/doc/modules/cassandra/pages/architecture/dynamo.adoc
@@ -1,7 +1,7 @@
= Dynamo
Apache Cassandra relies on a number of techniques from Amazon's
-http://courses.cse.tamu.edu/caverlee/csce438/readings/dynamo-paper.pdf[Dynamo]
+https://www.cs.cornell.edu/courses/cs5414/2017fa/papers/dynamo.pdf[Dynamo]
distributed storage key-value system. Each node in the Dynamo system has
three main components:
@@ -22,10 +22,10 @@
Cassandra was designed this way to meet large-scale (PiB+)
business-critical storage requirements. In particular, as applications
-demanded full global replication of petabyte scale datasets along with
+demanded full global replication of petabyte-scale datasets along with
always available low-latency reads and writes, it became imperative to
design a new kind of database model as the relational database systems
-of the time struggled to meet the new requirements of global scale
+of the time struggled to meet the new requirements of global-scale
applications.
== Dataset Partitioning: Consistent Hashing
@@ -38,11 +38,11 @@
mutations to every key that it owns, every key must be versioned. Unlike
in the original Dynamo paper where deterministic versions and vector
clocks were used to reconcile concurrent updates to a key, Cassandra
-uses a simpler last write wins model where every mutation is timestamped
+uses a simpler last-write-wins model where every mutation is timestamped
(including deletes) and then the latest version of data is the "winning"
value. Formally speaking, Cassandra uses a Last-Write-Wins Element-Set
conflict-free replicated data type for each CQL row, or
-https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type LWW-Element-Set_(Last-Write-Wins-Element-Set)[LWW-Element-Set
+https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type#LWW-Element-Set_(Last-Write-Wins-Element-Set)[LWW-Element-Set
CRDT], to resolve conflicting mutations on replica sets.
=== Consistent Hashing using a Token Ring
@@ -76,14 +76,14 @@
image::ring.svg[image]
-You can see that in a Dynamo like system, ranges of keys, also known as
+You can see that in a Dynamo-like system, ranges of keys, also known as
*token ranges*, map to the same physical set of nodes. In this example,
all keys that fall in the token range excluding token 1 and including
token 2 (grange(t1, t2]) are stored on nodes 2, 3 and 4.
=== Multiple Tokens per Physical Node (vnodes)
-Simple single token consistent hashing works well if you have many
+Simple single-token consistent hashing works well if you have many
physical nodes to spread data over, but with evenly spaced tokens and a
small number of physical nodes, incremental scaling (adding just a few
nodes of capacity) is difficult because there are no token selections
@@ -104,8 +104,7 @@
Cassandra introduces some nomenclature to handle these concepts:
-* *Token*: A single position on the dynamo style hash
-ring.
+* *Token*: A single position on the Dynamo-style hash ring.
* *Endpoint*: A single physical IP and port on the network.
* *Host ID*: A unique identifier for a single "physical" node, usually
present at one gEndpoint and containing one or more
@@ -131,7 +130,7 @@
. When a node is decommissioned, it loses data roughly equally to other
members of the ring, again keeping equal distribution of data across the
cluster.
-. If a node becomes unavailable, query load (especially token aware
+. If a node becomes unavailable, query load (especially token-aware
query load), is evenly distributed across many other nodes.
Multiple tokens, however, can also have disadvantages:
@@ -152,7 +151,7 @@
available was picking random tokens, which meant that to keep balance
the default number of tokens per node had to be quite high, at `256`.
This had the effect of coupling many physical endpoints together,
-increasing the risk of unavailability. That is why in `3.x +` the new
+increasing the risk of unavailability. That is why in `3.x +` a new
deterministic token allocator was added which intelligently picks tokens
such that the ring is optimally balanced while requiring a much lower
number of tokens per physical node.
@@ -256,7 +255,7 @@
Transient replication is an experimental feature that is not ready
for production use. The expected audience is experienced users of
Cassandra capable of fully validating a deployment of their particular
-application. That means being able check that operations like reads,
+application. That means you have the experience to check that operations like reads,
writes, decommission, remove, rebuild, repair, and replace all work with
your queries, data, configuration, operational practices, and
availability requirements.
@@ -269,18 +268,18 @@
Cassandra uses mutation timestamp versioning to guarantee eventual
consistency of data. Specifically all mutations that enter the system do
so with a timestamp provided either from a client clock or, absent a
-client provided timestamp, from the coordinator node's clock. Updates
+client-provided timestamp, from the coordinator node's clock. Updates
resolve according to the conflict resolution rule of last write wins.
Cassandra's correctness does depend on these clocks, so make sure a
proper time synchronization process is running such as NTP.
Cassandra applies separate mutation timestamps to every column of every
row within a CQL partition. Rows are guaranteed to be unique by primary
-key, and each column in a row resolve concurrent mutations according to
+key, and each column in a row resolves concurrent mutations according to
last-write-wins conflict resolution. This means that updates to
different primary keys within a partition can actually resolve without
conflict! Furthermore the CQL collection types such as maps and sets use
-this same conflict free mechanism, meaning that concurrent updates to
+this same conflict-free mechanism, meaning that concurrent updates to
maps and sets are guaranteed to resolve as well.
==== Replica Synchronization
@@ -293,7 +292,7 @@
These techniques are only best-effort, however, and to guarantee
eventual consistency Cassandra implements `anti-entropy
-repair <repair>` where replicas calculate hierarchical hash-trees over
+repair <repair>` where replicas calculate hierarchical hash trees over
their datasets called https://en.wikipedia.org/wiki/Merkle_tree[Merkle
trees] that can then be compared across replicas to identify mismatched
data. Like the original Dynamo paper Cassandra supports full repairs
@@ -340,7 +339,7 @@
A majority of the replicas in each datacenter must respond.
`LOCAL_ONE`::
Only a single replica must respond. In a multi-datacenter cluster,
- this also gaurantees that read requests are not sent to replicas in a
+ this also guarantees that read requests are not sent to replicas in a
remote datacenter.
`ANY`::
A single replica may respond, or the coordinator may store a hint. If
@@ -400,7 +399,7 @@
not only about themselves but also about other nodes they know about.
This information is versioned with a vector clock of
`(generation, version)` tuples, where the generation is a monotonic
-timestamp and version is a logical clock the increments roughly every
+timestamp and version is a logical clock that increments roughly every
second. These logical clocks allow Cassandra gossip to ignore old
versions of cluster state just by inspecting the logical clocks
presented with gossip messages.
@@ -417,10 +416,10 @@
one exists)
. Gossips with a seed node if that didn't happen in step 2.
-When an operator first bootstraps a Cassandra cluster they designate
-certain nodes as seed nodes. Any node can be a seed node and the only
-difference between seed and non-seed nodes is seed nodes are allowed to
-bootstrap into the ring without seeing any other seed nodes.
+When an operator first bootstraps a Cassandra cluster, they designate
+certain nodes as seed nodes. Any node can be a seed node, and the only
+difference between seed and non-seed nodes is that seed nodes are allowed
+to bootstrap into the ring without seeing any other seed nodes.
Furthermore, once a cluster is bootstrapped, seed nodes become
hotspots for gossip due to step 4 above.
@@ -435,7 +434,7 @@
Nodes do not have to agree on the seed nodes, and indeed once a cluster
is bootstrapped, newly launched nodes can be configured to use any
existing nodes as seeds. The only advantage to picking the same nodes
-as seeds is it increases their usefullness as gossip hotspots.
+as seeds is that it increases their usefulness as gossip hotspots.
====
Currently, gossip also propagates token metadata and schema
@@ -488,7 +487,7 @@
storage. In contrast, scaling-up implies adding more capacity to the
existing database nodes. Cassandra is also capable of scale-up, and in
certain environments it may be preferable depending on the deployment.
-Cassandra gives operators the flexibility to chose either scale-out or
+Cassandra gives operators the flexibility to choose either scale-out or
scale-up.
One key aspect of Dynamo that Cassandra follows is to attempt to run on
@@ -507,7 +506,7 @@
multi-partition transactions spanning multiple nodes are notoriously
difficult to implement and typically very latent.
-Instead, Cassanda chooses to offer fast, consistent, latency at any
+Instead, Cassandra chooses to offer fast, consistent, latency at any
scale for single partition operations, allowing retrieval of entire
partitions or only subsets of partitions based on primary key filters.
Furthermore, Cassandra does support single partition compare and swap
@@ -516,7 +515,7 @@
=== Simple Interface for Storing Records
Cassandra, in a slight departure from Dynamo, chooses a storage
-interface that is more sophisticated then "simple key value" stores but
+interface that is more sophisticated than "simple key-value" stores but
significantly less complex than SQL relational data models. Cassandra
presents a wide-column store interface, where partitions of data contain
multiple rows, each of which contains a flexible set of individually
diff --git a/doc/modules/cassandra/pages/architecture/guarantees.adoc b/doc/modules/cassandra/pages/architecture/guarantees.adoc
index 3313a11..a5f09b9 100644
--- a/doc/modules/cassandra/pages/architecture/guarantees.adoc
+++ b/doc/modules/cassandra/pages/architecture/guarantees.adoc
@@ -1,17 +1,17 @@
= Guarantees
Apache Cassandra is a highly scalable and reliable database. Cassandra
-is used in web based applications that serve large number of clients and
+is used in web-based applications that serve large number of clients and
the quantity of data processed is web-scale (Petabyte) large. Cassandra
makes some guarantees about its scalability, availability and
reliability. To fully understand the inherent limitations of a storage
system in an environment in which a certain level of network partition
failure is to be expected and taken into account when designing the
-system it is important to first briefly introduce the CAP theorem.
+system, it is important to first briefly introduce the CAP theorem.
== What is CAP?
-According to the CAP theorem it is not possible for a distributed data
+According to the CAP theorem, it is not possible for a distributed data
store to provide more than two of the following guarantees
simultaneously.
@@ -24,7 +24,7 @@
storage system to failure of a network partition. Even if some of the
messages are dropped or delayed the system continues to operate.
-CAP theorem implies that when using a network partition, with the
+The CAP theorem implies that when using a network partition, with the
inherent risk of partition failure, one has to choose between
consistency and availability and both cannot be guaranteed at the same
time. CAP theorem is illustrated in Figure 1.
@@ -33,7 +33,7 @@
Figure 1. CAP Theorem
-High availability is a priority in web based applications and to this
+High availability is a priority in web-based applications and to this
objective Cassandra chooses Availability and Partition Tolerance from
the CAP guarantees, compromising on data Consistency to some extent.
@@ -47,19 +47,19 @@
* Batched writes across multiple tables are guaranteed to succeed
completely or not at all
* Secondary indexes are guaranteed to be consistent with their local
-replicas data
+replicas' data
== High Scalability
Cassandra is a highly scalable storage system in which nodes may be
-added/removed as needed. Using gossip-based protocol a unified and
+added/removed as needed. Using gossip-based protocol, a unified and
consistent membership list is kept at each node.
== High Availability
Cassandra guarantees high availability of data by implementing a
-fault-tolerant storage system. Failure detection in a node is detected
-using a gossip-based protocol.
+fault-tolerant storage system. Failure of a node is detected using
+a gossip-based protocol.
== Durability
@@ -67,26 +67,26 @@
multiple copies of a data stored on different nodes in a cluster. In a
multi-datacenter environment the replicas may be stored on different
datacenters. If one replica is lost due to unrecoverable node/datacenter
-failure the data is not completely lost as replicas are still available.
+failure, the data is not completely lost, as replicas are still available.
== Eventual Consistency
Meeting the requirements of performance, reliability, scalability and
-high availability in production Cassandra is an eventually consistent
-storage system. Eventually consistent implies that all updates reach all
+high availability in production, Cassandra is an eventually consistent
+storage system. Eventually consistency implies that all updates reach all
replicas eventually. Divergent versions of the same data may exist
-temporarily but they are eventually reconciled to a consistent state.
-Eventual consistency is a tradeoff to achieve high availability and it
+temporarily, but they are eventually reconciled to a consistent state.
+Eventual consistency is a tradeoff to achieve high availability, and it
involves some read and write latencies.
== Lightweight transactions with linearizable consistency
-Data must be read and written in a sequential order. Paxos consensus
-protocol is used to implement lightweight transactions. Paxos protocol
+Data must be read and written in a sequential order. The Paxos consensus
+protocol is used to implement lightweight transactions. The Paxos protocol
implements lightweight transactions that are able to handle concurrent
operations using linearizable consistency. Linearizable consistency is
-sequential consistency with real-time constraints and it ensures
-transaction isolation with compare and set (CAS) transaction. With CAS
+sequential consistency with real-time constraints, and it ensures
+transaction isolation with compare-and-set (CAS) transactions. With CAS
replica data is compared and data that is found to be out of date is set
to the most consistent value. Reads with linearizable consistency allow
reading the current state of the data, which may possibly be
@@ -97,12 +97,12 @@
The guarantee for batched writes across multiple tables is that they
will eventually succeed, or none will. Batch data is first written to
batchlog system data, and when the batch data has been successfully
-stored in the cluster the batchlog data is removed. The batch is
-replicated to another node to ensure the full batch completes in the
-event the coordinator node fails.
+stored in the cluster, the batchlog data is removed. The batch is
+replicated to another node to ensure that the full batch completes in
+the event if coordinator node fails.
== Secondary Indexes
-A secondary index is an index on a column and is used to query a table
-that is normally not queryable. Secondary indexes when built are
+A secondary index is an index on a column, and it's used to query a table
+that is normally not queryable. Secondary indexes, when built, are
guaranteed to be consistent with their local replicas.
diff --git a/doc/modules/cassandra/pages/architecture/index.adoc b/doc/modules/cassandra/pages/architecture/index.adoc
index c4bef05..9e674d9 100644
--- a/doc/modules/cassandra/pages/architecture/index.adoc
+++ b/doc/modules/cassandra/pages/architecture/index.adoc
@@ -4,6 +4,6 @@
* xref:architecture/overview.adoc[Overview]
* xref:architecture/dynamo.adoc[Dynamo]
-* xref:architecture/storage_engine.adoc[Storage Engine]
+* xref:architecture/storage-engine.adoc[Storage Engine]
* xref:architecture/guarantees.adoc[Guarantees]
* xref:architecture/snitch.adoc[Snitches]
diff --git a/doc/modules/cassandra/pages/architecture/overview.adoc b/doc/modules/cassandra/pages/architecture/overview.adoc
index 58db6b1..2c86ce4 100644
--- a/doc/modules/cassandra/pages/architecture/overview.adoc
+++ b/doc/modules/cassandra/pages/architecture/overview.adoc
@@ -1,7 +1,7 @@
= Overview
:exper: experimental
-Apache Cassandra is an open source, distributed, NoSQL database. It
+Apache Cassandra is an open-source, distributed, NoSQL database. It
presents a partitioned wide column storage model with eventually
consistent semantics.
@@ -10,7 +10,7 @@
using a staged event-driven architecture
(http://www.sosp.org/2001/papers/welsh.pdf[SEDA]) to implement a
combination of Amazon’s
-http://courses.cse.tamu.edu/caverlee/csce438/readings/dynamo-paper.pdf[Dynamo]
+https://www.cs.cornell.edu/courses/cs5414/2017fa/papers/dynamo.pdf[Dynamo]
distributed storage and replication techniques and Google's
https://static.googleusercontent.com/media/research.google.com/en//archive/bigtable-osdi06.pdf[Bigtable]
data and storage engine model. Dynamo and Bigtable were both developed
@@ -23,7 +23,7 @@
replication and always available low-latency reads and writes, it became
imperative to design a new kind of database model as the relational
database systems of the time struggled to meet the new requirements of
-global scale applications.
+global-scale applications.
Systems like Cassandra are designed for these challenges and seek the
following design objectives:
@@ -59,19 +59,19 @@
CQL supports numerous advanced features over a partitioned dataset such
as:
-* Single partition lightweight transactions with atomic compare and set
-semantics.
+* Single-partition lightweight transactions with atomic compare and set
+semantics
* User-defined types, functions and aggregates
-* Collection types including sets, maps, and lists.
+* Collection types including sets, maps, and lists
* Local secondary indices
* (Experimental) materialized views
Cassandra explicitly chooses not to implement operations that require
-cross partition coordination as they are typically slow and hard to
+cross-partition coordination as they are typically slow and hard to
provide highly available global semantics. For example Cassandra does
not support:
-* Cross partition transactions
+* Cross-partition transactions
* Distributed joins
* Foreign keys or referential integrity.
diff --git a/doc/modules/cassandra/pages/architecture/snitch.adoc b/doc/modules/cassandra/pages/architecture/snitch.adoc
index 3ae066d..cd59f98 100644
--- a/doc/modules/cassandra/pages/architecture/snitch.adoc
+++ b/doc/modules/cassandra/pages/architecture/snitch.adoc
@@ -12,7 +12,7 @@
== Dynamic snitching
-The dynamic snitch monitor read latencies to avoid reading from hosts
+The dynamic snitch monitors read latencies to avoid reading from hosts
that have slowed down. The dynamic snitch is configured with the
following properties on `cassandra.yaml`:
diff --git a/doc/modules/cassandra/pages/architecture/storage-engine.adoc b/doc/modules/cassandra/pages/architecture/storage-engine.adoc
new file mode 100644
index 0000000..52158c6
--- /dev/null
+++ b/doc/modules/cassandra/pages/architecture/storage-engine.adoc
@@ -0,0 +1,228 @@
+= Storage Engine
+
+[[commit-log]]
+== CommitLog
+
+Commitlogs are an append-only log of all mutations local to a Cassandra
+node. Any data written to Cassandra will first be written to a commit
+log before being written to a memtable. This provides durability in the
+case of unexpected shutdown. On startup, any mutations in the commit log
+will be applied to memtables.
+
+All mutations are write-optimized by storing in commitlog segments, reducing
+the number of seeks needed to write to disk. Commitlog segments are
+limited by the `commitlog_segment_size` option. Once the size is
+reached, a new commitlog segment is created. Commitlog segments can be
+archived, deleted, or recycled once all the data has been flushed to
+SSTables. Commitlog segments are truncated when Cassandra has written
+data older than a certain point to the SSTables. Running "nodetool
+drain" before stopping Cassandra will write everything in the memtables
+to SSTables and remove the need to sync with the commitlogs on startup.
+
+* `commitlog_segment_size`: The default size is 32MiB, which is
+almost always fine, but if you are archiving commitlog segments (see
+commitlog_archiving.properties), then you probably want a finer
+granularity of archiving; 8 or 16 MiB is reasonable.
+`commitlog_segment_size` also determines the default value of
+`max_mutation_size` in `cassandra.yaml`. By default,
+`max_mutation_size` is a half the size of `commitlog_segment_size`.
+
+[NOTE]
+.Note
+====
+If `max_mutation_size` is set explicitly then
+`commitlog_segment_size` must be set to at least twice the size of
+`max_mutation_size`.
+====
+
+* `commitlog_sync`: may be either _periodic_ or _batch_.
+** `batch`: In batch mode, Cassandra won’t ack writes until the commit
+log has been fsynced to disk. It will wait
+"commitlog_sync_batch_window_in_ms" milliseconds between fsyncs. This
+window should be kept short because the writer threads will be unable to
+do extra work while waiting. You may need to increase concurrent_writes
+for the same reason.
++
+- `commitlog_sync_batch_window_in_ms`: Time to wait between "batch"
+fsyncs _Default Value:_ 2
+** `periodic`: In periodic mode, writes are immediately ack'ed, and the
+CommitLog is simply synced every "commitlog_sync_period"
+milliseconds.
++
+- `commitlog_sync_period`: Time to wait between "periodic" fsyncs
+_Default Value:_ 10000ms
+
+_Default Value:_ batch
+
+[NOTE]
+.Note
+====
+In the event of an unexpected shutdown, Cassandra can lose up
+to the sync period or more if the sync is delayed. If using "batch"
+mode, it is recommended to store commitlogs in a separate, dedicated
+device.
+====
+
+* `commitlog_directory`: This option is commented out by default. When
+running on magnetic HDD, this should be a separate spindle than the data
+directories. If not set, the default directory is
+`$CASSANDRA_HOME/data/commitlog`.
+
+_Default Value:_ `/var/lib/cassandra/commitlog`
+
+* `commitlog_compression`: Compression to apply to the commitlog. If
+omitted, the commit log will be written uncompressed. LZ4, Snappy,
+Deflate and Zstd compressors are supported.
+
+_Default Value:_ (complex option):
+
+[source, yaml]
+----
+# - class_name: LZ4Compressor
+# parameters:
+----
+
+* `commitlog_total_space`: Total space to use for commit logs on
+disk.
+
+If space gets above this value, Cassandra will flush every dirty CF in
+the oldest segment and remove it. So a small total commitlog space will
+tend to cause more flush activity on less-active columnfamilies.
+
+The default value is the smallest between 8192 and 1/4 of the total
+space of the commitlog volume.
+
+_Default Value:_ 8192MiB
+
+== Memtables
+
+Memtables are in-memory structures where Cassandra buffers writes. In
+general, there is one active memtable per table. Eventually, memtables
+are flushed onto disk and become immutable link:#sstables[SSTables].
+This can be triggered in several ways:
+
+* The memory usage of the memtables exceeds the configured threshold
+(see `memtable_cleanup_threshold`)
+* The `commit-log` approaches its maximum size, and forces memtable
+flushes in order to allow commitlog segments to be freed
+
+Memtables may be stored entirely on-heap or partially off-heap,
+depending on `memtable_allocation_type`.
+
+== SSTables
+
+SSTables are the immutable data files that Cassandra uses for persisting
+data on disk.
+
+As SSTables are flushed to disk from `memtables` or are streamed from
+other nodes, Cassandra triggers compactions which combine multiple
+SSTables into one. Once the new SSTable has been written, the old
+SSTables can be removed.
+
+Each SSTable is comprised of multiple components stored in separate
+files:
+
+`Data.db`::
+ The actual data, i.e. the contents of rows.
+`Index.db`::
+ An index from partition keys to positions in the `Data.db` file. For
+ wide partitions, this may also include an index to rows within a
+ partition.
+`Summary.db`::
+ A sampling of (by default) every 128th entry in the `Index.db` file.
+`Filter.db`::
+ A Bloom Filter of the partition keys in the SSTable.
+`CompressionInfo.db`::
+ Metadata about the offsets and lengths of compression chunks in the
+ `Data.db` file.
+`Statistics.db`::
+ Stores metadata about the SSTable, including information about
+ timestamps, tombstones, clustering keys, compaction, repair,
+ compression, TTLs, and more.
+`Digest.crc32`::
+ A CRC-32 digest of the `Data.db` file.
+`TOC.txt`::
+ A plain text list of the component files for the SSTable.
+
+Within the `Data.db` file, rows are organized by partition. These
+partitions are sorted in token order (i.e. by a hash of the partition
+key when the default partitioner, `Murmur3Partition`, is used). Within a
+partition, rows are stored in the order of their clustering keys.
+
+SSTables can be optionally compressed using block-based compression.
+
+== SSTable Versions
+
+This section was created using the following
+https://gist.github.com/shyamsalimkumar/49a61e5bc6f403d20c55[gist] which
+utilized this original
+http://www.bajb.net/2013/03/cassandra-sstable-format-version-numbers/[source].
+
+The version numbers, to date are:
+
+=== Version 0
+
+* b (0.7.0): added version to sstable filenames
+* c (0.7.0): bloom filter component computes hashes over raw key bytes
+instead of strings
+* d (0.7.0): row size in data component becomes a long instead of int
+* e (0.7.0): stores undecorated keys in data and index components
+* f (0.7.0): switched bloom filter implementations in data component
+* g (0.8): tracks flushed-at context in metadata component
+
+=== Version 1
+
+* h (1.0): tracks max client timestamp in metadata component
+* hb (1.0.3): records compression ration in metadata component
+* hc (1.0.4): records partitioner in metadata component
+* hd (1.0.10): includes row tombstones in maxtimestamp
+* he (1.1.3): includes ancestors generation in metadata component
+* hf (1.1.6): marker that replay position corresponds to 1.1.5+
+millis-based id (see CASSANDRA-4782)
+* ia (1.2.0):
+** column indexes are promoted to the index file
+** records estimated histogram of deletion times in tombstones
+** bloom filter (keys and columns) upgraded to Murmur3
+* ib (1.2.1): tracks min client timestamp in metadata component
+* ic (1.2.5): omits per-row bloom filter of column names
+
+=== Version 2
+
+* ja (2.0.0):
+** super columns are serialized as composites (note that there is no
+real format change, this is mostly a marker to know if we should expect
+super columns or not. We do need a major version bump however, because
+we should not allow streaming of super columns into this new format)
+** tracks max local deletiontime in sstable metadata
+** records bloom_filter_fp_chance in metadata component
+** remove data size and column count from data file (CASSANDRA-4180)
+** tracks max/min column values (according to comparator)
+* jb (2.0.1):
+** switch from crc32 to adler32 for compression checksums
+** checksum the compressed data
+* ka (2.1.0):
+** new Statistics.db file format
+** index summaries can be downsampled and the sampling level is
+persisted
+** switch uncompressed checksums to adler32
+** tracks presence of legacy (local and remote) counter shards
+* la (2.2.0): new file name format
+* lb (2.2.7): commit log lower bound included
+
+=== Version 3
+
+* ma (3.0.0):
+** swap bf hash order
+** store rows natively
+* mb (3.0.7, 3.7): commit log lower bound included
+* mc (3.0.8, 3.9): commit log intervals included
+
+=== Example Code
+
+The following example is useful for finding all sstables that do not
+match the "ib" SSTable version
+
+[source,bash]
+----
+include::example$BASH/find_sstables.sh[]
+----
diff --git a/doc/modules/cassandra/pages/architecture/storage_engine.adoc b/doc/modules/cassandra/pages/architecture/storage_engine.adoc
deleted file mode 100644
index 9a0c37a..0000000
--- a/doc/modules/cassandra/pages/architecture/storage_engine.adoc
+++ /dev/null
@@ -1,225 +0,0 @@
-= Storage Engine
-
-[[commit-log]]
-== CommitLog
-
-Commitlogs are an append only log of all mutations local to a Cassandra
-node. Any data written to Cassandra will first be written to a commit
-log before being written to a memtable. This provides durability in the
-case of unexpected shutdown. On startup, any mutations in the commit log
-will be applied to memtables.
-
-All mutations write optimized by storing in commitlog segments, reducing
-the number of seeks needed to write to disk. Commitlog Segments are
-limited by the `commitlog_segment_size` option, once the size is
-reached, a new commitlog segment is created. Commitlog segments can be
-archived, deleted, or recycled once all its data has been flushed to
-SSTables. Commitlog segments are truncated when Cassandra has written
-data older than a certain point to the SSTables. Running "nodetool
-drain" before stopping Cassandra will write everything in the memtables
-to SSTables and remove the need to sync with the commitlogs on startup.
-
-* `commitlog_segment_size`: The default size is 32MiB, which is
-almost always fine, but if you are archiving commitlog segments (see
-commitlog_archiving.properties), then you probably want a finer
-granularity of archiving; 8 or 16 MiB is reasonable. `commitlog_segment_size`
-also determines the default value of `max_mutation_size` in cassandra.yaml.
-By default, max_mutation_size is half the size of `commitlog_segment_size`.
-
-**NOTE: If `max_mutation_size` is set explicitly then
-`commitlog_segment_size` must be set to at least twice the size of
-`max_mutation_size`**.
-
-Commitlogs are an append only log of all mutations local to a Cassandra
-node. Any data written to Cassandra will first be written to a commit
-log before being written to a memtable. This provides durability in the
-case of unexpected shutdown. On startup, any mutations in the commit log
-will be applied.
-
-* `commitlog_sync`: may be either _periodic_ or _batch_.
-** `batch`: In batch mode, Cassandra won’t ack writes until the commit
-log has been fsynced to disk. It will wait
-"commitlog_sync_batch_window_in_ms" milliseconds between fsyncs. This
-window should be kept short because the writer threads will be unable to
-do extra work while waiting. You may need to increase concurrent_writes
-for the same reason.
-+
-- `commitlog_sync_batch_window_in_ms`: Time to wait between "batch"
-fsyncs _Default Value:_ 2
-** `periodic`: In periodic mode, writes are immediately ack'ed, and the
-CommitLog is simply synced every "commitlog_sync_period"
-milliseconds.
-+
-- `commitlog_sync_period`: Time to wait between "periodic" fsyncs
-_Default Value:_ 10000ms
-
-_Default Value:_ batch
-
-** NOTE: In the event of an unexpected shutdown, Cassandra can lose up
-to the sync period or more if the sync is delayed. If using "batch"
-mode, it is recommended to store commitlogs in a separate, dedicated
-device.*
-
-* `commitlog_directory`: This option is commented out by default When
-running on magnetic HDD, this should be a separate spindle than the data
-directories. If not set, the default directory is
-$CASSANDRA_HOME/data/commitlog.
-
-_Default Value:_ /var/lib/cassandra/commitlog
-
-* `commitlog_compression`: Compression to apply to the commitlog. If
-omitted, the commit log will be written uncompressed. LZ4, Snappy,
-Deflate and Zstd compressors are supported.
-
-(Default Value: (complex option):
-
-[source, yaml]
-----
-# - class_name: LZ4Compressor
-# parameters:
-----
-
-* `commitlog_total_space`: Total space to use for commit logs on
-disk.
-
-If space gets above this value, Cassandra will flush every dirty CF in
-the oldest segment and remove it. So a small total commitlog space will
-tend to cause more flush activity on less-active columnfamilies.
-
-The default value is the smaller of 8192, and 1/4 of the total space of
-the commitlog volume.
-
-_Default Value:_ 8192MiB
-
-== Memtables
-
-Memtables are in-memory structures where Cassandra buffers writes. In
-general, there is one active memtable per table. Eventually, memtables
-are flushed onto disk and become immutable link:#sstables[SSTables].
-This can be triggered in several ways:
-
-* The memory usage of the memtables exceeds the configured threshold
-(see `memtable_cleanup_threshold`)
-* The `commit-log` approaches its maximum size, and forces memtable
-flushes in order to allow commitlog segments to be freed
-
-Memtables may be stored entirely on-heap or partially off-heap,
-depending on `memtable_allocation_type`.
-
-== SSTables
-
-SSTables are the immutable data files that Cassandra uses for persisting
-data on disk.
-
-As SSTables are flushed to disk from `memtables` or are streamed from
-other nodes, Cassandra triggers compactions which combine multiple
-SSTables into one. Once the new SSTable has been written, the old
-SSTables can be removed.
-
-Each SSTable is comprised of multiple components stored in separate
-files:
-
-`Data.db`::
- The actual data, i.e. the contents of rows.
-`Index.db`::
- An index from partition keys to positions in the `Data.db` file. For
- wide partitions, this may also include an index to rows within a
- partition.
-`Summary.db`::
- A sampling of (by default) every 128th entry in the `Index.db` file.
-`Filter.db`::
- A Bloom Filter of the partition keys in the SSTable.
-`CompressionInfo.db`::
- Metadata about the offsets and lengths of compression chunks in the
- `Data.db` file.
-`Statistics.db`::
- Stores metadata about the SSTable, including information about
- timestamps, tombstones, clustering keys, compaction, repair,
- compression, TTLs, and more.
-`Digest.crc32`::
- A CRC-32 digest of the `Data.db` file.
-`TOC.txt`::
- A plain text list of the component files for the SSTable.
-
-Within the `Data.db` file, rows are organized by partition. These
-partitions are sorted in token order (i.e. by a hash of the partition
-key when the default partitioner, `Murmur3Partition`, is used). Within a
-partition, rows are stored in the order of their clustering keys.
-
-SSTables can be optionally compressed using block-based compression.
-
-== SSTable Versions
-
-This section was created using the following
-https://gist.github.com/shyamsalimkumar/49a61e5bc6f403d20c55[gist] which
-utilized this original
-http://www.bajb.net/2013/03/cassandra-sstable-format-version-numbers/[source].
-
-The version numbers, to date are:
-
-=== Version 0
-
-* b (0.7.0): added version to sstable filenames
-* c (0.7.0): bloom filter component computes hashes over raw key bytes
-instead of strings
-* d (0.7.0): row size in data component becomes a long instead of int
-* e (0.7.0): stores undecorated keys in data and index components
-* f (0.7.0): switched bloom filter implementations in data component
-* g (0.8): tracks flushed-at context in metadata component
-
-=== Version 1
-
-* h (1.0): tracks max client timestamp in metadata component
-* hb (1.0.3): records compression ration in metadata component
-* hc (1.0.4): records partitioner in metadata component
-* hd (1.0.10): includes row tombstones in maxtimestamp
-* he (1.1.3): includes ancestors generation in metadata component
-* hf (1.1.6): marker that replay position corresponds to 1.1.5+
-millis-based id (see CASSANDRA-4782)
-* ia (1.2.0):
-** column indexes are promoted to the index file
-** records estimated histogram of deletion times in tombstones
-** bloom filter (keys and columns) upgraded to Murmur3
-* ib (1.2.1): tracks min client timestamp in metadata component
-* ic (1.2.5): omits per-row bloom filter of column names
-
-=== Version 2
-
-* ja (2.0.0):
-** super columns are serialized as composites (note that there is no
-real format change, this is mostly a marker to know if we should expect
-super columns or not. We do need a major version bump however, because
-we should not allow streaming of super columns into this new format)
-** tracks max local deletiontime in sstable metadata
-** records bloom_filter_fp_chance in metadata component
-** remove data size and column count from data file (CASSANDRA-4180)
-** tracks max/min column values (according to comparator)
-* jb (2.0.1):
-** switch from crc32 to adler32 for compression checksums
-** checksum the compressed data
-* ka (2.1.0):
-** new Statistics.db file format
-** index summaries can be downsampled and the sampling level is
-persisted
-** switch uncompressed checksums to adler32
-** tracks presense of legacy (local and remote) counter shards
-* la (2.2.0): new file name format
-* lb (2.2.7): commit log lower bound included
-
-=== Version 3
-
-* ma (3.0.0):
-** swap bf hash order
-** store rows natively
-* mb (3.0.7, 3.7): commit log lower bound included
-* mc (3.0.8, 3.9): commit log intervals included
-
-=== Example Code
-
-The following example is useful for finding all sstables that do not
-match the "ib" SSTable version
-
-[source,bash]
-----
-include:example$find_sstables.sh[]
-----
diff --git a/doc/modules/cassandra/pages/configuration/cass_logback_xml_file.adoc b/doc/modules/cassandra/pages/configuration/cass_logback_xml_file.adoc
deleted file mode 100644
index e673622..0000000
--- a/doc/modules/cassandra/pages/configuration/cass_logback_xml_file.adoc
+++ /dev/null
@@ -1,166 +0,0 @@
-= logback.xml file
-
-The `logback.xml` configuration file can optionally set logging levels
-for the logs written to `system.log` and `debug.log`. The logging levels
-can also be set using `nodetool setlogginglevels`.
-
-== Options
-
-=== `appender name="<appender_choice>"...</appender>`
-
-Specify log type and settings. Possible appender names are: `SYSTEMLOG`,
-`DEBUGLOG`, `ASYNCDEBUGLOG`, and `STDOUT`. `SYSTEMLOG` ensures that WARN
-and ERROR message are written synchronously to the specified file.
-`DEBUGLOG` and `ASYNCDEBUGLOG` ensure that DEBUG messages are written
-either synchronously or asynchronously, respectively, to the specified
-file. `STDOUT` writes all messages to the console in a human-readable
-format.
-
-*Example:* <appender name="SYSTEMLOG"
-class="ch.qos.logback.core.rolling.RollingFileAppender">
-
-=== `<file> <filename> </file>`
-
-Specify the filename for a log.
-
-*Example:* <file>$\{cassandra.logdir}/system.log</file>
-
-=== `<level> <log_level> </level>`
-
-Specify the level for a log. Part of the filter. Levels are: `ALL`,
-`TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `OFF`. `TRACE` creates the
-most verbose log, `ERROR` the least.
-
-[NOTE]
-.Note
-====
-Note: Increasing logging levels can generate heavy logging output on
-a moderately trafficked cluster. You can use the
-`nodetool getlogginglevels` command to see the current logging
-configuration.
-====
-
-*Default:* INFO
-
-*Example:* <level>INFO</level>
-
-=== `<rollingPolicy class="<rolling_policy_choice>" <fileNamePattern><pattern_info></fileNamePattern> ... </rollingPolicy>`
-
-Specify the policy for rolling logs over to an archive.
-
-*Example:* <rollingPolicy
-class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
-=== `<fileNamePattern> <pattern_info> </fileNamePattern>`
-
-Specify the pattern information for rolling over the log to archive.
-Part of the rolling policy.
-
-*Example:*
-<fileNamePattern>$\{cassandra.logdir}/system.log.%d\{yyyy-MM-dd}.%i.zip</fileNamePattern>
-
-=== `<maxFileSize> <size> </maxFileSize>`
-
-Specify the maximum file size to trigger rolling a log. Part of the
-rolling policy.
-
-*Example:* <maxFileSize>50MB</maxFileSize>
-
-=== `<maxHistory> <number_of_days> </maxHistory>`
-
-Specify the maximum history in days to trigger rolling a log. Part of
-the rolling policy.
-
-*Example:* <maxHistory>7</maxHistory>
-
-=== `<encoder> <pattern>...</pattern> </encoder>`
-
-Specify the format of the message. Part of the rolling policy.
-
-*Example:* <maxHistory>7</maxHistory> *Example:* <encoder>
-<pattern>%-5level [%thread] %date\{ISO8601} %F:%L - %msg%n</pattern>
-</encoder>
-
-=== Contents of default `logback.xml`
-
-[source,XML]
-----
-<configuration scan="true" scanPeriod="60 seconds">
- <jmxConfigurator />
-
- <!-- No shutdown hook; we run it ourselves in StorageService after shutdown -->
-
- <!-- SYSTEMLOG rolling file appender to system.log (INFO level) -->
-
- <appender name="SYSTEMLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <file>${cassandra.logdir}/system.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
- <!-- rollover daily -->
- <fileNamePattern>${cassandra.logdir}/system.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
- <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
- <maxFileSize>50MB</maxFileSize>
- <maxHistory>7</maxHistory>
- <totalSizeCap>5GB</totalSizeCap>
- </rollingPolicy>
- <encoder>
- <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
- </encoder>
- </appender>
-
- <!-- DEBUGLOG rolling file appender to debug.log (all levels) -->
-
- <appender name="DEBUGLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${cassandra.logdir}/debug.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
- <!-- rollover daily -->
- <fileNamePattern>${cassandra.logdir}/debug.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
- <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
- <maxFileSize>50MB</maxFileSize>
- <maxHistory>7</maxHistory>
- <totalSizeCap>5GB</totalSizeCap>
- </rollingPolicy>
- <encoder>
- <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
- </encoder>
- </appender>
-
- <!-- ASYNCLOG assynchronous appender to debug.log (all levels) -->
-
- <appender name="ASYNCDEBUGLOG" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <discardingThreshold>0</discardingThreshold>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="DEBUGLOG" />
- </appender>
-
- <!-- STDOUT console appender to stdout (INFO level) -->
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <encoder>
- <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
- </encoder>
- </appender>
-
- <!-- Uncomment bellow and corresponding appender-ref to activate logback metrics
- <appender name="LogbackMetrics" class="com.codahale.metrics.logback.InstrumentedAppender" />
- -->
-
- <root level="INFO">
- <appender-ref ref="SYSTEMLOG" />
- <appender-ref ref="STDOUT" />
- <appender-ref ref="ASYNCDEBUGLOG" /> <!-- Comment this line to disable debug.log -->
- <!--
- <appender-ref ref="LogbackMetrics" />
- -->
- </root>
-
- <logger name="org.apache.cassandra" level="DEBUG"/>
- <logger name="com.thinkaurelius.thrift" level="ERROR"/>
-</configuration>
-----
diff --git a/doc/modules/cassandra/pages/cql/appendices.adoc b/doc/modules/cassandra/pages/cql/appendices.adoc
deleted file mode 100644
index 7e17266..0000000
--- a/doc/modules/cassandra/pages/cql/appendices.adoc
+++ /dev/null
@@ -1,179 +0,0 @@
-= Appendices
-
-[[appendix-A]]
-== Appendix A: CQL Keywords
-
-CQL distinguishes between _reserved_ and _non-reserved_ keywords.
-Reserved keywords cannot be used as identifier, they are truly reserved
-for the language (but one can enclose a reserved keyword by
-double-quotes to use it as an identifier). Non-reserved keywords however
-only have a specific meaning in certain context but can used as
-identifier otherwise. The only _raison d’être_ of these non-reserved
-keywords is convenience: some keyword are non-reserved when it was
-always easy for the parser to decide whether they were used as keywords
-or not.
-
-[width="48%",cols="60%,40%",options="header",]
-|===
-|Keyword |Reserved?
-|`ADD` |yes
-|`AGGREGATE` |no
-|`ALL` |no
-|`ALLOW` |yes
-|`ALTER` |yes
-|`AND` |yes
-|`APPLY` |yes
-|`AS` |no
-|`ASC` |yes
-|`ASCII` |no
-|`AUTHORIZE` |yes
-|`BATCH` |yes
-|`BEGIN` |yes
-|`BIGINT` |no
-|`BLOB` |no
-|`BOOLEAN` |no
-|`BY` |yes
-|`CALLED` |no
-|`CLUSTERING` |no
-|`COLUMNFAMILY` |yes
-|`COMPACT` |no
-|`CONTAINS` |no
-|`COUNT` |no
-|`COUNTER` |no
-|`CREATE` |yes
-|`CUSTOM` |no
-|`DATE` |no
-|`DECIMAL` |no
-|`DELETE` |yes
-|`DESC` |yes
-|`DESCRIBE` |yes
-|`DISTINCT` |no
-|`DOUBLE` |no
-|`DROP` |yes
-|`ENTRIES` |yes
-|`EXECUTE` |yes
-|`EXISTS` |no
-|`FILTERING` |no
-|`FINALFUNC` |no
-|`FLOAT` |no
-|`FROM` |yes
-|`FROZEN` |no
-|`FULL` |yes
-|`FUNCTION` |no
-|`FUNCTIONS` |no
-|`GRANT` |yes
-|`IF` |yes
-|`IN` |yes
-|`INDEX` |yes
-|`INET` |no
-|`INFINITY` |yes
-|`INITCOND` |no
-|`INPUT` |no
-|`INSERT` |yes
-|`INT` |no
-|`INTO` |yes
-|`JSON` |no
-|`KEY` |no
-|`KEYS` |no
-|`KEYSPACE` |yes
-|`KEYSPACES` |no
-|`LANGUAGE` |no
-|`LIMIT` |yes
-|`LIST` |no
-|`LOGIN` |no
-|`MAP` |no
-|`MODIFY` |yes
-|`NAN` |yes
-|`NOLOGIN` |no
-|`NORECURSIVE` |yes
-|`NOSUPERUSER` |no
-|`NOT` |yes
-|`NULL` |yes
-|`OF` |yes
-|`ON` |yes
-|`OPTIONS` |no
-|`OR` |yes
-|`ORDER` |yes
-|`PASSWORD` |no
-|`PERMISSION` |no
-|`PERMISSIONS` |no
-|`PRIMARY` |yes
-|`RENAME` |yes
-|`REPLACE` |yes
-|`RETURNS` |no
-|`REVOKE` |yes
-|`ROLE` |no
-|`ROLES` |no
-|`SCHEMA` |yes
-|`SELECT` |yes
-|`SET` |yes
-|`SFUNC` |no
-|`SMALLINT` |no
-|`STATIC` |no
-|`STORAGE` |no
-|`STYPE` |no
-|`SUPERUSER` |no
-|`TABLE` |yes
-|`TEXT` |no
-|`TIME` |no
-|`TIMESTAMP` |no
-|`TIMEUUID` |no
-|`TINYINT` |no
-|`TO` |yes
-|`TOKEN` |yes
-|`TRIGGER` |no
-|`TRUNCATE` |yes
-|`TTL` |no
-|`TUPLE` |no
-|`TYPE` |no
-|`UNLOGGED` |yes
-|`UPDATE` |yes
-|`USE` |yes
-|`USER` |no
-|`USERS` |no
-|`USING` |yes
-|`UUID` |no
-|`VALUES` |no
-|`VARCHAR` |no
-|`VARINT` |no
-|`WHERE` |yes
-|`WITH` |yes
-|`WRITETIME` |no
-|===
-
-== Appendix B: CQL Reserved Types
-
-The following type names are not currently used by CQL, but are reserved
-for potential future use. User-defined types may not use reserved type
-names as their name.
-
-[width="25%",cols="100%",options="header",]
-|===
-|type
-|`bitstring`
-|`byte`
-|`complex`
-|`enum`
-|`interval`
-|`macaddr`
-|===
-
-== Appendix C: Dropping Compact Storage
-
-Starting version 4.0, Thrift and COMPACT STORAGE is no longer supported.
-
-`ALTER ... DROP COMPACT STORAGE` statement makes Compact Tables
-CQL-compatible, exposing internal structure of Thrift/Compact Tables:
-
-* CQL-created Compact Tables that have no clustering columns, will
-expose an additional clustering column `column1` with `UTF8Type`.
-* CQL-created Compact Tables that had no regular columns, will expose a
-regular column `value` with `BytesType`.
-* For CQL-Created Compact Tables, all columns originally defined as
-`regular` will be come `static`
-* CQL-created Compact Tables that have clustering but have no regular
-columns will have an empty value column (of `EmptyType`)
-* SuperColumn Tables (can only be created through Thrift) will expose a
-compact value map with an empty name.
-* Thrift-created Compact Tables will have types corresponding to their
-Thrift definition.
diff --git a/doc/modules/cassandra/pages/cql/changes.adoc b/doc/modules/cassandra/pages/cql/changes.adoc
deleted file mode 100644
index df99a39..0000000
--- a/doc/modules/cassandra/pages/cql/changes.adoc
+++ /dev/null
@@ -1,225 +0,0 @@
-= Changes
-
-The following describes the changes in each version of CQL.
-
-== 3.4.6
-
-* Add support for IF EXISTS and IF NOT EXISTS in ALTER statements (`16916`)
-* Allow GRANT/REVOKE multiple permissions in a single statement (`17030`)
-* Pre hashed passwords in CQL (`17334`)
-* Add support for type casting in WHERE clause components and in the values of INSERT/UPDATE statements (`14337`)
-* Add support for CONTAINS and CONTAINS KEY in conditional UPDATE and DELETE statement (`10537`)
-* Allow to grant permission for all tables in a keyspace (`17027`)
-* Allow to aggregate by time intervals (`11871`)
-
-== 3.4.5
-
-* Adds support for arithmetic operators (`11935`)
-* Adds support for `+` and `-` operations on dates (`11936`)
-* Adds `currentTimestamp`, `currentDate`, `currentTime` and
-`currentTimeUUID` functions (`13132`)
-
-== 3.4.4
-
-* `ALTER TABLE` `ALTER` has been removed; a column's type may not be
-changed after creation (`12443`).
-* `ALTER TYPE` `ALTER` has been removed; a field's type may not be
-changed after creation (`12443`).
-
-== 3.4.3
-
-* Adds a new `duration` `data types <data-types>` (`11873`).
-* Support for `GROUP BY` (`10707`).
-* Adds a `DEFAULT UNSET` option for `INSERT JSON` to ignore omitted
-columns (`11424`).
-* Allows `null` as a legal value for TTL on insert and update. It will
-be treated as equivalent to inserting a 0 (`12216`).
-
-== 3.4.2
-
-* If a table has a non zero `default_time_to_live`, then explicitly
-specifying a TTL of 0 in an `INSERT` or `UPDATE` statement will result
-in the new writes not having any expiration (that is, an explicit TTL of
-0 cancels the `default_time_to_live`). This wasn't the case before and
-the `default_time_to_live` was applied even though a TTL had been
-explicitly set.
-* `ALTER TABLE` `ADD` and `DROP` now allow multiple columns to be
-added/removed.
-* New `PER PARTITION LIMIT` option for `SELECT` statements (see
-https://issues.apache.org/jira/browse/CASSANDRA-7017)[CASSANDRA-7017].
-* `User-defined functions <cql-functions>` can now instantiate
-`UDTValue` and `TupleValue` instances via the new `UDFContext` interface
-(see
-https://issues.apache.org/jira/browse/CASSANDRA-10818)[CASSANDRA-10818].
-* `User-defined types <udts>` may now be stored in a non-frozen form,
-allowing individual fields to be updated and deleted in `UPDATE`
-statements and `DELETE` statements, respectively.
-(https://issues.apache.org/jira/browse/CASSANDRA-7423)[CASSANDRA-7423]).
-
-== 3.4.1
-
-* Adds `CAST` functions.
-
-== 3.4.0
-
-* Support for `materialized views <materialized-views>`.
-* `DELETE` support for inequality expressions and `IN` restrictions on
-any primary key columns.
-* `UPDATE` support for `IN` restrictions on any primary key columns.
-
-== 3.3.1
-
-* The syntax `TRUNCATE TABLE X` is now accepted as an alias for
-`TRUNCATE X`.
-
-== 3.3.0
-
-* `User-defined functions and aggregates <cql-functions>` are now
-supported.
-* Allows double-dollar enclosed strings literals as an alternative to
-single-quote enclosed strings.
-* Introduces Roles to supersede user based authentication and access
-control
-* New `date`, `time`, `tinyint` and `smallint` `data types <data-types>`
-have been added.
-* `JSON support <cql-json>` has been added
-* Adds new time conversion functions and deprecate `dateOf` and
-`unixTimestampOf`.
-
-== 3.2.0
-
-* `User-defined types <udts>` supported.
-* `CREATE INDEX` now supports indexing collection columns, including
-indexing the keys of map collections through the `keys()` function
-* Indexes on collections may be queried using the new `CONTAINS` and
-`CONTAINS KEY` operators
-* `Tuple types <tuples>` were added to hold fixed-length sets of typed
-positional fields.
-* `DROP INDEX` now supports optionally specifying a keyspace.
-
-== 3.1.7
-
-* `SELECT` statements now support selecting multiple rows in a single
-partition using an `IN` clause on combinations of clustering columns.
-* `IF NOT EXISTS` and `IF EXISTS` syntax is now supported by
-`CREATE USER` and `DROP USER` statements, respectively.
-
-== 3.1.6
-
-* A new `uuid()` method has been added.
-* Support for `DELETE ... IF EXISTS` syntax.
-
-== 3.1.5
-
-* It is now possible to group clustering columns in a relation, see
-`WHERE <where-clause>` clauses.
-* Added support for `static columns <static-columns>`.
-
-== 3.1.4
-
-* `CREATE INDEX` now allows specifying options when creating CUSTOM
-indexes.
-
-== 3.1.3
-
-* Millisecond precision formats have been added to the
-`timestamp <timestamps>` parser.
-
-== 3.1.2
-
-* `NaN` and `Infinity` has been added as valid float constants. They are
-now reserved keywords. In the unlikely case you we using them as a
-column identifier (or keyspace/table one), you will now need to double
-quote them.
-
-== 3.1.1
-
-* `SELECT` statement now allows listing the partition keys (using the
-`DISTINCT` modifier). See
-https://issues.apache.org/jira/browse/CASSANDRA-4536[CASSANDRA-4536].
-* The syntax `c IN ?` is now supported in `WHERE` clauses. In that case,
-the value expected for the bind variable will be a list of whatever type
-`c` is.
-* It is now possible to use named bind variables (using `:name` instead
-of `?`).
-
-== 3.1.0
-
-* `ALTER TABLE` `DROP` option added.
-* `SELECT` statement now supports aliases in select clause. Aliases in
-WHERE and ORDER BY clauses are not supported.
-* `CREATE` statements for `KEYSPACE`, `TABLE` and `INDEX` now supports
-an `IF NOT EXISTS` condition. Similarly, `DROP` statements support a
-`IF EXISTS` condition.
-* `INSERT` statements optionally supports a `IF NOT EXISTS` condition
-and `UPDATE` supports `IF` conditions.
-
-== 3.0.5
-
-* `SELECT`, `UPDATE`, and `DELETE` statements now allow empty `IN`
-relations (see
-https://issues.apache.org/jira/browse/CASSANDRA-5626)[CASSANDRA-5626].
-
-== 3.0.4
-
-* Updated the syntax for custom `secondary indexes <secondary-indexes>`.
-* Non-equal condition on the partition key are now never supported, even
-for ordering partitioner as this was not correct (the order was *not*
-the one of the type of the partition key). Instead, the `token` method
-should always be used for range queries on the partition key (see
-`WHERE clauses <where-clause>`).
-
-== 3.0.3
-
-* Support for custom `secondary indexes <secondary-indexes>` has been
-added.
-
-== 3.0.2
-
-* Type validation for the `constants <constants>` has been fixed. For
-instance, the implementation used to allow `'2'` as a valid value for an
-`int` column (interpreting it has the equivalent of `2`), or `42` as a
-valid `blob` value (in which case `42` was interpreted as an hexadecimal
-representation of the blob). This is no longer the case, type validation
-of constants is now more strict. See the `data types <data-types>`
-section for details on which constant is allowed for which type.
-* The type validation fixed of the previous point has lead to the
-introduction of blobs constants to allow the input of blobs. Do note
-that while the input of blobs as strings constant is still supported by
-this version (to allow smoother transition to blob constant), it is now
-deprecated and will be removed by a future version. If you were using
-strings as blobs, you should thus update your client code ASAP to switch
-blob constants.
-* A number of functions to convert native types to blobs have also been
-introduced. Furthermore the token function is now also allowed in select
-clauses. See the `section on functions <cql-functions>` for details.
-
-== 3.0.1
-
-* Date strings (and timestamps) are no longer accepted as valid
-`timeuuid` values. Doing so was a bug in the sense that date string are
-not valid `timeuuid`, and it was thus resulting in
-https://issues.apache.org/jira/browse/CASSANDRA-4936[confusing
-behaviors]. However, the following new methods have been added to help
-working with `timeuuid`: `now`, `minTimeuuid`, `maxTimeuuid` , `dateOf`
-and `unixTimestampOf`.
-* Float constants now support the exponent notation. In other words,
-`4.2E10` is now a valid floating point value.
-
-== Versioning
-
-Versioning of the CQL language adheres to the http://semver.org[Semantic
-Versioning] guidelines. Versions take the form X.Y.Z where X, Y, and Z
-are integer values representing major, minor, and patch level
-respectively. There is no correlation between Cassandra release versions
-and the CQL language version.
-
-[cols=",",options="header",]
-|===
-|version |description
-| Major | The major version _must_ be bumped when backward incompatible changes
-are introduced. This should rarely occur.
-| Minor | Minor version increments occur when new, but backward compatible,
-functionality is introduced.
-| Patch | The patch version is incremented when bugs are fixed.
-|===
diff --git a/doc/modules/cassandra/pages/cql/cql_singlefile.adoc b/doc/modules/cassandra/pages/cql/cql_singlefile.adoc
deleted file mode 100644
index d99e12b..0000000
--- a/doc/modules/cassandra/pages/cql/cql_singlefile.adoc
+++ /dev/null
@@ -1,3921 +0,0 @@
-== Cassandra Query Language (CQL) v3.4.3
-
-\{toc:maxLevel=3}
-
-=== CQL Syntax
-
-==== Preamble
-
-This document describes the Cassandra Query Language (CQL) version 3.
-CQL v3 is not backward compatible with CQL v2 and differs from it in
-numerous ways. Note that this document describes the last version of the
-languages. However, the link:#changes[changes] section provides the diff
-between the different versions of CQL v3.
-
-CQL v3 offers a model very close to SQL in the sense that data is put in
-_tables_ containing _rows_ of _columns_. For that reason, when used in
-this document, these terms (tables, rows and columns) have the same
-definition than they have in SQL. But please note that as such, they do
-*not* refer to the concept of rows and columns found in the internal
-implementation of Cassandra and in the thrift and CQL v2 API.
-
-==== Conventions
-
-To aid in specifying the CQL syntax, we will use the following
-conventions in this document:
-
-* Language rules will be given in a
-http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form[BNF] -like
-notation:
-
-bc(syntax). ::= TERMINAL
-
-* Nonterminal symbols will have `<angle brackets>`.
-* As additional shortcut notations to BNF, we’ll use traditional regular
-expression’s symbols (`?`, `+` and `*`) to signify that a given symbol
-is optional and/or can be repeated. We’ll also allow parentheses to
-group symbols and the `[<characters>]` notation to represent any one of
-`<characters>`.
-* The grammar is provided for documentation purposes and leave some
-minor details out. For instance, the last column definition in a
-`CREATE TABLE` statement is optional but supported if present even
-though the provided grammar in this document suggest it is not
-supported.
-* Sample code will be provided in a code block:
-
-bc(sample). SELECT sample_usage FROM cql;
-
-* References to keywords or pieces of CQL code in running text will be
-shown in a `fixed-width font`.
-
-[[identifiers]]
-==== Identifiers and keywords
-
-The CQL language uses _identifiers_ (or _names_) to identify tables,
-columns and other objects. An identifier is a token matching the regular
-expression `[a-zA-Z]``[a-zA-Z0-9_]``*`.
-
-A number of such identifiers, like `SELECT` or `WITH`, are _keywords_.
-They have a fixed meaning for the language and most are reserved. The
-list of those keywords can be found in link:#appendixA[Appendix A].
-
-Identifiers and (unquoted) keywords are case insensitive. Thus `SELECT`
-is the same than `select` or `sElEcT`, and `myId` is the same than
-`myid` or `MYID` for instance. A convention often used (in particular by
-the samples of this documentation) is to use upper case for keywords and
-lower case for other identifiers.
-
-There is a second kind of identifiers called _quoted identifiers_
-defined by enclosing an arbitrary sequence of characters in
-double-quotes(`"`). Quoted identifiers are never keywords. Thus
-`"select"` is not a reserved keyword and can be used to refer to a
-column, while `select` would raise a parse error. Also, contrarily to
-unquoted identifiers and keywords, quoted identifiers are case sensitive
-(`"My Quoted Id"` is _different_ from `"my quoted id"`). A fully
-lowercase quoted identifier that matches `[a-zA-Z]``[a-zA-Z0-9_]``*` is
-equivalent to the unquoted identifier obtained by removing the
-double-quote (so `"myid"` is equivalent to `myid` and to `myId` but
-different from `"myId"`). Inside a quoted identifier, the double-quote
-character can be repeated to escape it, so `"foo "" bar"` is a valid
-identifier.
-
-*Warning*: _quoted identifiers_ allows to declare columns with arbitrary
-names, and those can sometime clash with specific names used by the
-server. For instance, when using conditional update, the server will
-respond with a result-set containing a special result named
-`"[applied]"`. If you’ve declared a column with such a name, this could
-potentially confuse some tools and should be avoided. In general,
-unquoted identifiers should be preferred but if you use quoted
-identifiers, it is strongly advised to avoid any name enclosed by
-squared brackets (like `"[applied]"`) and any name that looks like a
-function call (like `"f(x)"`).
-
-==== Constants
-
-CQL defines the following kind of _constants_: strings, integers,
-floats, booleans, uuids and blobs:
-
-* A string constant is an arbitrary sequence of characters characters
-enclosed by single-quote(`'`). One can include a single-quote in a
-string by repeating it, e.g. `'It''s raining today'`. Those are not to
-be confused with quoted identifiers that use double-quotes.
-* An integer constant is defined by `'-'?[0-9]+`.
-* A float constant is defined by
-`'-'?[0-9]+('.'[0-9]*)?([eE][+-]?[0-9+])?`. On top of that, `NaN` and
-`Infinity` are also float constants.
-* A boolean constant is either `true` or `false` up to
-case-insensitivity (i.e. `True` is a valid boolean constant).
-* A http://en.wikipedia.org/wiki/Universally_unique_identifier[UUID]
-constant is defined by `hex{8}-hex{4}-hex{4}-hex{4}-hex{12}` where `hex`
-is an hexadecimal character, e.g. `[0-9a-fA-F]` and `{4}` is the number
-of such characters.
-* A blob constant is an hexadecimal number defined by `0[xX](hex)+`
-where `hex` is an hexadecimal character, e.g. `[0-9a-fA-F]`.
-
-For how these constants are typed, see the link:#types[data types
-section].
-
-==== Comments
-
-A comment in CQL is a line beginning by either double dashes (`--`) or
-double slash (`//`).
-
-Multi-line comments are also supported through enclosure within `/*` and
-`*/` (but nesting is not supported).
-
-bc(sample). +
-— This is a comment +
-// This is a comment too +
-/* This is +
-a multi-line comment */
-
-==== Statements
-
-CQL consists of statements. As in SQL, these statements can be divided
-in 3 categories:
-
-* Data definition statements, that allow to set and change the way data
-is stored.
-* Data manipulation statements, that allow to change data
-* Queries, to look up data
-
-All statements end with a semicolon (`;`) but that semicolon can be
-omitted when dealing with a single statement. The supported statements
-are described in the following sections. When describing the grammar of
-said statements, we will reuse the non-terminal symbols defined below:
-
-bc(syntax).. +
-::= any quoted or unquoted identifier, excluding reserved keywords +
-::= ( `.')?
-
-::= a string constant +
-::= an integer constant +
-::= a float constant +
-::= | +
-::= a uuid constant +
-::= a boolean constant +
-::= a blob constant
-
-::= +
-| +
-| +
-| +
-| +
-::= `?' +
-| `:' +
-::= +
-| +
-| +
-| `(' ( (`,' )*)? `)'
-
-::= +
-| +
-| +
-::= `\{' ( `:' ( `,' `:' )* )? `}' +
-::= `\{' ( ( `,' )* )? `}' +
-::= `[' ( ( `,' )* )? `]'
-
-::=
-
-::= (AND )* +
-::= `=' ( | | ) +
-p. +
-Please note that not every possible productions of the grammar above
-will be valid in practice. Most notably, `<variable>` and nested
-`<collection-literal>` are currently not allowed inside
-`<collection-literal>`.
-
-A `<variable>` can be either anonymous (a question mark (`?`)) or named
-(an identifier preceded by `:`). Both declare a bind variables for
-link:#preparedStatement[prepared statements]. The only difference
-between an anymous and a named variable is that a named one will be
-easier to refer to (how exactly depends on the client driver used).
-
-The `<properties>` production is use by statement that create and alter
-keyspaces and tables. Each `<property>` is either a _simple_ one, in
-which case it just has a value, or a _map_ one, in which case it’s value
-is a map grouping sub-options. The following will refer to one or the
-other as the _kind_ (_simple_ or _map_) of the property.
-
-A `<tablename>` will be used to identify a table. This is an identifier
-representing the table name that can be preceded by a keyspace name. The
-keyspace name, if provided, allow to identify a table in another
-keyspace than the currently active one (the currently active keyspace is
-set through the `USE` statement).
-
-For supported `<function>`, see the section on
-link:#functions[functions].
-
-Strings can be either enclosed with single quotes or two dollar
-characters. The second syntax has been introduced to allow strings that
-contain single quotes. Typical candidates for such strings are source
-code fragments for user-defined functions.
-
-_Sample:_
-
-bc(sample).. +
-`some string value'
-
-$$double-dollar string can contain single ’ quotes$$ +
-p.
-
-[[preparedStatement]]
-==== Prepared Statement
-
-CQL supports _prepared statements_. Prepared statement is an
-optimization that allows to parse a query only once but execute it
-multiple times with different concrete values.
-
-In a statement, each time a column value is expected (in the data
-manipulation and query statements), a `<variable>` (see above) can be
-used instead. A statement with bind variables must then be _prepared_.
-Once it has been prepared, it can executed by providing concrete values
-for the bind variables. The exact procedure to prepare a statement and
-execute a prepared statement depends on the CQL driver used and is
-beyond the scope of this document.
-
-In addition to providing column values, bind markers may be used to
-provide values for `LIMIT`, `TIMESTAMP`, and `TTL` clauses. If anonymous
-bind markers are used, the names for the query parameters will be
-`[limit]`, `[timestamp]`, and `[ttl]`, respectively.
-
-[[dataDefinition]]
-=== Data Definition
-
-[[createKeyspaceStmt]]
-==== CREATE KEYSPACE
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE KEYSPACE (IF NOT EXISTS)? WITH +
-p. +
-_Sample:_
-
-bc(sample).. +
-CREATE KEYSPACE Excelsior +
-WITH replication = \{’class’: `SimpleStrategy', `replication_factor' :
-3};
-
-CREATE KEYSPACE Excalibur +
-WITH replication = \{’class’: `NetworkTopologyStrategy', `DC1' : 1,
-`DC2' : 3} +
-AND durable_writes = false; +
-p. +
-The `CREATE KEYSPACE` statement creates a new top-level _keyspace_. A
-keyspace is a namespace that defines a replication strategy and some
-options for a set of tables. Valid keyspaces names are identifiers
-composed exclusively of alphanumerical characters and whose length is
-lesser or equal to 32. Note that as identifiers, keyspace names are case
-insensitive: use a quoted identifier for case sensitive keyspace names.
-
-The supported `<properties>` for `CREATE KEYSPACE` are:
-
-[cols=",,,,",options="header",]
-|===
-|name |kind |mandatory |default |description
-|`replication` |_map_ |yes | |The replication strategy and options to
-use for the keyspace.
-
-|`durable_writes` |_simple_ |no |true |Whether to use the commit log for
-updates on this keyspace (disable this option at your own risk!).
-|===
-
-The `replication` `<property>` is mandatory. It must at least contains
-the `'class'` sub-option which defines the replication strategy class to
-use. The rest of the sub-options depends on that replication strategy
-class. By default, Cassandra support the following `'class'`:
-
-* `'SimpleStrategy'`: A simple strategy that defines a simple
-replication factor for the whole cluster. The only sub-options supported
-is `'replication_factor'` to define that replication factor and is
-mandatory.
-* `'NetworkTopologyStrategy'`: A replication strategy that allows to set
-the replication factor independently for each data-center. The rest of
-the sub-options are key-value pairs where each time the key is the name
-of a datacenter and the value the replication factor for that
-data-center.
-
-Attempting to create an already existing keyspace will return an error
-unless the `IF NOT EXISTS` option is used. If it is used, the statement
-will be a no-op if the keyspace already exists.
-
-[[useStmt]]
-==== USE
-
-_Syntax:_
-
-bc(syntax). ::= USE
-
-_Sample:_
-
-bc(sample). USE myApp;
-
-The `USE` statement takes an existing keyspace name as argument and set
-it as the per-connection current working keyspace. All subsequent
-keyspace-specific actions will be performed in the context of the
-selected keyspace, unless link:#statements[otherwise specified], until
-another USE statement is issued or the connection terminates.
-
-[[alterKeyspaceStmt]]
-==== ALTER KEYSPACE
-
-_Syntax:_
-
-bc(syntax).. +
-::= ALTER KEYSPACE (IF EXISTS)? WITH +
-p. +
-_Sample:_
-
-bc(sample).. +
-ALTER KEYSPACE Excelsior +
-WITH replication = \{’class’: `SimpleStrategy', `replication_factor' :
-4};
-
-The `ALTER KEYSPACE` statement alters the properties of an existing
-keyspace. The supported `<properties>` are the same as for the
-link:#createKeyspaceStmt[`CREATE KEYSPACE`] statement.
-
-[[dropKeyspaceStmt]]
-==== DROP KEYSPACE
-
-_Syntax:_
-
-bc(syntax). ::= DROP KEYSPACE ( IF EXISTS )?
-
-_Sample:_
-
-bc(sample). DROP KEYSPACE myApp;
-
-A `DROP KEYSPACE` statement results in the immediate, irreversible
-removal of an existing keyspace, including all column families in it,
-and all data contained in those column families.
-
-If the keyspace does not exists, the statement will return an error,
-unless `IF EXISTS` is used in which case the operation is a no-op.
-
-[[createTableStmt]]
-==== CREATE TABLE
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE ( TABLE | COLUMNFAMILY ) ( IF NOT EXISTS )? +
-`(' ( `,' )* `)' +
-( WITH ( AND )* )?
-
-::= ( STATIC )? ( PRIMARY KEY )? +
-| PRIMARY KEY `(' ( `,' )* `)'
-
-::= +
-| `(' (`,' )* `)'
-
-::= +
-| COMPACT STORAGE +
-| CLUSTERING ORDER +
-p. +
-_Sample:_
-
-bc(sample).. +
-CREATE TABLE monkeySpecies ( +
-species text PRIMARY KEY, +
-common_name text, +
-population varint, +
-average_size int +
-) WITH comment=`Important biological records';
-
-CREATE TABLE timeline ( +
-userid uuid, +
-posted_month int, +
-posted_time uuid, +
-body text, +
-posted_by text, +
-PRIMARY KEY (userid, posted_month, posted_time) +
-) WITH compaction = \{ `class' : `LeveledCompactionStrategy' }; +
-p. +
-The `CREATE TABLE` statement creates a new table. Each such table is a
-set of _rows_ (usually representing related entities) for which it
-defines a number of properties. A table is defined by a
-link:#createTableName[name], it defines the columns composing rows of
-the table and have a number of link:#createTableOptions[options]. Note
-that the `CREATE COLUMNFAMILY` syntax is supported as an alias for
-`CREATE TABLE` (for historical reasons).
-
-Attempting to create an already existing table will return an error
-unless the `IF NOT EXISTS` option is used. If it is used, the statement
-will be a no-op if the table already exists.
-
-[[createTableName]]
-===== `<tablename>`
-
-Valid table names are the same as valid
-link:#createKeyspaceStmt[keyspace names] (up to 32 characters long
-alphanumerical identifiers). If the table name is provided alone, the
-table is created within the current keyspace (see `USE`), but if it is
-prefixed by an existing keyspace name (see
-link:#statements[`<tablename>`] grammar), it is created in the specified
-keyspace (but does *not* change the current keyspace).
-
-[[createTableColumn]]
-===== `<column-definition>`
-
-A `CREATE TABLE` statement defines the columns that rows of the table
-can have. A _column_ is defined by its name (an identifier) and its type
-(see the link:#types[data types] section for more details on allowed
-types and their properties).
-
-Within a table, a row is uniquely identified by its `PRIMARY KEY` (or
-more simply the key), and hence all table definitions *must* define a
-PRIMARY KEY (and only one). A `PRIMARY KEY` is composed of one or more
-of the columns defined in the table. If the `PRIMARY KEY` is only one
-column, this can be specified directly after the column definition.
-Otherwise, it must be specified by following `PRIMARY KEY` by the
-comma-separated list of column names composing the key within
-parenthesis. Note that:
-
-bc(sample). +
-CREATE TABLE t ( +
-k int PRIMARY KEY, +
-other text +
-)
-
-is equivalent to
-
-bc(sample). +
-CREATE TABLE t ( +
-k int, +
-other text, +
-PRIMARY KEY (k) +
-)
-
-[[createTablepartitionClustering]]
-===== Partition key and clustering columns
-
-In CQL, the order in which columns are defined for the `PRIMARY KEY`
-matters. The first column of the key is called the _partition key_. It
-has the property that all the rows sharing the same partition key (even
-across table in fact) are stored on the same physical node. Also,
-insertion/update/deletion on rows sharing the same partition key for a
-given table are performed _atomically_ and in _isolation_. Note that it
-is possible to have a composite partition key, i.e. a partition key
-formed of multiple columns, using an extra set of parentheses to define
-which columns forms the partition key.
-
-The remaining columns of the `PRIMARY KEY` definition, if any, are
-called __clustering columns. On a given physical node, rows for a given
-partition key are stored in the order induced by the clustering columns,
-making the retrieval of rows in that clustering order particularly
-efficient (see `SELECT`).
-
-[[createTableStatic]]
-===== `STATIC` columns
-
-Some columns can be declared as `STATIC` in a table definition. A column
-that is static will be ``shared'' by all the rows belonging to the same
-partition (having the same partition key). For instance, in:
-
-bc(sample). +
-CREATE TABLE test ( +
-pk int, +
-t int, +
-v text, +
-s text static, +
-PRIMARY KEY (pk, t) +
-); +
-INSERT INTO test(pk, t, v, s) VALUES (0, 0, `val0', `static0'); +
-INSERT INTO test(pk, t, v, s) VALUES (0, 1, `val1', `static1'); +
-SELECT * FROM test WHERE pk=0 AND t=0;
-
-the last query will return `'static1'` as value for `s`, since `s` is
-static and thus the 2nd insertion modified this ``shared'' value. Note
-however that static columns are only static within a given partition,
-and if in the example above both rows where from different partitions
-(i.e. if they had different value for `pk`), then the 2nd insertion
-would not have modified the value of `s` for the first row.
-
-A few restrictions applies to when static columns are allowed:
-
-* tables with the `COMPACT STORAGE` option (see below) cannot have them
-* a table without clustering columns cannot have static columns (in a
-table without clustering columns, every partition has only one row, and
-so every column is inherently static).
-* only non `PRIMARY KEY` columns can be static
-
-[[createTableOptions]]
-===== `<option>`
-
-The `CREATE TABLE` statement supports a number of options that controls
-the configuration of a new table. These options can be specified after
-the `WITH` keyword.
-
-The first of these option is `COMPACT STORAGE`. This option is mainly
-targeted towards backward compatibility for definitions created before
-CQL3 (see
-http://www.datastax.com/dev/blog/thrift-to-cql3[www.datastax.com/dev/blog/thrift-to-cql3]
-for more details). The option also provides a slightly more compact
-layout of data on disk but at the price of diminished flexibility and
-extensibility for the table. Most notably, `COMPACT STORAGE` tables
-cannot have collections nor static columns and a `COMPACT STORAGE` table
-with at least one clustering column supports exactly one (as in not 0
-nor more than 1) column not part of the `PRIMARY KEY` definition (which
-imply in particular that you cannot add nor remove columns after
-creation). For those reasons, `COMPACT STORAGE` is not recommended
-outside of the backward compatibility reason evoked above.
-
-Another option is `CLUSTERING ORDER`. It allows to define the ordering
-of rows on disk. It takes the list of the clustering column names with,
-for each of them, the on-disk order (Ascending or descending). Note that
-this option affects link:#selectOrderBy[what `ORDER BY` are allowed
-during `SELECT`].
-
-Table creation supports the following other `<property>`:
-
-[cols=",,,",options="header",]
-|===
-|option |kind |default |description
-|`comment` |_simple_ |none |A free-form, human-readable comment.
-
-|`gc_grace_seconds` |_simple_ |864000 |Time to wait before garbage
-collecting tombstones (deletion markers).
-
-|`bloom_filter_fp_chance` |_simple_ |0.00075 |The target probability of
-false positive of the sstable bloom filters. Said bloom filters will be
-sized to provide the provided probability (thus lowering this value
-impact the size of bloom filters in-memory and on-disk)
-
-|`default_time_to_live` |_simple_ |0 |The default expiration time
-(``TTL'') in seconds for a table.
-
-|`compaction` |_map_ |_see below_ |Compaction options, see
-link:#compactionOptions[below].
-
-|`compression` |_map_ |_see below_ |Compression options, see
-link:#compressionOptions[below].
-
-|`caching` |_map_ |_see below_ |Caching options, see
-link:#cachingOptions[below].
-|===
-
-[[compactionOptions]]
-===== Compaction options
-
-The `compaction` property must at least define the `'class'` sub-option,
-that defines the compaction strategy class to use. The default supported
-class are `'SizeTieredCompactionStrategy'`,
-`'LeveledCompactionStrategy'`, `'DateTieredCompactionStrategy'` and
-`'TimeWindowCompactionStrategy'`. Custom strategy can be provided by
-specifying the full class name as a link:#constants[string constant].
-The rest of the sub-options depends on the chosen class. The sub-options
-supported by the default classes are:
-
-[cols=",,,",options="header",]
-|===
-|option |supported compaction strategy |default |description
-|`enabled` |_all_ |true |A boolean denoting whether compaction should be
-enabled or not.
-
-|`tombstone_threshold` |_all_ |0.2 |A ratio such that if a sstable has
-more than this ratio of gcable tombstones over all contained columns,
-the sstable will be compacted (with no other sstables) for the purpose
-of purging those tombstones.
-
-|`tombstone_compaction_interval` |_all_ |1 day |The minimum time to wait
-after an sstable creation time before considering it for ``tombstone
-compaction'', where ``tombstone compaction'' is the compaction triggered
-if the sstable has more gcable tombstones than `tombstone_threshold`.
-
-|`unchecked_tombstone_compaction` |_all_ |false |Setting this to true
-enables more aggressive tombstone compactions - single sstable tombstone
-compactions will run without checking how likely it is that they will be
-successful.
-
-|`min_sstable_size` |SizeTieredCompactionStrategy |50MB |The size tiered
-strategy groups SSTables to compact in buckets. A bucket groups SSTables
-that differs from less than 50% in size. However, for small sizes, this
-would result in a bucketing that is too fine grained. `min_sstable_size`
-defines a size threshold (in bytes) below which all SSTables belong to
-one unique bucket
-
-|`min_threshold` |SizeTieredCompactionStrategy |4 |Minimum number of
-SSTables needed to start a minor compaction.
-
-|`max_threshold` |SizeTieredCompactionStrategy |32 |Maximum number of
-SSTables processed by one minor compaction.
-
-|`bucket_low` |SizeTieredCompactionStrategy |0.5 |Size tiered consider
-sstables to be within the same bucket if their size is within
-[average_size * `bucket_low`, average_size * `bucket_high` ] (i.e the
-default groups sstable whose sizes diverges by at most 50%)
-
-|`bucket_high` |SizeTieredCompactionStrategy |1.5 |Size tiered consider
-sstables to be within the same bucket if their size is within
-[average_size * `bucket_low`, average_size * `bucket_high` ] (i.e the
-default groups sstable whose sizes diverges by at most 50%).
-
-|`sstable_size_in_mb` |LeveledCompactionStrategy |5MB |The target size
-(in MB) for sstables in the leveled strategy. Note that while sstable
-sizes should stay less or equal to `sstable_size_in_mb`, it is possible
-to exceptionally have a larger sstable as during compaction, data for a
-given partition key are never split into 2 sstables
-
-|`timestamp_resolution` |DateTieredCompactionStrategy |MICROSECONDS |The
-timestamp resolution used when inserting data, could be MILLISECONDS,
-MICROSECONDS etc (should be understandable by Java TimeUnit) - don’t
-change this unless you do mutations with USING TIMESTAMP (or equivalent
-directly in the client)
-
-|`base_time_seconds` |DateTieredCompactionStrategy |60 |The base size of
-the time windows.
-
-|`max_sstable_age_days` |DateTieredCompactionStrategy |365 |SSTables
-only containing data that is older than this will never be compacted.
-
-|`timestamp_resolution` |TimeWindowCompactionStrategy |MICROSECONDS |The
-timestamp resolution used when inserting data, could be MILLISECONDS,
-MICROSECONDS etc (should be understandable by Java TimeUnit) - don’t
-change this unless you do mutations with USING TIMESTAMP (or equivalent
-directly in the client)
-
-|`compaction_window_unit` |TimeWindowCompactionStrategy |DAYS |The Java
-TimeUnit used for the window size, set in conjunction with
-`compaction_window_size`. Must be one of DAYS, HOURS, MINUTES
-
-|`compaction_window_size` |TimeWindowCompactionStrategy |1 |The number
-of `compaction_window_unit` units that make up a time window.
-
-|`unsafe_aggressive_sstable_expiration` |TimeWindowCompactionStrategy
-|false |Expired sstables will be dropped without checking its data is
-shadowing other sstables. This is a potentially risky option that can
-lead to data loss or deleted data re-appearing, going beyond what
-`unchecked_tombstone_compaction` does for single sstable compaction. Due
-to the risk the jvm must also be started with
-`-Dcassandra.unsafe_aggressive_sstable_expiration=true`.
-|===
-
-[[compressionOptions]]
-===== Compression options
-
-For the `compression` property, the following sub-options are available:
-
-[cols=",,,,,",options="header",]
-|===
-|option |default |description | | |
-|`class` |LZ4Compressor |The compression algorithm to use. Default
-compressor are: LZ4Compressor, SnappyCompressor and DeflateCompressor.
-Use `'enabled' : false` to disable compression. Custom compressor can be
-provided by specifying the full class name as a link:#constants[string
-constant]. | | |
-
-|`enabled` |true |By default compression is enabled. To disable it, set
-`enabled` to `false` |`chunk_length_in_kb` |64KB |On disk SSTables are
-compressed by block (to allow random reads). This defines the size (in
-KB) of said block. Bigger values may improve the compression rate, but
-increases the minimum size of data to be read from disk for a read
-
-|`crc_check_chance` |1.0 |When compression is enabled, each compressed
-block includes a checksum of that block for the purpose of detecting
-disk bitrot and avoiding the propagation of corruption to other replica.
-This option defines the probability with which those checksums are
-checked during read. By default they are always checked. Set to 0 to
-disable checksum checking and to 0.5 for instance to check them every
-other read | | |
-|===
-
-[[cachingOptions]]
-===== Caching options
-
-For the `caching` property, the following sub-options are available:
-
-[cols=",,",options="header",]
-|===
-|option |default |description
-|`keys` |ALL |Whether to cache keys (``key cache'') for this table.
-Valid values are: `ALL` and `NONE`.
-
-|`rows_per_partition` |NONE |The amount of rows to cache per partition
-(``row cache''). If an integer `n` is specified, the first `n` queried
-rows of a partition will be cached. Other possible options are `ALL`, to
-cache all rows of a queried partition, or `NONE` to disable row caching.
-|===
-
-===== Other considerations:
-
-* When link:#insertStmt[inserting] / link:#updateStmt[updating] a given
-row, not all columns needs to be defined (except for those part of the
-key), and missing columns occupy no space on disk. Furthermore, adding
-new columns (see `ALTER TABLE`) is a constant time operation. There is
-thus no need to try to anticipate future usage (or to cry when you
-haven’t) when creating a table.
-
-[[alterTableStmt]]
-==== ALTER TABLE
-
-_Syntax:_
-
-bc(syntax).. +
-::= ALTER (TABLE | COLUMNFAMILY) (IF EXISTS)?
-
-::= ADD (IF NOT EXISTS)? +
-| ADD (IF NOT EXISTS)? ( ( , )* ) +
-| DROP (IF EXISTS)? +
-| DROP (IF EXISTS)? ( ( , )* ) +
-| RENAME (IF EXISTS)? TO (AND TO)* +
-| WITH ( AND )* +
-p. +
-_Sample:_
-
-bc(sample).. +
-ALTER TABLE addamsFamily
-
-ALTER TABLE addamsFamily +
-ADD gravesite varchar;
-
-ALTER TABLE addamsFamily +
-WITH comment = `A most excellent and useful column family'; +
-p. +
-The `ALTER` statement is used to manipulate table definitions. It allows
-for adding new columns, dropping existing ones, or updating the table
-options. As with table creation, `ALTER COLUMNFAMILY` is allowed as an
-alias for `ALTER TABLE`.
-If the table does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-The `<tablename>` is the table name optionally preceded by the keyspace
-name. The `<instruction>` defines the alteration to perform:
-
-* `ADD`: Adds a new column to the table. The `<identifier>` for the new
-column must not conflict with an existing column. Moreover, columns
-cannot be added to tables defined with the `COMPACT STORAGE` option.
-If the new column already exists, the statement will return an error, unless `IF NOT EXISTS` is used in which case the operation is a no-op.
-* `DROP`: Removes a column from the table. Dropped columns will
-immediately become unavailable in the queries and will not be included
-in compacted sstables in the future. If a column is readded, queries
-won’t return values written before the column was last dropped. It is
-assumed that timestamps represent actual time, so if this is not your
-case, you should NOT read previously dropped columns. Columns can’t be
-dropped from tables defined with the `COMPACT STORAGE` option.
-If the dropped column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-* `RENAME` a primary key column of a table. Non primary key columns cannot be renamed.
-Furthermore, renaming a column to another name which already exists isn't allowed.
-It's important to keep in mind that renamed columns shouldn't have dependent seconday indexes.
-If the renamed column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-* `WITH`: Allows to update the options of the table. The
-link:#createTableOptions[supported `<option>`] (and syntax) are the same
-as for the `CREATE TABLE` statement except that `COMPACT STORAGE` is not
-supported. Note that setting any `compaction` sub-options has the effect
-of erasing all previous `compaction` options, so you need to re-specify
-all the sub-options if you want to keep them. The same note applies to
-the set of `compression` sub-options.
-
-===== CQL type compatibility:
-
-CQL data types may be converted only as the following table.
-
-[cols=",",options="header",]
-|===
-|Data type may be altered to: |Data type
-|timestamp |bigint
-
-|ascii, bigint, boolean, date, decimal, double, float, inet, int,
-smallint, text, time, timestamp, timeuuid, tinyint, uuid, varchar,
-varint |blob
-
-|int |date
-
-|ascii, varchar |text
-
-|bigint |time
-
-|bigint |timestamp
-
-|timeuuid |uuid
-
-|ascii, text |varchar
-
-|bigint, int, timestamp |varint
-|===
-
-Clustering columns have stricter requirements, only the below
-conversions are allowed.
-
-[cols=",",options="header",]
-|===
-|Data type may be altered to: |Data type
-|ascii, text, varchar |blob
-|ascii, varchar |text
-|ascii, text |varchar
-|===
-
-[[dropTableStmt]]
-==== DROP TABLE
-
-_Syntax:_
-
-bc(syntax). ::= DROP TABLE ( IF EXISTS )?
-
-_Sample:_
-
-bc(sample). DROP TABLE worldSeriesAttendees;
-
-The `DROP TABLE` statement results in the immediate, irreversible
-removal of a table, including all data contained in it. As for table
-creation, `DROP COLUMNFAMILY` is allowed as an alias for `DROP TABLE`.
-
-If the table does not exist, the statement will return an error, unless
-`IF EXISTS` is used in which case the operation is a no-op.
-
-[[truncateStmt]]
-==== TRUNCATE
-
-_Syntax:_
-
-bc(syntax). ::= TRUNCATE ( TABLE | COLUMNFAMILY )?
-
-_Sample:_
-
-bc(sample). TRUNCATE superImportantData;
-
-The `TRUNCATE` statement permanently removes all data from a table.
-
-[[createIndexStmt]]
-==== CREATE INDEX
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE ( CUSTOM )? INDEX ( IF NOT EXISTS )? ( )? +
-ON `(' `)' +
-( USING ( WITH OPTIONS = )? )?
-
-::= +
-| keys( ) +
-p. +
-_Sample:_
-
-bc(sample). +
-CREATE INDEX userIndex ON NerdMovies (user); +
-CREATE INDEX ON Mutants (abilityId); +
-CREATE INDEX ON users (keys(favs)); +
-CREATE CUSTOM INDEX ON users (email) USING `path.to.the.IndexClass'; +
-CREATE CUSTOM INDEX ON users (email) USING `path.to.the.IndexClass' WITH
-OPTIONS = \{’storage’: `/mnt/ssd/indexes/'};
-
-The `CREATE INDEX` statement is used to create a new (automatic)
-secondary index for a given (existing) column in a given table. A name
-for the index itself can be specified before the `ON` keyword, if
-desired. If data already exists for the column, it will be indexed
-asynchronously. After the index is created, new data for the column is
-indexed automatically at insertion time.
-
-Attempting to create an already existing index will return an error
-unless the `IF NOT EXISTS` option is used. If it is used, the statement
-will be a no-op if the index already exists.
-
-[[keysIndex]]
-===== Indexes on Map Keys
-
-When creating an index on a link:#map[map column], you may index either
-the keys or the values. If the column identifier is placed within the
-`keys()` function, the index will be on the map keys, allowing you to
-use `CONTAINS KEY` in `WHERE` clauses. Otherwise, the index will be on
-the map values.
-
-[[dropIndexStmt]]
-==== DROP INDEX
-
-_Syntax:_
-
-bc(syntax). ::= DROP INDEX ( IF EXISTS )? ( `.' )?
-
-_Sample:_
-
-bc(sample).. +
-DROP INDEX userIndex;
-
-DROP INDEX userkeyspace.address_index; +
-p. +
-The `DROP INDEX` statement is used to drop an existing secondary index.
-The argument of the statement is the index name, which may optionally
-specify the keyspace of the index.
-
-If the index does not exists, the statement will return an error, unless
-`IF EXISTS` is used in which case the operation is a no-op.
-
-[[createMVStmt]]
-==== CREATE MATERIALIZED VIEW
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE MATERIALIZED VIEW ( IF NOT EXISTS )? AS +
-SELECT ( `(' ( `,' ) * `)' | `*' ) +
-FROM +
-( WHERE )? +
-PRIMARY KEY `(' ( `,' )* `)' +
-( WITH ( AND )* )? +
-p. +
-_Sample:_
-
-bc(sample).. +
-CREATE MATERIALIZED VIEW monkeySpecies_by_population AS +
-SELECT * +
-FROM monkeySpecies +
-WHERE population IS NOT NULL AND species IS NOT NULL +
-PRIMARY KEY (population, species) +
-WITH comment=`Allow query by population instead of species'; +
-p. +
-The `CREATE MATERIALIZED VIEW` statement creates a new materialized
-view. Each such view is a set of _rows_ which corresponds to rows which
-are present in the underlying, or base, table specified in the `SELECT`
-statement. A materialized view cannot be directly updated, but updates
-to the base table will cause corresponding updates in the view.
-
-Attempting to create an already existing materialized view will return
-an error unless the `IF NOT EXISTS` option is used. If it is used, the
-statement will be a no-op if the materialized view already exists.
-
-[[createMVWhere]]
-===== `WHERE` Clause
-
-The `<where-clause>` is similar to the link:#selectWhere[where clause of
-a `SELECT` statement], with a few differences. First, the where clause
-must contain an expression that disallows `NULL` values in columns in
-the view’s primary key. If no other restriction is desired, this can be
-accomplished with an `IS NOT NULL` expression. Second, only columns
-which are in the base table’s primary key may be restricted with
-expressions other than `IS NOT NULL`. (Note that this second restriction
-may be lifted in the future.)
-
-[[alterMVStmt]]
-==== ALTER MATERIALIZED VIEW
-
-_Syntax:_
-
-bc(syntax). ::= ALTER MATERIALIZED VIEW +
-WITH ( AND )*
-
-The `ALTER MATERIALIZED VIEW` statement allows options to be update;
-these options are the same as `CREATE TABLE`’s options.
-
-[[dropMVStmt]]
-==== DROP MATERIALIZED VIEW
-
-_Syntax:_
-
-bc(syntax). ::= DROP MATERIALIZED VIEW ( IF EXISTS )?
-
-_Sample:_
-
-bc(sample). DROP MATERIALIZED VIEW monkeySpecies_by_population;
-
-The `DROP MATERIALIZED VIEW` statement is used to drop an existing
-materialized view.
-
-If the materialized view does not exists, the statement will return an
-error, unless `IF EXISTS` is used in which case the operation is a
-no-op.
-
-[[createTypeStmt]]
-==== CREATE TYPE
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE TYPE ( IF NOT EXISTS )? +
-`(' ( `,' )* `)'
-
-::= ( `.' )?
-
-::=
-
-_Sample:_
-
-bc(sample).. +
-CREATE TYPE address ( +
-street_name text, +
-street_number int, +
-city text, +
-state text, +
-zip int +
-)
-
-CREATE TYPE work_and_home_addresses ( +
-home_address address, +
-work_address address +
-) +
-p. +
-The `CREATE TYPE` statement creates a new user-defined type. Each type
-is a set of named, typed fields. Field types may be any valid type,
-including collections and other existing user-defined types.
-
-Attempting to create an already existing type will result in an error
-unless the `IF NOT EXISTS` option is used. If it is used, the statement
-will be a no-op if the type already exists.
-
-[[createTypeName]]
-===== `<typename>`
-
-Valid type names are identifiers. The names of existing CQL types and
-link:#appendixB[reserved type names] may not be used.
-
-If the type name is provided alone, the type is created with the current
-keyspace (see `USE`). If it is prefixed by an existing keyspace name,
-the type is created within the specified keyspace instead of the current
-keyspace.
-
-[[alterTypeStmt]]
-==== ALTER TYPE
-
-_Syntax:_
-
-bc(syntax).. +
-::= ALTER TYPE (IF EXISTS)?
-
-::= ADD (IF NOT EXISTS)? +
-| RENAME (IF EXISTS)? TO ( AND TO )* +
-p. +
-_Sample:_
-
-bc(sample).. +
-ALTER TYPE address ADD country text
-
-ALTER TYPE address RENAME zip TO zipcode AND street_name TO street +
-p. +
-The `ALTER TYPE` statement is used to manipulate type definitions. It
-allows for adding new fields, renaming existing fields, or changing the
-type of existing fields. If the type does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-[[dropTypeStmt]]
-==== DROP TYPE
-
-_Syntax:_
-
-bc(syntax).. +
-::= DROP TYPE ( IF EXISTS )? +
-p. +
-The `DROP TYPE` statement results in the immediate, irreversible removal
-of a type. Attempting to drop a type that is still in use by another
-type or a table will result in an error.
-
-If the type does not exist, an error will be returned unless `IF EXISTS`
-is used, in which case the operation is a no-op.
-
-[[createTriggerStmt]]
-==== CREATE TRIGGER
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE TRIGGER ( IF NOT EXISTS )? ( )? +
-ON +
-USING
-
-_Sample:_
-
-bc(sample). +
-CREATE TRIGGER myTrigger ON myTable USING
-`org.apache.cassandra.triggers.InvertedIndex';
-
-The actual logic that makes up the trigger can be written in any Java
-(JVM) language and exists outside the database. You place the trigger
-code in a `lib/triggers` subdirectory of the Cassandra installation
-directory, it loads during cluster startup, and exists on every node
-that participates in a cluster. The trigger defined on a table fires
-before a requested DML statement occurs, which ensures the atomicity of
-the transaction.
-
-[[dropTriggerStmt]]
-==== DROP TRIGGER
-
-_Syntax:_
-
-bc(syntax).. +
-::= DROP TRIGGER ( IF EXISTS )? ( )? +
-ON +
-p. +
-_Sample:_
-
-bc(sample). +
-DROP TRIGGER myTrigger ON myTable;
-
-`DROP TRIGGER` statement removes the registration of a trigger created
-using `CREATE TRIGGER`.
-
-[[createFunctionStmt]]
-==== CREATE FUNCTION
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE ( OR REPLACE )? +
-FUNCTION ( IF NOT EXISTS )? +
-( `.' )? +
-`(' ( `,' )* `)' +
-( CALLED | RETURNS NULL ) ON NULL INPUT +
-RETURNS +
-LANGUAGE +
-AS
-
-_Sample:_
-
-bc(sample). +
-CREATE OR REPLACE FUNCTION somefunction +
-( somearg int, anotherarg text, complexarg frozen, listarg list ) +
-RETURNS NULL ON NULL INPUT +
-RETURNS text +
-LANGUAGE java +
-AS $$ +
-// some Java code +
-$$; +
-CREATE FUNCTION akeyspace.fname IF NOT EXISTS +
-( someArg int ) +
-CALLED ON NULL INPUT +
-RETURNS text +
-LANGUAGE java +
-AS $$ +
-// some Java code +
-$$;
-
-`CREATE FUNCTION` creates or replaces a user-defined function.
-
-[[functionSignature]]
-===== Function Signature
-
-Signatures are used to distinguish individual functions. The signature
-consists of:
-
-. The fully qualified function name - i.e _keyspace_ plus
-_function-name_
-. The concatenated list of all argument types
-
-Note that keyspace names, function names and argument types are subject
-to the default naming conventions and case-sensitivity rules.
-
-`CREATE FUNCTION` with the optional `OR REPLACE` keywords either creates
-a function or replaces an existing one with the same signature. A
-`CREATE FUNCTION` without `OR REPLACE` fails if a function with the same
-signature already exists.
-
-Behavior on invocation with `null` values must be defined for each
-function. There are two options:
-
-. `RETURNS NULL ON NULL INPUT` declares that the function will always
-return `null` if any of the input arguments is `null`.
-. `CALLED ON NULL INPUT` declares that the function will always be
-executed.
-
-If the optional `IF NOT EXISTS` keywords are used, the function will
-only be created if another function with the same signature does not
-exist.
-
-`OR REPLACE` and `IF NOT EXIST` cannot be used together.
-
-Functions belong to a keyspace. If no keyspace is specified in
-`<function-name>`, the current keyspace is used (i.e. the keyspace
-specified using the link:#useStmt[`USE`] statement). It is not possible
-to create a user-defined function in one of the system keyspaces.
-
-See the section on link:#udfs[user-defined functions] for more
-information.
-
-[[dropFunctionStmt]]
-==== DROP FUNCTION
-
-_Syntax:_
-
-bc(syntax).. +
-::= DROP FUNCTION ( IF EXISTS )? +
-( `.' )? +
-( `(' ( `,' )* `)' )?
-
-_Sample:_
-
-bc(sample). +
-DROP FUNCTION myfunction; +
-DROP FUNCTION mykeyspace.afunction; +
-DROP FUNCTION afunction ( int ); +
-DROP FUNCTION afunction ( text );
-
-`DROP FUNCTION` statement removes a function created using
-`CREATE FUNCTION`. +
-You must specify the argument types (link:#functionSignature[signature]
-) of the function to drop if there are multiple functions with the same
-name but a different signature (overloaded functions).
-
-`DROP FUNCTION` with the optional `IF EXISTS` keywords drops a function
-if it exists.
-
-[[createAggregateStmt]]
-==== CREATE AGGREGATE
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE ( OR REPLACE )? +
-AGGREGATE ( IF NOT EXISTS )? +
-( `.' )? +
-`(' ( `,' )* `)' +
-SFUNC +
-STYPE +
-( FINALFUNC )? +
-( INITCOND )? +
-p. +
-_Sample:_
-
-bc(sample). +
-CREATE AGGREGATE myaggregate ( val text ) +
-SFUNC myaggregate_state +
-STYPE text +
-FINALFUNC myaggregate_final +
-INITCOND `foo';
-
-See the section on link:#udas[user-defined aggregates] for a complete
-example.
-
-`CREATE AGGREGATE` creates or replaces a user-defined aggregate.
-
-`CREATE AGGREGATE` with the optional `OR REPLACE` keywords either
-creates an aggregate or replaces an existing one with the same
-signature. A `CREATE AGGREGATE` without `OR REPLACE` fails if an
-aggregate with the same signature already exists.
-
-`CREATE AGGREGATE` with the optional `IF NOT EXISTS` keywords either
-creates an aggregate if it does not already exist.
-
-`OR REPLACE` and `IF NOT EXIST` cannot be used together.
-
-Aggregates belong to a keyspace. If no keyspace is specified in
-`<aggregate-name>`, the current keyspace is used (i.e. the keyspace
-specified using the link:#useStmt[`USE`] statement). It is not possible
-to create a user-defined aggregate in one of the system keyspaces.
-
-Signatures for user-defined aggregates follow the
-link:#functionSignature[same rules] as for user-defined functions.
-
-`STYPE` defines the type of the state value and must be specified.
-
-The optional `INITCOND` defines the initial state value for the
-aggregate. It defaults to `null`. A non-`null` `INITCOND` must be
-specified for state functions that are declared with
-`RETURNS NULL ON NULL INPUT`.
-
-`SFUNC` references an existing function to be used as the state
-modifying function. The type of first argument of the state function
-must match `STYPE`. The remaining argument types of the state function
-must match the argument types of the aggregate function. State is not
-updated for state functions declared with `RETURNS NULL ON NULL INPUT`
-and called with `null`.
-
-The optional `FINALFUNC` is called just before the aggregate result is
-returned. It must take only one argument with type `STYPE`. The return
-type of the `FINALFUNC` may be a different type. A final function
-declared with `RETURNS NULL ON NULL INPUT` means that the aggregate’s
-return value will be `null`, if the last state is `null`.
-
-If no `FINALFUNC` is defined, the overall return type of the aggregate
-function is `STYPE`. If a `FINALFUNC` is defined, it is the return type
-of that function.
-
-See the section on link:#udas[user-defined aggregates] for more
-information.
-
-[[dropAggregateStmt]]
-==== DROP AGGREGATE
-
-_Syntax:_
-
-bc(syntax).. +
-::= DROP AGGREGATE ( IF EXISTS )? +
-( `.' )? +
-( `(' ( `,' )* `)' )? +
-p.
-
-_Sample:_
-
-bc(sample). +
-DROP AGGREGATE myAggregate; +
-DROP AGGREGATE myKeyspace.anAggregate; +
-DROP AGGREGATE someAggregate ( int ); +
-DROP AGGREGATE someAggregate ( text );
-
-The `DROP AGGREGATE` statement removes an aggregate created using
-`CREATE AGGREGATE`. You must specify the argument types of the aggregate
-to drop if there are multiple aggregates with the same name but a
-different signature (overloaded aggregates).
-
-`DROP AGGREGATE` with the optional `IF EXISTS` keywords drops an
-aggregate if it exists, and does nothing if a function with the
-signature does not exist.
-
-Signatures for user-defined aggregates follow the
-link:#functionSignature[same rules] as for user-defined functions.
-
-[[dataManipulation]]
-=== Data Manipulation
-
-[[insertStmt]]
-==== INSERT
-
-_Syntax:_
-
-bc(syntax).. +
-::= INSERT INTO +
-( ( VALUES ) +
-| ( JSON )) +
-( IF NOT EXISTS )? +
-( USING ( AND )* )?
-
-::= `(' ( `,' )* `)'
-
-::= `(' ( `,' )* `)'
-
-::= TIMESTAMP +
-| TTL +
-p. +
-_Sample:_
-
-bc(sample).. +
-INSERT INTO NerdMovies (movie, director, main_actor, year) +
-VALUES (`Serenity', `Joss Whedon', `Nathan Fillion', 2005) +
-USING TTL 86400;
-
-INSERT INTO NerdMovies JSON `\{``movie'': ``Serenity'', ``director'':
-``Joss Whedon'', ``year'': 2005}' +
-p. +
-The `INSERT` statement writes one or more columns for a given row in a
-table. Note that since a row is identified by its `PRIMARY KEY`, at
-least the columns composing it must be specified. The list of columns to
-insert to must be supplied when using the `VALUES` syntax. When using
-the `JSON` syntax, they are optional. See the section on
-link:#insertJson[`INSERT JSON`] for more details.
-
-Note that unlike in SQL, `INSERT` does not check the prior existence of
-the row by default: the row is created if none existed before, and
-updated otherwise. Furthermore, there is no mean to know which of
-creation or update happened.
-
-It is however possible to use the `IF NOT EXISTS` condition to only
-insert if the row does not exist prior to the insertion. But please note
-that using `IF NOT EXISTS` will incur a non negligible performance cost
-(internally, Paxos will be used) so this should be used sparingly.
-
-All updates for an `INSERT` are applied atomically and in isolation.
-
-Please refer to the link:#updateOptions[`UPDATE`] section for
-information on the `<option>` available and to the
-link:#collections[collections] section for use of
-`<collection-literal>`. Also note that `INSERT` does not support
-counters, while `UPDATE` does.
-
-[[updateStmt]]
-==== UPDATE
-
-_Syntax:_
-
-bc(syntax).. +
-::= UPDATE +
-( USING ( AND )* )? +
-SET ( `,' )* +
-WHERE +
-( IF ( AND condition )* )?
-
-::= `=' +
-| `=' (`+' | `-') ( | | ) +
-| `=' `+' +
-| `[' `]' `=' +
-| `.' `='
-
-::= +
-| CONTAINS (KEY)? +
-| IN +
-| `[' `]' +
-| `[' `]' IN +
-| `.' +
-| `.' IN
-
-::= `<' | `<=' | `=' | `!=' | `>=' | `>' +
-::= ( | `(' ( ( `,' )* )? `)')
-
-::= ( AND )*
-
-::= `=' +
-| `(' (`,' )* `)' `=' +
-| IN `(' ( ( `,' )* )? `)' +
-| IN +
-| `(' (`,' )* `)' IN `(' ( ( `,' )* )? `)' +
-| `(' (`,' )* `)' IN
-
-::= TIMESTAMP +
-| TTL +
-p. +
-_Sample:_
-
-bc(sample).. +
-UPDATE NerdMovies USING TTL 400 +
-SET director = `Joss Whedon', +
-main_actor = `Nathan Fillion', +
-year = 2005 +
-WHERE movie = `Serenity';
-
-UPDATE UserActions SET total = total + 2 WHERE user =
-B70DE1D0-9908-4AE3-BE34-5573E5B09F14 AND action = `click'; +
-p. +
-The `UPDATE` statement writes one or more columns for a given row in a
-table. The `<where-clause>` is used to select the row to update and must
-include all columns composing the `PRIMARY KEY`. Other columns values
-are specified through `<assignment>` after the `SET` keyword.
-
-Note that unlike in SQL, `UPDATE` does not check the prior existence of
-the row by default (except through the use of `<condition>`, see below):
-the row is created if none existed before, and updated otherwise.
-Furthermore, there are no means to know whether a creation or update
-occurred.
-
-It is however possible to use the conditions on some columns through
-`IF`, in which case the row will not be updated unless the conditions
-are met. But, please note that using `IF` conditions will incur a
-non-negligible performance cost (internally, Paxos will be used) so this
-should be used sparingly.
-
-In an `UPDATE` statement, all updates within the same partition key are
-applied atomically and in isolation.
-
-The `c = c + 3` form of `<assignment>` is used to increment/decrement
-counters. The identifier after the `=' sign *must* be the same than the
-one before the `=' sign (Only increment/decrement is supported on
-counters, not the assignment of a specific value).
-
-The `id = id + <collection-literal>` and `id[value1] = value2` forms of
-`<assignment>` are for collections. Please refer to the
-link:#collections[relevant section] for more details.
-
-The `id.field = <term>` form of `<assignemt>` is for setting the value
-of a single field on a non-frozen user-defined types.
-
-[[updateOptions]]
-===== `<options>`
-
-The `UPDATE` and `INSERT` statements support the following options:
-
-* `TIMESTAMP`: sets the timestamp for the operation. If not specified,
-the coordinator will use the current time (in microseconds) at the start
-of statement execution as the timestamp. This is usually a suitable
-default.
-* `TTL`: specifies an optional Time To Live (in seconds) for the
-inserted values. If set, the inserted values are automatically removed
-from the database after the specified time. Note that the TTL concerns
-the inserted values, not the columns themselves. This means that any
-subsequent update of the column will also reset the TTL (to whatever TTL
-is specified in that update). By default, values never expire. A TTL of
-0 is equivalent to no TTL. If the table has a default_time_to_live, a
-TTL of 0 will remove the TTL for the inserted or updated values.
-
-[[deleteStmt]]
-==== DELETE
-
-_Syntax:_
-
-bc(syntax).. +
-::= DELETE ( ( `,' )* )? +
-FROM +
-( USING TIMESTAMP )? +
-WHERE +
-( IF ( EXISTS | ( ( AND )*) ) )?
-
-::= +
-| `[' `]' +
-| `.'
-
-::= ( AND )*
-
-::= +
-| `(' (`,' )* `)' +
-| IN `(' ( ( `,' )* )? `)' +
-| IN +
-| `(' (`,' )* `)' IN `(' ( ( `,' )* )? `)' +
-| `(' (`,' )* `)' IN
-
-::= `=' | `<' | `>' | `<=' | `>=' +
-::= ( | `(' ( ( `,' )* )? `)')
-
-::= ( | `!=') +
-| CONTAINS (KEY)? +
-| IN +
-| `[' `]' ( | `!=') +
-| `[' `]' IN +
-| `.' ( | `!=') +
-| `.' IN
-
-_Sample:_
-
-bc(sample).. +
-DELETE FROM NerdMovies USING TIMESTAMP 1240003134 WHERE movie =
-`Serenity';
-
-DELETE phone FROM Users WHERE userid IN
-(C73DE1D3-AF08-40F3-B124-3FF3E5109F22,
-B70DE1D0-9908-4AE3-BE34-5573E5B09F14); +
-p. +
-The `DELETE` statement deletes columns and rows. If column names are
-provided directly after the `DELETE` keyword, only those columns are
-deleted from the row indicated by the `<where-clause>`. The `id[value]`
-syntax in `<selection>` is for non-frozen collections (please refer to
-the link:#collections[collection section] for more details). The
-`id.field` syntax is for the deletion of non-frozen user-defined types.
-Otherwise, whole rows are removed. The `<where-clause>` specifies which
-rows are to be deleted. Multiple rows may be deleted with one statement
-by using an `IN` clause. A range of rows may be deleted using an
-inequality operator (such as `>=`).
-
-`DELETE` supports the `TIMESTAMP` option with the same semantics as the
-link:#updateStmt[`UPDATE`] statement.
-
-In a `DELETE` statement, all deletions within the same partition key are
-applied atomically and in isolation.
-
-A `DELETE` operation can be conditional through the use of an `IF`
-clause, similar to `UPDATE` and `INSERT` statements. However, as with
-`INSERT` and `UPDATE` statements, this will incur a non-negligible
-performance cost (internally, Paxos will be used) and so should be used
-sparingly.
-
-[[batchStmt]]
-==== BATCH
-
-_Syntax:_
-
-bc(syntax).. +
-::= BEGIN ( UNLOGGED | COUNTER ) BATCH +
-( USING ( AND )* )? +
-( `;' )* +
-APPLY BATCH
-
-::= +
-| +
-|
-
-::= TIMESTAMP +
-p. +
-_Sample:_
-
-bc(sample). +
-BEGIN BATCH +
-INSERT INTO users (userid, password, name) VALUES (`user2', `ch@ngem3b',
-`second user'); +
-UPDATE users SET password = `ps22dhds' WHERE userid = `user3'; +
-INSERT INTO users (userid, password) VALUES (`user4', `ch@ngem3c'); +
-DELETE name FROM users WHERE userid = `user1'; +
-APPLY BATCH;
-
-The `BATCH` statement group multiple modification statements
-(insertions/updates and deletions) into a single statement. It serves
-several purposes:
-
-. It saves network round-trips between the client and the server (and
-sometimes between the server coordinator and the replicas) when batching
-multiple updates.
-. All updates in a `BATCH` belonging to a given partition key are
-performed in isolation.
-. By default, all operations in the batch are performed as `LOGGED`, to
-ensure all mutations eventually complete (or none will). See the notes
-on link:#unloggedBatch[`UNLOGGED`] for more details.
-
-Note that:
-
-* `BATCH` statements may only contain `UPDATE`, `INSERT` and `DELETE`
-statements.
-* Batches are _not_ a full analogue for SQL transactions.
-* If a timestamp is not specified for each operation, then all
-operations will be applied with the same timestamp. Due to Cassandra’s
-conflict resolution procedure in the case of
-http://wiki.apache.org/cassandra/FAQ#clocktie[timestamp ties],
-operations may be applied in an order that is different from the order
-they are listed in the `BATCH` statement. To force a particular
-operation ordering, you must specify per-operation timestamps.
-
-[[unloggedBatch]]
-===== `UNLOGGED`
-
-By default, Cassandra uses a batch log to ensure all operations in a
-batch eventually complete or none will (note however that operations are
-only isolated within a single partition).
-
-There is a performance penalty for batch atomicity when a batch spans
-multiple partitions. If you do not want to incur this penalty, you can
-tell Cassandra to skip the batchlog with the `UNLOGGED` option. If the
-`UNLOGGED` option is used, a failed batch might leave the patch only
-partly applied.
-
-[[counterBatch]]
-===== `COUNTER`
-
-Use the `COUNTER` option for batched counter updates. Unlike other
-updates in Cassandra, counter updates are not idempotent.
-
-[[batchOptions]]
-===== `<option>`
-
-`BATCH` supports both the `TIMESTAMP` option, with similar semantic to
-the one described in the link:#updateOptions[`UPDATE`] statement (the
-timestamp applies to all the statement inside the batch). However, if
-used, `TIMESTAMP` *must not* be used in the statements within the batch.
-
-=== Queries
-
-[[selectStmt]]
-==== SELECT
-
-_Syntax:_
-
-bc(syntax).. +
-::= SELECT ( JSON )? +
-FROM +
-( WHERE )? +
-( GROUP BY )? +
-( ORDER BY )? +
-( PER PARTITION LIMIT )? +
-( LIMIT )? +
-( ALLOW FILTERING )?
-
-::= DISTINCT?
-
-::= (AS )? ( `,' (AS )? )* +
-| `*'
-
-::= +
-| +
-| WRITETIME `(' `)' +
-| COUNT `(' `*' `)' +
-| TTL `(' `)' +
-| CAST `(' AS `)' +
-| `(' ( (`,' )*)? `)' +
-| `.' +
-| `[' `]' +
-| `[' ? .. ? `]'
-
-::= ( AND )*
-
-::= +
-| `(' (`,' )* `)' +
-| IN `(' ( ( `,' )* )? `)' +
-| `(' (`,' )* `)' IN `(' ( ( `,' )* )? `)' +
-| TOKEN `(' ( `,' )* `)'
-
-::= `=' | `<' | `>' | `<=' | `>=' | CONTAINS | CONTAINS KEY +
-::= (`,' )* +
-::= ( `,' )* +
-::= ( ASC | DESC )? +
-::= `(' (`,' )* `)' +
-p. +
-_Sample:_
-
-bc(sample).. +
-SELECT name, occupation FROM users WHERE userid IN (199, 200, 207);
-
-SELECT JSON name, occupation FROM users WHERE userid = 199;
-
-SELECT name AS user_name, occupation AS user_occupation FROM users;
-
-SELECT time, value +
-FROM events +
-WHERE event_type = `myEvent' +
-AND time > `2011-02-03' +
-AND time <= `2012-01-01'
-
-SELECT COUNT (*) FROM users;
-
-SELECT COUNT (*) AS user_count FROM users;
-
-The `SELECT` statements reads one or more columns for one or more rows
-in a table. It returns a result-set of rows, where each row contains the
-collection of columns corresponding to the query. If the `JSON` keyword
-is used, the results for each row will contain only a single column
-named ``json''. See the section on link:#selectJson[`SELECT JSON`] for
-more details.
-
-[[selectSelection]]
-===== `<select-clause>`
-
-The `<select-clause>` determines which columns needs to be queried and
-returned in the result-set. It consists of either the comma-separated
-list of or the wildcard character (`*`) to select all the columns
-defined for the table. Please note that for wildcard `SELECT` queries
-the order of columns returned is not specified and is not guaranteed to
-be stable between Cassandra versions.
-
-A `<selector>` is either a column name to retrieve or a `<function>` of
-one or more `<term>`s. The function allowed are the same as for `<term>`
-and are described in the link:#functions[function section]. In addition
-to these generic functions, the `WRITETIME` (resp. `TTL`) function
-allows to select the timestamp of when the column was inserted (resp.
-the time to live (in seconds) for the column (or null if the column has
-no expiration set)) and the link:#castFun[`CAST`] function can be used
-to convert one data type to another. The `WRITETIME` and `TTL` functions
-can't be used on multi-cell columns such as non-frozen collections or
-non-frozen user-defined types.
-
-Additionally, individual values of maps and sets can be selected using
-`[ <term> ]`. For maps, this will return the value corresponding to the
-key, if such entry exists. For sets, this will return the key that is
-selected if it exists and is thus mainly a way to check element
-existence. It is also possible to select a slice of a set or map with
-`[ <term> ... <term> `], where both bound can be omitted.
-
-Any `<selector>` can be aliased using `AS` keyword (see examples).
-Please note that `<where-clause>` and `<order-by>` clause should refer
-to the columns by their original names and not by their aliases.
-
-The `COUNT` keyword can be used with parenthesis enclosing `*`. If so,
-the query will return a single result: the number of rows matching the
-query. Note that `COUNT(1)` is supported as an alias.
-
-[[selectWhere]]
-===== `<where-clause>`
-
-The `<where-clause>` specifies which rows must be queried. It is
-composed of relations on the columns that are part of the `PRIMARY KEY`
-and/or have a link:#createIndexStmt[secondary index] defined on them.
-
-Not all relations are allowed in a query. For instance, non-equal
-relations (where `IN` is considered as an equal relation) on a partition
-key are not supported (but see the use of the `TOKEN` method below to do
-non-equal queries on the partition key). Moreover, for a given partition
-key, the clustering columns induce an ordering of rows and relations on
-them is restricted to the relations that allow to select a *contiguous*
-(for the ordering) set of rows. For instance, given
-
-bc(sample). +
-CREATE TABLE posts ( +
-userid text, +
-blog_title text, +
-posted_at timestamp, +
-entry_title text, +
-content text, +
-category int, +
-PRIMARY KEY (userid, blog_title, posted_at) +
-)
-
-The following query is allowed:
-
-bc(sample). +
-SELECT entry_title, content FROM posts WHERE userid=`john doe' AND
-blog_title=`John'`s Blog' AND posted_at >= `2012-01-01' AND posted_at <
-`2012-01-31'
-
-But the following one is not, as it does not select a contiguous set of
-rows (and we suppose no secondary indexes are set):
-
-bc(sample). +
-// Needs a blog_title to be set to select ranges of posted_at +
-SELECT entry_title, content FROM posts WHERE userid=`john doe' AND
-posted_at >= `2012-01-01' AND posted_at < `2012-01-31'
-
-When specifying relations, the `TOKEN` function can be used on the
-`PARTITION KEY` column to query. In that case, rows will be selected
-based on the token of their `PARTITION_KEY` rather than on the value.
-Note that the token of a key depends on the partitioner in use, and that
-in particular the RandomPartitioner won’t yield a meaningful order. Also
-note that ordering partitioners always order token values by bytes (so
-even if the partition key is of type int, `token(-1) > token(0)` in
-particular). Example:
-
-bc(sample). +
-SELECT * FROM posts WHERE token(userid) > token(`tom') AND token(userid)
-< token(`bob')
-
-Moreover, the `IN` relation is only allowed on the last column of the
-partition key and on the last column of the full primary key.
-
-It is also possible to ``group'' `CLUSTERING COLUMNS` together in a
-relation using the tuple notation. For instance:
-
-bc(sample). +
-SELECT * FROM posts WHERE userid=`john doe' AND (blog_title, posted_at)
-> (`John'`s Blog', `2012-01-01')
-
-will request all rows that sorts after the one having ``John’s Blog'' as
-`blog_tile` and `2012-01-01' for `posted_at` in the clustering order. In
-particular, rows having a `post_at <= '2012-01-01'` will be returned as
-long as their `blog_title > 'John''s Blog'`, which wouldn’t be the case
-for:
-
-bc(sample). +
-SELECT * FROM posts WHERE userid=`john doe' AND blog_title > `John'`s
-Blog' AND posted_at > `2012-01-01'
-
-The tuple notation may also be used for `IN` clauses on
-`CLUSTERING COLUMNS`:
-
-bc(sample). +
-SELECT * FROM posts WHERE userid=`john doe' AND (blog_title, posted_at)
-IN ((`John'`s Blog', `2012-01-01), (’Extreme Chess', `2014-06-01'))
-
-The `CONTAINS` operator may only be used on collection columns (lists,
-sets, and maps). In the case of maps, `CONTAINS` applies to the map
-values. The `CONTAINS KEY` operator may only be used on map columns and
-applies to the map keys.
-
-[[selectOrderBy]]
-===== `<order-by>`
-
-The `ORDER BY` option allows to select the order of the returned
-results. It takes as argument a list of column names along with the
-order for the column (`ASC` for ascendant and `DESC` for descendant,
-omitting the order being equivalent to `ASC`). Currently the possible
-orderings are limited (which depends on the table
-link:#createTableOptions[`CLUSTERING ORDER`] ):
-
-* if the table has been defined without any specific `CLUSTERING ORDER`,
-then then allowed orderings are the order induced by the clustering
-columns and the reverse of that one.
-* otherwise, the orderings allowed are the order of the
-`CLUSTERING ORDER` option and the reversed one.
-
-[[selectGroupBy]]
-===== `<group-by>`
-
-The `GROUP BY` option allows to condense into a single row all selected
-rows that share the same values for a set of columns.
-
-Using the `GROUP BY` option, it is only possible to group rows at the
-partition key level or at a clustering column level. By consequence, the
-`GROUP BY` option only accept as arguments primary key column names in
-the primary key order. If a primary key column is restricted by an
-equality restriction it is not required to be present in the `GROUP BY`
-clause.
-
-Aggregate functions will produce a separate value for each group. If no
-`GROUP BY` clause is specified, aggregates functions will produce a
-single value for all the rows.
-
-If a column is selected without an aggregate function, in a statement
-with a `GROUP BY`, the first value encounter in each group will be
-returned.
-
-[[selectLimit]]
-===== `LIMIT` and `PER PARTITION LIMIT`
-
-The `LIMIT` option to a `SELECT` statement limits the number of rows
-returned by a query, while the `PER PARTITION LIMIT` option limits the
-number of rows returned for a given partition by the query. Note that
-both type of limit can used in the same statement.
-
-[[selectAllowFiltering]]
-===== `ALLOW FILTERING`
-
-By default, CQL only allows select queries that don’t involve
-``filtering'' server side, i.e. queries where we know that all (live)
-record read will be returned (maybe partly) in the result set. The
-reasoning is that those ``non filtering'' queries have predictable
-performance in the sense that they will execute in a time that is
-proportional to the amount of data *returned* by the query (which can be
-controlled through `LIMIT`).
-
-The `ALLOW FILTERING` option allows to explicitly allow (some) queries
-that require filtering. Please note that a query using `ALLOW FILTERING`
-may thus have unpredictable performance (for the definition above), i.e.
-even a query that selects a handful of records *may* exhibit performance
-that depends on the total amount of data stored in the cluster.
-
-For instance, considering the following table holding user profiles with
-their year of birth (with a secondary index on it) and country of
-residence:
-
-bc(sample).. +
-CREATE TABLE users ( +
-username text PRIMARY KEY, +
-firstname text, +
-lastname text, +
-birth_year int, +
-country text +
-)
-
-CREATE INDEX ON users(birth_year); +
-p.
-
-Then the following queries are valid:
-
-bc(sample). +
-SELECT * FROM users; +
-SELECT firstname, lastname FROM users WHERE birth_year = 1981;
-
-because in both case, Cassandra guarantees that these queries
-performance will be proportional to the amount of data returned. In
-particular, if no users are born in 1981, then the second query
-performance will not depend of the number of user profile stored in the
-database (not directly at least: due to secondary index implementation
-consideration, this query may still depend on the number of node in the
-cluster, which indirectly depends on the amount of data stored.
-Nevertheless, the number of nodes will always be multiple number of
-magnitude lower than the number of user profile stored). Of course, both
-query may return very large result set in practice, but the amount of
-data returned can always be controlled by adding a `LIMIT`.
-
-However, the following query will be rejected:
-
-bc(sample). +
-SELECT firstname, lastname FROM users WHERE birth_year = 1981 AND
-country = `FR';
-
-because Cassandra cannot guarantee that it won’t have to scan large
-amount of data even if the result to those query is small. Typically, it
-will scan all the index entries for users born in 1981 even if only a
-handful are actually from France. However, if you ``know what you are
-doing'', you can force the execution of this query by using
-`ALLOW FILTERING` and so the following query is valid:
-
-bc(sample). +
-SELECT firstname, lastname FROM users WHERE birth_year = 1981 AND
-country = `FR' ALLOW FILTERING;
-
-[[databaseRoles]]
-=== Database Roles
-
-[[createRoleStmt]]
-==== CREATE ROLE
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE ROLE ( IF NOT EXISTS )? ( WITH ( AND )* )?
-
-::= PASSWORD = +
-| LOGIN = +
-| SUPERUSER = +
-| OPTIONS = +
-p.
-
-_Sample:_
-
-bc(sample). +
-CREATE ROLE new_role; +
-CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true; +
-CREATE ROLE bob WITH PASSWORD = `password_b' AND LOGIN = true AND
-SUPERUSER = true; +
-CREATE ROLE carlos WITH OPTIONS = \{ `custom_option1' : `option1_value',
-`custom_option2' : 99 };
-
-By default roles do not possess `LOGIN` privileges or `SUPERUSER`
-status.
-
-link:#permissions[Permissions] on database resources are granted to
-roles; types of resources include keyspaces, tables, functions and roles
-themselves. Roles may be granted to other roles to create hierarchical
-permissions structures; in these hierarchies, permissions and
-`SUPERUSER` status are inherited, but the `LOGIN` privilege is not.
-
-If a role has the `LOGIN` privilege, clients may identify as that role
-when connecting. For the duration of that connection, the client will
-acquire any roles and privileges granted to that role.
-
-Only a client with with the `CREATE` permission on the database roles
-resource may issue `CREATE ROLE` requests (see the
-link:#permissions[relevant section] below), unless the client is a
-`SUPERUSER`. Role management in Cassandra is pluggable and custom
-implementations may support only a subset of the listed options.
-
-Role names should be quoted if they contain non-alphanumeric characters.
-
-[[createRolePwd]]
-===== Setting credentials for internal authentication
-
-Use the `WITH PASSWORD` clause to set a password for internal
-authentication, enclosing the password in single quotation marks. +
-If internal authentication has not been set up or the role does not have
-`LOGIN` privileges, the `WITH PASSWORD` clause is not necessary.
-
-[[createRoleConditional]]
-===== Creating a role conditionally
-
-Attempting to create an existing role results in an invalid query
-condition unless the `IF NOT EXISTS` option is used. If the option is
-used and the role exists, the statement is a no-op.
-
-bc(sample). +
-CREATE ROLE other_role; +
-CREATE ROLE IF NOT EXISTS other_role;
-
-[[alterRoleStmt]]
-==== ALTER ROLE
-
-_Syntax:_
-
-bc(syntax).. +
-::= ALTER ROLE (IF EXISTS)? ( WITH ( AND )* )?
-
-::= PASSWORD = +
-| LOGIN = +
-| SUPERUSER = +
-| OPTIONS = +
-p.
-
-_Sample:_
-
-bc(sample). +
-ALTER ROLE bob WITH PASSWORD = `PASSWORD_B' AND SUPERUSER = false;
-
-If the role does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-Conditions on executing `ALTER ROLE` statements:
-
-* A client must have `SUPERUSER` status to alter the `SUPERUSER` status
-of another role
-* A client cannot alter the `SUPERUSER` status of any role it currently
-holds
-* A client can only modify certain properties of the role with which it
-identified at login (e.g. `PASSWORD`)
-* To modify properties of a role, the client must be granted `ALTER`
-link:#permissions[permission] on that role
-
-[[dropRoleStmt]]
-==== DROP ROLE
-
-_Syntax:_
-
-bc(syntax).. +
-::= DROP ROLE ( IF EXISTS )? +
-p.
-
-_Sample:_
-
-bc(sample). +
-DROP ROLE alice; +
-DROP ROLE IF EXISTS bob;
-
-`DROP ROLE` requires the client to have `DROP`
-link:#permissions[permission] on the role in question. In addition,
-client may not `DROP` the role with which it identified at login.
-Finaly, only a client with `SUPERUSER` status may `DROP` another
-`SUPERUSER` role. +
-Attempting to drop a role which does not exist results in an invalid
-query condition unless the `IF EXISTS` option is used. If the option is
-used and the role does not exist the statement is a no-op.
-
-[[grantRoleStmt]]
-==== GRANT ROLE
-
-_Syntax:_
-
-bc(syntax). +
-::= GRANT TO
-
-_Sample:_
-
-bc(sample). +
-GRANT report_writer TO alice;
-
-This statement grants the `report_writer` role to `alice`. Any
-permissions granted to `report_writer` are also acquired by `alice`. +
-Roles are modelled as a directed acyclic graph, so circular grants are
-not permitted. The following examples result in error conditions:
-
-bc(sample). +
-GRANT role_a TO role_b; +
-GRANT role_b TO role_a;
-
-bc(sample). +
-GRANT role_a TO role_b; +
-GRANT role_b TO role_c; +
-GRANT role_c TO role_a;
-
-[[revokeRoleStmt]]
-==== REVOKE ROLE
-
-_Syntax:_
-
-bc(syntax). +
-::= REVOKE FROM
-
-_Sample:_
-
-bc(sample). +
-REVOKE report_writer FROM alice;
-
-This statement revokes the `report_writer` role from `alice`. Any
-permissions that `alice` has acquired via the `report_writer` role are
-also revoked.
-
-[[listRolesStmt]]
-===== LIST ROLES
-
-_Syntax:_
-
-bc(syntax). +
-::= LIST ROLES ( OF )? ( NORECURSIVE )?
-
-_Sample:_
-
-bc(sample). +
-LIST ROLES;
-
-Return all known roles in the system, this requires `DESCRIBE`
-permission on the database roles resource.
-
-bc(sample). +
-LIST ROLES OF `alice`;
-
-Enumerate all roles granted to `alice`, including those transitively
-aquired.
-
-bc(sample). +
-LIST ROLES OF `bob` NORECURSIVE
-
-List all roles directly granted to `bob`.
-
-[[createUserStmt]]
-==== CREATE USER
-
-Prior to the introduction of roles in Cassandra 2.2, authentication and
-authorization were based around the concept of a `USER`. For backward
-compatibility, the legacy syntax has been preserved with `USER` centric
-statments becoming synonyms for the `ROLE` based equivalents.
-
-_Syntax:_
-
-bc(syntax).. +
-::= CREATE USER ( IF NOT EXISTS )? ( WITH PASSWORD )? ()?
-
-::= SUPERUSER +
-| NOSUPERUSER +
-p.
-
-_Sample:_
-
-bc(sample). +
-CREATE USER alice WITH PASSWORD `password_a' SUPERUSER; +
-CREATE USER bob WITH PASSWORD `password_b' NOSUPERUSER;
-
-`CREATE USER` is equivalent to `CREATE ROLE` where the `LOGIN` option is
-`true`. So, the following pairs of statements are equivalent:
-
-bc(sample).. +
-CREATE USER alice WITH PASSWORD `password_a' SUPERUSER; +
-CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true AND
-SUPERUSER = true;
-
-CREATE USER IF NOT EXISTS alice WITH PASSWORD `password_a' SUPERUSER; +
-CREATE ROLE IF NOT EXISTS alice WITH PASSWORD = `password_a' AND LOGIN =
-true AND SUPERUSER = true;
-
-CREATE USER alice WITH PASSWORD `password_a' NOSUPERUSER; +
-CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true AND
-SUPERUSER = false;
-
-CREATE USER alice WITH PASSWORD `password_a' NOSUPERUSER; +
-CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true;
-
-CREATE USER alice WITH PASSWORD `password_a'; +
-CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true; +
-p.
-
-[[alterUserStmt]]
-==== ALTER USER
-
-_Syntax:_
-
-bc(syntax).. +
-::= ALTER USER (IF EXISTS)? ( WITH PASSWORD )? ( )?
-
-::= SUPERUSER +
-| NOSUPERUSER +
-p.
-
-bc(sample). +
-ALTER USER alice WITH PASSWORD `PASSWORD_A'; +
-ALTER USER bob SUPERUSER;
-
-If the user does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-[[dropUserStmt]]
-==== DROP USER
-
-_Syntax:_
-
-bc(syntax).. +
-::= DROP USER ( IF EXISTS )? +
-p.
-
-_Sample:_
-
-bc(sample). +
-DROP USER alice; +
-DROP USER IF EXISTS bob;
-
-[[listUsersStmt]]
-==== LIST USERS
-
-_Syntax:_
-
-bc(syntax). +
-::= LIST USERS;
-
-_Sample:_
-
-bc(sample). +
-LIST USERS;
-
-This statement is equivalent to
-
-bc(sample). +
-LIST ROLES;
-
-but only roles with the `LOGIN` privilege are included in the output.
-
-[[dataControl]]
-=== Data Control
-
-==== Permissions
-
-Permissions on resources are granted to roles; there are several
-different types of resources in Cassandra and each type is modelled
-hierarchically:
-
-* The hierarchy of Data resources, Keyspaces and Tables has the
-structure `ALL KEYSPACES` -> `KEYSPACE` -> `TABLE`
-* Function resources have the structure `ALL FUNCTIONS` -> `KEYSPACE` ->
-`FUNCTION`
-* Resources representing roles have the structure `ALL ROLES` -> `ROLE`
-* Resources representing JMX ObjectNames, which map to sets of
-MBeans/MXBeans, have the structure `ALL MBEANS` -> `MBEAN`
-
-Permissions can be granted at any level of these hierarchies and they
-flow downwards. So granting a permission on a resource higher up the
-chain automatically grants that same permission on all resources lower
-down. For example, granting `SELECT` on a `KEYSPACE` automatically
-grants it on all `TABLES` in that `KEYSPACE`. Likewise, granting a
-permission on `ALL FUNCTIONS` grants it on every defined function,
-regardless of which keyspace it is scoped in. It is also possible to
-grant permissions on all functions scoped to a particular keyspace.
-
-Modifications to permissions are visible to existing client sessions;
-that is, connections need not be re-established following permissions
-changes.
-
-The full set of available permissions is:
-
-* `CREATE`
-* `ALTER`
-* `DROP`
-* `SELECT`
-* `MODIFY`
-* `AUTHORIZE`
-* `DESCRIBE`
-* `EXECUTE`
-
-Not all permissions are applicable to every type of resource. For
-instance, `EXECUTE` is only relevant in the context of functions or
-mbeans; granting `EXECUTE` on a resource representing a table is
-nonsensical. Attempting to `GRANT` a permission on resource to which it
-cannot be applied results in an error response. The following
-illustrates which permissions can be granted on which types of resource,
-and which statements are enabled by that permission.
-
-[cols=",,,,,",options="header",]
-|===
-|permission |resource |operations | | |
-|`CREATE` |`ALL KEYSPACES` |`CREATE KEYSPACE` <br> `CREATE TABLE` in any
-keyspace | | |
-
-|`CREATE` |`KEYSPACE` |`CREATE TABLE` in specified keyspace | | |
-
-|`CREATE` |`ALL FUNCTIONS` |`CREATE FUNCTION` in any keyspace <br>
-`CREATE AGGREGATE` in any keyspace | | |
-
-|`CREATE` |`ALL FUNCTIONS IN KEYSPACE` |`CREATE FUNCTION` in keyspace
-<br> `CREATE AGGREGATE` in keyspace | | |
-
-|`CREATE` |`ALL ROLES` |`CREATE ROLE` | | |
-
-|`ALTER` |`ALL KEYSPACES` |`ALTER KEYSPACE` <br> `ALTER TABLE` in any
-keyspace | | |
-
-|`ALTER` |`KEYSPACE` |`ALTER KEYSPACE` <br> `ALTER TABLE` in keyspace |
-| |
-
-|`ALTER` |`TABLE` |`ALTER TABLE` | | |
-
-|`ALTER` |`ALL FUNCTIONS` |`CREATE FUNCTION` replacing any existing <br>
-`CREATE AGGREGATE` replacing any existing | | |
-
-|`ALTER` |`ALL FUNCTIONS IN KEYSPACE` |`CREATE FUNCTION` replacing
-existing in keyspace <br> `CREATE AGGREGATE` replacing any existing in
-keyspace | | |
-
-|`ALTER` |`FUNCTION` |`CREATE FUNCTION` replacing existing <br>
-`CREATE AGGREGATE` replacing existing | | |
-
-|`ALTER` |`ALL ROLES` |`ALTER ROLE` on any role | | |
-
-|`ALTER` |`ROLE` |`ALTER ROLE` | | |
-
-|`DROP` |`ALL KEYSPACES` |`DROP KEYSPACE` <br> `DROP TABLE` in any
-keyspace | | |
-
-|`DROP` |`KEYSPACE` |`DROP TABLE` in specified keyspace | | |
-
-|`DROP` |`TABLE` |`DROP TABLE` | | |
-
-|`DROP` |`ALL FUNCTIONS` |`DROP FUNCTION` in any keyspace <br>
-`DROP AGGREGATE` in any existing | | |
-
-|`DROP` |`ALL FUNCTIONS IN KEYSPACE` |`DROP FUNCTION` in keyspace <br>
-`DROP AGGREGATE` in existing | | |
-
-|`DROP` |`FUNCTION` |`DROP FUNCTION` | | |
-
-|`DROP` |`ALL ROLES` |`DROP ROLE` on any role | | |
-
-|`DROP` |`ROLE` |`DROP ROLE` | | |
-
-|`SELECT` |`ALL KEYSPACES` |`SELECT` on any table | | |
-
-|`SELECT` |`KEYSPACE` |`SELECT` on any table in keyspace | | |
-
-|`SELECT` |`TABLE` |`SELECT` on specified table | | |
-
-|`SELECT` |`ALL MBEANS` |Call getter methods on any mbean | | |
-
-|`SELECT` |`MBEANS` |Call getter methods on any mbean matching a
-wildcard pattern | | |
-
-|`SELECT` |`MBEAN` |Call getter methods on named mbean | | |
-
-|`MODIFY` |`ALL KEYSPACES` |`INSERT` on any table <br> `UPDATE` on any
-table <br> `DELETE` on any table <br> `TRUNCATE` on any table | | |
-
-|`MODIFY` |`KEYSPACE` |`INSERT` on any table in keyspace <br> `UPDATE`
-on any table in keyspace <br> `DELETE` on any table in keyspace <br>
-`TRUNCATE` on any table in keyspace |`MODIFY` |`TABLE` |`INSERT` <br>
-`UPDATE` <br> `DELETE` <br> `TRUNCATE`
-
-|`MODIFY` |`ALL MBEANS` |Call setter methods on any mbean | | |
-
-|`MODIFY` |`MBEANS` |Call setter methods on any mbean matching a
-wildcard pattern | | |
-
-|`MODIFY` |`MBEAN` |Call setter methods on named mbean | | |
-
-|`AUTHORIZE` |`ALL KEYSPACES` |`GRANT PERMISSION` on any table <br>
-`REVOKE PERMISSION` on any table | | |
-
-|`AUTHORIZE` |`KEYSPACE` |`GRANT PERMISSION` on table in keyspace <br>
-`REVOKE PERMISSION` on table in keyspace | | |
-
-|`AUTHORIZE` |`TABLE` |`GRANT PERMISSION` <br> `REVOKE PERMISSION` | | |
-
-|`AUTHORIZE` |`ALL FUNCTIONS` |`GRANT PERMISSION` on any function <br>
-`REVOKE PERMISSION` on any function | | |
-
-|`AUTHORIZE` |`ALL FUNCTIONS IN KEYSPACE` |`GRANT PERMISSION` in
-keyspace <br> `REVOKE PERMISSION` in keyspace | | |
-
-|`AUTHORIZE` |`ALL FUNCTIONS IN KEYSPACE` |`GRANT PERMISSION` in
-keyspace <br> `REVOKE PERMISSION` in keyspace | | |
-
-|`AUTHORIZE` |`FUNCTION` |`GRANT PERMISSION` <br> `REVOKE PERMISSION` |
-| |
-
-|`AUTHORIZE` |`ALL MBEANS` |`GRANT PERMISSION` on any mbean <br>
-`REVOKE PERMISSION` on any mbean | | |
-
-|`AUTHORIZE` |`MBEANS` |`GRANT PERMISSION` on any mbean matching a
-wildcard pattern <br> `REVOKE PERMISSION` on any mbean matching a
-wildcard pattern | | |
-
-|`AUTHORIZE` |`MBEAN` |`GRANT PERMISSION` on named mbean <br>
-`REVOKE PERMISSION` on named mbean | | |
-
-|`AUTHORIZE` |`ALL ROLES` |`GRANT ROLE` grant any role <br>
-`REVOKE ROLE` revoke any role | | |
-
-|`AUTHORIZE` |`ROLES` |`GRANT ROLE` grant role <br> `REVOKE ROLE` revoke
-role | | |
-
-|`DESCRIBE` |`ALL ROLES` |`LIST ROLES` all roles or only roles granted
-to another, specified role | | |
-
-|`DESCRIBE` |@ALL MBEANS |Retrieve metadata about any mbean from the
-platform’s MBeanServer | | |
-
-|`DESCRIBE` |@MBEANS |Retrieve metadata about any mbean matching a
-wildcard patter from the platform’s MBeanServer | | |
-
-|`DESCRIBE` |@MBEAN |Retrieve metadata about a named mbean from the
-platform’s MBeanServer | | |
-
-|`EXECUTE` |`ALL FUNCTIONS` |`SELECT`, `INSERT`, `UPDATE` using any
-function <br> use of any function in `CREATE AGGREGATE` | | |
-
-|`EXECUTE` |`ALL FUNCTIONS IN KEYSPACE` |`SELECT`, `INSERT`, `UPDATE`
-using any function in keyspace <br> use of any function in keyspace in
-`CREATE AGGREGATE` | | |
-
-|`EXECUTE` |`FUNCTION` |`SELECT`, `INSERT`, `UPDATE` using function <br>
-use of function in `CREATE AGGREGATE` | | |
-
-|`EXECUTE` |`ALL MBEANS` |Execute operations on any mbean | | |
-
-|`EXECUTE` |`MBEANS` |Execute operations on any mbean matching a
-wildcard pattern | | |
-
-|`EXECUTE` |`MBEAN` |Execute operations on named mbean | | |
-|===
-
-[[grantPermissionsStmt]]
-==== GRANT PERMISSION
-
-_Syntax:_
-
-bc(syntax).. +
-::= GRANT ( ALL ( PERMISSIONS )? | ( PERMISSION )? ) ON TO
-
-::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE |
-EXECUTE
-
-::= ALL KEYSPACES +
-| KEYSPACE +
-| ( TABLE )? +
-| ALL ROLES +
-| ROLE +
-| ALL FUNCTIONS ( IN KEYSPACE )? +
-| FUNCTION +
-| ALL MBEANS +
-| ( MBEAN | MBEANS ) +
-p.
-
-_Sample:_
-
-bc(sample). +
-GRANT SELECT ON ALL KEYSPACES TO data_reader;
-
-This gives any user with the role `data_reader` permission to execute
-`SELECT` statements on any table across all keyspaces
-
-bc(sample). +
-GRANT MODIFY ON KEYSPACE keyspace1 TO data_writer;
-
-This give any user with the role `data_writer` permission to perform
-`UPDATE`, `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` queries on all
-tables in the `keyspace1` keyspace
-
-bc(sample). +
-GRANT DROP ON keyspace1.table1 TO schema_owner;
-
-This gives any user with the `schema_owner` role permissions to `DROP`
-`keyspace1.table1`.
-
-bc(sample). +
-GRANT EXECUTE ON FUNCTION keyspace1.user_function( int ) TO
-report_writer;
-
-This grants any user with the `report_writer` role permission to execute
-`SELECT`, `INSERT` and `UPDATE` queries which use the function
-`keyspace1.user_function( int )`
-
-bc(sample). +
-GRANT DESCRIBE ON ALL ROLES TO role_admin;
-
-This grants any user with the `role_admin` role permission to view any
-and all roles in the system with a `LIST ROLES` statement
-
-[[grantAll]]
-===== GRANT ALL
-
-When the `GRANT ALL` form is used, the appropriate set of permissions is
-determined automatically based on the target resource.
-
-[[autoGrantPermissions]]
-===== Automatic Granting
-
-When a resource is created, via a `CREATE KEYSPACE`, `CREATE TABLE`,
-`CREATE FUNCTION`, `CREATE AGGREGATE` or `CREATE ROLE` statement, the
-creator (the role the database user who issues the statement is
-identified as), is automatically granted all applicable permissions on
-the new resource.
-
-[[revokePermissionsStmt]]
-==== REVOKE PERMISSION
-
-_Syntax:_
-
-bc(syntax).. +
-::= REVOKE ( ALL ( PERMISSIONS )? | ( PERMISSION )? ) ON FROM
-
-::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE |
-EXECUTE
-
-::= ALL KEYSPACES +
-| KEYSPACE +
-| ( TABLE )? +
-| ALL ROLES +
-| ROLE +
-| ALL FUNCTIONS ( IN KEYSPACE )? +
-| FUNCTION +
-| ALL MBEANS +
-| ( MBEAN | MBEANS ) +
-p.
-
-_Sample:_
-
-bc(sample).. +
-REVOKE SELECT ON ALL KEYSPACES FROM data_reader; +
-REVOKE MODIFY ON KEYSPACE keyspace1 FROM data_writer; +
-REVOKE DROP ON keyspace1.table1 FROM schema_owner; +
-REVOKE EXECUTE ON FUNCTION keyspace1.user_function( int ) FROM
-report_writer; +
-REVOKE DESCRIBE ON ALL ROLES FROM role_admin; +
-p.
-
-[[listPermissionsStmt]]
-===== LIST PERMISSIONS
-
-_Syntax:_
-
-bc(syntax).. +
-::= LIST ( ALL ( PERMISSIONS )? | ) +
-( ON )? +
-( OF ( NORECURSIVE )? )?
-
-::= ALL KEYSPACES +
-| KEYSPACE +
-| ( TABLE )? +
-| ALL ROLES +
-| ROLE +
-| ALL FUNCTIONS ( IN KEYSPACE )? +
-| FUNCTION +
-| ALL MBEANS +
-| ( MBEAN | MBEANS ) +
-p.
-
-_Sample:_
-
-bc(sample). +
-LIST ALL PERMISSIONS OF alice;
-
-Show all permissions granted to `alice`, including those acquired
-transitively from any other roles.
-
-bc(sample). +
-LIST ALL PERMISSIONS ON keyspace1.table1 OF bob;
-
-Show all permissions on `keyspace1.table1` granted to `bob`, including
-those acquired transitively from any other roles. This also includes any
-permissions higher up the resource hierarchy which can be applied to
-`keyspace1.table1`. For example, should `bob` have `ALTER` permission on
-`keyspace1`, that would be included in the results of this query. Adding
-the `NORECURSIVE` switch restricts the results to only those permissions
-which were directly granted to `bob` or one of `bob`’s roles.
-
-bc(sample). +
-LIST SELECT PERMISSIONS OF carlos;
-
-Show any permissions granted to `carlos` or any of `carlos`’s roles,
-limited to `SELECT` permissions on any resource.
-
-[[types]]
-=== Data Types
-
-CQL supports a rich set of data types for columns defined in a table,
-including collection types. On top of those native +
-and collection types, users can also provide custom types (through a
-JAVA class extending `AbstractType` loadable by +
-Cassandra). The syntax of types is thus:
-
-bc(syntax).. +
-::= +
-| +
-| +
-| // Used for custom types. The fully-qualified name of a JAVA class
-
-::= ascii +
-| bigint +
-| blob +
-| boolean +
-| counter +
-| date +
-| decimal +
-| double +
-| float +
-| inet +
-| int +
-| smallint +
-| text +
-| time +
-| timestamp +
-| timeuuid +
-| tinyint +
-| uuid +
-| varchar +
-| varint
-
-::= list `<' `>' +
-| set `<' `>' +
-| map `<' `,' `>' +
-::= tuple `<' (`,' )* `>' +
-p. Note that the native types are keywords and as such are
-case-insensitive. They are however not reserved ones.
-
-The following table gives additional informations on the native data
-types, and on which kind of link:#constants[constants] each type
-supports:
-
-[cols=",,",options="header",]
-|===
-|type |constants supported |description
-|`ascii` |strings |ASCII character string
-
-|`bigint` |integers |64-bit signed long
-
-|`blob` |blobs |Arbitrary bytes (no validation)
-
-|`boolean` |booleans |true or false
-
-|`counter` |integers |Counter column (64-bit signed value). See
-link:#counters[Counters] for details
-
-|`date` |integers, strings |A date (with no corresponding time value).
-See link:#usingdates[Working with dates] below for more information.
-
-|`decimal` |integers, floats |Variable-precision decimal
-
-|`double` |integers |64-bit IEEE-754 floating point
-
-|`float` |integers, floats |32-bit IEEE-754 floating point
-
-|`inet` |strings |An IP address. It can be either 4 bytes long (IPv4) or
-16 bytes long (IPv6). There is no `inet` constant, IP address should be
-inputed as strings
-
-|`int` |integers |32-bit signed int
-
-|`smallint` |integers |16-bit signed int
-
-|`text` |strings |UTF8 encoded string
-
-|`time` |integers, strings |A time with nanosecond precision. See
-link:#usingtime[Working with time] below for more information.
-
-|`timestamp` |integers, strings |A timestamp. Strings constant are allow
-to input timestamps as dates, see link:#usingtimestamps[Working with
-timestamps] below for more information.
-
-|`timeuuid` |uuids |Type 1 UUID. This is generally used as a
-``conflict-free'' timestamp. Also see the link:#timeuuidFun[functions on
-Timeuuid]
-
-|`tinyint` |integers |8-bit signed int
-
-|`uuid` |uuids |Type 1 or type 4 UUID
-
-|`varchar` |strings |UTF8 encoded string
-
-|`varint` |integers |Arbitrary-precision integer
-|===
-
-For more information on how to use the collection types, see the
-link:#collections[Working with collections] section below.
-
-[[usingtimestamps]]
-==== Working with timestamps
-
-Values of the `timestamp` type are encoded as 64-bit signed integers
-representing a number of milliseconds since the standard base time known
-as ``the epoch'': January 1 1970 at 00:00:00 GMT.
-
-Timestamp can be input in CQL as simple long integers, giving the number
-of milliseconds since the epoch, as defined above.
-
-They can also be input as string literals in any of the following ISO
-8601 formats, each representing the time and date Mar 2, 2011, at
-04:05:00 AM, GMT.:
-
-* `2011-02-03 04:05+0000`
-* `2011-02-03 04:05:00+0000`
-* `2011-02-03 04:05:00.000+0000`
-* `2011-02-03T04:05+0000`
-* `2011-02-03T04:05:00+0000`
-* `2011-02-03T04:05:00.000+0000`
-
-The `+0000` above is an RFC 822 4-digit time zone specification; `+0000`
-refers to GMT. US Pacific Standard Time is `-0800`. The time zone may be
-omitted if desired— the date will be interpreted as being in the time
-zone under which the coordinating Cassandra node is configured.
-
-* `2011-02-03 04:05`
-* `2011-02-03 04:05:00`
-* `2011-02-03 04:05:00.000`
-* `2011-02-03T04:05`
-* `2011-02-03T04:05:00`
-* `2011-02-03T04:05:00.000`
-
-There are clear difficulties inherent in relying on the time zone
-configuration being as expected, though, so it is recommended that the
-time zone always be specified for timestamps when feasible.
-
-The time of day may also be omitted, if the date is the only piece that
-matters:
-
-* `2011-02-03`
-* `2011-02-03+0000`
-
-In that case, the time of day will default to 00:00:00, in the specified
-or default time zone.
-
-[[usingdates]]
-==== Working with dates
-
-Values of the `date` type are encoded as 32-bit unsigned integers
-representing a number of days with ``the epoch'' at the center of the
-range (2^31). Epoch is January 1st, 1970
-
-A date can be input in CQL as an unsigned integer as defined above.
-
-They can also be input as string literals in the following format:
-
-* `2014-01-01`
-
-[[usingtime]]
-==== Working with time
-
-Values of the `time` type are encoded as 64-bit signed integers
-representing the number of nanoseconds since midnight.
-
-A time can be input in CQL as simple long integers, giving the number of
-nanoseconds since midnight.
-
-They can also be input as string literals in any of the following
-formats:
-
-* `08:12:54`
-* `08:12:54.123`
-* `08:12:54.123456`
-* `08:12:54.123456789`
-
-==== Counters
-
-The `counter` type is used to define _counter columns_. A counter column
-is a column whose value is a 64-bit signed integer and on which 2
-operations are supported: incrementation and decrementation (see
-link:#updateStmt[`UPDATE`] for syntax). Note the value of a counter
-cannot be set. A counter doesn’t exist until first
-incremented/decremented, and the first incrementation/decrementation is
-made as if the previous value was 0. Deletion of counter columns is
-supported but have some limitations (see the
-http://wiki.apache.org/cassandra/Counters[Cassandra Wiki] for more
-information).
-
-The use of the counter type is limited in the following way:
-
-* It cannot be used for column that is part of the `PRIMARY KEY` of a
-table.
-* A table that contains a counter can only contain counters. In other
-words, either all the columns of a table outside the `PRIMARY KEY` have
-the counter type, or none of them have it.
-
-[[collections]]
-==== Working with collections
-
-===== Noteworthy characteristics
-
-Collections are meant for storing/denormalizing relatively small amount
-of data. They work well for things like ``the phone numbers of a given
-user'', ``labels applied to an email'', etc. But when items are expected
-to grow unbounded (``all the messages sent by a given user'', ``events
-registered by a sensor'', …), then collections are not appropriate
-anymore and a specific table (with clustering columns) should be used.
-Concretely, collections have the following limitations:
-
-* Collections are always read in their entirety (and reading one is not
-paged internally).
-* Collections cannot have more than 65535 elements. More precisely,
-while it may be possible to insert more than 65535 elements, it is not
-possible to read more than the 65535 first elements (see
-https://issues.apache.org/jira/browse/CASSANDRA-5428[CASSANDRA-5428] for
-details).
-* While insertion operations on sets and maps never incur a
-read-before-write internally, some operations on lists do (see the
-section on lists below for details). It is thus advised to prefer sets
-over lists when possible.
-
-Please note that while some of those limitations may or may not be
-loosen in the future, the general rule that collections are for
-denormalizing small amount of data is meant to stay.
-
-[[map]]
-===== Maps
-
-A `map` is a link:#types[typed] set of key-value pairs, where keys are
-unique. Furthermore, note that the map are internally sorted by their
-keys and will thus always be returned in that order. To create a column
-of type `map`, use the `map` keyword suffixed with comma-separated key
-and value types, enclosed in angle brackets. For example:
-
-bc(sample). +
-CREATE TABLE users ( +
-id text PRIMARY KEY, +
-given text, +
-surname text, +
-favs map<text, text> // A map of text keys, and text values +
-)
-
-Writing `map` data is accomplished with a JSON-inspired syntax. To write
-a record using `INSERT`, specify the entire map as a JSON-style
-associative array. _Note: This form will always replace the entire map._
-
-bc(sample). +
-// Inserting (or Updating) +
-INSERT INTO users (id, given, surname, favs) +
-VALUES (`jsmith', `John', `Smith', \{ `fruit' : `apple', `band' :
-`Beatles' })
-
-Adding or updating key-values of a (potentially) existing map can be
-accomplished either by subscripting the map column in an `UPDATE`
-statement or by adding a new map literal:
-
-bc(sample). +
-// Updating (or inserting) +
-UPDATE users SET favs[`author'] = `Ed Poe' WHERE id = `jsmith' +
-UPDATE users SET favs = favs + \{ `movie' : `Cassablanca' } WHERE id =
-`jsmith'
-
-Note that TTLs are allowed for both `INSERT` and `UPDATE`, but in both
-case the TTL set only apply to the newly inserted/updated _values_. In
-other words,
-
-bc(sample). +
-// Updating (or inserting) +
-UPDATE users USING TTL 10 SET favs[`color'] = `green' WHERE id =
-`jsmith'
-
-will only apply the TTL to the `{ 'color' : 'green' }` record, the rest
-of the map remaining unaffected.
-
-Deleting a map record is done with:
-
-bc(sample). +
-DELETE favs[`author'] FROM users WHERE id = `jsmith'
-
-[[set]]
-===== Sets
-
-A `set` is a link:#types[typed] collection of unique values. Sets are
-ordered by their values. To create a column of type `set`, use the `set`
-keyword suffixed with the value type enclosed in angle brackets. For
-example:
-
-bc(sample). +
-CREATE TABLE images ( +
-name text PRIMARY KEY, +
-owner text, +
-date timestamp, +
-tags set +
-);
-
-Writing a `set` is accomplished by comma separating the set values, and
-enclosing them in curly braces. _Note: An `INSERT` will always replace
-the entire set._
-
-bc(sample). +
-INSERT INTO images (name, owner, date, tags) +
-VALUES (`cat.jpg', `jsmith', `now', \{ `kitten', `cat', `pet' });
-
-Adding and removing values of a set can be accomplished with an `UPDATE`
-by adding/removing new set values to an existing `set` column.
-
-bc(sample). +
-UPDATE images SET tags = tags + \{ `cute', `cuddly' } WHERE name =
-`cat.jpg'; +
-UPDATE images SET tags = tags - \{ `lame' } WHERE name = `cat.jpg';
-
-As with link:#map[maps], TTLs if used only apply to the newly
-inserted/updated _values_.
-
-[[list]]
-===== Lists
-
-A `list` is a link:#types[typed] collection of non-unique values where
-elements are ordered by there position in the list. To create a column
-of type `list`, use the `list` keyword suffixed with the value type
-enclosed in angle brackets. For example:
-
-bc(sample). +
-CREATE TABLE plays ( +
-id text PRIMARY KEY, +
-game text, +
-players int, +
-scores list +
-)
-
-Do note that as explained below, lists have some limitations and
-performance considerations to take into account, and it is advised to
-prefer link:#set[sets] over lists when this is possible.
-
-Writing `list` data is accomplished with a JSON-style syntax. To write a
-record using `INSERT`, specify the entire list as a JSON array. _Note:
-An `INSERT` will always replace the entire list._
-
-bc(sample). +
-INSERT INTO plays (id, game, players, scores) +
-VALUES (`123-afde', `quake', 3, [17, 4, 2]);
-
-Adding (appending or prepending) values to a list can be accomplished by
-adding a new JSON-style array to an existing `list` column.
-
-bc(sample). +
-UPDATE plays SET players = 5, scores = scores + [ 14, 21 ] WHERE id =
-`123-afde'; +
-UPDATE plays SET players = 5, scores = [ 12 ] + scores WHERE id =
-`123-afde';
-
-It should be noted that append and prepend are not idempotent
-operations. This means that if during an append or a prepend the
-operation timeout, it is not always safe to retry the operation (as this
-could result in the record appended or prepended twice).
-
-Lists also provides the following operation: setting an element by its
-position in the list, removing an element by its position in the list
-and remove all the occurrence of a given value in the list. _However,
-and contrarily to all the other collection operations, these three
-operations induce an internal read before the update, and will thus
-typically have slower performance characteristics_. Those operations
-have the following syntax:
-
-bc(sample). +
-UPDATE plays SET scores[1] = 7 WHERE id = `123-afde'; // sets the 2nd
-element of scores to 7 (raises an error is scores has less than 2
-elements) +
-DELETE scores[1] FROM plays WHERE id = `123-afde'; // deletes the 2nd
-element of scores (raises an error is scores has less than 2 elements) +
-UPDATE plays SET scores = scores - [ 12, 21 ] WHERE id = `123-afde'; //
-removes all occurrences of 12 and 21 from scores
-
-As with link:#map[maps], TTLs if used only apply to the newly
-inserted/updated _values_.
-
-=== Functions
-
-CQL3 distinguishes between built-in functions (so called `native
-functions') and link:#udfs[user-defined functions]. CQL3 includes
-several native functions, described below:
-
-[[castFun]]
-==== Cast
-
-The `cast` function can be used to converts one native datatype to
-another.
-
-The following table describes the conversions supported by the `cast`
-function. Cassandra will silently ignore any cast converting a datatype
-into its own datatype.
-
-[cols=",",options="header",]
-|===
-|from |to
-|`ascii` |`text`, `varchar`
-
-|`bigint` |`tinyint`, `smallint`, `int`, `float`, `double`, `decimal`,
-`varint`, `text`, `varchar`
-
-|`boolean` |`text`, `varchar`
-
-|`counter` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
-`decimal`, `varint`, `text`, `varchar`
-
-|`date` |`timestamp`
-
-|`decimal` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
-`varint`, `text`, `varchar`
-
-|`double` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `decimal`,
-`varint`, `text`, `varchar`
-
-|`float` |`tinyint`, `smallint`, `int`, `bigint`, `double`, `decimal`,
-`varint`, `text`, `varchar`
-
-|`inet` |`text`, `varchar`
-
-|`int` |`tinyint`, `smallint`, `bigint`, `float`, `double`, `decimal`,
-`varint`, `text`, `varchar`
-
-|`smallint` |`tinyint`, `int`, `bigint`, `float`, `double`, `decimal`,
-`varint`, `text`, `varchar`
-
-|`time` |`text`, `varchar`
-
-|`timestamp` |`date`, `text`, `varchar`
-
-|`timeuuid` |`timestamp`, `date`, `text`, `varchar`
-
-|`tinyint` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
-`decimal`, `varint`, `text`, `varchar`
-
-|`uuid` |`text`, `varchar`
-
-|`varint` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
-`decimal`, `text`, `varchar`
-|===
-
-The conversions rely strictly on Java’s semantics. For example, the
-double value 1 will be converted to the text value `1.0'.
-
-bc(sample). +
-SELECT avg(cast(count as double)) FROM myTable
-
-[[tokenFun]]
-==== Token
-
-The `token` function allows to compute the token for a given partition
-key. The exact signature of the token function depends on the table
-concerned and of the partitioner used by the cluster.
-
-The type of the arguments of the `token` depend on the type of the
-partition key columns. The return type depend on the partitioner in use:
-
-* For Murmur3Partitioner, the return type is `bigint`.
-* For RandomPartitioner, the return type is `varint`.
-* For ByteOrderedPartitioner, the return type is `blob`.
-
-For instance, in a cluster using the default Murmur3Partitioner, if a
-table is defined by
-
-bc(sample). +
-CREATE TABLE users ( +
-userid text PRIMARY KEY, +
-username text, +
-… +
-)
-
-then the `token` function will take a single argument of type `text` (in
-that case, the partition key is `userid` (there is no clustering columns
-so the partition key is the same than the primary key)), and the return
-type will be `bigint`.
-
-[[uuidFun]]
-==== Uuid
-
-The `uuid` function takes no parameters and generates a random type 4
-uuid suitable for use in INSERT or SET statements.
-
-[[timeuuidFun]]
-==== Timeuuid functions
-
-===== `now`
-
-The `now` function takes no arguments and generates, on the coordinator
-node, a new unique timeuuid (at the time where the statement using it is
-executed). Note that this method is useful for insertion but is largely
-non-sensical in `WHERE` clauses. For instance, a query of the form
-
-bc(sample). +
-SELECT * FROM myTable WHERE t = now()
-
-will never return any result by design, since the value returned by
-`now()` is guaranteed to be unique.
-
-===== `minTimeuuid` and `maxTimeuuid`
-
-The `minTimeuuid` (resp. `maxTimeuuid`) function takes a `timestamp`
-value `t` (which can be link:#usingtimestamps[either a timestamp or a
-date string] ) and return a _fake_ `timeuuid` corresponding to the
-_smallest_ (resp. _biggest_) possible `timeuuid` having for timestamp
-`t`. So for instance:
-
-bc(sample). +
-SELECT * FROM myTable WHERE t > maxTimeuuid(`2013-01-01 00:05+0000') AND
-t < minTimeuuid(`2013-02-02 10:00+0000')
-
-will select all rows where the `timeuuid` column `t` is strictly older
-than `2013-01-01 00:05+0000' but strictly younger than `2013-02-02
-10:00+0000'. Please note that
-`t >= maxTimeuuid('2013-01-01 00:05+0000')` would still _not_ select a
-`timeuuid` generated exactly at `2013-01-01 00:05+0000' and is
-essentially equivalent to `t > maxTimeuuid('2013-01-01 00:05+0000')`.
-
-_Warning_: We called the values generated by `minTimeuuid` and
-`maxTimeuuid` _fake_ UUID because they do no respect the Time-Based UUID
-generation process specified by the
-http://www.ietf.org/rfc/rfc4122.txt[RFC 4122]. In particular, the value
-returned by these 2 methods will not be unique. This means you should
-only use those methods for querying (as in the example above). Inserting
-the result of those methods is almost certainly _a bad idea_.
-
-[[timeFun]]
-==== Time conversion functions
-
-A number of functions are provided to ``convert'' a `timeuuid`, a
-`timestamp` or a `date` into another `native` type.
-
-[cols=",,",options="header",]
-|===
-|function name |input type |description
-|`toDate` |`timeuuid` |Converts the `timeuuid` argument into a `date`
-type
-
-|`toDate` |`timestamp` |Converts the `timestamp` argument into a `date`
-type
-
-|`toTimestamp` |`timeuuid` |Converts the `timeuuid` argument into a
-`timestamp` type
-
-|`toTimestamp` |`date` |Converts the `date` argument into a `timestamp`
-type
-
-|`toUnixTimestamp` |`timeuuid` |Converts the `timeuuid` argument into a
-`bigInt` raw value
-
-|`toUnixTimestamp` |`timestamp` |Converts the `timestamp` argument into
-a `bigInt` raw value
-
-|`toUnixTimestamp` |`date` |Converts the `date` argument into a `bigInt`
-raw value
-
-|`dateOf` |`timeuuid` |Similar to `toTimestamp(timeuuid)` (DEPRECATED)
-
-|`unixTimestampOf` |`timeuuid` |Similar to `toUnixTimestamp(timeuuid)`
-(DEPRECATED)
-|===
-
-[[blobFun]]
-==== Blob conversion functions
-
-A number of functions are provided to ``convert'' the native types into
-binary data (`blob`). For every `<native-type>` `type` supported by CQL3
-(a notable exceptions is `blob`, for obvious reasons), the function
-`typeAsBlob` takes a argument of type `type` and return it as a `blob`.
-Conversely, the function `blobAsType` takes a 64-bit `blob` argument and
-convert it to a `bigint` value. And so for instance, `bigintAsBlob(3)`
-is `0x0000000000000003` and `blobAsBigint(0x0000000000000003)` is `3`.
-
-=== Aggregates
-
-Aggregate functions work on a set of rows. They receive values for each
-row and returns one value for the whole set. +
-If `normal` columns, `scalar functions`, `UDT` fields, `writetime` or
-`ttl` are selected together with aggregate functions, the values
-returned for them will be the ones of the first row matching the query.
-
-CQL3 distinguishes between built-in aggregates (so called `native
-aggregates') and link:#udas[user-defined aggregates]. CQL3 includes
-several native aggregates, described below:
-
-[[countFct]]
-==== Count
-
-The `count` function can be used to count the rows returned by a query.
-Example:
-
-bc(sample). +
-SELECT COUNT (*) FROM plays; +
-SELECT COUNT (1) FROM plays;
-
-It also can be used to count the non null value of a given column.
-Example:
-
-bc(sample). +
-SELECT COUNT (scores) FROM plays;
-
-[[maxMinFcts]]
-==== Max and Min
-
-The `max` and `min` functions can be used to compute the maximum and the
-minimum value returned by a query for a given column.
-
-bc(sample). +
-SELECT MIN (players), MAX (players) FROM plays WHERE game = `quake';
-
-[[sumFct]]
-==== Sum
-
-The `sum` function can be used to sum up all the values returned by a
-query for a given column.
-
-bc(sample). +
-SELECT SUM (players) FROM plays;
-
-[[avgFct]]
-==== Avg
-
-The `avg` function can be used to compute the average of all the values
-returned by a query for a given column.
-
-bc(sample). +
-SELECT AVG (players) FROM plays;
-
-[[udfs]]
-=== User-Defined Functions
-
-User-defined functions allow execution of user-provided code in
-Cassandra. By default, Cassandra supports defining functions in _Java_
-and _JavaScript_. Support for other JSR 223 compliant scripting
-languages (such as Python, Ruby, and Scala) has been removed in 3.0.11.
-
-UDFs are part of the Cassandra schema. As such, they are automatically
-propagated to all nodes in the cluster.
-
-UDFs can be _overloaded_ - i.e. multiple UDFs with different argument
-types but the same function name. Example:
-
-bc(sample). +
-CREATE FUNCTION sample ( arg int ) …; +
-CREATE FUNCTION sample ( arg text ) …;
-
-User-defined functions are susceptible to all of the normal problems
-with the chosen programming language. Accordingly, implementations
-should be safe against null pointer exceptions, illegal arguments, or
-any other potential source of exceptions. An exception during function
-execution will result in the entire statement failing.
-
-It is valid to use _complex_ types like collections, tuple types and
-user-defined types as argument and return types. Tuple types and
-user-defined types are handled by the conversion functions of the
-DataStax Java Driver. Please see the documentation of the Java Driver
-for details on handling tuple types and user-defined types.
-
-Arguments for functions can be literals or terms. Prepared statement
-placeholders can be used, too.
-
-Note that you can use the double-quoted string syntax to enclose the UDF
-source code. For example:
-
-bc(sample).. +
-CREATE FUNCTION some_function ( arg int ) +
-RETURNS NULL ON NULL INPUT +
-RETURNS int +
-LANGUAGE java +
-AS $$ return arg; $$;
-
-SELECT some_function(column) FROM atable …; +
-UPDATE atable SET col = some_function(?) …; +
-p.
-
-bc(sample). +
-CREATE TYPE custom_type (txt text, i int); +
-CREATE FUNCTION fct_using_udt ( udtarg frozen ) +
-RETURNS NULL ON NULL INPUT +
-RETURNS text +
-LANGUAGE java +
-AS $$ return udtarg.getString(``txt''); $$;
-
-User-defined functions can be used in link:#selectStmt[`SELECT`],
-link:#insertStmt[`INSERT`] and link:#updateStmt[`UPDATE`] statements.
-
-The implicitly available `udfContext` field (or binding for script UDFs)
-provides the neccessary functionality to create new UDT and tuple
-values.
-
-bc(sample). +
-CREATE TYPE custom_type (txt text, i int); +
-CREATE FUNCTION fct_using_udt ( somearg int ) +
-RETURNS NULL ON NULL INPUT +
-RETURNS custom_type +
-LANGUAGE java +
-AS $$ +
-UDTValue udt = udfContext.newReturnUDTValue(); +
-udt.setString(``txt'', ``some string''); +
-udt.setInt(``i'', 42); +
-return udt; +
-$$;
-
-The definition of the `UDFContext` interface can be found in the Apache
-Cassandra source code for
-`org.apache.cassandra.cql3.functions.UDFContext`.
-
-bc(sample). +
-public interface UDFContext +
-\{ +
-UDTValue newArgUDTValue(String argName); +
-UDTValue newArgUDTValue(int argNum); +
-UDTValue newReturnUDTValue(); +
-UDTValue newUDTValue(String udtName); +
-TupleValue newArgTupleValue(String argName); +
-TupleValue newArgTupleValue(int argNum); +
-TupleValue newReturnTupleValue(); +
-TupleValue newTupleValue(String cqlDefinition); +
-}
-
-Java UDFs already have some imports for common interfaces and classes
-defined. These imports are: +
-Please note, that these convenience imports are not available for script
-UDFs.
-
-bc(sample). +
-import java.nio.ByteBuffer; +
-import java.util.List; +
-import java.util.Map; +
-import java.util.Set; +
-import org.apache.cassandra.cql3.functions.UDFContext; +
-import com.datastax.driver.core.TypeCodec; +
-import com.datastax.driver.core.TupleValue; +
-import com.datastax.driver.core.UDTValue;
-
-See link:#createFunctionStmt[`CREATE FUNCTION`] and
-link:#dropFunctionStmt[`DROP FUNCTION`].
-
-[[udas]]
-=== User-Defined Aggregates
-
-User-defined aggregates allow creation of custom aggregate functions
-using link:#udfs[UDFs]. Common examples of aggregate functions are
-_count_, _min_, and _max_.
-
-Each aggregate requires an _initial state_ (`INITCOND`, which defaults
-to `null`) of type `STYPE`. The first argument of the state function
-must have type `STYPE`. The remaining arguments of the state function
-must match the types of the user-defined aggregate arguments. The state
-function is called once for each row, and the value returned by the
-state function becomes the new state. After all rows are processed, the
-optional `FINALFUNC` is executed with last state value as its argument.
-
-`STYPE` is mandatory in order to be able to distinguish possibly
-overloaded versions of the state and/or final function (since the
-overload can appear after creation of the aggregate).
-
-User-defined aggregates can be used in link:#selectStmt[`SELECT`]
-statement.
-
-A complete working example for user-defined aggregates (assuming that a
-keyspace has been selected using the link:#useStmt[`USE`] statement):
-
-bc(sample).. +
-CREATE OR REPLACE FUNCTION averageState ( state tuple<int,bigint>, val
-int ) +
-CALLED ON NULL INPUT +
-RETURNS tuple<int,bigint> +
-LANGUAGE java +
-AS ’ +
-if (val != null) \{ +
-state.setInt(0, state.getInt(0)+1); +
-state.setLong(1, state.getLong(1)+val.intValue()); +
-} +
-return state; +
-’;
-
-CREATE OR REPLACE FUNCTION averageFinal ( state tuple<int,bigint> ) +
-CALLED ON NULL INPUT +
-RETURNS double +
-LANGUAGE java +
-AS ’ +
-double r = 0; +
-if (state.getInt(0) == 0) return null; +
-r = state.getLong(1); +
-r /= state.getInt(0); +
-return Double.valueOf®; +
-’;
-
-CREATE OR REPLACE AGGREGATE average ( int ) +
-SFUNC averageState +
-STYPE tuple<int,bigint> +
-FINALFUNC averageFinal +
-INITCOND (0, 0);
-
-CREATE TABLE atable ( +
-pk int PRIMARY KEY, +
-val int); +
-INSERT INTO atable (pk, val) VALUES (1,1); +
-INSERT INTO atable (pk, val) VALUES (2,2); +
-INSERT INTO atable (pk, val) VALUES (3,3); +
-INSERT INTO atable (pk, val) VALUES (4,4); +
-SELECT average(val) FROM atable; +
-p.
-
-See link:#createAggregateStmt[`CREATE AGGREGATE`] and
-link:#dropAggregateStmt[`DROP AGGREGATE`].
-
-[[json]]
-=== JSON Support
-
-Cassandra 2.2 introduces JSON support to link:#selectStmt[`SELECT`] and
-link:#insertStmt[`INSERT`] statements. This support does not
-fundamentally alter the CQL API (for example, the schema is still
-enforced), it simply provides a convenient way to work with JSON
-documents.
-
-[[selectJson]]
-==== SELECT JSON
-
-With `SELECT` statements, the new `JSON` keyword can be used to return
-each row as a single `JSON` encoded map. The remainder of the `SELECT`
-statment behavior is the same.
-
-The result map keys are the same as the column names in a normal result
-set. For example, a statement like ```SELECT JSON a, ttl(b) FROM ...`''
-would result in a map with keys `"a"` and `"ttl(b)"`. However, this is
-one notable exception: for symmetry with `INSERT JSON` behavior,
-case-sensitive column names with upper-case letters will be surrounded
-with double quotes. For example, ```SELECT JSON myColumn FROM ...`''
-would result in a map key `"\"myColumn\""` (note the escaped quotes).
-
-The map values will `JSON`-encoded representations (as described below)
-of the result set values.
-
-[[insertJson]]
-==== INSERT JSON
-
-With `INSERT` statements, the new `JSON` keyword can be used to enable
-inserting a `JSON` encoded map as a single row. The format of the `JSON`
-map should generally match that returned by a `SELECT JSON` statement on
-the same table. In particular, case-sensitive column names should be
-surrounded with double quotes. For example, to insert into a table with
-two columns named ``myKey'' and ``value'', you would do the following:
-
-bc(sample). +
-INSERT INTO mytable JSON `\{``\''myKey\``'': 0, ``value'': 0}'
-
-Any columns which are ommitted from the `JSON` map will be defaulted to
-a `NULL` value (which will result in a tombstone being created).
-
-[[jsonEncoding]]
-==== JSON Encoding of Cassandra Data Types
-
-Where possible, Cassandra will represent and accept data types in their
-native `JSON` representation. Cassandra will also accept string
-representations matching the CQL literal format for all single-field
-types. For example, floats, ints, UUIDs, and dates can be represented by
-CQL literal strings. However, compound types, such as collections,
-tuples, and user-defined types must be represented by native `JSON`
-collections (maps and lists) or a JSON-encoded string representation of
-the collection.
-
-The following table describes the encodings that Cassandra will accept
-in `INSERT JSON` values (and `fromJson()` arguments) as well as the
-format Cassandra will use when returning data for `SELECT JSON`
-statements (and `fromJson()`):
-
-[cols=",,,",options="header",]
-|===
-|type |formats accepted |return format |notes
-|`ascii` |string |string |Uses JSON’s `\u` character escape
-
-|`bigint` |integer, string |integer |String must be valid 64 bit integer
-
-|`blob` |string |string |String should be 0x followed by an even number
-of hex digits
-
-|`boolean` |boolean, string |boolean |String must be ``true'' or
-``false''
-
-|`date` |string |string |Date in format `YYYY-MM-DD`, timezone UTC
-
-|`decimal` |integer, float, string |float |May exceed 32 or 64-bit
-IEEE-754 floating point precision in client-side decoder
-
-|`double` |integer, float, string |float |String must be valid integer
-or float
-
-|`float` |integer, float, string |float |String must be valid integer or
-float
-
-|`inet` |string |string |IPv4 or IPv6 address
-
-|`int` |integer, string |integer |String must be valid 32 bit integer
-
-|`list` |list, string |list |Uses JSON’s native list representation
-
-|`map` |map, string |map |Uses JSON’s native map representation
-
-|`smallint` |integer, string |integer |String must be valid 16 bit
-integer
-
-|`set` |list, string |list |Uses JSON’s native list representation
-
-|`text` |string |string |Uses JSON’s `\u` character escape
-
-|`time` |string |string |Time of day in format `HH-MM-SS[.fffffffff]`
-
-|`timestamp` |integer, string |string |A timestamp. Strings constant are
-allow to input timestamps as dates, see link:#usingdates[Working with
-dates] below for more information. Datestamps with format
-`YYYY-MM-DD HH:MM:SS.SSS` are returned.
-
-|`timeuuid` |string |string |Type 1 UUID. See link:#constants[Constants]
-for the UUID format
-
-|`tinyint` |integer, string |integer |String must be valid 8 bit integer
-
-|`tuple` |list, string |list |Uses JSON’s native list representation
-
-|`UDT` |map, string |map |Uses JSON’s native map representation with
-field names as keys
-
-|`uuid` |string |string |See link:#constants[Constants] for the UUID
-format
-
-|`varchar` |string |string |Uses JSON’s `\u` character escape
-
-|`varint` |integer, string |integer |Variable length; may overflow 32 or
-64 bit integers in client-side decoder
-|===
-
-[[fromJson]]
-==== The fromJson() Function
-
-The `fromJson()` function may be used similarly to `INSERT JSON`, but
-for a single column value. It may only be used in the `VALUES` clause of
-an `INSERT` statement or as one of the column values in an `UPDATE`,
-`DELETE`, or `SELECT` statement. For example, it cannot be used in the
-selection clause of a `SELECT` statement.
-
-[[toJson]]
-==== The toJson() Function
-
-The `toJson()` function may be used similarly to `SELECT JSON`, but for
-a single column value. It may only be used in the selection clause of a
-`SELECT` statement.
-
-[[appendixA]]
-=== Appendix A: CQL Keywords
-
-CQL distinguishes between _reserved_ and _non-reserved_ keywords.
-Reserved keywords cannot be used as identifier, they are truly reserved
-for the language (but one can enclose a reserved keyword by
-double-quotes to use it as an identifier). Non-reserved keywords however
-only have a specific meaning in certain context but can used as
-identifer otherwise. The only _raison d’être_ of these non-reserved
-keywords is convenience: some keyword are non-reserved when it was
-always easy for the parser to decide whether they were used as keywords
-or not.
-
-[cols=",",options="header",]
-|===
-|Keyword |Reserved?
-|`ADD` |yes
-|`AGGREGATE` |no
-|`ALL` |no
-|`ALLOW` |yes
-|`ALTER` |yes
-|`AND` |yes
-|`APPLY` |yes
-|`AS` |no
-|`ASC` |yes
-|`ASCII` |no
-|`AUTHORIZE` |yes
-|`BATCH` |yes
-|`BEGIN` |yes
-|`BIGINT` |no
-|`BLOB` |no
-|`BOOLEAN` |no
-|`BY` |yes
-|`CALLED` |no
-|`CAST` |no
-|`CLUSTERING` |no
-|`COLUMNFAMILY` |yes
-|`COMPACT` |no
-|`CONTAINS` |no
-|`COUNT` |no
-|`COUNTER` |no
-|`CREATE` |yes
-|`CUSTOM` |no
-|`DATE` |no
-|`DECIMAL` |no
-|`DEFAULT` |yes
-|`DELETE` |yes
-|`DESC` |yes
-|`DESCRIBE` |yes
-|`DISTINCT` |no
-|`DOUBLE` |no
-|`DROP` |yes
-|`DURATION` |no
-|`ENTRIES` |yes
-|`EXECUTE` |yes
-|`EXISTS` |no
-|`FILTERING` |no
-|`FINALFUNC` |no
-|`FLOAT` |no
-|`FROM` |yes
-|`FROZEN` |no
-|`FULL` |yes
-|`FUNCTION` |no
-|`FUNCTIONS` |no
-|`GRANT` |yes
-|`GROUP` |no
-|`IF` |yes
-|`IN` |yes
-|`INDEX` |yes
-|`INET` |no
-|`INFINITY` |yes
-|`INITCOND` |no
-|`INPUT` |no
-|`INSERT` |yes
-|`INT` |no
-|`INTO` |yes
-|`IS` |yes
-|`JSON` |no
-|`KEY` |no
-|`KEYS` |no
-|`KEYSPACE` |yes
-|`KEYSPACES` |no
-|`LANGUAGE` |no
-|`LIKE` |no
-|`LIMIT` |yes
-|`LIST` |no
-|`LOGIN` |no
-|`MAP` |no
-|`MATERIALIZED` |yes
-|`MBEAN` |yes
-|`MBEANS` |yes
-|`MODIFY` |yes
-|`NAN` |yes
-|`NOLOGIN` |no
-|`NORECURSIVE` |yes
-|`NOSUPERUSER` |no
-|`NOT` |yes
-|`NULL` |yes
-|`OF` |yes
-|`ON` |yes
-|`OPTIONS` |no
-|`OR` |yes
-|`ORDER` |yes
-|`PARTITION` |no
-|`PASSWORD` |no
-|`PER` |no
-|`PERMISSION` |no
-|`PERMISSIONS` |no
-|`PRIMARY` |yes
-|`RENAME` |yes
-|`REPLACE` |yes
-|`RETURNS` |no
-|`REVOKE` |yes
-|`ROLE` |no
-|`ROLES` |no
-|`SCHEMA` |yes
-|`SELECT` |yes
-|`SET` |yes
-|`SFUNC` |no
-|`SMALLINT` |no
-|`STATIC` |no
-|`STORAGE` |no
-|`STYPE` |no
-|`SUPERUSER` |no
-|`TABLE` |yes
-|`TEXT` |no
-|`TIME` |no
-|`TIMESTAMP` |no
-|`TIMEUUID` |no
-|`TINYINT` |no
-|`TO` |yes
-|`TOKEN` |yes
-|`TRIGGER` |no
-|`TRUNCATE` |yes
-|`TTL` |no
-|`TUPLE` |no
-|`TYPE` |no
-|`UNLOGGED` |yes
-|`UNSET` |yes
-|`UPDATE` |yes
-|`USE` |yes
-|`USER` |no
-|`USERS` |no
-|`USING` |yes
-|`UUID` |no
-|`VALUES` |no
-|`VARCHAR` |no
-|`VARINT` |no
-|`VIEW` |yes
-|`WHERE` |yes
-|`WITH` |yes
-|`WRITETIME` |no
-|===
-
-[[appendixB]]
-=== Appendix B: CQL Reserved Types
-
-The following type names are not currently used by CQL, but are reserved
-for potential future use. User-defined types may not use reserved type
-names as their name.
-
-[cols="",options="header",]
-|===
-|type
-|`bitstring`
-|`byte`
-|`complex`
-|`date`
-|`enum`
-|`interval`
-|`macaddr`
-|===
-
-=== Changes
-
-The following describes the changes in each version of CQL.
-
-==== 3.4.3
-
-* Support for `GROUP BY`. See link:#selectGroupBy[`<group-by>`] (see
-https://issues.apache.org/jira/browse/CASSANDRA-10707)[CASSANDRA-10707].
-
-==== 3.4.2
-
-* Support for selecting elements and slices of a collection
-(https://issues.apache.org/jira/browse/CASSANDRA-7396)[CASSANDRA-7396].
-
-==== 3.4.2
-
-* link:#updateOptions[`INSERT/UPDATE options`] for tables having a
-default_time_to_live specifying a TTL of 0 will remove the TTL from the
-inserted or updated values
-* link:#alterTableStmt[`ALTER TABLE`] `ADD` and `DROP` now allow mutiple
-columns to be added/removed
-* New link:#selectLimit[`PER PARTITION LIMIT`] option (see
-https://issues.apache.org/jira/browse/CASSANDRA-7017)[CASSANDRA-7017].
-* link:#udfs[User-defined functions] can now instantiate `UDTValue` and
-`TupleValue` instances via the new `UDFContext` interface (see
-https://issues.apache.org/jira/browse/CASSANDRA-10818)[CASSANDRA-10818].
-* ``User-defined types''#createTypeStmt may now be stored in a
-non-frozen form, allowing individual fields to be updated and deleted in
-link:#updateStmt[`UPDATE` statements] and link:#deleteStmt[`DELETE`
-statements], respectively.
-(https://issues.apache.org/jira/browse/CASSANDRA-7423)[CASSANDRA-7423]
-
-==== 3.4.1
-
-* Adds `CAST` functions. See link:#castFun[`Cast`].
-
-==== 3.4.0
-
-* Support for link:#createMVStmt[materialized views]
-* link:#deleteStmt[`DELETE`] support for inequality expressions and `IN`
-restrictions on any primary key columns
-* link:#updateStmt[`UPDATE`] support for `IN` restrictions on any
-primary key columns
-
-==== 3.3.1
-
-* The syntax `TRUNCATE TABLE X` is now accepted as an alias for
-`TRUNCATE X`
-
-==== 3.3.0
-
-* Adds new link:#aggregates[aggregates]
-* User-defined functions are now supported through
-link:#createFunctionStmt[`CREATE FUNCTION`] and
-link:#dropFunctionStmt[`DROP FUNCTION`].
-* User-defined aggregates are now supported through
-link:#createAggregateStmt[`CREATE AGGREGATE`] and
-link:#dropAggregateStmt[`DROP AGGREGATE`].
-* Allows double-dollar enclosed strings literals as an alternative to
-single-quote enclosed strings.
-* Introduces Roles to supercede user based authentication and access
-control
-* link:#usingdates[`Date`] and link:usingtime[`Time`] data types have
-been added
-* link:#json[`JSON`] support has been added
-* `Tinyint` and `Smallint` data types have been added
-* Adds new time conversion functions and deprecate `dateOf` and
-`unixTimestampOf`. See link:#timeFun[`Time conversion functions`]
-
-==== 3.2.0
-
-* User-defined types are now supported through
-link:#createTypeStmt[`CREATE TYPE`], link:#alterTypeStmt[`ALTER TYPE`],
-and link:#dropTypeStmt[`DROP TYPE`]
-* link:#createIndexStmt[`CREATE INDEX`] now supports indexing collection
-columns, including indexing the keys of map collections through the
-`keys()` function
-* Indexes on collections may be queried using the new `CONTAINS` and
-`CONTAINS KEY` operators
-* Tuple types were added to hold fixed-length sets of typed positional
-fields (see the section on link:#types[types] )
-* link:#dropIndexStmt[`DROP INDEX`] now supports optionally specifying a
-keyspace
-
-==== 3.1.7
-
-* `SELECT` statements now support selecting multiple rows in a single
-partition using an `IN` clause on combinations of clustering columns.
-See link:#selectWhere[SELECT WHERE] clauses.
-* `IF NOT EXISTS` and `IF EXISTS` syntax is now supported by
-`CREATE USER` and `DROP USER` statmenets, respectively.
-
-==== 3.1.6
-
-* A new link:#uuidFun[`uuid` method] has been added.
-* Support for `DELETE ... IF EXISTS` syntax.
-
-==== 3.1.5
-
-* It is now possible to group clustering columns in a relatiion, see
-link:#selectWhere[SELECT WHERE] clauses.
-* Added support for `STATIC` columns, see link:#createTableStatic[static
-in CREATE TABLE].
-
-==== 3.1.4
-
-* `CREATE INDEX` now allows specifying options when creating CUSTOM
-indexes (see link:#createIndexStmt[CREATE INDEX reference] ).
-
-==== 3.1.3
-
-* Millisecond precision formats have been added to the timestamp parser
-(see link:#usingtimestamps[working with dates] ).
-
-==== 3.1.2
-
-* `NaN` and `Infinity` has been added as valid float contants. They are
-now reserved keywords. In the unlikely case you we using them as a
-column identifier (or keyspace/table one), you will noew need to double
-quote them (see link:#identifiers[quote identifiers] ).
-
-==== 3.1.1
-
-* `SELECT` statement now allows listing the partition keys (using the
-`DISTINCT` modifier). See
-https://issues.apache.org/jira/browse/CASSANDRA-4536[CASSANDRA-4536].
-* The syntax `c IN ?` is now supported in `WHERE` clauses. In that case,
-the value expected for the bind variable will be a list of whatever type
-`c` is.
-* It is now possible to use named bind variables (using `:name` instead
-of `?`).
-
-==== 3.1.0
-
-* link:#alterTableStmt[ALTER TABLE] `DROP` option has been reenabled for
-CQL3 tables and has new semantics now: the space formerly used by
-dropped columns will now be eventually reclaimed (post-compaction). You
-should not readd previously dropped columns unless you use timestamps
-with microsecond precision (see
-https://issues.apache.org/jira/browse/CASSANDRA-3919[CASSANDRA-3919] for
-more details).
-* `SELECT` statement now supports aliases in select clause. Aliases in
-WHERE and ORDER BY clauses are not supported. See the
-link:#selectStmt[section on select] for details.
-* `CREATE` statements for `KEYSPACE`, `TABLE` and `INDEX` now supports
-an `IF NOT EXISTS` condition. Similarly, `DROP` statements support a
-`IF EXISTS` condition.
-* `INSERT` statements optionally supports a `IF NOT EXISTS` condition
-and `UPDATE` supports `IF` conditions.
-
-==== 3.0.5
-
-* `SELECT`, `UPDATE`, and `DELETE` statements now allow empty `IN`
-relations (see
-https://issues.apache.org/jira/browse/CASSANDRA-5626)[CASSANDRA-5626].
-
-==== 3.0.4
-
-* Updated the syntax for custom link:#createIndexStmt[secondary
-indexes].
-* Non-equal condition on the partition key are now never supported, even
-for ordering partitioner as this was not correct (the order was *not*
-the one of the type of the partition key). Instead, the `token` method
-should always be used for range queries on the partition key (see
-link:#selectWhere[WHERE clauses] ).
-
-==== 3.0.3
-
-* Support for custom link:#createIndexStmt[secondary indexes] has been
-added.
-
-==== 3.0.2
-
-* Type validation for the link:#constants[constants] has been fixed. For
-instance, the implementation used to allow `'2'` as a valid value for an
-`int` column (interpreting it has the equivalent of `2`), or `42` as a
-valid `blob` value (in which case `42` was interpreted as an hexadecimal
-representation of the blob). This is no longer the case, type validation
-of constants is now more strict. See the link:#types[data types] section
-for details on which constant is allowed for which type.
-* The type validation fixed of the previous point has lead to the
-introduction of link:#constants[blobs constants] to allow inputing
-blobs. Do note that while inputing blobs as strings constant is still
-supported by this version (to allow smoother transition to blob
-constant), it is now deprecated (in particular the link:#types[data
-types] section does not list strings constants as valid blobs) and will
-be removed by a future version. If you were using strings as blobs, you
-should thus update your client code ASAP to switch blob constants.
-* A number of functions to convert native types to blobs have also been
-introduced. Furthermore the token function is now also allowed in select
-clauses. See the link:#functions[section on functions] for details.
-
-==== 3.0.1
-
-* link:#usingtimestamps[Date strings] (and timestamps) are no longer
-accepted as valid `timeuuid` values. Doing so was a bug in the sense
-that date string are not valid `timeuuid`, and it was thus resulting in
-https://issues.apache.org/jira/browse/CASSANDRA-4936[confusing
-behaviors]. However, the following new methods have been added to help
-working with `timeuuid`: `now`, `minTimeuuid`, `maxTimeuuid` , `dateOf`
-and `unixTimestampOf`. See the link:#timeuuidFun[section dedicated to
-these methods] for more detail.
-* ``Float constants''#constants now support the exponent notation. In
-other words, `4.2E10` is now a valid floating point value.
-
-=== Versioning
-
-Versioning of the CQL language adheres to the http://semver.org[Semantic
-Versioning] guidelines. Versions take the form X.Y.Z where X, Y, and Z
-are integer values representing major, minor, and patch level
-respectively. There is no correlation between Cassandra release versions
-and the CQL language version.
-
-[cols=",",options="header",]
-|===
-|version |description
-|Major |The major version _must_ be bumped when backward incompatible
-changes are introduced. This should rarely occur.
-
-|Minor |Minor version increments occur when new, but backward
-compatible, functionality is introduced.
-
-|Patch |The patch version is incremented when bugs are fixed.
-|===
diff --git a/doc/modules/cassandra/pages/cql/ddl.adoc b/doc/modules/cassandra/pages/cql/ddl.adoc
deleted file mode 100644
index 36cce45..0000000
--- a/doc/modules/cassandra/pages/cql/ddl.adoc
+++ /dev/null
@@ -1,807 +0,0 @@
-= Data Definition
-:tabs:
-
-CQL stores data in _tables_, whose schema defines the layout of the
-data in the table. Tables are located in _keyspaces_.
-A keyspace defines options that apply to all the keyspace's tables.
-The xref:cql/ddl.adoc#replication-strategy[replication strategy] is an important keyspace option, as is the replication factor.
-A good general rule is one keyspace per application.
-It is common for a cluster to define only one keyspace for an actie application.
-
-This section describes the statements used to create, modify, and remove
-those keyspace and tables.
-
-== Common definitions
-
-The names of the keyspaces and tables are defined by the following
-grammar:
-
-[source,bnf]
-----
-include::example$BNF/ks_table.bnf[]
-----
-
-Both keyspace and table name should be comprised of only alphanumeric
-characters, cannot be empty and are limited in size to 48 characters
-(that limit exists mostly to avoid filenames (which may include the
-keyspace and table name) to go over the limits of certain file systems).
-By default, keyspace and table names are case-insensitive (`myTable` is
-equivalent to `mytable`) but case sensitivity can be forced by using
-double-quotes (`"myTable"` is different from `mytable`).
-
-Further, a table is always part of a keyspace and a table name can be
-provided fully-qualified by the keyspace it is part of. If is is not
-fully-qualified, the table is assumed to be in the _current_ keyspace
-(see xref:cql/ddl.adoc#use-statement[USE] statement.
-
-Further, the valid names for columns are defined as:
-
-[source,bnf]
-----
-include::example$BNF/column.bnf[]
-----
-
-We also define the notion of statement options for use in the following
-section:
-
-[source,bnf]
-----
-include::example$BNF/options.bnf[]
-----
-
-[[create-keyspace-statement]]
-== CREATE KEYSPACE
-
-A keyspace is created with a `CREATE KEYSPACE` statement:
-
-[source,bnf]
-----
-include::example$BNF/create_ks.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/create_ks.cql[]
-----
-
-Attempting to create a keyspace that already exists will return an error
-unless the `IF NOT EXISTS` option is used. If it is used, the statement
-will be a no-op if the keyspace already exists.
-
-The supported `options` are:
-
-[cols=",,,,",options="header",]
-|===
-|name | kind | mandatory | default | description
-|`replication` | _map_ | yes | n/a | The replication strategy and options to use for the keyspace (see
-details below).
-|`durable_writes` | _simple_ | no | true | Whether to use the commit log for updates on this keyspace (disable this
-option at your own risk!).
-|===
-
-The `replication` property is mandatory and must contain the `'class'` sub-option that defines the desired
-xref:cql/ddl.adoc#replication-strategy[replication strategy] class.
-The rest of the sub-options depend on which replication strategy is used.
-By default, Cassandra supports the following `'class'` values:
-
-[[replication-strategy]]
-=== `SimpleStrategy`
-
-A simple strategy that defines a replication factor for data to be
-spread across the entire cluster. This is generally not a wise choice
-for production, as it does not respect datacenter layouts and can
-lead to wildly varying query latency. For production, use
-`NetworkTopologyStrategy`. `SimpleStrategy` supports a single
-mandatory argument:
-
-[cols=",,,",options="header",]
-|===
-|sub-option |type |since |description
-|`'replication_factor'` | int | all | The number of replicas to store per range
-|===
-
-=== `NetworkTopologyStrategy`
-
-A production-ready replication strategy that sets the
-replication factor independently for each data-center. The rest of the
-sub-options are key-value pairs, with a key set to a data-center name and
-its value set to the associated replication factor. Options:
-
-[cols=",,,",options="header",]
-|===
-|sub-option |type |description
-|`'<datacenter>'` | int | The number of replicas to store per range in the provided datacenter.
-|`'replication_factor'` | int | The number of replicas to use as a default per datacenter if not
-specifically provided. Note that this always defers to existing
-definitions or explicit datacenter settings. For example, to have three
-replicas per datacenter, set a value of 3.
-|===
-
-When later altering keyspaces and changing the `replication_factor`,
-auto-expansion will only _add_ new datacenters for safety, it will not
-alter existing datacenters or remove any, even if they are no longer in
-the cluster. If you want to remove datacenters while setting the
-`replication_factor`, explicitly zero out the datacenter you want to
-have zero replicas.
-
-An example of auto-expanding datacenters with two datacenters: `DC1` and
-`DC2`:
-
-[source,cql]
-----
-include::example$CQL/autoexpand_ks.cql[]
-----
-will result in:
-[source,plaintext]
-----
-include::example$RESULTS/autoexpand_ks.result[]
-----
-
-An example of auto-expanding and overriding a datacenter:
-
-[source,cql]
-----
-include::example$CQL/autoexpand_ks_override.cql[]
-----
-will result in:
-[source,plaintext]
-----
-include::example$RESULTS/autoexpand_ks_override.result[]
-----
-
-An example that excludes a datacenter while using `replication_factor`:
-
-[source,cql]
-----
-include::example$CQL/autoexpand_exclude_dc.cql[]
-----
-will result in:
-[source,plaintext]
-----
-include::example$RESULTS/autoexpand_exclude_dc.result[]
-----
-
-If xref:new/transientreplication.adoc[transient replication] has been enabled, transient replicas can be
-configured for both `SimpleStrategy` and `NetworkTopologyStrategy` by
-defining replication factors in the format
-`'<total_replicas>/<transient_replicas>'`
-
-For instance, this keyspace will have 3 replicas in DC1, 1 of which is
-transient, and 5 replicas in DC2, 2 of which are transient:
-
-[source,cql]
-----
-include::example$CQL/create_ks_trans_repl.cql[]
-----
-
-[[use-statement]]
-== USE
-
-The `USE` statement changes the _current_ keyspace to the specified keyspace.
-A number of objects in CQL are bound to a keyspace (tables, user-defined types, functions, etc.) and the
-current keyspace is the default keyspace used when those objects are
-referred to in a query without a fully-qualified name (without a prefixed keyspace name).
-A `USE` statement specifies the keyspace to use as an argument:
-
-[source,bnf]
-----
-include::example$BNF/use_ks.bnf[]
-----
-Using CQL:
-[source,cql]
-----
-include::example$CQL/use_ks.cql[]
-----
-
-[[alter-keyspace-statement]]
-== ALTER KEYSPACE
-
-An `ALTER KEYSPACE` statement modifies the options of a keyspace:
-
-[source,bnf]
-----
-include::example$BNF/alter_ks.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/alter_ks.cql[]
-----
-If the keyspace does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-The supported options are the same as for xref:cql/ddl.adoc#create-keyspace-statement[creating a keyspace].
-
-[[drop-keyspace-statement]]
-== DROP KEYSPACE
-
-Dropping a keyspace is done with the `DROP KEYSPACE` statement:
-
-[source,bnf]
-----
-include::example$BNF/drop_ks.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/drop_ks.cql[]
-----
-
-Dropping a keyspace results in the immediate, irreversible removal of
-that keyspace, including all the tables, user-defined types, user-defined functions, and
-all the data contained in those tables.
-
-If the keyspace does not exists, the statement will return an error,
-unless `IF EXISTS` is used in which case the operation is a no-op.
-
-[[create-table-statement]]
-== CREATE TABLE
-
-Creating a new table uses the `CREATE TABLE` statement:
-
-[source,bnf]
-----
-include::example$BNF/create_table.bnf[]
-----
-
-For example, here are some CQL statements to create tables:
-
-[source,cql]
-----
-include::example$CQL/create_table.cql[]
-----
-
-A CQL table has a name and is composed of a set of _rows_.
-Creating a table amounts to defining which xref:cql/ddl.adoc#column-definition[columns] each rows will have,
-which of those columns comprise the xref:cql/ddl.adoc#primary-key[primary key], as well as defined
-xref:cql/ddl.adoc#create-table-options[options] for the table.
-
-Attempting to create an already existing table will return an error
-unless the `IF NOT EXISTS` directive is used. If it is used, the
-statement will be a no-op if the table already exists.
-
-[[column-definition]]
-=== Column definitions
-
-Every row in a CQL table will have the predefined columns defined at table creation.
-Columns can be added later using an xref:cql/ddl.adoc#alter-table-statement[alter statement].
-
-A `column_definition` is comprised of the name of the column and its xref:cql/ddl.adoc#data-type[type],
-restricting the values that are accepted for that column. Additionally, a column definition can have the
-following modifiers:
-
-* `STATIC`: declares the column as a xref:cql/ddl.adoc#static-column[static column]
-* `PRIMARY KEY`: declares the column as the sole component of the xref:cql/ddl.adoc#primary-key[primary key] of the table
-
-[[static-column]]
-==== Static columns
-
-Some columns can be declared as `STATIC` in a table definition. A column
-that is static will be “shared” by all the rows belonging to the same
-partition (having the same xref:cql/ddl.adoc#partition-key[partition key].
-
-For example:
-
-[{tabs}]
-====
-Code::
-+
---
-[source,cql]
-----
-include::example$CQL/create_static_column.cql[]
-include::example$CQL/insert_static_data.cql[]
-include::example$CQL/select_static_data.cql[]
-----
---
-
-Results::
-+
---
-[source,cql]
-----
-include::example$RESULTS/select_static_data.result[]
-----
---
-====
-
-As can be seen, the `s` value is the same (`static1`) for both of the
-rows in the partition (the partition key being `pk`, and both
-rows are in the same partition): the second insertion overrides the
-value for `s`.
-
-The use of static columns has the following restrictions:
-
-* A table without clustering columns cannot have static columns.
-In a table without clustering columns, every partition has only one row, and
-so every column is inherently static)
-* Only non-primary key columns can be static.
-
-[[primary-key]]
-=== The Primary key
-
-Within a table, a row is uniquely identified by its `PRIMARY KEY`, and
-hence all tables *must* define a single PRIMARY KEY.
-A `PRIMARY KEY` is composed of one or more of the defined columns in the table.
-Syntactically, the primary key is defined with the phrase `PRIMARY KEY`
-followed by a comma-separated list of the column names within parenthesis.
-If the primary key has only one column, you can alternatively add the `PRIMARY KEY` phrase to
-that column in the table definition.
-The order of the columns in the primary key definition defines the partition key and
-clustering columns.
-
-A CQL primary key is composed of two parts:
-
-xref:cql/ddl.adoc#partition-key[partition key]::
-* It is the first component of the primary key definition.
-It can be a single column or, using an additional set of parenthesis, can be multiple columns.
-A table must have at least one partition key, the smallest possible table definition is:
-+
-[source,cql]
-----
-include::example$CQL/create_table_single_pk.cql[]
-----
-xref:cql/ddl.adoc#clustering-columns[clustering columns]::
-* The columns are the columns that follow the partition key in the primary key definition.
-The order of those columns define the _clustering order_.
-
-Some examples of primary key definition are:
-
-* `PRIMARY KEY (a)`: `a` is the single partition key and there are no clustering columns
-* `PRIMARY KEY (a, b, c)` : `a` is the single partition key and `b` and `c` are the clustering columns
-* `PRIMARY KEY ((a, b), c)` : `a` and `b` compose the _composite_ partition key and `c` is the clustering column
-
-[IMPORTANT]
-====
-The primary key uniquely identifies a row in the table, as described above.
-A consequence of this uniqueness is that if another row is inserted using the same primary key,
-then an `UPSERT` occurs and an existing row with the same primary key is replaced.
-Columns that are not part of the primary key cannot define uniqueness.
-====
-
-[[partition-key]]
-==== Partition key
-
-Within a table, CQL defines the notion of a _partition_ that defines the location of data within a Cassandra cluster.
-A partition is the set of rows that share the same value for their partition key.
-
-Note that if the partition key is composed of multiple columns, then rows belong to the same partition
-when they have the same values for all those partition key columns.
-A hash is computed from the partition key columns and that hash value defines the partition location.
-So, for instance, given the following table definition and content:
-
-[source,cql]
-----
-include::example$CQL/create_table_compound_pk.cql[]
-include::example$CQL/insert_table_compound_pk.cql[]
-include::example$CQL/select_table_compound_pk.cql[]
-----
-
-will result in
-[source,cql]
-----
-include::example$RESULTS/select_table_compound_pk.result[]
-----
-<1> Rows 1 and 2 are in the same partition, because both columns `a` and `b` are zero.
-<2> Rows 3 and 4 are in the same partition, but a different one, because column `a` is zero and column `b` is 1 in both rows.
-<3> Row 5 is in a third partition by itself, because both columns `a` and `b` are 1.
-
-Note that a table always has a partition key, and that if the table has
-no `clustering columns`, then every partition of that table has a single row.
-because the partition key, compound or otherwise, identifies a single location.
-
-The most important property of partition is that all the rows belonging
-to the same partition are guaranteed to be stored on the same set of
-replica nodes.
-In other words, the partition key of a table defines which rows will be localized on the same
-node in the cluster.
-The localization of data is important to the efficient retrieval of data, requiring the Cassandra coordinator
-to contact as few nodes as possible.
-However, there is a flip-side to this guarantee, and all rows sharing a partition key will be stored on the same
-node, creating a hotspot for both reading and writing.
-While selecting a primary key that groups table rows assists batch updates and can ensure that the updates are
-_atomic_ and done in _isolation_, the partitions must be sized "just right, not too big nor too small".
-
-Data modeling that considers the querying patterns and assigns primary keys based on the queries will have the lowest
-latency in fetching data.
-
-[[clustering-columns]]
-==== Clustering columns
-
-The clustering columns of a table define the clustering order for the partition of that table.
-For a given `partition`, all rows are ordered by that clustering order. Clustering columns also add uniqueness to
-a row in a table.
-
-For instance, given:
-
-[source,cql]
-----
-include::example$CQL/create_table_clustercolumn.cql[]
-include::example$CQL/insert_table_clustercolumn.cql[]
-include::example$CQL/select_table_clustercolumn.cql[]
-----
-
-will result in
-[source,cql]
-----
-include::example$RESULTS/select_table_clustercolumn.result[]
-----
-<1> Row 1 is in one partition, and Rows 2-5 are in a different one. The display order is also different.
-
-Looking more closely at the four rows in the same partition, the `b` clustering column defines the order in which those rows
-are displayed.
-Whereas the partition key of the table groups rows on the same node, the clustering columns control
-how those rows are stored on the node.
-
-That sorting allows the very efficient retrieval of a range of rows within a partition:
-
-[source,cql]
-----
-include::example$CQL/select_range.cql[]
-----
-
-will result in
-[source,cql]
-----
-include::example$RESULTS/select_range.result[]
-----
-
-[[create-table-options]]
-=== Table options
-
-A CQL table has a number of options that can be set at creation (and,
-for most of them, altered later). These options are specified after the
-`WITH` keyword.
-
-One important option that cannot be changed after creation, `CLUSTERING ORDER BY`, influences how queries can be done against the table. It is worth discussing in more detail here.
-
-[[clustering-order]]
-==== Clustering order
-
-The clustering order of a table is defined by the clustering columns.
-By default, the clustering order is ascending for the clustering column's data types.
-For example, integers order from 1, 2, ... n, while text orders from A to Z.
-
-The `CLUSTERING ORDER BY` table option uses a comma-separated list of the
-clustering columns, each set for either `ASC` (for _ascending_ order) or `DESC` (for _descending order).
-The default is ascending for all clustering columns if the `CLUSTERING ORDER BY` option is not set.
-
-This option is basically a hint for the storage engine that changes the order in which it stores the row.
-Beware of the consequences of setting this option:
-
-* It changes the default ascending order of results when queried with a `SELECT` statement with no `ORDER BY` clause.
-
-* It limits how the `ORDER BY` clause is used in `SELECT` statements on that table.
-Results can only be ordered with either the original clustering order or the reverse clustering order.
-Suppose you create a table with two clustering columns `a` and `b`, defined `WITH CLUSTERING ORDER BY (a DESC, b ASC)`.
-Queries on the table can use `ORDER BY (a DESC, b ASC)` or `ORDER BY (a ASC, b DESC)`.
-Mixed order, such as `ORDER BY (a ASC, b ASC)` or `ORDER BY (a DESC, b DESC)` will not return expected order.
-
-* It has a performance impact on queries. Queries in reverse clustering order are slower than the default ascending order.
-If you plan to query mostly in descending order, declare the clustering order in the table schema using `WITH CLUSTERING ORDER BY ()`.
-This optimization is common for time series, to retrieve the data from newest to oldest.
-
-[[create-table-general-options]]
-==== Other table options
-
-A table supports the following options:
-
-[width="100%",cols="30%,9%,11%,50%",options="header",]
-|===
-|option | kind | default | description
-
-| `comment` | _simple_ | none | A free-form, human-readable comment
-| xref:cql/ddl.adoc#spec_retry[`speculative_retry`] | _simple_ | 99PERCENTILE | Speculative retry options
-| `cdc` |_boolean_ |false |Create a Change Data Capture (CDC) log on the table
-| `additional_write_policy` |_simple_ |99PERCENTILE | Same as `speculative_retry`
-| `gc_grace_seconds` |_simple_ |864000 |Time to wait before garbage collecting tombstones (deletion markers)
-| `bloom_filter_fp_chance` |_simple_ |0.00075 |The target probability of
-false positive of the sstable bloom filters. Said bloom filters will be
-sized to provide the provided probability, thus lowering this value
-impact the size of bloom filters in-memory and on-disk.
-| `default_time_to_live` |_simple_ |0 |Default expiration time (“TTL”) in seconds for a table
-| `compaction` |_map_ |_see below_ | xref:operating/compaction/index.adoc#cql-compaction-options[Compaction options]
-| `compression` |_map_ |_see below_ | xref:operating/compression/index.adoc#cql-compression-options[Compression options]
-| `caching` |_map_ |_see below_ |Caching options
-| `memtable_flush_period_in_ms` |_simple_ |0 |Time (in ms) before Cassandra flushes memtables to disk
-| `read_repair` |_simple_ |BLOCKING |Sets read repair behavior (see below)
-|===
-
-[[spec_retry]]
-===== Speculative retry options
-
-By default, Cassandra read coordinators only query as many replicas as
-necessary to satisfy consistency levels: one for consistency level
-`ONE`, a quorum for `QUORUM`, and so on. `speculative_retry` determines
-when coordinators may query additional replicas, a useful action when
-replicas are slow or unresponsive. Speculative retries reduce the latency.
-The speculative_retry option configures rapid read protection, where a coordinator sends more
-requests than needed to satisfy the consistency level.
-
-[IMPORTANT]
-====
-Frequently reading from additional replicas can hurt cluster
-performance. When in doubt, keep the default `99PERCENTILE`.
-====
-
-Pre-Cassandra 4.0 speculative retry policy takes a single string as a parameter:
-
-* `NONE`
-* `ALWAYS`
-* `99PERCENTILE` (PERCENTILE)
-* `50MS` (CUSTOM)
-
-An example of setting speculative retry sets a custom value:
-
-[source,cql]
-----
-include::example$CQL/alter_table_spec_retry.cql[]
-----
-
-This example uses a percentile for the setting:
-
-[source,cql]
-----
-include::example$CQL/alter_table_spec_retry_percent.cql[]
-----
-
-A percentile setting can backfire. If a single host becomes unavailable, it can
-force up the percentiles. A value of `p99` will not speculate as intended because the
-value at the specified percentile has increased too much. If the consistency level is set to `ALL`, all
-replicas are queried regardless of the speculative retry setting.
-
-Cassandra 4.0 supports case-insensitivity for speculative retry values (https://issues.apache.org/jira/browse/CASSANDRA-14293[CASSANDRA-14293]). For example, assigning the value as `none`, `None`, or `NONE` has the same effect.
-
-Additionally, the following values are added:
-
-[cols=",,",options="header",]
-|===
-|Format |Example |Description
-| `XPERCENTILE` | 90.5PERCENTILE | Coordinators record average per-table response times
-for all replicas. If a replica takes longer than `X` percent of this
-table's average response time, the coordinator queries an additional
-replica. `X` must be between 0 and 100.
-| `XP` | 90.5P | Same as `XPERCENTILE`
-| `Yms` | 25ms | If a replica takes more than `Y` milliseconds to respond, the
-coordinator queries an additional replica.
-| `MIN(XPERCENTILE,YMS)` | MIN(99PERCENTILE,35MS) | A hybrid policy that uses either the
-specified percentile or fixed milliseconds depending on which value is
-lower at the time of calculation. Parameters are `XPERCENTILE`, `XP`, or
-`Yms`. This setting helps protect against a single slow instance.
-
-| `MAX(XPERCENTILE,YMS)` `ALWAYS` `NEVER` | MAX(90.5P,25ms) | A hybrid policy that uses either the specified
-percentile or fixed milliseconds depending on which value is higher at
-the time of calculation.
-|===
-
-Cassandra 4.0 adds support for hybrid `MIN()` and `MAX()` speculative retry policies, with a mix and match of either `MIN(), MAX()`, `MIN(), MIN()`, or `MAX(), MAX()` (https://issues.apache.org/jira/browse/CASSANDRA-14293[CASSANDRA-14293]).
-The hybrid mode will still speculate if the normal `p99` for the table is < 50ms, the minimum value.
-But if the `p99` level goes higher than the maximum value, then that value can be used.
-In a hybrid value, one value must be a fixed time (ms) value and the other a percentile value.
-
-To illustrate variations, the following examples are all valid:
-
-[source,cql]
-----
-include::example$CQL/spec_retry_values.cql[]
-----
-
-The `additional_write_policy` setting specifies the threshold at which a cheap
-quorum write will be upgraded to include transient replicas.
-
-[[cql-compaction-options]]
-===== Compaction options
-
-The `compaction` options must minimally define the `'class'` sub-option,
-to specify the compaction strategy class to use.
-The supported classes are:
-
-* `'SizeTieredCompactionStrategy'`, xref:operating/compaction/stcs.adoc#stcs[STCS] (Default)
-* `'LeveledCompactionStrategy'`, xref:operating/compaction/lcs.adoc#lcs[LCS]
-* `'TimeWindowCompactionStrategy'`, xref:operating/compaction/twcs.adoc#twcs[TWCS]
-
-The `'DateTieredCompactionStrategy'` is also supported but deprecated;
-`'TimeWindowCompactionStrategy'` should be used.
-If a custom strategies is required, specify the full class name as a xref:cql/definitions.adoc#constants[string constant].
-
-All default strategies support a number of xref:operating/compaction/index.adoc#compaction-options[common options], as well as options specific to the strategy chosen. See the section corresponding to your strategy for details: xref:operating/compaction/stcs.adoc#stcs_options[STCS], xref:operating/compaction/lcs.adoc#lcs_options[LCS], xref:operating/compaction/twcs.adoc#twcs_options[TWCS].
-
-[[cql-compression-options]]
-===== Compression options
-
-The `compression` options define if and how the SSTables of the table
-are compressed. Compression is configured on a per-table basis as an
-optional argument to `CREATE TABLE` or `ALTER TABLE`. The following
-sub-options are available:
-
-[cols=",,",options="header",]
-|===
-|Option |Default |Description
-| `class` | LZ4Compressor | The compression algorithm to use. Default compressor are: LZ4Compressor,
-SnappyCompressor, DeflateCompressor and ZstdCompressor.
-Use `'enabled' : false` to disable compression.
-Custom compressor can be provided by specifying the full class name as a xref:cql/definitions.adoc#constants[string constant].
-
-| `enabled` | true | Enable/disable sstable compression.
-If the `enabled` option is set to `false`, no other options must be specified.
-
-| `chunk_length_in_kb` | 64 | On disk SSTables are compressed by block (to allow random reads).
-This option defines the size (in KB) of said block. See xref:cql/ddl.adoc#chunk_note[note] for further information.
-
-| `crc_check_chance` | 1.0 | Determines how likely Cassandra is to verify the checksum on each
-compression chunk during reads.
-
-| `compression_level` | 3 | Compression level. Only applicable for `ZstdCompressor`.
-Accepts values between `-131072` and `22`.
-|===
-
-[[chunk_note]]
-[NOTE]
-====
-Bigger values may improve the compression rate, but will increase the minimum size of data to be read from
-disk for a read.
-The default value is an optimal value for compressing tables.
-Chunk length must be a power of 2 when computing the chunk number from an uncompressed file offset.
-Block size may be adjusted based on read/write access patterns such as:
-
-* How much data is typically requested at once
-* Average size of rows in the table
-====
-
-For instance, to create a table with LZ4Compressor and a `chunk_length_in_kb` of 4 KB:
-
-[source,cql]
-----
-include::example$CQL/chunk_length.cql[]
-----
-
-[[cql-caching-options]]
-===== Caching options
-
-Caching optimizes the use of cache memory of a table. The cached data is
-weighed by size and access frequency.
-The `caching` options can configure both the `key cache` and the `row cache` for the table.
-The following sub-options are available:
-
-[cols=",,",options="header",]
-|===
-|Option |Default |Description
-| `keys` | ALL | Whether to cache keys (key cache) for this table. Valid values are: `ALL` and `NONE`.
-
-| `rows_per_partition` | NONE | The amount of rows to cache per partition (row cache).
-If an integer `n` is specified, the first `n` queried rows of a partition will be cached.
-Valid values are: `ALL`, to cache all rows of a queried partition, or `NONE` to disable row caching.
-|===
-
-For instance, to create a table with both a key cache and 10 rows cached per partition:
-
-[source,cql]
-----
-include::example$CQL/caching_option.cql[]
-----
-
-[[read-repair-options]]
-===== Read Repair options
-
-The `read_repair` options configure the read repair behavior, tuning for various performance and consistency behaviors.
-
-The values are:
-[cols=",,",options="header",]
-|===
-|Option |Default |Description
-|`BLOCKING` | yes | If a read repair is triggered, the read blocks writes sent to other replicas until the consistency level is reached by the writes.
-
-|`NONE` | no | If set, the coordinator reconciles any differences between replicas, but doesn't attempt to repair them.
-|===
-
-Two consistency properties are affected by read repair behavior.
-
-* Monotonic quorum reads: Monotonic quorum reads
-prevents reads from appearing to go back in time in some circumstances.
-When monotonic quorum reads are not provided and a write fails to reach
-a quorum of replicas, the read values may be visible in one read, and then disappear
-in a subsequent read. `BLOCKING` provides this behavior.
-* Write atomicity: Write atomicity prevents reads
-from returning partially-applied writes. Cassandra attempts to provide
-partition-level write atomicity, but since only the data covered by a
-SELECT statement is repaired by a read repair, read repair can break
-write atomicity when data is read at a more granular level than it is
-written. For example, read repair can break write atomicity if you write
-multiple rows to a clustered partition in a batch, but then select a
-single row by specifying the clustering column in a SELECT statement.
-`NONE` provides this behavior.
-
-===== Other considerations:
-
-* Adding new columns (see `ALTER TABLE` below) is a constant time
-operation. Thus, there is no need to anticipate future usage while initially creating a table.
-
-[[alter-table-statement]]
-== ALTER TABLE
-
-Altering an existing table uses the `ALTER TABLE` statement:
-
-[source,bnf]
-----
-include::example$BNF/alter_table.bnf[]
-----
-If the table does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/alter_table_add_column.cql[]
-include::example$CQL/alter_table_with_comment.cql[]
-----
-
-The `ALTER TABLE` statement can:
-
-* `ADD` a new column to a table. The primary key of a table cannot ever be altered.
-A new column, thus, cannot be part of the primary key.
-Adding a column is a constant-time operation based on the amount of data in the table.
-If the new column already exists, the statement will return an error, unless `IF NOT EXISTS` is used in which case the operation is a no-op.
-* `DROP` a column from a table. This command drops both the column and all
-its content. Be aware that, while the column becomes immediately
-unavailable, its content are removed lazily during compaction. Because of this lazy removal,
-the command is a constant-time operation based on the amount of data in the table.
-Also, it is important to know that once a column is dropped, a column with the same name can be re-added,
-unless the dropped column was a non-frozen column like a collection.
-If the dropped column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-[WARNING]
-.Warning
-====
-Dropping a column assumes that the timestamps used for the value of this
-column are "real" timestamp in microseconds. Using "real" timestamps in
-microseconds is the default is and is *strongly* recommended but as
-Cassandra allows the client to provide any timestamp on any table, it is
-theoretically possible to use another convention. Please be aware that
-if you do so, dropping a column will not correctly execute.
-====
-
-* `RENAME` a primary key column of a table. Non primary key columns cannot be renamed.
-Furthermore, renaming a column to another name which already exists isn't allowed.
-It's important to keep in mind that renamed columns shouldn't have dependent seconday indexes.
-If the renamed column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-* Use `WITH` to change a table option. The xref:CQL/ddl.adoc#create-table-options[supported options]
-are the same as those used when creating a table, with the exception of `CLUSTERING ORDER`.
-However, setting any `compaction` sub-options will erase *ALL* previous `compaction` options, so you need to re-specify
-all the sub-options you wish to keep. The same is true for `compression` sub-options.
-
-[[drop-table-statement]]
-== DROP TABLE
-
-Dropping a table uses the `DROP TABLE` statement:
-
-[source,bnf]
-----
-include::example$BNF/drop_table.bnf[]
-----
-
-Dropping a table results in the immediate, irreversible removal of the
-table, including all data it contains.
-
-If the table does not exist, the statement will return an error, unless
-`IF EXISTS` is used, when the operation is a no-op.
-
-[[truncate-statement]]
-== TRUNCATE
-
-A table can be truncated using the `TRUNCATE` statement:
-
-[source,bnf]
-----
-include::example$BNF/truncate_table.bnf[]
-----
-
-`TRUNCATE TABLE foo` is the preferred syntax for consistency with other DDL
-statements.
-However, tables are the only object that can be truncated currently, and the `TABLE` keyword can be omitted.
-
-Truncating a table permanently removes all existing data from the table, but without removing the table itself.
diff --git a/doc/modules/cassandra/pages/cql/definitions.adoc b/doc/modules/cassandra/pages/cql/definitions.adoc
deleted file mode 100644
index 95be20f..0000000
--- a/doc/modules/cassandra/pages/cql/definitions.adoc
+++ /dev/null
@@ -1,187 +0,0 @@
-= Definitions
-
-== Conventions
-
-To aid in specifying the CQL syntax, we will use the following
-conventions in this document:
-
-* Language rules will be given in an informal
-http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form#Variants[BNF
-variant] notation. In particular, we'll use square brakets (`[ item ]`)
-for optional items, `*` and `+` for repeated items (where `+` imply at
-least one).
-* The grammar will also use the following convention for convenience:
-non-terminal term will be lowercase (and link to their definition) while
-terminal keywords will be provided "all caps". Note however that
-keywords are `identifiers` and are thus case insensitive in practice. We
-will also define some early construction using regexp, which we'll
-indicate with `re(<some regular expression>)`.
-* The grammar is provided for documentation purposes and leave some
-minor details out. For instance, the comma on the last column definition
-in a `CREATE TABLE` statement is optional but supported if present even
-though the grammar in this document suggests otherwise. Also, not
-everything accepted by the grammar is necessarily valid CQL.
-* References to keywords or pieces of CQL code in running text will be
-shown in a `fixed-width font`.
-
-[[identifiers]]
-== Identifiers and keywords
-
-The CQL language uses _identifiers_ (or _names_) to identify tables,
-columns and other objects. An identifier is a token matching the regular
-expression `[a-zA-Z][a-zA-Z0-9_]*`.
-
-A number of such identifiers, like `SELECT` or `WITH`, are _keywords_.
-They have a fixed meaning for the language and most are reserved. The
-list of those keywords can be found in xref:cql/appendices.adoc#appendix-A[Appendix A].
-
-Identifiers and (unquoted) keywords are case insensitive. Thus `SELECT`
-is the same than `select` or `sElEcT`, and `myId` is the same than
-`myid` or `MYID`. A convention often used (in particular by the samples
-of this documentation) is to use uppercase for keywords and lowercase
-for other identifiers.
-
-There is a second kind of identifier called a _quoted identifier_
-defined by enclosing an arbitrary sequence of characters (non-empty) in
-double-quotes(`"`). Quoted identifiers are never keywords. Thus
-`"select"` is not a reserved keyword and can be used to refer to a
-column (note that using this is particularly ill-advised), while `select`
-would raise a parsing error. Also, unlike unquoted identifiers
-and keywords, quoted identifiers are case sensitive (`"My Quoted Id"` is
-_different_ from `"my quoted id"`). A fully lowercase quoted identifier
-that matches `[a-zA-Z][a-zA-Z0-9_]*` is however _equivalent_ to the
-unquoted identifier obtained by removing the double-quote (so `"myid"`
-is equivalent to `myid` and to `myId` but different from `"myId"`).
-Inside a quoted identifier, the double-quote character can be repeated
-to escape it, so `"foo "" bar"` is a valid identifier.
-
-[NOTE]
-.Note
-====
-The _quoted identifier_ can declare columns with arbitrary names, and
-these can sometime clash with specific names used by the server. For
-instance, when using conditional update, the server will respond with a
-result set containing a special result named `"[applied]"`. If you’ve
-declared a column with such a name, this could potentially confuse some
-tools and should be avoided. In general, unquoted identifiers should be
-preferred but if you use quoted identifiers, it is strongly advised that you
-avoid any name enclosed by squared brackets (like `"[applied]"`) and any
-name that looks like a function call (like `"f(x)"`).
-====
-
-More formally, we have:
-
-[source, bnf]
-----
-include::example$BNF/identifier.bnf[]
-----
-
-[[constants]]
-== Constants
-
-CQL defines the following _constants_:
-
-[source, bnf]
-----
-include::example$BNF/constant.bnf[]
-----
-
-In other words:
-
-* A string constant is an arbitrary sequence of characters enclosed by
-single-quote(`'`). A single-quote can be included by repeating it, e.g.
-`'It''s raining today'`. Those are not to be confused with quoted
-`identifiers` that use double-quotes. Alternatively, a string can be
-defined by enclosing the arbitrary sequence of characters by two dollar
-characters, in which case single-quote can be used without escaping
-(`$$It's raining today$$`). That latter form is often used when defining
-xref:cql/functions.adoc#udfs[user-defined functions] to avoid having to escape single-quote
-characters in function body (as they are more likely to occur than
-`$$`).
-* Integer, float and boolean constant are defined as expected. Note
-however than float allows the special `NaN` and `Infinity` constants.
-* CQL supports
-https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID]
-constants.
-* Blobs content are provided in hexadecimal and prefixed by `0x`.
-* The special `NULL` constant denotes the absence of value.
-
-For how these constants are typed, see the xref:cql/types.adoc[Data types] section.
-
-== Terms
-
-CQL has the notion of a _term_, which denotes the kind of values that
-CQL support. Terms are defined by:
-
-[source, bnf]
-----
-include::example$BNF/term.bnf[]
-----
-
-A term is thus one of:
-
-* A xref:cql/defintions.adoc#constants[constant]
-* A literal for either a xref:cql/types.adoc#collections[collection],
-a xref:cql/types.adoc#udts[user-defined type] or a xref:cql/types.adoc#tuples[tuple]
-* A xref:cql/functions.adoc#cql-functions[function] call, either a xref:cql/functions.adoc#scalar-native-functions[native function]
-or a xref:cql/functions.adoc#user-defined-scalar-functions[user-defined function]
-* An xref:cql/operators.adoc#arithmetic_operators[arithmetic operation] between terms
-* A type hint
-* A bind marker, which denotes a variable to be bound at execution time.
-See the section on `prepared-statements` for details. A bind marker can
-be either anonymous (`?`) or named (`:some_name`). The latter form
-provides a more convenient way to refer to the variable for binding it
-and should generally be preferred.
-
-== Comments
-
-A comment in CQL is a line beginning by either double dashes (`--`) or
-double slash (`//`).
-
-Multi-line comments are also supported through enclosure within `/*` and
-`*/` (but nesting is not supported).
-
-[source,cql]
-----
--- This is a comment
-// This is a comment too
-/* This is
- a multi-line comment */
-----
-
-== Statements
-
-CQL consists of statements that can be divided in the following
-categories:
-
-* `data-definition` statements, to define and change how the data is
-stored (keyspaces and tables).
-* `data-manipulation` statements, for selecting, inserting and deleting
-data.
-* `secondary-indexes` statements.
-* `materialized-views` statements.
-* `cql-roles` statements.
-* `cql-permissions` statements.
-* `User-Defined Functions (UDFs)` statements.
-* `udts` statements.
-* `cql-triggers` statements.
-
-All the statements are listed below and are described in the rest of
-this documentation (see links above):
-
-[source, bnf]
-----
-include::example$BNF/cql_statement.bnf[]
-----
-
-== Prepared Statements
-
-CQL supports _prepared statements_. Prepared statements are an
-optimization that allows to parse a query only once but execute it
-multiple times with different concrete values.
-
-Any statement that uses at least one bind marker (see `bind_marker`)
-will need to be _prepared_. After which the statement can be _executed_
-by provided concrete values for each of its marker. The exact details of
-how a statement is prepared and then executed depends on the CQL driver
-used and you should refer to your driver documentation.
diff --git a/doc/modules/cassandra/pages/cql/dml.adoc b/doc/modules/cassandra/pages/cql/dml.adoc
deleted file mode 100644
index d0517aa..0000000
--- a/doc/modules/cassandra/pages/cql/dml.adoc
+++ /dev/null
@@ -1,461 +0,0 @@
-= Data Manipulation
-
-This section describes the statements supported by CQL to insert,
-update, delete and query data.
-
-[[select-statement]]
-== SELECT
-
-Querying data from data is done using a `SELECT` statement:
-
-[source,bnf]
-----
-include::example$BNF/select_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/select_statement.cql[]
-----
-
-The `SELECT` statements reads one or more columns for one or more rows
-in a table. It returns a result-set of the rows matching the request,
-where each row contains the values for the selection corresponding to
-the query. Additionally, xref:cql/functions.adoc#cql-functions[functions] including
-xref:cql/functions.adoc#aggregate-functions[aggregations] can be applied to the result.
-
-A `SELECT` statement contains at least a xref:cql/dml.adoc#selection-clause[selection clause] and the name of the table on which
-the selection is executed.
-CQL does *not* execute joins or sub-queries and a select statement only apply to a single table.
-A select statement can also have a xref:cql/dml.adoc#where-clause[where clause] that can further narrow the query results.
-Additional clauses can xref:cql/dml.adoc#ordering-clause[order] or xref:cql/dml.adoc#limit-clause[limit] the results.
-Lastly, xref:cql/dml.adoc#allow-filtering[queries that require full cluster filtering] can append `ALLOW FILTERING` to any query.
-
-[[selection-clause]]
-=== Selection clause
-
-The `select_clause` determines which columns will be queried and returned in the result set.
-This clause can also apply transformations to apply to the result before returning.
-The selection clause consists of a comma-separated list of specific _selectors_ or, alternatively, the wildcard character (`*`) to select all the columns defined in the table.
-
-==== Selectors
-
-A `selector` can be one of:
-
-* A column name of the table selected, to retrieve the values for that
-column.
-* A term, which is usually used nested inside other selectors like
-functions (if a term is selected directly, then the corresponding column
-of the result-set will simply have the value of this term for every row
-returned).
-* A casting, which allows to convert a nested selector to a (compatible)
-type.
-* A function call, where the arguments are selector themselves. See the
-section on xref:cql/functions.adoc#cql-functions[functions] for more details.
-* The special call `COUNT(*)` to the xref:cql/functions.adoc#count-function[COUNT function],
-which counts all non-null results.
-
-==== Aliases
-
-Every _top-level_ selector can also be aliased (using AS).
-If so, the name of the corresponding column in the result set will be
-that of the alias. For instance:
-
-[source,cql]
-----
-include::example$CQL/as.cql[]
-----
-
-[NOTE]
-====
-Currently, aliases aren't recognized in the `WHERE` or `ORDER BY` clauses in the statement.
-You must use the orignal column name instead.
-====
-
-[[writetime-and-ttl-function]]
-==== `WRITETIME` and `TTL` function
-
-Selection supports two special functions that aren't allowed anywhere
-else: `WRITETIME` and `TTL`.
-Both functions take only one argument, a column name.
-These functions retrieve meta-information that is stored internally for each column:
-
-* `WRITETIME` stores the timestamp of the value of the column
-* `TTL` stores the remaining time to live (in seconds) for the value of the column if it is set to expire; otherwise the value is `null`.
-
-The `WRITETIME` and `TTL` functions can't be used on multi-cell columns such as non-frozen
-collections or non-frozen user-defined types.
-
-[[where-clause]]
-=== The `WHERE` clause
-
-The `WHERE` clause specifies which rows are queried. It specifies
-a relationship for `PRIMARY KEY` columns or a column that has
-a xref:cql/indexes.adoc#create-index-statement[secondary index] defined, along with a set value.
-
-Not all relationships are allowed in a query. For instance, only an equality
-is allowed on a partition key. The `IN` clause is considered an equality for one or more values.
-The `TOKEN` clause can be used to query for partition key non-equalities.
-A partition key must be specified before clustering columns in the `WHERE` clause. The relationship
-for clustering columns must specify a *contiguous* set of rows to order.
-
-For instance, given:
-
-[source,cql]
-----
-include::example$CQL/table_for_where.cql[]
-----
-
-The following query is allowed:
-
-[source,cql]
-----
-include::example$CQL/where.cql[]
-----
-
-But the following one is not, as it does not select a contiguous set of
-rows (and we suppose no secondary indexes are set):
-
-[source,cql]
-----
-include::example$CQL/where_fail.cql[]
-----
-
-When specifying relationships, the `TOKEN` function can be applied to the `PARTITION KEY` column to query.
-Rows will be selected based on the token of the `PARTITION_KEY` rather than on the value.
-[IMPORTANT]
-====
-The token of a key depends on the partitioner in use, and that
-in particular the `RandomPartitioner` won't yield a meaningful order.
-Also note that ordering partitioners always order token values by bytes (so
-even if the partition key is of type int, `token(-1) > token(0)` in
-particular).
-====
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/token.cql[]
-----
-
-The `IN` relationship is only allowed on the last column of the
-partition key or on the last column of the full primary key.
-
-It is also possible to “group” `CLUSTERING COLUMNS` together in a
-relation using the tuple notation.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/where_group_cluster_columns.cql[]
-----
-
-This query will return all rows that sort after the one having “John's Blog” as
-`blog_tile` and '2012-01-01' for `posted_at` in the clustering order. In
-particular, rows having a `post_at <= '2012-01-01'` will be returned, as
-long as their `blog_title > 'John''s Blog'`.
-
-That would not be the case for this example:
-
-[source,cql]
-----
-include::example$CQL/where_no_group_cluster_columns.cql[]
-----
-
-The tuple notation may also be used for `IN` clauses on clustering columns:
-
-[source,cql]
-----
-include::example$CQL/where_in_tuple.cql[]
-----
-
-The `CONTAINS` operator may only be used for collection columns (lists,
-sets, and maps). In the case of maps, `CONTAINS` applies to the map
-values. The `CONTAINS KEY` operator may only be used on map columns and
-applies to the map keys.
-
-[[group-by-clause]]
-=== Grouping results
-
-The `GROUP BY` option can condense all selected
-rows that share the same values for a set of columns into a single row.
-
-Using the `GROUP BY` option, rows can be grouped at the partition key or clustering column level.
-Consequently, the `GROUP BY` option only accepts primary key columns in defined order as arguments.
-If a primary key column is restricted by an equality restriction, it is not included in the `GROUP BY` clause.
-
-Aggregate functions will produce a separate value for each group.
-If no `GROUP BY` clause is specified, aggregates functions will produce a single value for all the rows.
-
-If a column is selected without an aggregate function, in a statement
-with a `GROUP BY`, the first value encounter in each group will be
-returned.
-
-[[ordering-clause]]
-=== Ordering results
-
-The `ORDER BY` clause selects the order of the returned results.
-The argument is a list of column names and each column's order
-(`ASC` for ascendant and `DESC` for descendant,
-The possible orderings are limited by the xref:cql/ddl.adoc#clustering-order[clustering order] defined on the table:
-
-* if the table has been defined without any specific `CLUSTERING ORDER`, then the order is as defined by the clustering columns
-or the reverse
-* otherwise, the order is defined by the `CLUSTERING ORDER` option and the reversed one.
-
-[[limit-clause]]
-=== Limiting results
-
-The `LIMIT` option to a `SELECT` statement limits the number of rows
-returned by a query. The `PER PARTITION LIMIT` option limits the
-number of rows returned for a given partition by the query. Both types of limits can used in the same statement.
-
-[[allow-filtering]]
-=== Allowing filtering
-
-By default, CQL only allows select queries that don't involve a full scan of all partitions.
-If all partitions are scanned, then returning the results may experience a significant latency proportional to the
-amount of data in the table. The `ALLOW FILTERING` option explicitly executes a full scan. Thus, the performance of
-the query can be unpredictable.
-
-For example, consider the following table of user profiles with birth year and country of residence.
-The birth year has a secondary index defined.
-
-[source,cql]
-----
-include::example$CQL/allow_filtering.cql[]
-----
-
-The following queries are valid:
-
-[source,cql]
-----
-include::example$CQL/query_allow_filtering.cql[]
-----
-
-In both cases, the query performance is proportional to the amount of data returned.
-The first query returns all rows, because all users are selected.
-The second query returns only the rows defined by the secondary index, a per-node implementation; the results will
-depend on the number of nodes in the cluster, and is indirectly proportional to the amount of data stored.
-The number of nodes will always be multiple number of magnitude lower than the number of user profiles stored.
-Both queries may return very large result sets, but the addition of a `LIMIT` clause can reduced the latency.
-
-The following query will be rejected:
-
-[source,cql]
-----
-include::example$CQL/query_fail_allow_filtering.cql[]
-----
-
-Cassandra cannot guarantee that large amounts of data won't have to scanned amount of data, even if the result is small.
-If you know that the dataset is small, and the performance will be reasonable, add `ALLOW FILTERING` to allow the query to
-execute:
-
-[source,cql]
-----
-include::example$CQL/query_nofail_allow_filtering.cql[]
-----
-
-[[insert-statement]]
-== INSERT
-
-Inserting data for a row is done using an `INSERT` statement:
-
-[source,bnf]
-----
-include::example$BNF/insert_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/insert_statement.cql[]
-----
-
-The `INSERT` statement writes one or more columns for a given row in a
-table.
-Since a row is identified by its `PRIMARY KEY`, at least one columns must be specified.
-The list of columns to insert must be supplied with the `VALUES` syntax.
-When using the `JSON` syntax, `VALUES` are optional.
-See the section on xref:cql/dml.adoc#cql-json[JSON support] for more detail.
-All updates for an `INSERT` are applied atomically and in isolation.
-
-Unlike in SQL, `INSERT` does not check the prior existence of the row by default.
-The row is created if none existed before, and updated otherwise.
-Furthermore, there is no means of knowing which action occurred.
-
-The `IF NOT EXISTS` condition can restrict the insertion if the row does not exist.
-However, note that using `IF NOT EXISTS` will incur a non-negligible performance cost, because Paxos is used,
-so this should be used sparingly.
-
-Please refer to the xref:cql/dml.adoc#update-parameters[UPDATE] section for informations on the `update_parameter`.
-Also note that `INSERT` does not support counters, while `UPDATE` does.
-
-[[update-statement]]
-== UPDATE
-
-Updating a row is done using an `UPDATE` statement:
-
-[source, bnf]
-----
-include::example$BNF/update_statement.bnf[]
-----
-
-For instance:
-
-[source,cql]
-----
-include::example$CQL/update_statement.cql[]
-----
-
-The `UPDATE` statement writes one or more columns for a given row in a
-table.
-The `WHERE`clause is used to select the row to update and must include all columns of the `PRIMARY KEY`.
-Non-primary key columns are set using the `SET` keyword.
-In an `UPDATE` statement, all updates within the same partition key are applied atomically and in isolation.
-
-Unlike in SQL, `UPDATE` does not check the prior existence of the row by default.
-The row is created if none existed before, and updated otherwise.
-Furthermore, there is no means of knowing which action occurred.
-
-The `IF` condition can be used to choose whether the row is updated or not if a particular condition is met.
-However, like the `IF NOT EXISTS` condition, a non-negligible performance cost can be incurred.
-
-Regarding the `SET` assignment:
-
-* `c = c + 3` will increment/decrement counters, the only operation allowed.
-The column name after the '=' sign *must* be the same than the one before the '=' sign.
-Increment/decrement is only allowed on counters.
-See the section on xref:cql/dml.adoc#counters[counters] for details.
-* `id = id + <some-collection>` and `id[value1] = value2` are for collections.
-See the xref:cql/types.adoc#collections[collections] for details.
-* `id.field = 3` is for setting the value of a field on a non-frozen user-defined types.
-See the xref:cql/types.adoc#udts[UDTs] for details.
-
-=== Update parameters
-
-`UPDATE` and `INSERT` statements support the following parameters:
-
-* `TTL`: specifies an optional Time To Live (in seconds) for the
-inserted values. If set, the inserted values are automatically removed
-from the database after the specified time. Note that the TTL concerns
-the inserted values, not the columns themselves. This means that any
-subsequent update of the column will also reset the TTL (to whatever TTL
-is specified in that update). By default, values never expire. A TTL of
-0 is equivalent to no TTL. If the table has a default_time_to_live, a
-TTL of 0 will remove the TTL for the inserted or updated values. A TTL
-of `null` is equivalent to inserting with a TTL of 0.
-
-`UPDATE`, `INSERT`, `DELETE` and `BATCH` statements support the following parameters:
-
-* `TIMESTAMP`: sets the timestamp for the operation. If not specified,
-the coordinator will use the current time (in microseconds) at the start
-of statement execution as the timestamp. This is usually a suitable
-default.
-
-[[delete_statement]]
-== DELETE
-
-Deleting rows or parts of rows uses the `DELETE` statement:
-
-[source,bnf]
-----
-include::example$BNF/delete_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/delete_statement.cql[]
-----
-
-The `DELETE` statement deletes columns and rows. If column names are
-provided directly after the `DELETE` keyword, only those columns are
-deleted from the row indicated by the `WHERE` clause. Otherwise, whole
-rows are removed.
-
-The `WHERE` clause specifies which rows are to be deleted. Multiple rows
-may be deleted with one statement by using an `IN` operator. A range of
-rows may be deleted using an inequality operator (such as `>=`).
-
-`DELETE` supports the `TIMESTAMP` option with the same semantics as in
-xref:cql/dml.adoc#update-parameters[updates].
-
-In a `DELETE` statement, all deletions within the same partition key are
-applied atomically and in isolation.
-
-A `DELETE` operation can be conditional through the use of an `IF`
-clause, similar to `UPDATE` and `INSERT` statements. However, as with
-`INSERT` and `UPDATE` statements, this will incur a non-negligible
-performance cost because Paxos is used, and should be used sparingly.
-
-[[batch_statement]]
-== BATCH
-
-Multiple `INSERT`, `UPDATE` and `DELETE` can be executed in a single
-statement by grouping them through a `BATCH` statement:
-
-[source, bnf]
-----
-include::example$BNF/batch_statement.bnf[]
-----
-
-For instance:
-
-[source,cql]
-----
-include::example$CQL/batch_statement.cql[]
-----
-
-The `BATCH` statement group multiple modification statements
-(insertions/updates and deletions) into a single statement. It serves
-several purposes:
-
-* It saves network round-trips between the client and the server (and
-sometimes between the server coordinator and the replicas) when batching
-multiple updates.
-* All updates in a `BATCH` belonging to a given partition key are
-performed in isolation.
-* By default, all operations in the batch are performed as _logged_, to
-ensure all mutations eventually complete (or none will). See the notes
-on xref:cql/dml.adoc#unlogged-batches[UNLOGGED batches] for more details.
-
-Note that:
-
-* `BATCH` statements may only contain `UPDATE`, `INSERT` and `DELETE`
-statements (not other batches for instance).
-* Batches are _not_ a full analogue for SQL transactions.
-* If a timestamp is not specified for each operation, then all
-operations will be applied with the same timestamp (either one generated
-automatically, or the timestamp provided at the batch level). Due to
-Cassandra's conflict resolution procedure in the case of
-http://wiki.apache.org/cassandra/FAQ#clocktie[timestamp ties],
-operations may be applied in an order that is different from the order
-they are listed in the `BATCH` statement. To force a particular
-operation ordering, you must specify per-operation timestamps.
-* A LOGGED batch to a single partition will be converted to an UNLOGGED
-batch as an optimization.
-
-[[unlogged-batches]]
-=== `UNLOGGED` batches
-
-By default, Cassandra uses a batch log to ensure all operations in a
-batch eventually complete or none will (note however that operations are
-only isolated within a single partition).
-
-There is a performance penalty for batch atomicity when a batch spans
-multiple partitions. If you do not want to incur this penalty, you can
-tell Cassandra to skip the batchlog with the `UNLOGGED` option. If the
-`UNLOGGED` option is used, a failed batch might leave the patch only
-partly applied.
-
-=== `COUNTER` batches
-
-Use the `COUNTER` option for batched counter updates. Unlike other
-updates in Cassandra, counter updates are not idempotent.
diff --git a/doc/modules/cassandra/pages/cql/functions.adoc b/doc/modules/cassandra/pages/cql/functions.adoc
deleted file mode 100644
index 93439a3..0000000
--- a/doc/modules/cassandra/pages/cql/functions.adoc
+++ /dev/null
@@ -1,512 +0,0 @@
-// Need some intro for UDF and native functions in general and point those to it.
-// [[cql-functions]][[native-functions]]
-
-== Functions
-
-CQL supports 2 main categories of functions:
-
-* xref:cql/functions.adoc#scalar-functions[scalar functions] that take a number of values and produce an output
-* xref:cql/functions.adoc#aggregate-functions[aggregate functions] that aggregate multiple rows resulting from a `SELECT` statement
-
-In both cases, CQL provides a number of native "hard-coded" functions as
-well as the ability to create new user-defined functions.
-
-[NOTE]
-.Note
-====
-By default, the use of user-defined functions is disabled by default for
-security concerns (even when enabled, the execution of user-defined
-functions is sandboxed and a "rogue" function should not be allowed to
-do evil, but no sandbox is perfect so using user-defined functions is
-opt-in). See the `user_defined_functions_enabled` in `cassandra.yaml` to
-enable them.
-====
-
-A function is identifier by its name:
-
-[source, bnf]
-----
-include::example$BNF/function.bnf[]
-----
-
-=== Scalar functions
-
-[[scalar-native-functions]]
-==== Native functions
-
-===== Cast
-
-The `cast` function can be used to converts one native datatype to
-another.
-
-The following table describes the conversions supported by the `cast`
-function. Cassandra will silently ignore any cast converting a datatype
-into its own datatype.
-
-[cols=",",options="header",]
-|===
-|From |To
-
-| `ascii` | `text`, `varchar`
-
-| `bigint` | `tinyint`, `smallint`, `int`, `float`, `double`, `decimal`, `varint`,
-`text`, `varchar`
-
-| `boolean` | `text`, `varchar`
-
-| `counter` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `decimal`,
-`varint`, `text`, `varchar`
-
-| `date` | `timestamp`
-
-| `decimal` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `varint`,
-`text`, `varchar`
-
-| `double` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `decimal`, `varint`,
-`text`, `varchar`
-
-| `float` | `tinyint`, `smallint`, `int`, `bigint`, `double`, `decimal`, `varint`,
-`text`, `varchar`
-
-| `inet` | `text`, `varchar`
-
-| `int` | `tinyint`, `smallint`, `bigint`, `float`, `double`, `decimal`, `varint`,
-`text`, `varchar`
-
-| `smallint` | `tinyint`, `int`, `bigint`, `float`, `double`, `decimal`, `varint`,
-`text`, `varchar`
-
-| `time` | `text`, `varchar`
-
-| `timestamp` | `date`, `text`, `varchar`
-
-| `timeuuid` | `timestamp`, `date`, `text`, `varchar`
-
-| `tinyint` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `decimal`,
-`varint`, `text`, `varchar`
-
-| `uuid` | `text`, `varchar`
-
-| `varint` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `decimal`,
-`text`, `varchar`
-|===
-
-The conversions rely strictly on Java's semantics. For example, the
-double value 1 will be converted to the text value '1.0'. For instance:
-
-[source,cql]
-----
-SELECT avg(cast(count as double)) FROM myTable
-----
-
-===== Token
-
-The `token` function computes the token for a given partition key.
-The exact signature of the token function depends on the table concerned and the partitioner used by the cluster.
-
-The type of the arguments of the `token` depend on the partition key column type. The returned type depends on the defined partitioner:
-
-[cols=",",options="header",]
-|===
-|Partitioner | Returned type
-| Murmur3Partitioner | `bigint`
-| RandomPartitioner | `varint`
-| ByteOrderedPartitioner | `blob`
-|===
-
-For example, consider the following table:
-
-[source,cql]
-----
-include::example$CQL/create_table_simple.cql[]
-----
-
-The table uses the default Murmur3Partitioner.
-The `token` function uses the single argument `text`, because the partition key is `userid` of text type.
-The returned type will be `bigint`.
-
-===== Uuid
-
-The `uuid` function takes no parameters and generates a random type 4
-uuid suitable for use in `INSERT` or `UPDATE` statements.
-
-===== Timeuuid functions
-
-====== `now`
-
-The `now` function takes no arguments and generates, on the coordinator
-node, a new unique timeuuid at the time the function is invoked. Note
-that this method is useful for insertion but is largely non-sensical in
-`WHERE` clauses.
-
-For example, a query of the form:
-
-[source,cql]
-----
-include::example$CQL/timeuuid_now.cql[]
-----
-
-will not return a result, by design, since the value returned by
-`now()` is guaranteed to be unique.
-
-`currentTimeUUID` is an alias of `now`.
-
-====== `minTimeuuid` and `maxTimeuuid`
-
-The `minTimeuuid` function takes a `timestamp` value `t`, either a timestamp or a date string.
-It returns a _fake_ `timeuuid` corresponding to the _smallest_ possible `timeuuid` for timestamp `t`.
-The `maxTimeuuid` works similarly, but returns the _largest_ possible `timeuuid`.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/timeuuid_min_max.cql[]
-----
-
-will select all rows where the `timeuuid` column `t` is later than `'2013-01-01 00:05+0000'` and earlier than `'2013-02-02 10:00+0000'`.
-The clause `t >= maxTimeuuid('2013-01-01 00:05+0000')` would still _not_ select a `timeuuid` generated exactly at '2013-01-01 00:05+0000', and is essentially equivalent to `t > maxTimeuuid('2013-01-01 00:05+0000')`.
-
-[NOTE]
-.Note
-====
-The values generated by `minTimeuuid` and `maxTimeuuid` are called _fake_ UUID because they do no respect the time-based UUID generation process
-specified by the http://www.ietf.org/rfc/rfc4122.txt[IETF RFC 4122].
-In particular, the value returned by these two methods will not be unique.
-Thus, only use these methods for *querying*, not for *insertion*, to prevent possible data overwriting.
-====
-
-===== Datetime functions
-
-====== Retrieving the current date/time
-
-The following functions can be used to retrieve the date/time at the
-time where the function is invoked:
-
-[cols=",",options="header",]
-|===
-|Function name |Output type
-
-| `currentTimestamp` | `timestamp`
-
-| `currentDate` | `date`
-
-| `currentTime` | `time`
-
-| `currentTimeUUID` | `timeUUID`
-|===
-
-For example the last two days of data can be retrieved using:
-
-[source,cql]
-----
-include::example$CQL/currentdate.cql[]
-----
-
-====== Time conversion functions
-
-A number of functions are provided to convert a `timeuuid`, a `timestamp` or a `date` into another `native` type.
-
-[cols=",,",options="header",]
-|===
-|Function name |Input type |Description
-
-| `toDate` | `timeuuid` | Converts the `timeuuid` argument into a `date` type
-
-| `toDate` | `timestamp` | Converts the `timestamp` argument into a `date` type
-
-| `toTimestamp` | `timeuuid` | Converts the `timeuuid` argument into a `timestamp` type
-
-| `toTimestamp` | `date` | Converts the `date` argument into a `timestamp` type
-
-| `toUnixTimestamp` | `timeuuid` | Converts the `timeuuid` argument into a `bigInt` raw value
-
-| `toUnixTimestamp` | `timestamp` | Converts the `timestamp` argument into a `bigInt` raw value
-
-| `toUnixTimestamp` | `date` | Converts the `date` argument into a `bigInt` raw value
-
-| `dateOf` | `timeuuid` | Similar to `toTimestamp(timeuuid)` (DEPRECATED)
-
-| `unixTimestampOf` | `timeuuid` | Similar to `toUnixTimestamp(timeuuid)` (DEPRECATED)
-|===
-
-===== Blob conversion functions
-
-A number of functions are provided to convert the native types into
-binary data, or a `blob`.
-For every xref:cql/types.adoc#native-types[type] supported by CQL, the function `typeAsBlob` takes a argument of type `type` and returns it as a `blob`.
-Conversely, the function `blobAsType` takes a 64-bit `blob` argument and converts it to a `bigint` value.
-For example, `bigintAsBlob(3)` returns `0x0000000000000003` and `blobAsBigint(0x0000000000000003)` returns `3`.
-
-[[user-defined-scalar-functions]]
-==== User-defined functions
-
-User-defined functions (UDFs) execute user-provided code in Cassandra.
-By default, Cassandra supports defining functions in _Java_ and _JavaScript_.
-Support for other JSR 223 compliant scripting languages, such as Python, Ruby, and Scala, is possible by adding a JAR to the classpath.
-
-UDFs are part of the Cassandra schema, and are automatically propagated to all nodes in the cluster.
-UDFs can be _overloaded_, so that multiple UDFs with different argument types can have the same function name.
-
-
-[NOTE]
-.Note
-====
-_JavaScript_ user-defined functions have been deprecated. They are planned for removal
-in the next major release.
-====
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/function_overload.cql[]
-----
-
-UDFs are susceptible to all of the normal problems with the chosen programming language.
-Accordingly, implementations should be safe against null pointer exceptions, illegal arguments, or any other potential source of exceptions.
-An exception during function execution will result in the entire statement failing.
-Valid queries for UDF use are `SELECT`, `INSERT` and `UPDATE` statements.
-
-_Complex_ types like collections, tuple types and user-defined types are valid argument and return types in UDFs.
-Tuple types and user-defined types use the DataStax Java Driver conversion functions.
-Please see the Java Driver documentation for details on handling tuple types and user-defined types.
-
-Arguments for functions can be literals or terms.
-Prepared statement placeholders can be used, too.
-
-Note the use the double dollar-sign syntax to enclose the UDF source code.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/function_dollarsign.cql[]
-----
-
-The implicitly available `udfContext` field (or binding for script UDFs) provides the necessary functionality to create new UDT and tuple values:
-
-[source,cql]
-----
-include::example$CQL/function_udfcontext.cql[]
-----
-
-The definition of the `UDFContext` interface can be found in the Apache Cassandra source code for `org.apache.cassandra.cql3.functions.UDFContext`.
-
-[source,java]
-----
-include::example$JAVA/udfcontext.java[]
-----
-
-Java UDFs already have some imports for common interfaces and classes defined. These imports are:
-
-[source,java]
-----
-include::example$JAVA/udf_imports.java[]
-----
-
-Please note, that these convenience imports are not available for script UDFs.
-
-[[create-function-statement]]
-==== CREATE FUNCTION statement
-
-Creating a new user-defined function uses the `CREATE FUNCTION` statement:
-
-[source,bnf]
-----
-include::example$BNF/create_function_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/create_function.cql[]
-----
-
-`CREATE FUNCTION` with the optional `OR REPLACE` keywords creates either a function or replaces an existing one with the same signature.
-A `CREATE FUNCTION` without `OR REPLACE` fails if a function with the same signature already exists.
-If the optional `IF NOT EXISTS` keywords are used, the function will only be created only if another function with the same signature does not
-exist.
-`OR REPLACE` and `IF NOT EXISTS` cannot be used together.
-
-Behavior for `null` input values must be defined for each function:
-
-* `RETURNS NULL ON NULL INPUT` declares that the function will always return `null` if any of the input arguments is `null`.
-* `CALLED ON NULL INPUT` declares that the function will always be executed.
-
-===== Function Signature
-
-Signatures are used to distinguish individual functions. The signature consists of a fully-qualified function name of the <keyspace>.<function_name> and a concatenated list of all the argument types.
-
-Note that keyspace names, function names and argument types are subject to the default naming conventions and case-sensitivity rules.
-
-Functions belong to a keyspace; if no keyspace is specified, the current keyspace is used.
-User-defined functions are not allowed in the system keyspaces.
-
-[[drop-function-statement]]
-==== DROP FUNCTION statement
-
-Dropping a function uses the `DROP FUNCTION` statement:
-
-[source, bnf]
-----
-include::example$BNF/drop_function_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/drop_function.cql[]
-----
-
-You must specify the argument types of the function, the arguments_signature, in the drop command if there are multiple overloaded functions with the same name but different signatures.
-`DROP FUNCTION` with the optional `IF EXISTS` keywords drops a function if it exists, but does not throw an error if it doesn't.
-
-[[aggregate-functions]]
-=== Aggregate functions
-
-Aggregate functions work on a set of rows.
-Values for each row are input, to return a single value for the set of rows aggregated.
-
-If `normal` columns, `scalar functions`, `UDT` fields, `writetime`, or `ttl` are selected together with aggregate functions, the values
-returned for them will be the ones of the first row matching the query.
-
-==== Native aggregates
-
-[[count-function]]
-===== Count
-
-The `count` function can be used to count the rows returned by a query.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/count.cql[]
-----
-
-It also can count the non-null values of a given column:
-
-[source,cql]
-----
-include::example$CQL/count_nonnull.cql[]
-----
-
-===== Max and Min
-
-The `max` and `min` functions compute the maximum and the minimum value returned by a query for a given column.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/min_max.cql[]
-----
-
-===== Sum
-
-The `sum` function sums up all the values returned by a query for a given column.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/sum.cql[]
-----
-
-===== Avg
-
-The `avg` function computes the average of all the values returned by a query for a given column.
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/avg.cql[]
-----
-
-[[user-defined-aggregates-functions]]
-==== User-Defined Aggregates (UDAs)
-
-User-defined aggregates allow the creation of custom aggregate functions.
-User-defined aggregates can be used in `SELECT` statement.
-
-Each aggregate requires an _initial state_ of type `STYPE` defined with the `INITCOND`value (default value: `null`).
-The first argument of the state function must have type `STYPE`.
-The remaining arguments of the state function must match the types of the user-defined aggregate arguments.
-The state function is called once for each row, and the value returned by the state function becomes the new state.
-After all rows are processed, the optional `FINALFUNC` is executed with last state value as its argument.
-
-The `STYPE` value is mandatory in order to distinguish possibly overloaded versions of the state and/or final function, since the
-overload can appear after creation of the aggregate.
-
-
-A complete working example for user-defined aggregates (assuming that a
-keyspace has been selected using the `USE` statement):
-
-[source,cql]
-----
-include::example$CQL/uda.cql[]
-----
-
-[[create-aggregate-statement]]
-==== CREATE AGGREGATE statement
-
-Creating (or replacing) a user-defined aggregate function uses the
-`CREATE AGGREGATE` statement:
-
-[source, bnf]
-----
-include::example$BNF/create_aggregate_statement.bnf[]
-----
-
-See above for a complete example.
-
-The `CREATE AGGREGATE` command with the optional `OR REPLACE` keywords creates either an aggregate or replaces an existing one with the same
-signature.
-A `CREATE AGGREGATE` without `OR REPLACE` fails if an aggregate with the same signature already exists.
-The `CREATE AGGREGATE` command with the optional `IF NOT EXISTS` keywords creates an aggregate if it does not already exist.
-The `OR REPLACE` and `IF NOT EXISTS` phrases cannot be used together.
-
-The `STYPE` value defines the type of the state value and must be specified.
-The optional `INITCOND` defines the initial state value for the aggregate; the default value is `null`.
-A non-null `INITCOND` must be specified for state functions that are declared with `RETURNS NULL ON NULL INPUT`.
-
-The `SFUNC` value references an existing function to use as the state-modifying function.
-The first argument of the state function must have type `STYPE`.
-The remaining arguments of the state function must match the types of the user-defined aggregate arguments.
-The state function is called once for each row, and the value returned by the state function becomes the new state.
-State is not updated for state functions declared with `RETURNS NULL ON NULL INPUT` and called with `null`.
-After all rows are processed, the optional `FINALFUNC` is executed with last state value as its argument.
-It must take only one argument with type `STYPE`, but the return type of the `FINALFUNC` may be a different type.
-A final function declared with `RETURNS NULL ON NULL INPUT` means that the aggregate's return value will be `null`, if the last state is `null`.
-
-If no `FINALFUNC` is defined, the overall return type of the aggregate function is `STYPE`.
-If a `FINALFUNC` is defined, it is the return type of that function.
-
-[[drop-aggregate-statement]]
-==== DROP AGGREGATE statement
-
-Dropping an user-defined aggregate function uses the `DROP AGGREGATE`
-statement:
-
-[source, bnf]
-----
-include::example$BNF/drop_aggregate_statement.bnf[]
-----
-
-For instance:
-
-[source,cql]
-----
-include::example$CQL/drop_aggregate.cql[]
-----
-
-The `DROP AGGREGATE` statement removes an aggregate created using `CREATE AGGREGATE`.
-You must specify the argument types of the aggregate to drop if there are multiple overloaded aggregates with the same name but a
-different signature.
-
-The `DROP AGGREGATE` command with the optional `IF EXISTS` keywords drops an aggregate if it exists, and does nothing if a function with the
-signature does not exist.
diff --git a/doc/modules/cassandra/pages/cql/index.adoc b/doc/modules/cassandra/pages/cql/index.adoc
deleted file mode 100644
index 4b43be3..0000000
--- a/doc/modules/cassandra/pages/cql/index.adoc
+++ /dev/null
@@ -1,24 +0,0 @@
-= The Cassandra Query Language (CQL)
-
-This document describes the Cassandra Query Language
-(CQL) version 3.
-Note that this document describes the last version of the language.
-However, the xref:cql/changes.adoc[changes] section provides the differences between the versions of CQL since version 3.0.
-
-CQL offers a model similar to SQL.
-The data is stored in *tables* containing *rows* of *columns*.
-For that reason, when used in this document, these terms (tables, rows and columns) have the same definition that they have in SQL.
-
-* xref:cql/definitions.adoc[Definitions]
-* xref:cql/types.adoc[Data types]
-* xref:cql/ddl.adoc[Data definition language]
-* xref:cql/dml.adoc[Data manipulation language]
-* xref:cql/operators.adoc[Operators]
-* xref:cql/indexes.adoc[Secondary indexes]
-* xref:cql/mvs.adoc[Materialized views]
-* xref:cql/functions.adoc[Functions]
-* xref:cql/json.adoc[JSON]
-* xref:cql/security.adoc[CQL security]
-* xref:cql/triggers.adoc[Triggers]
-* xref:cql/appendices.adoc[Appendices]
-* xref:cql/changes.adoc[Changes]
diff --git a/doc/modules/cassandra/pages/cql/json.adoc b/doc/modules/cassandra/pages/cql/json.adoc
deleted file mode 100644
index 7d0aa26..0000000
--- a/doc/modules/cassandra/pages/cql/json.adoc
+++ /dev/null
@@ -1,125 +0,0 @@
-= JSON Support
-
-Cassandra 2.2 introduces JSON support to `SELECT <select-statement>` and
-`INSERT <insert-statement>` statements.
-This support does not fundamentally alter the CQL API (for example, the schema is still
-enforced).
-It simply provides a convenient way to work with JSON documents.
-
-== SELECT JSON
-
-With `SELECT` statements, the `JSON` keyword is used to return each row as a single `JSON` encoded map.
-The remainder of the `SELECT` statement behavior is the same.
-
-The result map keys match the column names in a normal result set.
-For example, a statement like `SELECT JSON a, ttl(b) FROM ...` would result in a map with keys `"a"` and `"ttl(b)"`.
-However, there is one notable exception: for symmetry with `INSERT JSON` behavior, case-sensitive column names with upper-case letters will be surrounded with double quotes.
-For example, `SELECT JSON myColumn FROM ...` would result in a map key `"\"myColumn\""` with escaped quotes).
-
-The map values will JSON-encoded representations (as described below) of the result set values.
-
-== INSERT JSON
-
-With `INSERT` statements, the new `JSON` keyword can be used to enable
-inserting a `JSON` encoded map as a single row. The format of the `JSON`
-map should generally match that returned by a `SELECT JSON` statement on
-the same table. In particular, case-sensitive column names should be
-surrounded with double quotes. For example, to insert into a table with
-two columns named "myKey" and "value", you would do the following:
-
-[source,cql]
-----
-include::example$CQL/insert_json.cql[]
-----
-
-By default (or if `DEFAULT NULL` is explicitly used), a column omitted
-from the `JSON` map will be set to `NULL`, meaning that any pre-existing
-value for that column will be removed (resulting in a tombstone being
-created). Alternatively, if the `DEFAULT UNSET` directive is used after
-the value, omitted column values will be left unset, meaning that
-pre-existing values for those column will be preserved.
-
-== JSON Encoding of Cassandra Data Types
-
-Where possible, Cassandra will represent and accept data types in their
-native `JSON` representation. Cassandra will also accept string
-representations matching the CQL literal format for all single-field
-types. For example, floats, ints, UUIDs, and dates can be represented by
-CQL literal strings. However, compound types, such as collections,
-tuples, and user-defined types must be represented by native `JSON`
-collections (maps and lists) or a JSON-encoded string representation of
-the collection.
-
-The following table describes the encodings that Cassandra will accept
-in `INSERT JSON` values (and `fromJson()` arguments) as well as the
-format Cassandra will use when returning data for `SELECT JSON`
-statements (and `fromJson()`):
-
-[cols=",,,",options="header",]
-|===
-|Type |Formats accepted |Return format |Notes
-
-| `ascii` | string | string | Uses JSON's `\u` character escape
-
-| `bigint` | integer, string | integer | String must be valid 64 bit integer
-
-| `blob` | string | string | String should be 0x followed by an even number of hex digits
-
-| `boolean` | boolean, string | boolean | String must be "true" or "false"
-
-| `date` | string | string | Date in format `YYYY-MM-DD`, timezone UTC
-
-| `decimal` | integer, float, string | float | May exceed 32 or 64-bit IEEE-754 floating point precision in client-side decoder
-
-| `double` | integer, float, string | float | String must be valid integer or float
-
-| `float` | integer, float, string | float | String must be valid integer or float
-
-| `inet` | string | string | IPv4 or IPv6 address
-
-| `int` | integer, string | integer | String must be valid 32 bit integer
-
-| `list` | list, string | list | Uses JSON's native list representation
-
-| `map` | map, string | map | Uses JSON's native map representation
-
-| `smallint` | integer, string | integer | String must be valid 16 bit integer
-
-| `set` | list, string | list | Uses JSON's native list representation
-
-| `text` | string | string | Uses JSON's `\u` character escape
-
-| `time` | string | string | Time of day in format `HH-MM-SS[.fffffffff]`
-
-| `timestamp` | integer, string | string | A timestamp. Strings constant allows to input `timestamps
-as dates <timestamps>`. Datestamps with format `YYYY-MM-DD HH:MM:SS.SSS`
-are returned.
-
-| `timeuuid` | string | string | Type 1 UUID. See `constant` for the UUID format
-
-| `tinyint` | integer, string | integer | String must be valid 8 bit integer
-
-| `tuple` | list, string | list | Uses JSON's native list representation
-
-| `UDT` | map, string | map | Uses JSON's native map representation with field names as keys
-
-| `uuid` | string | string | See `constant` for the UUID format
-
-| `varchar` | string | string | Uses JSON's `\u` character escape
-
-| `varint` | integer, string | integer | Variable length; may overflow 32 or 64 bit integers in client-side decoder
-|===
-
-== The fromJson() Function
-
-The `fromJson()` function may be used similarly to `INSERT JSON`, but
-for a single column value. It may only be used in the `VALUES` clause of
-an `INSERT` statement or as one of the column values in an `UPDATE`,
-`DELETE`, or `SELECT` statement. For example, it cannot be used in the
-selection clause of a `SELECT` statement.
-
-== The toJson() Function
-
-The `toJson()` function may be used similarly to `SELECT JSON`, but for
-a single column value. It may only be used in the selection clause of a
-`SELECT` statement.
diff --git a/doc/modules/cassandra/pages/cql/security.adoc b/doc/modules/cassandra/pages/cql/security.adoc
deleted file mode 100644
index 904dea0..0000000
--- a/doc/modules/cassandra/pages/cql/security.adoc
+++ /dev/null
@@ -1,616 +0,0 @@
-role_name ::= identifier | string= Security
-
-[[cql-roles]]
-== Database Roles
-
-CQL uses database roles to represent users and group of users.
-Syntactically, a role is defined by:
-
-[source, bnf]
-----
-include::example$BNF/role_name.bnf[]
-----
-
-
-[[create-role-statement]]
-=== CREATE ROLE
-
-Creating a role uses the `CREATE ROLE` statement:
-
-[source, bnf]
-----
-include::example$BNF/create_role_statement.bnf[]
-----
-
-For instance:
-
-[source,cql]
-----
-include::example$CQL/create_role.cql[]
-----
-
-By default roles do not possess `LOGIN` privileges or `SUPERUSER`
-status.
-
-xref:cql/security.adoc#cql-permissions[Permissions] on database resources are granted to
-roles; types of resources include keyspaces, tables, functions and roles
-themselves. Roles may be granted to other roles to create hierarchical
-permissions structures; in these hierarchies, permissions and
-`SUPERUSER` status are inherited, but the `LOGIN` privilege is not.
-
-If a role has the `LOGIN` privilege, clients may identify as that role
-when connecting. For the duration of that connection, the client will
-acquire any roles and privileges granted to that role.
-
-Only a client with with the `CREATE` permission on the database roles
-resource may issue `CREATE ROLE` requests (see the
-xref:cql/security.adoc#cql-permissions[relevant section]), unless the client is a
-`SUPERUSER`. Role management in Cassandra is pluggable and custom
-implementations may support only a subset of the listed options.
-
-Role names should be quoted if they contain non-alphanumeric characters.
-
-==== Setting credentials for internal authentication
-
-Use the `WITH PASSWORD` clause to set a password for internal
-authentication, enclosing the password in single quotation marks.
-
-If internal authentication has not been set up or the role does not have
-`LOGIN` privileges, the `WITH PASSWORD` clause is not necessary.
-
-USE `WITH HASHED PASSWORD` to provide the jBcrypt hashed password directly. See the `hash_password` tool.
-
-==== Restricting connections to specific datacenters
-
-If a `network_authorizer` has been configured, you can restrict login
-roles to specific datacenters with the `ACCESS TO DATACENTERS` clause
-followed by a set literal of datacenters the user can access. Not
-specifiying datacenters implicitly grants access to all datacenters. The
-clause `ACCESS TO ALL DATACENTERS` can be used for explicitness, but
-there's no functional difference.
-
-==== Creating a role conditionally
-
-Attempting to create an existing role results in an invalid query
-condition unless the `IF NOT EXISTS` option is used. If the option is
-used and the role exists, the statement is a no-op:
-
-[source,cql]
-----
-include::example$CQL/create_role_ifnotexists.cql[]
-----
-
-[[alter-role-statement]]
-=== ALTER ROLE
-
-Altering a role options uses the `ALTER ROLE` statement:
-
-[source, bnf]
-----
-include::example$BNF/alter_role_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/alter_role.cql[]
-----
-If the role does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-USE `WITH HASHED PASSWORD` to provide the jBcrypt hashed password directly. See the `hash_password` tool.
-
-==== Restricting connections to specific datacenters
-
-If a `network_authorizer` has been configured, you can restrict login
-roles to specific datacenters with the `ACCESS TO DATACENTERS` clause
-followed by a set literal of datacenters the user can access. To remove
-any data center restrictions, use the `ACCESS TO ALL DATACENTERS`
-clause.
-
-Conditions on executing `ALTER ROLE` statements:
-
-* a client must have `SUPERUSER` status to alter the `SUPERUSER` status
-of another role
-* a client cannot alter the `SUPERUSER` status of any role it currently
-holds
-* a client can only modify certain properties of the role with which it
-identified at login (e.g. `PASSWORD`)
-* to modify properties of a role, the client must be granted `ALTER`
-`permission <cql-permissions>` on that role
-
-[[drop-role-statement]]
-=== DROP ROLE
-
-Dropping a role uses the `DROP ROLE` statement:
-
-[source, bnf]
-----
-include::example$BNF/drop_role_statement.bnf[]
-----
-
-`DROP ROLE` requires the client to have `DROP`
-`permission <cql-permissions>` on the role in question. In addition,
-client may not `DROP` the role with which it identified at login.
-Finally, only a client with `SUPERUSER` status may `DROP` another
-`SUPERUSER` role.
-
-Attempting to drop a role which does not exist results in an invalid
-query condition unless the `IF EXISTS` option is used. If the option is
-used and the role does not exist the statement is a no-op.
-
-[NOTE]
-.Note
-====
-DROP ROLE intentionally does not terminate any open user sessions.
-Currently connected sessions will remain connected and will retain the
-ability to perform any database actions which do not require
-xref:cql/security.adoc#authorization[authorization].
-However, if authorization is enabled, xref:cql/security.adoc#cql-permissions[permissions] of the dropped role are also revoked,
-subject to the xref:cql/security.adoc#auth-caching[caching options] configured in xref:cql/configuring.adoc#cassandra.yaml[cassandra-yaml] file.
-Should a dropped role be subsequently recreated and have new xref:security.adoc#grant-permission-statement[permissions] or
-xref:security.adoc#grant-role-statement[roles]` granted to it, any client sessions still
-connected will acquire the newly granted permissions and roles.
-====
-
-[[grant-role-statement]]
-=== GRANT ROLE
-
-Granting a role to another uses the `GRANT ROLE` statement:
-
-[source, bnf]
-----
-include::example$BNF/grant_role_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/grant_role.cql[]
-----
-
-This statement grants the `report_writer` role to `alice`. Any
-permissions granted to `report_writer` are also acquired by `alice`.
-
-Roles are modelled as a directed acyclic graph, so circular grants are
-not permitted. The following examples result in error conditions:
-
-[source,cql]
-----
-include::example$CQL/role_error.cql[]
-----
-
-[[revoke-role-statement]]
-=== REVOKE ROLE
-
-Revoking a role uses the `REVOKE ROLE` statement:
-
-[source, bnf]
-----
-include::example$BNF/revoke_role_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/revoke_role.cql[]
-----
-
-This statement revokes the `report_writer` role from `alice`. Any
-permissions that `alice` has acquired via the `report_writer` role are
-also revoked.
-
-[[list-roles-statement]]
-=== LIST ROLES
-
-All the known roles (in the system or granted to specific role) can be
-listed using the `LIST ROLES` statement:
-
-[source, bnf]
-----
-include::example$BNF/list_roles_statement.bnf[]
-----
-
-For instance:
-
-[source,cql]
-----
-include::example$CQL/list_roles.cql[]
-----
-
-returns all known roles in the system, this requires `DESCRIBE`
-permission on the database roles resource.
-
-This example enumerates all roles granted to `alice`, including those transitively
-acquired:
-
-[source,cql]
-----
-include::example$CQL/list_roles_of.cql[]
-----
-
-This example lists all roles directly granted to `bob` without including any of the
-transitively acquired ones:
-
-[source,cql]
-----
-include::example$CQL/list_roles_nonrecursive.cql[]
-----
-
-== Users
-
-Prior to the introduction of roles in Cassandra 2.2, authentication and
-authorization were based around the concept of a `USER`. For backward
-compatibility, the legacy syntax has been preserved with `USER` centric
-statements becoming synonyms for the `ROLE` based equivalents. In other
-words, creating/updating a user is just a different syntax for
-creating/updating a role.
-
-[[create-user-statement]]
-=== CREATE USER
-
-Creating a user uses the `CREATE USER` statement:
-
-[source, bnf]
-----
-include::example$BNF/create_user_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/create_user.cql[]
-----
-
-The `CREATE USER` command is equivalent to `CREATE ROLE` where the `LOGIN` option is `true`.
-So, the following pairs of statements are equivalent:
-
-[source,cql]
-----
-include::example$CQL/create_user_role.cql[]
-----
-
-[[alter-user-statement]]
-=== ALTER USER
-
-Altering the options of a user uses the `ALTER USER` statement:
-
-[source, bnf]
-----
-include::example$BNF/alter_user_statement.bnf[]
-----
-If the role does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-For example:
-
-[source,cql]
-----
-include::example$CQL/alter_user.cql[]
-----
-
-[[drop-user-statement]]
-=== DROP USER
-
-Dropping a user uses the `DROP USER` statement:
-
-[source, bnf]
-----
-include::example$BNF/drop_user_statement.bnf[]
-----
-
-[[list-users-statement]]
-=== LIST USERS
-
-Existing users can be listed using the `LIST USERS` statement:
-
-[source, bnf]
-----
-include::example$BNF/list_users_statement.bnf[]
-----
-
-Note that this statement is equivalent to xref:security.adoc#list-roles-statement[`LIST ROLES], but only roles with the `LOGIN` privilege are included in the output.
-
-== Data Control
-
-[[cql-permissions]]
-=== Permissions
-
-Permissions on resources are granted to roles; there are several
-different types of resources in Cassandra and each type is modelled
-hierarchically:
-
-* The hierarchy of Data resources, Keyspaces and Tables has the
-structure `ALL KEYSPACES` -> `KEYSPACE` -> `TABLE`.
-* Function resources have the structure `ALL FUNCTIONS` -> `KEYSPACE` ->
-`FUNCTION`
-* Resources representing roles have the structure `ALL ROLES` -> `ROLE`
-* Resources representing JMX ObjectNames, which map to sets of
-MBeans/MXBeans, have the structure `ALL MBEANS` -> `MBEAN`
-
-Permissions can be granted at any level of these hierarchies and they
-flow downwards. So granting a permission on a resource higher up the
-chain automatically grants that same permission on all resources lower
-down. For example, granting `SELECT` on a `KEYSPACE` automatically
-grants it on all `TABLES` in that `KEYSPACE`. Likewise, granting a
-permission on `ALL FUNCTIONS` grants it on every defined function,
-regardless of which keyspace it is scoped in. It is also possible to
-grant permissions on all functions scoped to a particular keyspace.
-
-Modifications to permissions are visible to existing client sessions;
-that is, connections need not be re-established following permissions
-changes.
-
-The full set of available permissions is:
-
-* `CREATE`
-* `ALTER`
-* `DROP`
-* `SELECT`
-* `MODIFY`
-* `AUTHORIZE`
-* `DESCRIBE`
-* `EXECUTE`
-
-Not all permissions are applicable to every type of resource. For
-instance, `EXECUTE` is only relevant in the context of functions or
-mbeans; granting `EXECUTE` on a resource representing a table is
-nonsensical. Attempting to `GRANT` a permission on resource to which it
-cannot be applied results in an error response. The following
-illustrates which permissions can be granted on which types of resource,
-and which statements are enabled by that permission.
-
-[cols=",,",options="header",]
-|===
-|Permission |Resource |Operations
-
-| `CREATE` | `ALL KEYSPACES` | `CREATE KEYSPACE` and `CREATE TABLE` in any keyspace
-
-| `CREATE` | `KEYSPACE` | `CREATE TABLE` in specified keyspace
-
-| `CREATE` | `ALL FUNCTIONS` | `CREATE FUNCTION` in any keyspace and `CREATE AGGREGATE` in any keyspace
-
-| `CREATE` | `ALL FUNCTIONS IN KEYSPACE` | `CREATE FUNCTION` and `CREATE AGGREGATE` in specified keyspace
-
-| `CREATE` | `ALL ROLES` | `CREATE ROLE`
-
-| `ALTER` | `ALL KEYSPACES` | `ALTER KEYSPACE` and `ALTER TABLE` in any keyspace
-
-| `ALTER` | `KEYSPACE` | `ALTER KEYSPACE` and `ALTER TABLE` in specified keyspace
-
-| `ALTER` | `TABLE` | `ALTER TABLE`
-
-| `ALTER` | `ALL FUNCTIONS` | `CREATE FUNCTION` and `CREATE AGGREGATE`: replacing any existing
-
-| `ALTER` | `ALL FUNCTIONS IN KEYSPACE` | `CREATE FUNCTION` and `CREATE AGGREGATE`: replacing existing in specified keyspace
-
-| `ALTER` | `FUNCTION` | `CREATE FUNCTION` and `CREATE AGGREGATE`: replacing existing
-
-| `ALTER` | `ALL ROLES` | `ALTER ROLE` on any role
-
-| `ALTER` | `ROLE` | `ALTER ROLE`
-
-| `DROP` | `ALL KEYSPACES` | `DROP KEYSPACE` and `DROP TABLE` in any keyspace
-
-| `DROP` | `KEYSPACE` | `DROP TABLE` in specified keyspace
-
-| `DROP` | `TABLE` | `DROP TABLE`
-
-| `DROP` | `ALL FUNCTIONS` | `DROP FUNCTION` and `DROP AGGREGATE` in any keyspace
-
-| `DROP` | `ALL FUNCTIONS IN KEYSPACE` | `DROP FUNCTION` and `DROP AGGREGATE` in specified keyspace
-
-| `DROP` | `FUNCTION` | `DROP FUNCTION`
-
-| `DROP` | `ALL ROLES` | `DROP ROLE` on any role
-
-| `DROP` | `ROLE` | `DROP ROLE`
-
-| `SELECT` | `ALL KEYSPACES` | `SELECT` on any table
-
-| `SELECT` | `KEYSPACE` | `SELECT` on any table in specified keyspace
-
-| `SELECT` | `TABLE` | `SELECT` on specified table
-
-| `SELECT` | `ALL MBEANS` | Call getter methods on any mbean
-
-| `SELECT` | `MBEANS` | Call getter methods on any mbean matching a wildcard pattern
-
-| `SELECT` | `MBEAN` | Call getter methods on named mbean
-
-| `MODIFY` | `ALL KEYSPACES` | `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` on any table
-
-| `MODIFY` | `KEYSPACE` | `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` on any table in specified
-keyspace
-
-| `MODIFY` | `TABLE` | `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` on specified table
-
-| `MODIFY` | `ALL MBEANS` | Call setter methods on any mbean
-
-| `MODIFY` | `MBEANS` | Call setter methods on any mbean matching a wildcard pattern
-
-| `MODIFY` | `MBEAN` | Call setter methods on named mbean
-
-| `AUTHORIZE` | `ALL KEYSPACES` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any table
-
-| `AUTHORIZE` | `KEYSPACE` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any table in specified keyspace
-
-| `AUTHORIZE` | `TABLE` | `GRANT PERMISSION` and `REVOKE PERMISSION` on specified table
-
-| `AUTHORIZE` | `ALL FUNCTIONS` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any function
-
-| `AUTHORIZE` | `ALL FUNCTIONS IN KEYSPACE` | `GRANT PERMISSION` and `REVOKE PERMISSION` in specified keyspace
-
-| `AUTHORIZE` | `FUNCTION` | `GRANT PERMISSION` and `REVOKE PERMISSION` on specified function
-
-| `AUTHORIZE` | `ALL MBEANS` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any mbean
-
-| `AUTHORIZE` | `MBEANS` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any mbean matching a wildcard pattern
-
-| `AUTHORIZE` | `MBEAN` | `GRANT PERMISSION` and `REVOKE PERMISSION` on named mbean
-
-| `AUTHORIZE` | `ALL ROLES` | `GRANT ROLE` and `REVOKE ROLE` on any role
-
-| `AUTHORIZE` | `ROLES` | `GRANT ROLE` and `REVOKE ROLE` on specified roles
-
-| `DESCRIBE` | `ALL ROLES` | `LIST ROLES` on all roles or only roles granted to another, specified role
-
-| `DESCRIBE` | `ALL MBEANS` | Retrieve metadata about any mbean from the platform's MBeanServer
-
-
-| `DESCRIBE` | `MBEANS` | Retrieve metadata about any mbean matching a wildcard patter from the
-platform's MBeanServer
-
-| `DESCRIBE` | `MBEAN` | Retrieve metadata about a named mbean from the platform's MBeanServer
-
-| `EXECUTE` | `ALL FUNCTIONS` | `SELECT`, `INSERT` and `UPDATE` using any function, and use of any
-function in `CREATE AGGREGATE`
-
-| `EXECUTE` | `ALL FUNCTIONS IN KEYSPACE` | `SELECT`, `INSERT` and `UPDATE` using any function in specified keyspace
-and use of any function in keyspace in `CREATE AGGREGATE`
-
-| `EXECUTE` | `FUNCTION` | `SELECT`, `INSERT` and `UPDATE` using specified function and use of the function in `CREATE AGGREGATE`
-
-| `EXECUTE` | `ALL MBEANS` | Execute operations on any mbean
-
-| `EXECUTE` | `MBEANS` | Execute operations on any mbean matching a wildcard pattern
-
-| `EXECUTE` | `MBEAN` | Execute operations on named mbean
-|===
-
-[[grant-permission-statement]]
-=== GRANT PERMISSION
-
-Granting a permission uses the `GRANT PERMISSION` statement:
-
-[source, bnf]
-----
-include::example$BNF/grant_permission_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/grant_perm.cql[]
-----
-
-This example gives any user with the role `data_reader` permission to execute
-`SELECT` statements on any table across all keyspaces:
-
-[source,cql]
-----
-include::example$CQL/grant_modify.cql[]
-----
-
-To give any user with the role `data_writer` permission to perform
-`UPDATE`, `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` queries on all
-tables in the `keyspace1` keyspace:
-
-[source,cql]
-----
-include::example$CQL/grant_drop.cql[]
-----
-
-To give any user with the `schema_owner` role permissions to `DROP` a specific
-`keyspace1.table1`:
-
-[source,cql]
-----
-include::example$CQL/grant_execute.cql[]
-----
-
-This command grants any user with the `report_writer` role permission to execute
-`SELECT`, `INSERT` and `UPDATE` queries which use the function
-`keyspace1.user_function( int )`:
-
-[source,cql]
-----
-include::example$CQL/grant_describe.cql[]
-----
-
-This grants any user with the `role_admin` role permission to view any
-and all roles in the system with a `LIST ROLES` statement.
-
-==== GRANT ALL
-
-When the `GRANT ALL` form is used, the appropriate set of permissions is
-determined automatically based on the target resource.
-
-==== Automatic Granting
-
-When a resource is created, via a `CREATE KEYSPACE`, `CREATE TABLE`,
-`CREATE FUNCTION`, `CREATE AGGREGATE` or `CREATE ROLE` statement, the
-creator (the role the database user who issues the statement is
-identified as), is automatically granted all applicable permissions on
-the new resource.
-
-[[revoke-permission-statement]]
-=== REVOKE PERMISSION
-
-Revoking a permission from a role uses the `REVOKE PERMISSION`
-statement:
-
-[source, bnf]
-----
-include::example$BNF/revoke_permission_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/revoke_perm.cql[]
-----
-
-Because of their function in normal driver operations, certain tables
-cannot have their `SELECT` permissions revoked. The
-following tables will be available to all authorized users regardless of
-their assigned role:
-
-[source,cql]
-----
-include::example$CQL/no_revoke.cql[]
-----
-
-[[list-permissions-statement]]
-=== LIST PERMISSIONS
-
-Listing granted permissions uses the `LIST PERMISSIONS` statement:
-
-[source, bnf]
-----
-include::example$BNF/list_permissions_statement.bnf[]
-----
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/list_perm.cql[]
-----
-
-Show all permissions granted to `alice`, including those acquired
-transitively from any other roles:
-
-[source,cql]
-----
-include::example$CQL/list_all_perm.cql[]
-----
-
-Show all permissions on `keyspace1.table1` granted to `bob`, including
-those acquired transitively from any other roles. This also includes any
-permissions higher up the resource hierarchy which can be applied to
-`keyspace1.table1`. For example, should `bob` have `ALTER` permission on
-`keyspace1`, that would be included in the results of this query. Adding
-the `NORECURSIVE` switch restricts the results to only those permissions
-which were directly granted to `bob` or one of `bob`'s roles:
-
-[source,cql]
-----
-include::example$CQL/list_select_perm.cql[]
-----
-
-Show any permissions granted to `carlos` or any of `carlos`'s roles,
-limited to `SELECT` permissions on any resource.
diff --git a/doc/modules/cassandra/pages/cql/types.adoc b/doc/modules/cassandra/pages/cql/types.adoc
deleted file mode 100644
index 17c78b5..0000000
--- a/doc/modules/cassandra/pages/cql/types.adoc
+++ /dev/null
@@ -1,539 +0,0 @@
-= Data Types
-
-CQL is a typed language and supports a rich set of data types, including
-xref:cql/types.adoc#native-types[native types], xref:cql/types.adoc#collections[collection types],
-xref:cql/types.adoc#udts[user-defined types], xref:cql/types.adoc#tuples[tuple types], and xref:cql/types.adoc#custom-types[custom
-types]:
-
-[source, bnf]
-----
-include::example$BNF/cql_type.bnf[]
-----
-
-== Native types
-
-The native types supported by CQL are:
-
-[source, bnf]
-----
-include::example$BNF/native_type.bnf[]
-----
-
-The following table gives additional informations on the native data
-types, and on which kind of xref:cql/definitions.adoc#constants[constants] each type supports:
-
-[cols=",,",options="header",]
-|===
-| Type | Constants supported | Description
-
-| `ascii` | `string` | ASCII character string
-| `bigint` | `integer` | 64-bit signed long
-| `blob` | `blob` | Arbitrary bytes (no validation)
-| `boolean` | `boolean` | Either `true` or `false`
-| `counter` | `integer` | Counter column (64-bit signed value). See `counters` for details.
-| `date` | `integer`, `string` | A date (with no corresponding time value). See `dates` below for details.
-| `decimal` | `integer`, `float` | Variable-precision decimal
-| `double` | `integer` `float` | 64-bit IEEE-754 floating point
-| `duration` | `duration`, | A duration with nanosecond precision. See `durations` below for details.
-| `float` | `integer`, `float` | 32-bit IEEE-754 floating point
-| `inet` | `string` | An IP address, either IPv4 (4 bytes long) or IPv6 (16 bytes long). Note
-that there is no `inet` constant, IP address should be input as strings.
-| `int` | `integer` | 32-bit signed int
-| `smallint` | `integer` | 16-bit signed int
-| `text` | `string` | UTF8 encoded string
-| `time` | `integer`, `string` | A time (with no corresponding date value) with nanosecond precision. See
-`times` below for details.
-| `timestamp` | `integer`, `string` | A timestamp (date and time) with millisecond precision. See `timestamps`
-below for details.
-| `timeuuid` | `uuid` | Version 1 https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID],
-generally used as a “conflict-free” timestamp. Also see `timeuuid-functions`.
-| `tinyint` | `integer` | 8-bit signed int
-| `uuid` | `uuid` | A https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] (of any version)
-| `varchar` | `string` | UTF8 encoded string
-| `varint` | `integer` | Arbitrary-precision integer
-|===
-
-=== Counters
-
-The `counter` type is used to define _counter columns_. A counter column
-is a column whose value is a 64-bit signed integer and on which 2
-operations are supported: incrementing and decrementing (see the
-xref:cql/dml.adoc#update-statement[UPDATE] statement for syntax).
-Note that the value of a counter cannot
-be set: a counter does not exist until first incremented/decremented,
-and that first increment/decrement is made as if the prior value was 0.
-
-[[counter-limitations]]
-Counters have a number of important limitations:
-
-* They cannot be used for columns part of the `PRIMARY KEY` of a table.
-* A table that contains a counter can only contain counters. In other
-words, either all the columns of a table outside the `PRIMARY KEY` have
-the `counter` type, or none of them have it.
-* Counters do not support xref:cql/dml.adoc#writetime-and-ttl-function[expiration].
-* The deletion of counters is supported, but is only guaranteed to work
-the first time you delete a counter. In other words, you should not
-re-update a counter that you have deleted (if you do, proper behavior is
-not guaranteed).
-* Counter updates are, by nature, not
-https://en.wikipedia.org/wiki/Idempotence[idemptotent]. An important
-consequence is that if a counter update fails unexpectedly (timeout or
-loss of connection to the coordinator node), the client has no way to
-know if the update has been applied or not. In particular, replaying the
-update may or may not lead to an over count.
-
-[[timestamps]]
-== Working with timestamps
-
-Values of the `timestamp` type are encoded as 64-bit signed integers
-representing a number of milliseconds since the standard base time known
-as https://en.wikipedia.org/wiki/Unix_time[the epoch]: January 1 1970 at
-00:00:00 GMT.
-
-Timestamps can be input in CQL either using their value as an `integer`,
-or using a `string` that represents an
-https://en.wikipedia.org/wiki/ISO_8601[ISO 8601] date. For instance, all
-of the values below are valid `timestamp` values for Mar 2, 2011, at
-04:05:00 AM, GMT:
-
-* `1299038700000`
-* `'2011-02-03 04:05+0000'`
-* `'2011-02-03 04:05:00+0000'`
-* `'2011-02-03 04:05:00.000+0000'`
-* `'2011-02-03T04:05+0000'`
-* `'2011-02-03T04:05:00+0000'`
-* `'2011-02-03T04:05:00.000+0000'`
-
-The `+0000` above is an RFC 822 4-digit time zone specification; `+0000`
-refers to GMT. US Pacific Standard Time is `-0800`. The time zone may be
-omitted if desired (`'2011-02-03 04:05:00'`), and if so, the date will
-be interpreted as being in the time zone under which the coordinating
-Cassandra node is configured. There are however difficulties inherent in
-relying on the time zone configuration being as expected, so it is
-recommended that the time zone always be specified for timestamps when
-feasible.
-
-The time of day may also be omitted (`'2011-02-03'` or
-`'2011-02-03+0000'`), in which case the time of day will default to
-00:00:00 in the specified or default time zone. However, if only the
-date part is relevant, consider using the xref:cql/types.adoc#dates[date] type.
-
-[[dates]]
-== Date type
-
-Values of the `date` type are encoded as 32-bit unsigned integers
-representing a number of days with “the epoch” at the center of the
-range (2^31). Epoch is January 1st, 1970
-
-For xref:cql/types.adoc#timestamps[timestamps], a date can be input either as an
-`integer` or using a date `string`. In the later case, the format should
-be `yyyy-mm-dd` (so `'2011-02-03'` for instance).
-
-[[times]]
-== Time type
-
-Values of the `time` type are encoded as 64-bit signed integers
-representing the number of nanoseconds since midnight.
-
-For xref:cql/types.adoc#timestamps[timestamps], a time can be input either as an
-`integer` or using a `string` representing the time. In the later case,
-the format should be `hh:mm:ss[.fffffffff]` (where the sub-second
-precision is optional and if provided, can be less than the nanosecond).
-So for instance, the following are valid inputs for a time:
-
-* `'08:12:54'`
-* `'08:12:54.123'`
-* `'08:12:54.123456'`
-* `'08:12:54.123456789'`
-
-[[durations]]
-== Duration type
-
-Values of the `duration` type are encoded as 3 signed integer of
-variable lengths. The first integer represents the number of months, the
-second the number of days and the third the number of nanoseconds. This
-is due to the fact that the number of days in a month can change, and a
-day can have 23 or 25 hours depending on the daylight saving.
-Internally, the number of months and days are decoded as 32 bits
-integers whereas the number of nanoseconds is decoded as a 64 bits
-integer.
-
-A duration can be input as:
-
-* `(quantity unit)+` like `12h30m` where the unit can be:
-** `y`: years (12 months)
-** `mo`: months (1 month)
-** `w`: weeks (7 days)
-** `d`: days (1 day)
-** `h`: hours (3,600,000,000,000 nanoseconds)
-** `m`: minutes (60,000,000,000 nanoseconds)
-** `s`: seconds (1,000,000,000 nanoseconds)
-** `ms`: milliseconds (1,000,000 nanoseconds)
-** `us` or `µs` : microseconds (1000 nanoseconds)
-** `ns`: nanoseconds (1 nanosecond)
-* ISO 8601 format: `P[n]Y[n]M[n]DT[n]H[n]M[n]S or P[n]W`
-* ISO 8601 alternative format: `P[YYYY]-[MM]-[DD]T[hh]:[mm]:[ss]`
-
-For example:
-
-[source,cql]
-----
-include::example$CQL/insert_duration.cql[]
-----
-
-[[duration-limitation]]
-Duration columns cannot be used in a table's `PRIMARY KEY`. This
-limitation is due to the fact that durations cannot be ordered. It is
-effectively not possible to know if `1mo` is greater than `29d` without
-a date context.
-
-A `1d` duration is not equal to a `24h` one as the duration type has
-been created to be able to support daylight saving.
-
-== Collections
-
-CQL supports three kinds of collections: `maps`, `sets` and `lists`. The
-types of those collections is defined by:
-
-[source,bnf]
-----
-include::example$BNF/collection_type.bnf[]
-----
-
-and their values can be inputd using collection literals:
-
-[source,bnf]
-----
-include::example$BNF/collection_literal.bnf[]
-----
-
-Note however that neither `bind_marker` nor `NULL` are supported inside
-collection literals.
-
-=== Noteworthy characteristics
-
-Collections are meant for storing/denormalizing relatively small amount
-of data. They work well for things like “the phone numbers of a given
-user”, “labels applied to an email”, etc. But when items are expected to
-grow unbounded (“all messages sent by a user”, “events registered by a
-sensor”...), then collections are not appropriate and a specific table
-(with clustering columns) should be used. Concretely, (non-frozen)
-collections have the following noteworthy characteristics and
-limitations:
-
-* Individual collections are not indexed internally. Which means that
-even to access a single element of a collection, the while collection
-has to be read (and reading one is not paged internally).
-* While insertion operations on sets and maps never incur a
-read-before-write internally, some operations on lists do. Further, some
-lists operations are not idempotent by nature (see the section on
-xref:cql/types.adoc#lists[lists] below for details), making their retry in case of
-timeout problematic. It is thus advised to prefer sets over lists when
-possible.
-
-Please note that while some of those limitations may or may not be
-removed/improved upon in the future, it is a anti-pattern to use a
-(single) collection to store large amounts of data.
-
-=== Maps
-
-A `map` is a (sorted) set of key-value pairs, where keys are unique and
-the map is sorted by its keys. You can define and insert a map with:
-
-[source,cql]
-----
-include::example$CQL/map.cql[]
-----
-
-Further, maps support:
-
-* Updating or inserting one or more elements:
-+
-[source,cql]
-----
-include::example$CQL/update_map.cql[]
-----
-* Removing one or more element (if an element doesn't exist, removing it
-is a no-op but no error is thrown):
-+
-[source,cql]
-----
-include::example$CQL/delete_map.cql[]
-----
-+
-Note that for removing multiple elements in a `map`, you remove from it
-a `set` of keys.
-
-Lastly, TTLs are allowed for both `INSERT` and `UPDATE`, but in both
-case the TTL set only apply to the newly inserted/updated elements. In
-other words:
-
-[source,cql]
-----
-include::example$CQL/update_ttl_map.cql[]
-----
-
-will only apply the TTL to the `{ 'color' : 'green' }` record, the rest
-of the map remaining unaffected.
-
-=== Sets
-
-A `set` is a (sorted) collection of unique values. You can define and
-insert a map with:
-
-[source,cql]
-----
-include::example$CQL/set.cql[]
-----
-
-Further, sets support:
-
-* Adding one or multiple elements (as this is a set, inserting an
-already existing element is a no-op):
-+
-[source,cql]
-----
-include::example$CQL/update_set.cql[]
-----
-* Removing one or multiple elements (if an element doesn't exist,
-removing it is a no-op but no error is thrown):
-+
-[source,cql]
-----
-include::example$CQL/delete_set.cql[]
-----
-
-Lastly, for xref:cql/types.adoc#sets[sets], TTLs are only applied to newly inserted values.
-
-=== Lists
-
-[NOTE]
-.Note
-====
-As mentioned above and further discussed at the end of this section,
-lists have limitations and specific performance considerations that you
-should take into account before using them. In general, if you can use a
-xref:cql/types.adoc#sets[set] instead of list, always prefer a set.
-====
-
-A `list` is a (sorted) collection of non-unique values where
-elements are ordered by there position in the list. You can define and
-insert a list with:
-
-[source,cql]
-----
-include::example$CQL/list.cql[]
-----
-
-Further, lists support:
-
-* Appending and prepending values to a list:
-+
-[source,cql]
-----
-include::example$CQL/update_list.cql[]
-----
-
-[WARNING]
-.Warning
-====
-The append and prepend operations are not idempotent by nature. So in
-particular, if one of these operation timeout, then retrying the
-operation is not safe and it may (or may not) lead to
-appending/prepending the value twice.
-====
-
-* Setting the value at a particular position in a list that has a pre-existing element for that position. An error
-will be thrown if the list does not have the position.:
-+
-[source,cql]
-----
-include::example$CQL/update_particular_list_element.cql[]
-----
-* Removing an element by its position in the list that has a pre-existing element for that position. An error
-will be thrown if the list does not have the position. Further, as the operation removes an
-element from the list, the list size will decrease by one element, shifting
-the position of all the following elements one forward:
-+
-[source,cql]
-----
-include::example$CQL/delete_element_list.cql[]
-----
-
-* Deleting _all_ the occurrences of particular values in the list (if a
-particular element doesn't occur at all in the list, it is simply
-ignored and no error is thrown):
-+
-[source,cql]
-----
-include::example$CQL/delete_all_elements_list.cql[]
-----
-
-[WARNING]
-.Warning
-====
-Setting and removing an element by position and removing occurences of
-particular values incur an internal _read-before-write_. These operations will
-run slowly and use more resources than usual updates (with the
-exclusion of conditional write that have their own cost).
-====
-
-Lastly, for xref:cql/types.adoc#lists[lists], TTLs only apply to newly inserted values.
-
-[[udts]]
-== User-Defined Types (UDTs)
-
-CQL support the definition of user-defined types (UDTs). Such a
-type can be created, modified and removed using the
-`create_type_statement`, `alter_type_statement` and
-`drop_type_statement` described below. But once created, a UDT is simply
-referred to by its name:
-
-[source, bnf]
-----
-include::example$BNF/udt.bnf[]
-----
-
-=== Creating a UDT
-
-Creating a new user-defined type is done using a `CREATE TYPE` statement
-defined by:
-
-[source, bnf]
-----
-include::example$BNF/create_type.bnf[]
-----
-
-A UDT has a name (used to declared columns of that type) and is a set of
-named and typed fields. Fields name can be any type, including
-collections or other UDT. For instance:
-
-[source,cql]
-----
-include::example$CQL/udt.cql[]
-----
-
-Things to keep in mind about UDTs:
-
-* Attempting to create an already existing type will result in an error
-unless the `IF NOT EXISTS` option is used. If it is used, the statement
-will be a no-op if the type already exists.
-* A type is intrinsically bound to the keyspace in which it is created,
-and can only be used in that keyspace. At creation, if the type name is
-prefixed by a keyspace name, it is created in that keyspace. Otherwise,
-it is created in the current keyspace.
-* As of Cassandra , UDT have to be frozen in most cases, hence the
-`frozen<address>` in the table definition above. Please see the section
-on xref:cql/types.adoc#frozen[frozen] for more details.
-
-=== UDT literals
-
-Once a used-defined type has been created, value can be input using a
-UDT literal:
-
-[source,bnf]
-----
-include::example$BNF/udt_literal.bnf[]
-----
-
-In other words, a UDT literal is like a xref:cql/types.adoc#maps[map]` literal but its
-keys are the names of the fields of the type. For instance, one could
-insert into the table define in the previous section using:
-
-[source,cql]
-----
-include::example$CQL/insert_udt.cql[]
-----
-
-To be valid, a UDT literal can only include fields defined by the
-type it is a literal of, but it can omit some fields (these will be set to `NULL`).
-
-=== Altering a UDT
-
-An existing user-defined type can be modified using an `ALTER TYPE`
-statement:
-
-[source,bnf]
-----
-include::example$BNF/alter_udt_statement.bnf[]
-----
-If the type does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-You can:
-
-* Add a new field to the type (`ALTER TYPE address ADD country text`).
-That new field will be `NULL` for any values of the type created before
-the addition. If the new field exists, the statement will return an error, unless `IF NOT EXISTS` is used in which case the operation is a no-op.
-* Rename the fields of the type. If the field(s) does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
-
-[source,cql]
-----
-include::example$CQL/rename_udt_field.cql[]
-----
-
-=== Dropping a UDT
-
-You can drop an existing user-defined type using a `DROP TYPE`
-statement:
-
-[source,bnf]
-----
-include::example$BNF/drop_udt_statement.bnf[]
-----
-
-Dropping a type results in the immediate, irreversible removal of that
-type. However, attempting to drop a type that is still in use by another
-type, table or function will result in an error.
-
-If the type dropped does not exist, an error will be returned unless
-`IF EXISTS` is used, in which case the operation is a no-op.
-
-== Tuples
-
-CQL also support tuples and tuple types (where the elements can be of
-different types). Functionally, tuples can be though as anonymous UDT
-with anonymous fields. Tuple types and tuple literals are defined by:
-
-[source,bnf]
-----
-include::example$BNF/tuple.bnf[]
-----
-
-and can be created:
-
-[source,cql]
-----
-include::example$CQL/tuple.cql[]
-----
-
-Unlike other composed types, like collections and UDTs, a tuple is always
-`frozen <frozen>` (without the need of the `frozen` keyword)
-and it is not possible to update only some elements of a tuple (without
-updating the whole tuple). Also, a tuple literal should always have the
-same number of value than declared in the type it is a tuple of (some of
-those values can be null but they need to be explicitly declared as so).
-
-== Custom Types
-
-[NOTE]
-.Note
-====
-Custom types exists mostly for backward compatibility purposes and their
-usage is discouraged. Their usage is complex, not user friendly and the
-other provided types, particularly xref:cql/types.adoc#udts[user-defined types], should
-almost always be enough.
-====
-
-A custom type is defined by:
-
-[source,bnf]
-----
-include::example$BNF/custom_type.bnf[]
-----
-
-A custom type is a `string` that contains the name of Java class that
-extends the server side `AbstractType` class and that can be loaded by
-Cassandra (it should thus be in the `CLASSPATH` of every node running
-Cassandra). That class will define what values are valid for the type
-and how the time sorts when used for a clustering column. For any other
-purpose, a value of a custom type is the same than that of a `blob`, and
-can in particular be input using the `blob` literal syntax.
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_conceptual.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_conceptual.adoc
deleted file mode 100644
index c1e1027..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_conceptual.adoc
+++ /dev/null
@@ -1,44 +0,0 @@
-= Conceptual Data Modeling
-
-First, let’s create a simple domain model that is easy to understand in
-the relational world, and then see how you might map it from a
-relational to a distributed hashtable model in Cassandra.
-
-Let's use an example that is complex enough to show the various data
-structures and design patterns, but not something that will bog you down
-with details. Also, a domain that’s familiar to everyone will allow you
-to concentrate on how to work with Cassandra, not on what the
-application domain is all about.
-
-For example, let's use a domain that is easily understood and that
-everyone can relate to: making hotel reservations.
-
-The conceptual domain includes hotels, guests that stay in the hotels, a
-collection of rooms for each hotel, the rates and availability of those
-rooms, and a record of reservations booked for guests. Hotels typically
-also maintain a collection of “points of interest,” which are parks,
-museums, shopping galleries, monuments, or other places near the hotel
-that guests might want to visit during their stay. Both hotels and
-points of interest need to maintain geolocation data so that they can be
-found on maps for mashups, and to calculate distances.
-
-The conceptual domain is depicted below using the entity–relationship
-model popularized by Peter Chen. This simple diagram represents the
-entities in the domain with rectangles, and attributes of those entities
-with ovals. Attributes that represent unique identifiers for items are
-underlined. Relationships between entities are represented as diamonds,
-and the connectors between the relationship and each entity show the
-multiplicity of the connection.
-
-image::data_modeling_hotel_erd.png[image]
-
-Obviously, in the real world, there would be many more considerations
-and much more complexity. For example, hotel rates are notoriously
-dynamic, and calculating them involves a wide array of factors. Here
-you’re defining something complex enough to be interesting and touch on
-the important points, but simple enough to maintain the focus on
-learning Cassandra.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_logical.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_logical.adoc
deleted file mode 100644
index bcbfa78..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_logical.adoc
+++ /dev/null
@@ -1,195 +0,0 @@
-= Logical Data Modeling
-
-Now that you have defined your queries, you’re ready to begin designing
-Cassandra tables. First, create a logical model containing a table for
-each query, capturing entities and relationships from the conceptual
-model.
-
-To name each table, you’ll identify the primary entity type for which
-you are querying and use that to start the entity name. If you are
-querying by attributes of other related entities, append those to the
-table name, separated with `_by_`. For example, `hotels_by_poi`.
-
-Next, you identify the primary key for the table, adding partition key
-columns based on the required query attributes, and clustering columns
-in order to guarantee uniqueness and support desired sort ordering.
-
-The design of the primary key is extremely important, as it will
-determine how much data will be stored in each partition and how that
-data is organized on disk, which in turn will affect how quickly
-Cassandra processes reads.
-
-Complete each table by adding any additional attributes identified by
-the query. If any of these additional attributes are the same for every
-instance of the partition key, mark the column as static.
-
-Now that was a pretty quick description of a fairly involved process, so
-it will be worthwhile to work through a detailed example. First, let’s
-introduce a notation that you can use to represent logical models.
-
-Several individuals within the Cassandra community have proposed
-notations for capturing data models in diagrammatic form. This document
-uses a notation popularized by Artem Chebotko which provides a simple,
-informative way to visualize the relationships between queries and
-tables in your designs. This figure shows the Chebotko notation for a
-logical data model.
-
-image::data_modeling_chebotko_logical.png[image]
-
-Each table is shown with its title and a list of columns. Primary key
-columns are identified via symbols such as *K* for partition key columns
-and **C**↑ or **C**↓ to represent clustering columns. Lines are shown
-entering tables or between tables to indicate the queries that each
-table is designed to support.
-
-== Hotel Logical Data Model
-
-The figure below shows a Chebotko logical data model for the queries
-involving hotels, points of interest, rooms, and amenities. One thing
-you'll notice immediately is that the Cassandra design doesn’t include
-dedicated tables for rooms or amenities, as you had in the relational
-design. This is because the workflow didn’t identify any queries
-requiring this direct access.
-
-image::data_modeling_hotel_logical.png[image]
-
-Let’s explore the details of each of these tables.
-
-The first query Q1 is to find hotels near a point of interest, so you’ll
-call this table `hotels_by_poi`. Searching by a named point of interest
-is a clue that the point of interest should be a part of the primary
-key. Let’s reference the point of interest by name, because according to
-the workflow that is how users will start their search.
-
-You’ll note that you certainly could have more than one hotel near a
-given point of interest, so you’ll need another component in the primary
-key in order to make sure you have a unique partition for each hotel. So
-you add the hotel key as a clustering column.
-
-An important consideration in designing your table’s primary key is
-making sure that it defines a unique data element. Otherwise you run the
-risk of accidentally overwriting data.
-
-Now for the second query (Q2), you’ll need a table to get information
-about a specific hotel. One approach would have been to put all of the
-attributes of a hotel in the `hotels_by_poi` table, but you added only
-those attributes that were required by the application workflow.
-
-From the workflow diagram, you know that the `hotels_by_poi` table is
-used to display a list of hotels with basic information on each hotel,
-and the application knows the unique identifiers of the hotels returned.
-When the user selects a hotel to view details, you can then use Q2,
-which is used to obtain details about the hotel. Because you already
-have the `hotel_id` from Q1, you use that as a reference to the hotel
-you’re looking for. Therefore the second table is just called `hotels`.
-
-Another option would have been to store a set of `poi_names` in the
-hotels table. This is an equally valid approach. You’ll learn through
-experience which approach is best for your application.
-
-Q3 is just a reverse of Q1—looking for points of interest near a hotel,
-rather than hotels near a point of interest. This time, however, you
-need to access the details of each point of interest, as represented by
-the `pois_by_hotel` table. As previously, you add the point of interest
-name as a clustering key to guarantee uniqueness.
-
-At this point, let’s now consider how to support query Q4 to help the
-user find available rooms at a selected hotel for the nights they are
-interested in staying. Note that this query involves both a start date
-and an end date. Because you’re querying over a range instead of a
-single date, you know that you’ll need to use the date as a clustering
-key. Use the `hotel_id` as a primary key to group room data for each
-hotel on a single partition, which should help searches be super fast.
-Let’s call this the `available_rooms_by_hotel_date` table.
-
-To support searching over a range, use `clustering columns
-<clustering-columns>` to store attributes that you need to access in a
-range query. Remember that the order of the clustering columns is
-important.
-
-The design of the `available_rooms_by_hotel_date` table is an instance
-of the *wide partition* pattern. This pattern is sometimes called the
-*wide row* pattern when discussing databases that support similar
-models, but wide partition is a more accurate description from a
-Cassandra perspective. The essence of the pattern is to group multiple
-related rows in a partition in order to support fast access to multiple
-rows within the partition in a single query.
-
-In order to round out the shopping portion of the data model, add the
-`amenities_by_room` table to support Q5. This will allow users to view
-the amenities of one of the rooms that is available for the desired stay
-dates.
-
-== Reservation Logical Data Model
-
-Now let's switch gears to look at the reservation queries. The figure
-shows a logical data model for reservations. You’ll notice that these
-tables represent a denormalized design; the same data appears in
-multiple tables, with differing keys.
-
-image::data_modeling_reservation_logical.png[image]
-
-In order to satisfy Q6, the `reservations_by_guest` table can be used to
-look up the reservation by guest name. You could envision query Q7 being
-used on behalf of a guest on a self-serve website or a call center agent
-trying to assist the guest. Because the guest name might not be unique,
-you include the guest ID here as a clustering column as well.
-
-Q8 and Q9 in particular help to remind you to create queries that
-support various stakeholders of the application, not just customers but
-staff as well, and perhaps even the analytics team, suppliers, and so
-on.
-
-The hotel staff might wish to see a record of upcoming reservations by
-date in order to get insight into how the hotel is performing, such as
-what dates the hotel is sold out or undersold. Q8 supports the retrieval
-of reservations for a given hotel by date.
-
-Finally, you create a `guests` table. This provides a single location
-that used to store guest information. In this case, you specify a
-separate unique identifier for guest records, as it is not uncommon for
-guests to have the same name. In many organizations, a customer database
-such as the `guests` table would be part of a separate customer
-management application, which is why other guest access patterns were
-omitted from the example.
-
-== Patterns and Anti-Patterns
-
-As with other types of software design, there are some well-known
-patterns and anti-patterns for data modeling in Cassandra. You’ve
-already used one of the most common patterns in this hotel model—the
-wide partition pattern.
-
-The *time series* pattern is an extension of the wide partition pattern.
-In this pattern, a series of measurements at specific time intervals are
-stored in a wide partition, where the measurement time is used as part
-of the partition key. This pattern is frequently used in domains
-including business analysis, sensor data management, and scientific
-experiments.
-
-The time series pattern is also useful for data other than measurements.
-Consider the example of a banking application. You could store each
-customer’s balance in a row, but that might lead to a lot of read and
-write contention as various customers check their balance or make
-transactions. You’d probably be tempted to wrap a transaction around
-writes just to protect the balance from being updated in error. In
-contrast, a time series–style design would store each transaction as a
-timestamped row and leave the work of calculating the current balance to
-the application.
-
-One design trap that many new users fall into is attempting to use
-Cassandra as a queue. Each item in the queue is stored with a timestamp
-in a wide partition. Items are appended to the end of the queue and read
-from the front, being deleted after they are read. This is a design that
-seems attractive, especially given its apparent similarity to the time
-series pattern. The problem with this approach is that the deleted items
-are now `tombstones <asynch-deletes>` that Cassandra must scan past in
-order to read from the front of the queue. Over time, a growing number
-of tombstones begins to degrade read performance.
-
-The queue anti-pattern serves as a reminder that any design that relies
-on the deletion of data is potentially a poorly performing design.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_physical.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_physical.adoc
deleted file mode 100644
index 0934067..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_physical.adoc
+++ /dev/null
@@ -1,96 +0,0 @@
-= Physical Data Modeling
-
-Once you have a logical data model defined, creating the physical model
-is a relatively simple process.
-
-You walk through each of the logical model tables, assigning types to
-each item. You can use any valid `CQL data type <data-types>`, including
-the basic types, collections, and user-defined types. You may identify
-additional user-defined types that can be created to simplify your
-design.
-
-After you’ve assigned data types, you analyze the model by performing
-size calculations and testing out how the model works. You may make some
-adjustments based on your findings. Once again let's cover the data
-modeling process in more detail by working through an example.
-
-Before getting started, let’s look at a few additions to the Chebotko
-notation for physical data models. To draw physical models, you need to
-be able to add the typing information for each column. This figure shows
-the addition of a type for each column in a sample table.
-
-image::data_modeling_chebotko_physical.png[image]
-
-The figure includes a designation of the keyspace containing each table
-and visual cues for columns represented using collections and
-user-defined types. Note the designation of static columns and secondary
-index columns. There is no restriction on assigning these as part of a
-logical model, but they are typically more of a physical data modeling
-concern.
-
-== Hotel Physical Data Model
-
-Now let’s get to work on the physical model. First, you need keyspaces
-to contain the tables. To keep the design relatively simple, create a
-`hotel` keyspace to contain tables for hotel and availability data, and
-a `reservation` keyspace to contain tables for reservation and guest
-data. In a real system, you might divide the tables across even more
-keyspaces in order to separate concerns.
-
-For the `hotels` table, use Cassandra’s `text` type to represent the
-hotel’s `id`. For the address, create an `address` user defined type.
-Use the `text` type to represent the phone number, as there is
-considerable variance in the formatting of numbers between countries.
-
-While it would make sense to use the `uuid` type for attributes such as
-the `hotel_id`, this document uses mostly `text` attributes as
-identifiers, to keep the samples simple and readable. For example, a
-common convention in the hospitality industry is to reference properties
-by short codes like "AZ123" or "NY229". This example uses these values
-for `hotel_ids`, while acknowledging they are not necessarily globally
-unique.
-
-You’ll find that it’s often helpful to use unique IDs to uniquely
-reference elements, and to use these `uuids` as references in tables
-representing other entities. This helps to minimize coupling between
-different entity types. This may prove especially effective if you are
-using a microservice architectural style for your application, in which
-there are separate services responsible for each entity type.
-
-As you work to create physical representations of various tables in the
-logical hotel data model, you use the same approach. The resulting
-design is shown in this figure:
-
-image::data_modeling_hotel_physical.png[image]
-
-Note that the `address` type is also included in the design. It is
-designated with an asterisk to denote that it is a user-defined type,
-and has no primary key columns identified. This type is used in the
-`hotels` and `hotels_by_poi` tables.
-
-User-defined types are frequently used to help reduce duplication of
-non-primary key columns, as was done with the `address` user-defined
-type. This can reduce complexity in the design.
-
-Remember that the scope of a UDT is the keyspace in which it is defined.
-To use `address` in the `reservation` keyspace defined below design,
-you’ll have to declare it again. This is just one of the many trade-offs
-you have to make in data model design.
-
-== Reservation Physical Data Model
-
-Now, let’s examine reservation tables in the design. Remember that the
-logical model contained three denormalized tables to support queries for
-reservations by confirmation number, guest, and hotel and date. For the
-first iteration of your physical data model design, assume you're going
-to manage this denormalization manually. Note that this design could be
-revised to use Cassandra’s (experimental) materialized view feature.
-
-image::data_modeling_reservation_physical.png[image]
-
-Note that the `address` type is reproduced in this keyspace and
-`guest_id` is modeled as a `uuid` type in all of the tables.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_queries.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_queries.adoc
deleted file mode 100644
index ed40fb8..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_queries.adoc
+++ /dev/null
@@ -1,60 +0,0 @@
-= Defining Application Queries
-
-Let’s try the query-first approach to start designing the data model for
-a hotel application. The user interface design for the application is
-often a great artifact to use to begin identifying queries. Let’s assume
-that you’ve talked with the project stakeholders and your UX designers
-have produced user interface designs or wireframes for the key use
-cases. You’ll likely have a list of shopping queries like the following:
-
-* Q1. Find hotels near a given point of interest.
-* Q2. Find information about a given hotel, such as its name and
-location.
-* Q3. Find points of interest near a given hotel.
-* Q4. Find an available room in a given date range.
-* Q5. Find the rate and amenities for a room.
-
-It is often helpful to be able to refer to queries by a shorthand number
-rather that explaining them in full. The queries listed here are
-numbered Q1, Q2, and so on, which is how they are referenced in diagrams
-throughout the example.
-
-Now if the application is to be a success, you’ll certainly want
-customers to be able to book reservations at hotels. This includes steps
-such as selecting an available room and entering their guest
-information. So clearly you will also need some queries that address the
-reservation and guest entities from the conceptual data model. Even
-here, however, you’ll want to think not only from the customer
-perspective in terms of how the data is written, but also in terms of
-how the data will be queried by downstream use cases.
-
-Your natural tendency might be to focus first on designing the tables
-to store reservation and guest records, and only then start thinking
-about the queries that would access them. You may have felt a similar
-tension already when discussing the shopping queries before, thinking
-“but where did the hotel and point of interest data come from?” Don’t
-worry, you will see soon enough. Here are some queries that describe how
-users will access reservations:
-
-* Q6. Lookup a reservation by confirmation number.
-* Q7. Lookup a reservation by hotel, date, and guest name.
-* Q8. Lookup all reservations by guest name.
-* Q9. View guest details.
-
-All of the queries are shown in the context of the workflow of the
-application in the figure below. Each box on the diagram represents a
-step in the application workflow, with arrows indicating the flows
-between steps and the associated query. If you’ve modeled the
-application well, each step of the workflow accomplishes a task that
-“unlocks” subsequent steps. For example, the “View hotels near POI” task
-helps the application learn about several hotels, including their unique
-keys. The key for a selected hotel may be used as part of Q2, in order
-to obtain detailed description of the hotel. The act of booking a room
-creates a reservation record that may be accessed by the guest and hotel
-staff at a later time through various additional queries.
-
-image::data_modeling_hotel_queries.png[image]
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_rdbms.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_rdbms.adoc
deleted file mode 100644
index 2acd6cc..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_rdbms.adoc
+++ /dev/null
@@ -1,144 +0,0 @@
-= RDBMS Design
-
-When you set out to build a new data-driven application that will use a
-relational database, you might start by modeling the domain as a set of
-properly normalized tables and use foreign keys to reference related
-data in other tables.
-
-The figure below shows how you might represent the data storage for your
-application using a relational database model. The relational model
-includes a couple of “join” tables in order to realize the many-to-many
-relationships from the conceptual model of hotels-to-points of interest,
-rooms-to-amenities, rooms-to-availability, and guests-to-rooms (via a
-reservation).
-
-image::data_modeling_hotel_relational.png[image]
-
-== Design Differences Between RDBMS and Cassandra
-
-Let’s take a minute to highlight some of the key differences in doing
-data modeling for Cassandra versus a relational database.
-
-=== No joins
-
-You cannot perform joins in Cassandra. If you have designed a data model
-and find that you need something like a join, you’ll have to either do
-the work on the client side, or create a denormalized second table that
-represents the join results for you. This latter option is preferred in
-Cassandra data modeling. Performing joins on the client should be a very
-rare case; you really want to duplicate (denormalize) the data instead.
-
-=== No referential integrity
-
-Although Cassandra supports features such as lightweight transactions
-and batches, Cassandra itself has no concept of referential integrity
-across tables. In a relational database, you could specify foreign keys
-in a table to reference the primary key of a record in another table.
-But Cassandra does not enforce this. It is still a common design
-requirement to store IDs related to other entities in your tables, but
-operations such as cascading deletes are not available.
-
-=== Denormalization
-
-In relational database design, you are often taught the importance of
-normalization. This is not an advantage when working with Cassandra
-because it performs best when the data model is denormalized. It is
-often the case that companies end up denormalizing data in relational
-databases as well. There are two common reasons for this. One is
-performance. Companies simply can’t get the performance they need when
-they have to do so many joins on years’ worth of data, so they
-denormalize along the lines of known queries. This ends up working, but
-goes against the grain of how relational databases are intended to be
-designed, and ultimately makes one question whether using a relational
-database is the best approach in these circumstances.
-
-A second reason that relational databases get denormalized on purpose is
-a business document structure that requires retention. That is, you have
-an enclosing table that refers to a lot of external tables whose data
-could change over time, but you need to preserve the enclosing document
-as a snapshot in history. The common example here is with invoices. You
-already have customer and product tables, and you’d think that you could
-just make an invoice that refers to those tables. But this should never
-be done in practice. Customer or price information could change, and
-then you would lose the integrity of the invoice document as it was on
-the invoice date, which could violate audits, reports, or laws, and
-cause other problems.
-
-In the relational world, denormalization violates Codd’s normal forms,
-and you try to avoid it. But in Cassandra, denormalization is, well,
-perfectly normal. It’s not required if your data model is simple. But
-don’t be afraid of it.
-
-Historically, denormalization in Cassandra has required designing and
-managing multiple tables using techniques described in this
-documentation. Beginning with the 3.0 release, Cassandra provides a
-feature known as `materialized views <materialized-views>` which allows
-you to create multiple denormalized views of data based on a base table
-design. Cassandra manages materialized views on the server, including
-the work of keeping the views in sync with the table.
-
-=== Query-first design
-
-Relational modeling, in simple terms, means that you start from the
-conceptual domain and then represent the nouns in the domain in tables.
-You then assign primary keys and foreign keys to model relationships.
-When you have a many-to-many relationship, you create the join tables
-that represent just those keys. The join tables don’t exist in the real
-world, and are a necessary side effect of the way relational models
-work. After you have all your tables laid out, you can start writing
-queries that pull together disparate data using the relationships
-defined by the keys. The queries in the relational world are very much
-secondary. It is assumed that you can always get the data you want as
-long as you have your tables modeled properly. Even if you have to use
-several complex subqueries or join statements, this is usually true.
-
-By contrast, in Cassandra you don’t start with the data model; you start
-with the query model. Instead of modeling the data first and then
-writing queries, with Cassandra you model the queries and let the data
-be organized around them. Think of the most common query paths your
-application will use, and then create the tables that you need to
-support them.
-
-Detractors have suggested that designing the queries first is overly
-constraining on application design, not to mention database modeling.
-But it is perfectly reasonable to expect that you should think hard
-about the queries in your application, just as you would, presumably,
-think hard about your relational domain. You may get it wrong, and then
-you’ll have problems in either world. Or your query needs might change
-over time, and then you’ll have to work to update your data set. But
-this is no different from defining the wrong tables, or needing
-additional tables, in an RDBMS.
-
-=== Designing for optimal storage
-
-In a relational database, it is frequently transparent to the user how
-tables are stored on disk, and it is rare to hear of recommendations
-about data modeling based on how the RDBMS might store tables on disk.
-However, that is an important consideration in Cassandra. Because
-Cassandra tables are each stored in separate files on disk, it’s
-important to keep related columns defined together in the same table.
-
-A key goal that you will see as you begin creating data models in
-Cassandra is to minimize the number of partitions that must be searched
-in order to satisfy a given query. Because the partition is a unit of
-storage that does not get divided across nodes, a query that searches a
-single partition will typically yield the best performance.
-
-=== Sorting is a design decision
-
-In an RDBMS, you can easily change the order in which records are
-returned to you by using `ORDER BY` in your query. The default sort
-order is not configurable; by default, records are returned in the order
-in which they are written. If you want to change the order, you just
-modify your query, and you can sort by any list of columns.
-
-In Cassandra, however, sorting is treated differently; it is a design
-decision. The sort order available on queries is fixed, and is
-determined entirely by the selection of clustering columns you supply in
-the `CREATE TABLE` command. The CQL `SELECT` statement does support
-`ORDER BY` semantics, but only in the order specified by the clustering
-columns.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_refining.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_refining.adoc
deleted file mode 100644
index 045a80c..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_refining.adoc
+++ /dev/null
@@ -1,202 +0,0 @@
-= Evaluating and Refining Data Models
-:stem: latexmath
-
-Once you’ve created a physical model, there are some steps you’ll want
-to take to evaluate and refine table designs to help ensure optimal
-performance.
-
-== Calculating Partition Size
-
-The first thing that you want to look for is whether your tables will
-have partitions that will be overly large, or to put it another way, too
-wide. Partition size is measured by the number of cells (values) that
-are stored in the partition. Cassandra’s hard limit is 2 billion cells
-per partition, but you’ll likely run into performance issues before
-reaching that limit.
-
-In order to calculate the size of partitions, use the following formula:
-
-[latexmath]
-++++
-\[N_v = N_r (N_c - N_{pk} - N_s) + N_s\]
-++++
-
-The number of values (or cells) in the partition (N~v~) is equal to the
-number of static columns (N~s~) plus the product of the number of rows
-(N~r~) and the number of of values per row. The number of values per row
-is defined as the number of columns (N~c~) minus the number of primary
-key columns (N~pk~) and static columns (N~s~).
-
-The number of columns tends to be relatively static, although it is
-possible to alter tables at runtime. For this reason, a primary driver
-of partition size is the number of rows in the partition. This is a key
-factor that you must consider in determining whether a partition has the
-potential to get too large. Two billion values sounds like a lot, but in
-a sensor system where tens or hundreds of values are measured every
-millisecond, the number of values starts to add up pretty fast.
-
-Let’s take a look at one of the tables to analyze the partition size.
-Because it has a wide partition design with one partition per hotel,
-look at the `available_rooms_by_hotel_date` table. The table has four
-columns total (N~c~ = 4), including three primary key columns (N~pk~ =
-3) and no static columns (N~s~ = 0). Plugging these values into the
-formula, the result is:
-
-[latexmath]
-++++
-\[N_v = N_r (4 - 3 - 0) + 0 = 1N_r\]
-++++
-
-Therefore the number of values for this table is equal to the number of
-rows. You still need to determine a number of rows. To do this, make
-estimates based on the application design. The table is storing a record
-for each room, in each of hotel, for every night. Let's assume the
-system will be used to store two years of inventory at a time, and there
-are 5,000 hotels in the system, with an average of 100 rooms in each
-hotel.
-
-Since there is a partition for each hotel, the estimated number of rows
-per partition is as follows:
-
-[latexmath]
-++++
-\[N_r = 100 rooms/hotel \times 730 days = 73,000 rows\]
-++++
-
-This relatively small number of rows per partition is not going to get
-you in too much trouble, but if you start storing more dates of
-inventory, or don’t manage the size of the inventory well using TTL, you
-could start having issues. You still might want to look at breaking up
-this large partition, which you'll see how to do shortly.
-
-When performing sizing calculations, it is tempting to assume the
-nominal or average case for variables such as the number of rows.
-Consider calculating the worst case as well, as these sorts of
-predictions have a way of coming true in successful systems.
-
-== Calculating Size on Disk
-
-In addition to calculating the size of a partition, it is also an
-excellent idea to estimate the amount of disk space that will be
-required for each table you plan to store in the cluster. In order to
-determine the size, use the following formula to determine the size S~t~
-of a partition:
-
-[latexmath]
-++++
-\[S_t = \displaystyle\sum_i sizeOf\big (c_{k_i}\big) + \displaystyle\sum_j sizeOf\big(c_{s_j}\big) + N_r\times \bigg(\displaystyle\sum_k sizeOf\big(c_{r_k}\big) + \displaystyle\sum_l sizeOf\big(c_{c_l}\big)\bigg) +\]
-++++
-
-[latexmath]
-++++
-\[N_v\times sizeOf\big(t_{avg}\big)\]
-++++
-
-This is a bit more complex than the previous formula, but let's break it
-down a bit at a time. Let’s take a look at the notation first:
-
-* In this formula, c~k~ refers to partition key columns, c~s~ to static
-columns, c~r~ to regular columns, and c~c~ to clustering columns.
-* The term t~avg~ refers to the average number of bytes of metadata
-stored per cell, such as timestamps. It is typical to use an estimate of
-8 bytes for this value.
-* You'll recognize the number of rows N~r~ and number of values N~v~
-from previous calculations.
-* The *sizeOf()* function refers to the size in bytes of the CQL data
-type of each referenced column.
-
-The first term asks you to sum the size of the partition key columns.
-For this example, the `available_rooms_by_hotel_date` table has a single
-partition key column, the `hotel_id`, which is of type `text`. Assuming
-that hotel identifiers are simple 5-character codes, you have a 5-byte
-value, so the sum of the partition key column sizes is 5 bytes.
-
-The second term asks you to sum the size of the static columns. This
-table has no static columns, so the size is 0 bytes.
-
-The third term is the most involved, and for good reason—it is
-calculating the size of the cells in the partition. Sum the size of the
-clustering columns and regular columns. The two clustering columns are
-the `date`, which is 4 bytes, and the `room_number`, which is a 2-byte
-short integer, giving a sum of 6 bytes. There is only a single regular
-column, the boolean `is_available`, which is 1 byte in size. Summing the
-regular column size (1 byte) plus the clustering column size (6 bytes)
-gives a total of 7 bytes. To finish up the term, multiply this value by
-the number of rows (73,000), giving a result of 511,000 bytes (0.51 MB).
-
-The fourth term is simply counting the metadata that that Cassandra
-stores for each cell. In the storage format used by Cassandra 3.0 and
-later, the amount of metadata for a given cell varies based on the type
-of data being stored, and whether or not custom timestamp or TTL values
-are specified for individual cells. For this table, reuse the number of
-values from the previous calculation (73,000) and multiply by 8, which
-gives 0.58 MB.
-
-Adding these terms together, you get a final estimate:
-
-[latexmath]
-++++
-\[Partition size = 16 bytes + 0 bytes + 0.51 MB + 0.58 MB = 1.1 MB\]
-++++
-
-This formula is an approximation of the actual size of a partition on
-disk, but is accurate enough to be quite useful. Remembering that the
-partition must be able to fit on a single node, it looks like the table
-design will not put a lot of strain on disk storage.
-
-Cassandra’s storage engine was re-implemented for the 3.0 release,
-including a new format for SSTable files. The previous format stored a
-separate copy of the clustering columns as part of the record for each
-cell. The newer format eliminates this duplication, which reduces the
-size of stored data and simplifies the formula for computing that size.
-
-Keep in mind also that this estimate only counts a single replica of
-data. You will need to multiply the value obtained here by the number of
-partitions and the number of replicas specified by the keyspace’s
-replication strategy in order to determine the total required total
-capacity for each table. This will come in handy when you plan your
-cluster.
-
-== Breaking Up Large Partitions
-
-As discussed previously, the goal is to design tables that can provide
-the data you need with queries that touch a single partition, or failing
-that, the minimum possible number of partitions. However, as shown in
-the examples, it is quite possible to design wide partition-style tables
-that approach Cassandra’s built-in limits. Performing sizing analysis on
-tables may reveal partitions that are potentially too large, either in
-number of values, size on disk, or both.
-
-The technique for splitting a large partition is straightforward: add an
-additional column to the partition key. In most cases, moving one of the
-existing columns into the partition key will be sufficient. Another
-option is to introduce an additional column to the table to act as a
-sharding key, but this requires additional application logic.
-
-Continuing to examine the available rooms example, if you add the `date`
-column to the partition key for the `available_rooms_by_hotel_date`
-table, each partition would then represent the availability of rooms at
-a specific hotel on a specific date. This will certainly yield
-partitions that are significantly smaller, perhaps too small, as the
-data for consecutive days will likely be on separate nodes.
-
-Another technique known as *bucketing* is often used to break the data
-into moderate-size partitions. For example, you could bucketize the
-`available_rooms_by_hotel_date` table by adding a `month` column to the
-partition key, perhaps represented as an integer. The comparision with
-the original design is shown in the figure below. While the `month`
-column is partially duplicative of the `date`, it provides a nice way of
-grouping related data in a partition that will not get too large.
-
-image::data_modeling_hotel_bucketing.png[image]
-
-If you really felt strongly about preserving a wide partition design,
-you could instead add the `room_id` to the partition key, so that each
-partition would represent the availability of the room across all dates.
-Because there was no query identified that involves searching
-availability of a specific room, the first or second design approach is
-most suitable to the application needs.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_schema.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_schema.adoc
deleted file mode 100644
index 7b0cf5c..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_schema.adoc
+++ /dev/null
@@ -1,130 +0,0 @@
-= Defining Database Schema
-
-Once you have finished evaluating and refining the physical model,
-you’re ready to implement the schema in CQL. Here is the schema for the
-`hotel` keyspace, using CQL’s comment feature to document the query
-pattern supported by each table:
-
-[source,cql]
-----
-CREATE KEYSPACE hotel WITH replication =
- {‘class’: ‘SimpleStrategy’, ‘replication_factor’ : 3};
-
-CREATE TYPE hotel.address (
- street text,
- city text,
- state_or_province text,
- postal_code text,
- country text );
-
-CREATE TABLE hotel.hotels_by_poi (
- poi_name text,
- hotel_id text,
- name text,
- phone text,
- address frozen<address>,
- PRIMARY KEY ((poi_name), hotel_id) )
- WITH comment = ‘Q1. Find hotels near given poi’
- AND CLUSTERING ORDER BY (hotel_id ASC) ;
-
-CREATE TABLE hotel.hotels (
- id text PRIMARY KEY,
- name text,
- phone text,
- address frozen<address>,
- pois set )
- WITH comment = ‘Q2. Find information about a hotel’;
-
-CREATE TABLE hotel.pois_by_hotel (
- poi_name text,
- hotel_id text,
- description text,
- PRIMARY KEY ((hotel_id), poi_name) )
- WITH comment = Q3. Find pois near a hotel’;
-
-CREATE TABLE hotel.available_rooms_by_hotel_date (
- hotel_id text,
- date date,
- room_number smallint,
- is_available boolean,
- PRIMARY KEY ((hotel_id), date, room_number) )
- WITH comment = ‘Q4. Find available rooms by hotel date’;
-
-CREATE TABLE hotel.amenities_by_room (
- hotel_id text,
- room_number smallint,
- amenity_name text,
- description text,
- PRIMARY KEY ((hotel_id, room_number), amenity_name) )
- WITH comment = ‘Q5. Find amenities for a room’;
-----
-
-Notice that the elements of the partition key are surrounded with
-parentheses, even though the partition key consists of the single column
-`poi_name`. This is a best practice that makes the selection of
-partition key more explicit to others reading your CQL.
-
-Similarly, here is the schema for the `reservation` keyspace:
-
-[source,cql]
-----
-CREATE KEYSPACE reservation WITH replication = {‘class’:
- ‘SimpleStrategy’, ‘replication_factor’ : 3};
-
-CREATE TYPE reservation.address (
- street text,
- city text,
- state_or_province text,
- postal_code text,
- country text );
-
-CREATE TABLE reservation.reservations_by_confirmation (
- confirm_number text,
- hotel_id text,
- start_date date,
- end_date date,
- room_number smallint,
- guest_id uuid,
- PRIMARY KEY (confirm_number) )
- WITH comment = ‘Q6. Find reservations by confirmation number’;
-
-CREATE TABLE reservation.reservations_by_hotel_date (
- hotel_id text,
- start_date date,
- end_date date,
- room_number smallint,
- confirm_number text,
- guest_id uuid,
- PRIMARY KEY ((hotel_id, start_date), room_number) )
- WITH comment = ‘Q7. Find reservations by hotel and date’;
-
-CREATE TABLE reservation.reservations_by_guest (
- guest_last_name text,
- hotel_id text,
- start_date date,
- end_date date,
- room_number smallint,
- confirm_number text,
- guest_id uuid,
- PRIMARY KEY ((guest_last_name), hotel_id) )
- WITH comment = ‘Q8. Find reservations by guest name’;
-
-CREATE TABLE reservation.guests (
- guest_id uuid PRIMARY KEY,
- first_name text,
- last_name text,
- title text,
- emails set,
- phone_numbers list,
- addresses map<text,
- frozen<address>,
- confirm_number text )
- WITH comment = ‘Q9. Find guest by ID’;
-----
-
-You now have a complete Cassandra schema for storing data for a hotel
-application.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/data_modeling_tools.adoc b/doc/modules/cassandra/pages/data_modeling/data_modeling_tools.adoc
deleted file mode 100644
index 0f3556f..0000000
--- a/doc/modules/cassandra/pages/data_modeling/data_modeling_tools.adoc
+++ /dev/null
@@ -1,44 +0,0 @@
-= Cassandra Data Modeling Tools
-
-There are several tools available to help you design and manage your
-Cassandra schema and build queries.
-
-* https://hackolade.com/nosqldb.html#cassandra[Hackolade] is a data
-modeling tool that supports schema design for Cassandra and many other
-NoSQL databases. Hackolade supports the unique concepts of CQL such as
-partition keys and clustering columns, as well as data types including
-collections and UDTs. It also provides the ability to create Chebotko
-diagrams.
-* http://kdm.dataview.org/[Kashlev Data Modeler] is a Cassandra data
-modeling tool that automates the data modeling methodology described in
-this documentation, including identifying access patterns, conceptual,
-logical, and physical data modeling, and schema generation. It also
-includes model patterns that you can optionally leverage as a starting
-point for your designs.
-* DataStax DevCenter is a tool for managing schema, executing queries
-and viewing results. While the tool is no longer actively supported, it
-is still popular with many developers and is available as a
-https://academy.datastax.com/downloads[free download]. DevCenter
-features syntax highlighting for CQL commands, types, and name literals.
-DevCenter provides command completion as you type out CQL commands and
-interprets the commands you type, highlighting any errors you make. The
-tool provides panes for managing multiple CQL scripts and connections to
-multiple clusters. The connections are used to run CQL commands against
-live clusters and view the results. The tool also has a query trace
-feature that is useful for gaining insight into the performance of your
-queries.
-* IDE Plugins - There are CQL plugins available for several Integrated
-Development Environments (IDEs), such as IntelliJ IDEA and Apache
-NetBeans. These plugins typically provide features such as schema
-management and query execution.
-
-Some IDEs and tools that claim to support Cassandra do not actually
-support CQL natively, but instead access Cassandra using a JDBC/ODBC
-driver and interact with Cassandra as if it were a relational database
-with SQL support. Wnen selecting tools for working with Cassandra you’ll
-want to make sure they support CQL and reinforce Cassandra best
-practices for data modeling as presented in this documentation.
-
-_Material adapted from Cassandra, The Definitive Guide. Published by
-O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
-rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/index.adoc b/doc/modules/cassandra/pages/data_modeling/index.adoc
deleted file mode 100644
index 105f5a3..0000000
--- a/doc/modules/cassandra/pages/data_modeling/index.adoc
+++ /dev/null
@@ -1,11 +0,0 @@
-= Data Modeling
-
-* xref:data_modeling/intro.adoc[Introduction]
-* xref:data_modeling/data_modeling_rdbms.adoc[RDBMS]
-* xref:data_modeling/data_modeling_conceptual.adoc[Conceptual]
-* xref:data_modeling/data_modeling_logical.adoc[Logical]
-* xref:data_modeling/data_modeling_physical.adoc[Physical]
-* xref:data_modeling/data_modeling_schema.adoc[Schema]
-* xref:data_modeling/data_modeling_queries.adoc[Queries]
-* xref:data_modeling/data_modeling_refining.adoc[Refining]
-* xref:data_modeling/data_modeling_tools.adoc[Tools]
diff --git a/doc/modules/cassandra/pages/cql/SASI.adoc b/doc/modules/cassandra/pages/developing/cql/SASI.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/cql/SASI.adoc
rename to doc/modules/cassandra/pages/developing/cql/SASI.adoc
diff --git a/doc/modules/cassandra/pages/developing/cql/appendices.adoc b/doc/modules/cassandra/pages/developing/cql/appendices.adoc
new file mode 100644
index 0000000..28b113b
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/appendices.adoc
@@ -0,0 +1,183 @@
+= Appendices
+
+[[appendix-A]]
+== Appendix A: CQL Keywords
+
+CQL distinguishes between _reserved_ and _non-reserved_ keywords.
+Reserved keywords cannot be used as identifier, they are truly reserved
+for the language (but one can enclose a reserved keyword by
+double-quotes to use it as an identifier). Non-reserved keywords however
+only have a specific meaning in certain context but can used as
+identifier otherwise. The only _raison d’être_ of these non-reserved
+keywords is convenience: some keyword are non-reserved when it was
+always easy for the parser to decide whether they were used as keywords
+or not.
+
+[width="48%",cols="60%,40%",options="header",]
+|===
+|Keyword |Reserved?
+|`ADD` |yes
+|`AGGREGATE` |no
+|`ALL` |no
+|`ALLOW` |yes
+|`ALTER` |yes
+|`AND` |yes
+|`APPLY` |yes
+|`AS` |no
+|`ASC` |yes
+|`ASCII` |no
+|`AUTHORIZE` |yes
+|`BATCH` |yes
+|`BEGIN` |yes
+|`BIGINT` |no
+|`BLOB` |no
+|`BOOLEAN` |no
+|`BY` |yes
+|`CALLED` |no
+|`CLUSTERING` |no
+|`COLUMNFAMILY` |yes
+|`COMPACT` |no
+|`CONTAINS` |no
+|`COUNT` |no
+|`COUNTER` |no
+|`CREATE` |yes
+|`CUSTOM` |no
+|`DATE` |no
+|`DECIMAL` |no
+|`DELETE` |yes
+|`DESC` |yes
+|`DESCRIBE` |yes
+|`DISTINCT` |no
+|`DOUBLE` |no
+|`DROP` |yes
+|`ENTRIES` |yes
+|`EXECUTE` |yes
+|`EXISTS` |no
+|`FILTERING` |no
+|`FINALFUNC` |no
+|`FLOAT` |no
+|`FROM` |yes
+|`FROZEN` |no
+|`FULL` |yes
+|`FUNCTION` |no
+|`FUNCTIONS` |no
+|`GRANT` |yes
+|`IF` |yes
+|`IN` |yes
+|`INDEX` |yes
+|`INET` |no
+|`INFINITY` |yes
+|`INITCOND` |no
+|`INPUT` |no
+|`INSERT` |yes
+|`INT` |no
+|`INTO` |yes
+|`JSON` |no
+|`KEY` |no
+|`KEYS` |no
+|`KEYSPACE` |yes
+|`KEYSPACES` |no
+|`LANGUAGE` |no
+|`LIMIT` |yes
+|`LIST` |no
+|`LOGIN` |no
+|`MAP` |no
+|`MASKED` |no
+|`MODIFY` |yes
+|`NAN` |yes
+|`NOLOGIN` |no
+|`NORECURSIVE` |yes
+|`NOSUPERUSER` |no
+|`NOT` |yes
+|`NULL` |yes
+|`OF` |yes
+|`ON` |yes
+|`OPTIONS` |no
+|`OR` |yes
+|`ORDER` |yes
+|`PASSWORD` |no
+|`PERMISSION` |no
+|`PERMISSIONS` |no
+|`PRIMARY` |yes
+|`RENAME` |yes
+|`REPLACE` |yes
+|`RETURNS` |no
+|`REVOKE` |yes
+|`ROLE` |no
+|`ROLES` |no
+|`SCHEMA` |yes
+|`SELECT` |yes
+|`SELECT_MASKED` |no
+|`SET` |yes
+|`SFUNC` |no
+|`SMALLINT` |no
+|`STATIC` |no
+|`STORAGE` |no
+|`STYPE` |no
+|`SUPERUSER` |no
+|`TABLE` |yes
+|`TEXT` |no
+|`TIME` |no
+|`TIMESTAMP` |no
+|`TIMEUUID` |no
+|`TINYINT` |no
+|`TO` |yes
+|`TOKEN` |yes
+|`TRIGGER` |no
+|`TRUNCATE` |yes
+|`TTL` |no
+|`TUPLE` |no
+|`TYPE` |no
+|`UNLOGGED` |yes
+|`UNMASK` |no
+|`UPDATE` |yes
+|`USE` |yes
+|`USER` |no
+|`USERS` |no
+|`USING` |yes
+|`UUID` |no
+|`VALUES` |no
+|`VARCHAR` |no
+|`VARINT` |no
+|`WHERE` |yes
+|`WITH` |yes
+|`WRITETIME` |no
+|`MAXWRITETIME` |no
+|===
+
+== Appendix B: CQL Reserved Types
+
+The following type names are not currently used by CQL, but are reserved
+for potential future use. User-defined types may not use reserved type
+names as their name.
+
+[width="25%",cols="100%",options="header",]
+|===
+|type
+|`bitstring`
+|`byte`
+|`complex`
+|`enum`
+|`interval`
+|`macaddr`
+|===
+
+== Appendix C: Dropping Compact Storage
+
+Starting version 4.0, Thrift and COMPACT STORAGE is no longer supported.
+
+`ALTER ... DROP COMPACT STORAGE` statement makes Compact Tables
+CQL-compatible, exposing internal structure of Thrift/Compact Tables:
+
+* CQL-created Compact Tables that have no clustering columns, will
+expose an additional clustering column `column1` with `UTF8Type`.
+* CQL-created Compact Tables that had no regular columns, will expose a
+regular column `value` with `BytesType`.
+* For CQL-Created Compact Tables, all columns originally defined as
+`regular` will be come `static`
+* CQL-created Compact Tables that have clustering but have no regular
+columns will have an empty value column (of `EmptyType`)
+* SuperColumn Tables (can only be created through Thrift) will expose a
+compact value map with an empty name.
+* Thrift-created Compact Tables will have types corresponding to their
+Thrift definition.
diff --git a/doc/modules/cassandra/pages/developing/cql/changes.adoc b/doc/modules/cassandra/pages/developing/cql/changes.adoc
new file mode 100644
index 0000000..452a182
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/changes.adoc
@@ -0,0 +1,234 @@
+= Changes
+
+The following describes the changes in each version of CQL.
+
+== 3.4.7
+
+* Remove deprecated functions `dateOf` and `unixTimestampOf`, replaced by `toTimestamp` and `toUnixTimestamp` (`18328`)
+* Added support for attaching masking functions to table columns (`18068`)
+* Add UNMASK permission (`18069`)
+* Add SELECT_MASKED permission (`18070`)
+* Add support for using UDFs as masking functions (`18071`)
+* Adopt snake_case function names, deprecating all previous camelCase or alltogetherwithoutspaces function names (`18037`)
+
+== 3.4.6
+
+* Add support for IF EXISTS and IF NOT EXISTS in ALTER statements (`16916`)
+* Allow GRANT/REVOKE multiple permissions in a single statement (`17030`)
+* Pre hashed passwords in CQL (`17334`)
+* Add support for type casting in WHERE clause components and in the values of INSERT/UPDATE statements (`14337`)
+* Add support for CONTAINS and CONTAINS KEY in conditional UPDATE and DELETE statement (`10537`)
+* Allow to grant permission for all tables in a keyspace (`17027`)
+* Allow to aggregate by time intervals (`11871`)
+
+== 3.4.5
+
+* Adds support for arithmetic operators (`11935`)
+* Adds support for `+` and `-` operations on dates (`11936`)
+* Adds `currentTimestamp`, `currentDate`, `currentTime` and
+`currentTimeUUID` functions (`13132`)
+
+== 3.4.4
+
+* `ALTER TABLE` `ALTER` has been removed; a column's type may not be
+changed after creation (`12443`).
+* `ALTER TYPE` `ALTER` has been removed; a field's type may not be
+changed after creation (`12443`).
+
+== 3.4.3
+
+* Adds a new `duration` `data types <data-types>` (`11873`).
+* Support for `GROUP BY` (`10707`).
+* Adds a `DEFAULT UNSET` option for `INSERT JSON` to ignore omitted
+columns (`11424`).
+* Allows `null` as a legal value for TTL on insert and update. It will
+be treated as equivalent to inserting a 0 (`12216`).
+
+== 3.4.2
+
+* If a table has a non zero `default_time_to_live`, then explicitly
+specifying a TTL of 0 in an `INSERT` or `UPDATE` statement will result
+in the new writes not having any expiration (that is, an explicit TTL of
+0 cancels the `default_time_to_live`). This wasn't the case before and
+the `default_time_to_live` was applied even though a TTL had been
+explicitly set.
+* `ALTER TABLE` `ADD` and `DROP` now allow multiple columns to be
+added/removed.
+* New `PER PARTITION LIMIT` option for `SELECT` statements (see
+https://issues.apache.org/jira/browse/CASSANDRA-7017)[CASSANDRA-7017].
+* `User-defined functions <cql-functions>` can now instantiate
+`UDTValue` and `TupleValue` instances via the new `UDFContext` interface
+(see
+https://issues.apache.org/jira/browse/CASSANDRA-10818)[CASSANDRA-10818].
+* `User-defined types <udts>` may now be stored in a non-frozen form,
+allowing individual fields to be updated and deleted in `UPDATE`
+statements and `DELETE` statements, respectively.
+(https://issues.apache.org/jira/browse/CASSANDRA-7423)[CASSANDRA-7423]).
+
+== 3.4.1
+
+* Adds `CAST` functions.
+
+== 3.4.0
+
+* Support for `materialized views <materialized-views>`.
+* `DELETE` support for inequality expressions and `IN` restrictions on
+any primary key columns.
+* `UPDATE` support for `IN` restrictions on any primary key columns.
+
+== 3.3.1
+
+* The syntax `TRUNCATE TABLE X` is now accepted as an alias for
+`TRUNCATE X`.
+
+== 3.3.0
+
+* `User-defined functions and aggregates <cql-functions>` are now
+supported.
+* Allows double-dollar enclosed strings literals as an alternative to
+single-quote enclosed strings.
+* Introduces Roles to supersede user based authentication and access
+control
+* New `date`, `time`, `tinyint` and `smallint` `data types <data-types>`
+have been added.
+* `JSON support <cql-json>` has been added
+* Adds new time conversion functions and deprecate `dateOf` and
+`unixTimestampOf`.
+
+== 3.2.0
+
+* `User-defined types <udts>` supported.
+* `CREATE INDEX` now supports indexing collection columns, including
+indexing the keys of map collections through the `keys()` function
+* Indexes on collections may be queried using the new `CONTAINS` and
+`CONTAINS KEY` operators
+* `Tuple types <tuples>` were added to hold fixed-length sets of typed
+positional fields.
+* `DROP INDEX` now supports optionally specifying a keyspace.
+
+== 3.1.7
+
+* `SELECT` statements now support selecting multiple rows in a single
+partition using an `IN` clause on combinations of clustering columns.
+* `IF NOT EXISTS` and `IF EXISTS` syntax is now supported by
+`CREATE USER` and `DROP USER` statements, respectively.
+
+== 3.1.6
+
+* A new `uuid()` method has been added.
+* Support for `DELETE ... IF EXISTS` syntax.
+
+== 3.1.5
+
+* It is now possible to group clustering columns in a relation, see
+`WHERE <where-clause>` clauses.
+* Added support for `static columns <static-columns>`.
+
+== 3.1.4
+
+* `CREATE INDEX` now allows specifying options when creating CUSTOM
+indexes.
+
+== 3.1.3
+
+* Millisecond precision formats have been added to the
+`timestamp <timestamps>` parser.
+
+== 3.1.2
+
+* `NaN` and `Infinity` has been added as valid float constants. They are
+now reserved keywords. In the unlikely case you we using them as a
+column identifier (or keyspace/table one), you will now need to double
+quote them.
+
+== 3.1.1
+
+* `SELECT` statement now allows listing the partition keys (using the
+`DISTINCT` modifier). See
+https://issues.apache.org/jira/browse/CASSANDRA-4536[CASSANDRA-4536].
+* The syntax `c IN ?` is now supported in `WHERE` clauses. In that case,
+the value expected for the bind variable will be a list of whatever type
+`c` is.
+* It is now possible to use named bind variables (using `:name` instead
+of `?`).
+
+== 3.1.0
+
+* `ALTER TABLE` `DROP` option added.
+* `SELECT` statement now supports aliases in select clause. Aliases in
+WHERE and ORDER BY clauses are not supported.
+* `CREATE` statements for `KEYSPACE`, `TABLE` and `INDEX` now supports
+an `IF NOT EXISTS` condition. Similarly, `DROP` statements support a
+`IF EXISTS` condition.
+* `INSERT` statements optionally supports a `IF NOT EXISTS` condition
+and `UPDATE` supports `IF` conditions.
+
+== 3.0.5
+
+* `SELECT`, `UPDATE`, and `DELETE` statements now allow empty `IN`
+relations (see
+https://issues.apache.org/jira/browse/CASSANDRA-5626)[CASSANDRA-5626].
+
+== 3.0.4
+
+* Updated the syntax for custom `secondary indexes <secondary-indexes>`.
+* Non-equal condition on the partition key are now never supported, even
+for ordering partitioner as this was not correct (the order was *not*
+the one of the type of the partition key). Instead, the `token` method
+should always be used for range queries on the partition key (see
+`WHERE clauses <where-clause>`).
+
+== 3.0.3
+
+* Support for custom `secondary indexes <secondary-indexes>` has been
+added.
+
+== 3.0.2
+
+* Type validation for the `constants <constants>` has been fixed. For
+instance, the implementation used to allow `'2'` as a valid value for an
+`int` column (interpreting it has the equivalent of `2`), or `42` as a
+valid `blob` value (in which case `42` was interpreted as an hexadecimal
+representation of the blob). This is no longer the case, type validation
+of constants is now more strict. See the `data types <data-types>`
+section for details on which constant is allowed for which type.
+* The type validation fixed of the previous point has lead to the
+introduction of blobs constants to allow the input of blobs. Do note
+that while the input of blobs as strings constant is still supported by
+this version (to allow smoother transition to blob constant), it is now
+deprecated and will be removed by a future version. If you were using
+strings as blobs, you should thus update your client code ASAP to switch
+blob constants.
+* A number of functions to convert native types to blobs have also been
+introduced. Furthermore the token function is now also allowed in select
+clauses. See the `section on functions <cql-functions>` for details.
+
+== 3.0.1
+
+* Date strings (and timestamps) are no longer accepted as valid
+`timeuuid` values. Doing so was a bug in the sense that date string are
+not valid `timeuuid`, and it was thus resulting in
+https://issues.apache.org/jira/browse/CASSANDRA-4936[confusing
+behaviors]. However, the following new methods have been added to help
+working with `timeuuid`: `now`, `minTimeuuid`, `maxTimeuuid` , `dateOf`
+and `unixTimestampOf`.
+* Float constants now support the exponent notation. In other words,
+`4.2E10` is now a valid floating point value.
+
+== Versioning
+
+Versioning of the CQL language adheres to the http://semver.org[Semantic
+Versioning] guidelines. Versions take the form X.Y.Z where X, Y, and Z
+are integer values representing major, minor, and patch level
+respectively. There is no correlation between Cassandra release versions
+and the CQL language version.
+
+[cols=",",options="header",]
+|===
+|version |description
+| Major | The major version _must_ be bumped when backward incompatible changes
+are introduced. This should rarely occur.
+| Minor | Minor version increments occur when new, but backward compatible,
+functionality is introduced.
+| Patch | The patch version is incremented when bugs are fixed.
+|===
diff --git a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
new file mode 100644
index 0000000..702ac3a
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
@@ -0,0 +1,3922 @@
+== Cassandra Query Language (CQL) v3.4.3
+
+\{toc:maxLevel=3}
+
+=== CQL Syntax
+
+==== Preamble
+
+This document describes the Cassandra Query Language (CQL) version 3.
+CQL v3 is not backward compatible with CQL v2 and differs from it in
+numerous ways. Note that this document describes the last version of the
+languages. However, the link:#changes[changes] section provides the diff
+between the different versions of CQL v3.
+
+CQL v3 offers a model very close to SQL in the sense that data is put in
+_tables_ containing _rows_ of _columns_. For that reason, when used in
+this document, these terms (tables, rows and columns) have the same
+definition than they have in SQL. But please note that as such, they do
+*not* refer to the concept of rows and columns found in the internal
+implementation of Cassandra and in the thrift and CQL v2 API.
+
+==== Conventions
+
+To aid in specifying the CQL syntax, we will use the following
+conventions in this document:
+
+* Language rules will be given in a
+http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form[BNF] -like
+notation:
+
+bc(syntax). ::= TERMINAL
+
+* Nonterminal symbols will have `<angle brackets>`.
+* As additional shortcut notations to BNF, we’ll use traditional regular
+expression’s symbols (`?`, `+` and `*`) to signify that a given symbol
+is optional and/or can be repeated. We’ll also allow parentheses to
+group symbols and the `[<characters>]` notation to represent any one of
+`<characters>`.
+* The grammar is provided for documentation purposes and leave some
+minor details out. For instance, the last column definition in a
+`CREATE TABLE` statement is optional but supported if present even
+though the provided grammar in this document suggest it is not
+supported.
+* Sample code will be provided in a code block:
+
+bc(sample). SELECT sample_usage FROM cql;
+
+* References to keywords or pieces of CQL code in running text will be
+shown in a `fixed-width font`.
+
+[[identifiers]]
+==== Identifiers and keywords
+
+The CQL language uses _identifiers_ (or _names_) to identify tables,
+columns and other objects. An identifier is a token matching the regular
+expression `[a-zA-Z]``[a-zA-Z0-9_]``*`.
+
+A number of such identifiers, like `SELECT` or `WITH`, are _keywords_.
+They have a fixed meaning for the language and most are reserved. The
+list of those keywords can be found in link:#appendixA[Appendix A].
+
+Identifiers and (unquoted) keywords are case insensitive. Thus `SELECT`
+is the same than `select` or `sElEcT`, and `myId` is the same than
+`myid` or `MYID` for instance. A convention often used (in particular by
+the samples of this documentation) is to use upper case for keywords and
+lower case for other identifiers.
+
+There is a second kind of identifiers called _quoted identifiers_
+defined by enclosing an arbitrary sequence of characters in
+double-quotes(`"`). Quoted identifiers are never keywords. Thus
+`"select"` is not a reserved keyword and can be used to refer to a
+column, while `select` would raise a parse error. Also, contrarily to
+unquoted identifiers and keywords, quoted identifiers are case sensitive
+(`"My Quoted Id"` is _different_ from `"my quoted id"`). A fully
+lowercase quoted identifier that matches `[a-zA-Z]``[a-zA-Z0-9_]``*` is
+equivalent to the unquoted identifier obtained by removing the
+double-quote (so `"myid"` is equivalent to `myid` and to `myId` but
+different from `"myId"`). Inside a quoted identifier, the double-quote
+character can be repeated to escape it, so `"foo "" bar"` is a valid
+identifier.
+
+*Warning*: _quoted identifiers_ allows to declare columns with arbitrary
+names, and those can sometime clash with specific names used by the
+server. For instance, when using conditional update, the server will
+respond with a result-set containing a special result named
+`"[applied]"`. If you’ve declared a column with such a name, this could
+potentially confuse some tools and should be avoided. In general,
+unquoted identifiers should be preferred but if you use quoted
+identifiers, it is strongly advised to avoid any name enclosed by
+squared brackets (like `"[applied]"`) and any name that looks like a
+function call (like `"f(x)"`).
+
+==== Constants
+
+CQL defines the following kind of _constants_: strings, integers,
+floats, booleans, uuids and blobs:
+
+* A string constant is an arbitrary sequence of characters characters
+enclosed by single-quote(`'`). One can include a single-quote in a
+string by repeating it, e.g. `'It''s raining today'`. Those are not to
+be confused with quoted identifiers that use double-quotes.
+* An integer constant is defined by `'-'?[0-9]+`.
+* A float constant is defined by
+`'-'?[0-9]+('.'[0-9]*)?([eE][+-]?[0-9+])?`. On top of that, `NaN` and
+`Infinity` are also float constants.
+* A boolean constant is either `true` or `false` up to
+case-insensitivity (i.e. `True` is a valid boolean constant).
+* A http://en.wikipedia.org/wiki/Universally_unique_identifier[UUID]
+constant is defined by `hex{8}-hex{4}-hex{4}-hex{4}-hex{12}` where `hex`
+is an hexadecimal character, e.g. `[0-9a-fA-F]` and `{4}` is the number
+of such characters.
+* A blob constant is an hexadecimal number defined by `0[xX](hex)+`
+where `hex` is an hexadecimal character, e.g. `[0-9a-fA-F]`.
+
+For how these constants are typed, see the link:#types[data types
+section].
+
+==== Comments
+
+A comment in CQL is a line beginning by either double dashes (`--`) or
+double slash (`//`).
+
+Multi-line comments are also supported through enclosure within `/*` and
+`*/` (but nesting is not supported).
+
+bc(sample). +
+— This is a comment +
+// This is a comment too +
+/* This is +
+a multi-line comment */
+
+==== Statements
+
+CQL consists of statements. As in SQL, these statements can be divided
+in 3 categories:
+
+* Data definition statements, that allow to set and change the way data
+is stored.
+* Data manipulation statements, that allow to change data
+* Queries, to look up data
+
+All statements end with a semicolon (`;`) but that semicolon can be
+omitted when dealing with a single statement. The supported statements
+are described in the following sections. When describing the grammar of
+said statements, we will reuse the non-terminal symbols defined below:
+
+bc(syntax).. +
+::= any quoted or unquoted identifier, excluding reserved keywords +
+::= ( `.')?
+
+::= a string constant +
+::= an integer constant +
+::= a float constant +
+::= | +
+::= a uuid constant +
+::= a boolean constant +
+::= a blob constant
+
+::= +
+| +
+| +
+| +
+| +
+::= `?' +
+| `:' +
+::= +
+| +
+| +
+| `(' ( (`,' )*)? `)'
+
+::= +
+| +
+| +
+::= `\{' ( `:' ( `,' `:' )* )? `}' +
+::= `\{' ( ( `,' )* )? `}' +
+::= `[' ( ( `,' )* )? `]'
+
+::=
+
+::= (AND )* +
+::= `=' ( | | ) +
+p. +
+Please note that not every possible productions of the grammar above
+will be valid in practice. Most notably, `<variable>` and nested
+`<collection-literal>` are currently not allowed inside
+`<collection-literal>`.
+
+A `<variable>` can be either anonymous (a question mark (`?`)) or named
+(an identifier preceded by `:`). Both declare a bind variables for
+link:#preparedStatement[prepared statements]. The only difference
+between an anymous and a named variable is that a named one will be
+easier to refer to (how exactly depends on the client driver used).
+
+The `<properties>` production is use by statement that create and alter
+keyspaces and tables. Each `<property>` is either a _simple_ one, in
+which case it just has a value, or a _map_ one, in which case it’s value
+is a map grouping sub-options. The following will refer to one or the
+other as the _kind_ (_simple_ or _map_) of the property.
+
+A `<tablename>` will be used to identify a table. This is an identifier
+representing the table name that can be preceded by a keyspace name. The
+keyspace name, if provided, allow to identify a table in another
+keyspace than the currently active one (the currently active keyspace is
+set through the `USE` statement).
+
+For supported `<function>`, see the section on
+link:#functions[functions].
+
+Strings can be either enclosed with single quotes or two dollar
+characters. The second syntax has been introduced to allow strings that
+contain single quotes. Typical candidates for such strings are source
+code fragments for user-defined functions.
+
+_Sample:_
+
+bc(sample).. +
+`some string value'
+
+$$double-dollar string can contain single ’ quotes$$ +
+p.
+
+[[preparedStatement]]
+==== Prepared Statement
+
+CQL supports _prepared statements_. Prepared statement is an
+optimization that allows to parse a query only once but execute it
+multiple times with different concrete values.
+
+In a statement, each time a column value is expected (in the data
+manipulation and query statements), a `<variable>` (see above) can be
+used instead. A statement with bind variables must then be _prepared_.
+Once it has been prepared, it can executed by providing concrete values
+for the bind variables. The exact procedure to prepare a statement and
+execute a prepared statement depends on the CQL driver used and is
+beyond the scope of this document.
+
+In addition to providing column values, bind markers may be used to
+provide values for `LIMIT`, `TIMESTAMP`, and `TTL` clauses. If anonymous
+bind markers are used, the names for the query parameters will be
+`[limit]`, `[timestamp]`, and `[ttl]`, respectively.
+
+[[dataDefinition]]
+=== Data Definition
+
+[[createKeyspaceStmt]]
+==== CREATE KEYSPACE
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE KEYSPACE (IF NOT EXISTS)? WITH +
+p. +
+_Sample:_
+
+bc(sample).. +
+CREATE KEYSPACE Excelsior +
+WITH replication = \{’class’: `SimpleStrategy', `replication_factor' :
+3};
+
+CREATE KEYSPACE Excalibur +
+WITH replication = \{’class’: `NetworkTopologyStrategy', `DC1' : 1,
+`DC2' : 3} +
+AND durable_writes = false; +
+p. +
+The `CREATE KEYSPACE` statement creates a new top-level _keyspace_. A
+keyspace is a namespace that defines a replication strategy and some
+options for a set of tables. Valid keyspaces names are identifiers
+composed exclusively of alphanumerical characters and whose length is
+lesser or equal to 32. Note that as identifiers, keyspace names are case
+insensitive: use a quoted identifier for case sensitive keyspace names.
+
+The supported `<properties>` for `CREATE KEYSPACE` are:
+
+[cols=",,,,",options="header",]
+|===
+|name |kind |mandatory |default |description
+|`replication` |_map_ |yes | |The replication strategy and options to
+use for the keyspace.
+
+|`durable_writes` |_simple_ |no |true |Whether to use the commit log for
+updates on this keyspace (disable this option at your own risk!).
+|===
+
+The `replication` `<property>` is mandatory. It must at least contains
+the `'class'` sub-option which defines the replication strategy class to
+use. The rest of the sub-options depends on that replication strategy
+class. By default, Cassandra support the following `'class'`:
+
+* `'SimpleStrategy'`: A simple strategy that defines a simple
+replication factor for the whole cluster. The only sub-options supported
+is `'replication_factor'` to define that replication factor and is
+mandatory.
+* `'NetworkTopologyStrategy'`: A replication strategy that allows to set
+the replication factor independently for each data-center. The rest of
+the sub-options are key-value pairs where each time the key is the name
+of a datacenter and the value the replication factor for that
+data-center.
+
+Attempting to create an already existing keyspace will return an error
+unless the `IF NOT EXISTS` option is used. If it is used, the statement
+will be a no-op if the keyspace already exists.
+
+[[useStmt]]
+==== USE
+
+_Syntax:_
+
+bc(syntax). ::= USE
+
+_Sample:_
+
+bc(sample). USE myApp;
+
+The `USE` statement takes an existing keyspace name as argument and set
+it as the per-connection current working keyspace. All subsequent
+keyspace-specific actions will be performed in the context of the
+selected keyspace, unless link:#statements[otherwise specified], until
+another USE statement is issued or the connection terminates.
+
+[[alterKeyspaceStmt]]
+==== ALTER KEYSPACE
+
+_Syntax:_
+
+bc(syntax).. +
+::= ALTER KEYSPACE (IF EXISTS)? WITH +
+p. +
+_Sample:_
+
+bc(sample).. +
+ALTER KEYSPACE Excelsior +
+WITH replication = \{’class’: `SimpleStrategy', `replication_factor' :
+4};
+
+The `ALTER KEYSPACE` statement alters the properties of an existing
+keyspace. The supported `<properties>` are the same as for the
+link:#createKeyspaceStmt[`CREATE KEYSPACE`] statement.
+
+[[dropKeyspaceStmt]]
+==== DROP KEYSPACE
+
+_Syntax:_
+
+bc(syntax). ::= DROP KEYSPACE ( IF EXISTS )?
+
+_Sample:_
+
+bc(sample). DROP KEYSPACE myApp;
+
+A `DROP KEYSPACE` statement results in the immediate, irreversible
+removal of an existing keyspace, including all column families in it,
+and all data contained in those column families.
+
+If the keyspace does not exists, the statement will return an error,
+unless `IF EXISTS` is used in which case the operation is a no-op.
+
+[[createTableStmt]]
+==== CREATE TABLE
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE ( TABLE | COLUMNFAMILY ) ( IF NOT EXISTS )? +
+`(' ( `,' )* `)' +
+( WITH ( AND )* )?
+
+::= ( STATIC )? ( PRIMARY KEY )? +
+| PRIMARY KEY `(' ( `,' )* `)'
+
+::= +
+| `(' (`,' )* `)'
+
+::= +
+| COMPACT STORAGE +
+| CLUSTERING ORDER +
+p. +
+_Sample:_
+
+bc(sample).. +
+CREATE TABLE monkeySpecies ( +
+species text PRIMARY KEY, +
+common_name text, +
+population varint, +
+average_size int +
+) WITH comment=`Important biological records';
+
+CREATE TABLE timeline ( +
+userid uuid, +
+posted_month int, +
+posted_time uuid, +
+body text, +
+posted_by text, +
+PRIMARY KEY (userid, posted_month, posted_time) +
+) WITH compaction = \{ `class' : `LeveledCompactionStrategy' }; +
+p. +
+The `CREATE TABLE` statement creates a new table. Each such table is a
+set of _rows_ (usually representing related entities) for which it
+defines a number of properties. A table is defined by a
+link:#createTableName[name], it defines the columns composing rows of
+the table and have a number of link:#createTableOptions[options]. Note
+that the `CREATE COLUMNFAMILY` syntax is supported as an alias for
+`CREATE TABLE` (for historical reasons).
+
+Attempting to create an already existing table will return an error
+unless the `IF NOT EXISTS` option is used. If it is used, the statement
+will be a no-op if the table already exists.
+
+[[createTableName]]
+===== `<tablename>`
+
+Valid table names are the same as valid
+link:#createKeyspaceStmt[keyspace names] (up to 32 characters long
+alphanumerical identifiers). If the table name is provided alone, the
+table is created within the current keyspace (see `USE`), but if it is
+prefixed by an existing keyspace name (see
+link:#statements[`<tablename>`] grammar), it is created in the specified
+keyspace (but does *not* change the current keyspace).
+
+[[createTableColumn]]
+===== `<column-definition>`
+
+A `CREATE TABLE` statement defines the columns that rows of the table
+can have. A _column_ is defined by its name (an identifier) and its type
+(see the link:#types[data types] section for more details on allowed
+types and their properties).
+
+Within a table, a row is uniquely identified by its `PRIMARY KEY` (or
+more simply the key), and hence all table definitions *must* define a
+PRIMARY KEY (and only one). A `PRIMARY KEY` is composed of one or more
+of the columns defined in the table. If the `PRIMARY KEY` is only one
+column, this can be specified directly after the column definition.
+Otherwise, it must be specified by following `PRIMARY KEY` by the
+comma-separated list of column names composing the key within
+parenthesis. Note that:
+
+bc(sample). +
+CREATE TABLE t ( +
+k int PRIMARY KEY, +
+other text +
+)
+
+is equivalent to
+
+bc(sample). +
+CREATE TABLE t ( +
+k int, +
+other text, +
+PRIMARY KEY (k) +
+)
+
+[[createTablepartitionClustering]]
+===== Partition key and clustering columns
+
+In CQL, the order in which columns are defined for the `PRIMARY KEY`
+matters. The first column of the key is called the _partition key_. It
+has the property that all the rows sharing the same partition key (even
+across table in fact) are stored on the same physical node. Also,
+insertion/update/deletion on rows sharing the same partition key for a
+given table are performed _atomically_ and in _isolation_. Note that it
+is possible to have a composite partition key, i.e. a partition key
+formed of multiple columns, using an extra set of parentheses to define
+which columns forms the partition key.
+
+The remaining columns of the `PRIMARY KEY` definition, if any, are
+called __clustering columns. On a given physical node, rows for a given
+partition key are stored in the order induced by the clustering columns,
+making the retrieval of rows in that clustering order particularly
+efficient (see `SELECT`).
+
+[[createTableStatic]]
+===== `STATIC` columns
+
+Some columns can be declared as `STATIC` in a table definition. A column
+that is static will be ``shared'' by all the rows belonging to the same
+partition (having the same partition key). For instance, in:
+
+bc(sample). +
+CREATE TABLE test ( +
+pk int, +
+t int, +
+v text, +
+s text static, +
+PRIMARY KEY (pk, t) +
+); +
+INSERT INTO test(pk, t, v, s) VALUES (0, 0, `val0', `static0'); +
+INSERT INTO test(pk, t, v, s) VALUES (0, 1, `val1', `static1'); +
+SELECT * FROM test WHERE pk=0 AND t=0;
+
+the last query will return `'static1'` as value for `s`, since `s` is
+static and thus the 2nd insertion modified this ``shared'' value. Note
+however that static columns are only static within a given partition,
+and if in the example above both rows where from different partitions
+(i.e. if they had different value for `pk`), then the 2nd insertion
+would not have modified the value of `s` for the first row.
+
+A few restrictions applies to when static columns are allowed:
+
+* tables with the `COMPACT STORAGE` option (see below) cannot have them
+* a table without clustering columns cannot have static columns (in a
+table without clustering columns, every partition has only one row, and
+so every column is inherently static).
+* only non `PRIMARY KEY` columns can be static
+
+[[createTableOptions]]
+===== `<option>`
+
+The `CREATE TABLE` statement supports a number of options that controls
+the configuration of a new table. These options can be specified after
+the `WITH` keyword.
+
+The first of these option is `COMPACT STORAGE`. This option is mainly
+targeted towards backward compatibility for definitions created before
+CQL3 (see
+http://www.datastax.com/dev/blog/thrift-to-cql3[www.datastax.com/dev/blog/thrift-to-cql3]
+for more details). The option also provides a slightly more compact
+layout of data on disk but at the price of diminished flexibility and
+extensibility for the table. Most notably, `COMPACT STORAGE` tables
+cannot have collections nor static columns and a `COMPACT STORAGE` table
+with at least one clustering column supports exactly one (as in not 0
+nor more than 1) column not part of the `PRIMARY KEY` definition (which
+imply in particular that you cannot add nor remove columns after
+creation). For those reasons, `COMPACT STORAGE` is not recommended
+outside of the backward compatibility reason evoked above.
+
+Another option is `CLUSTERING ORDER`. It allows to define the ordering
+of rows on disk. It takes the list of the clustering column names with,
+for each of them, the on-disk order (Ascending or descending). Note that
+this option affects link:#selectOrderBy[what `ORDER BY` are allowed
+during `SELECT`].
+
+Table creation supports the following other `<property>`:
+
+[cols=",,,",options="header",]
+|===
+|option |kind |default |description
+|`comment` |_simple_ |none |A free-form, human-readable comment.
+
+|`gc_grace_seconds` |_simple_ |864000 |Time to wait before garbage
+collecting tombstones (deletion markers).
+
+|`bloom_filter_fp_chance` |_simple_ |0.00075 |The target probability of
+false positive of the sstable bloom filters. Said bloom filters will be
+sized to provide the provided probability (thus lowering this value
+impact the size of bloom filters in-memory and on-disk)
+
+|`default_time_to_live` |_simple_ |0 |The default expiration time
+(``TTL'') in seconds for a table.
+
+|`compaction` |_map_ |_see below_ |Compaction options, see
+link:#compactionOptions[below].
+
+|`compression` |_map_ |_see below_ |Compression options, see
+link:#compressionOptions[below].
+
+|`caching` |_map_ |_see below_ |Caching options, see
+link:#cachingOptions[below].
+|===
+
+[[compactionOptions]]
+===== Compaction options
+
+The `compaction` property must at least define the `'class'` sub-option,
+that defines the compaction strategy class to use. The default supported
+class are `'SizeTieredCompactionStrategy'`,
+`'LeveledCompactionStrategy'` and
+`'TimeWindowCompactionStrategy'`. Custom strategy can be provided by
+specifying the full class name as a link:#constants[string constant].
+The rest of the sub-options depends on the chosen class. The sub-options
+supported by the default classes are:
+
+[cols=",,,",options="header",]
+|===
+|option |supported compaction strategy |default |description
+|`enabled` |_all_ |true |A boolean denoting whether compaction should be
+enabled or not.
+
+|`tombstone_threshold` |_all_ |0.2 |A ratio such that if a sstable has
+more than this ratio of gcable tombstones over all contained columns,
+the sstable will be compacted (with no other sstables) for the purpose
+of purging those tombstones.
+
+|`tombstone_compaction_interval` |_all_ |1 day |The minimum time to wait
+after an sstable creation time before considering it for ``tombstone
+compaction'', where ``tombstone compaction'' is the compaction triggered
+if the sstable has more gcable tombstones than `tombstone_threshold`.
+
+|`unchecked_tombstone_compaction` |_all_ |false |Setting this to true
+enables more aggressive tombstone compactions - single sstable tombstone
+compactions will run without checking how likely it is that they will be
+successful.
+
+|`min_sstable_size` |SizeTieredCompactionStrategy |50MB |The size tiered
+strategy groups SSTables to compact in buckets. A bucket groups SSTables
+that differs from less than 50% in size. However, for small sizes, this
+would result in a bucketing that is too fine grained. `min_sstable_size`
+defines a size threshold (in bytes) below which all SSTables belong to
+one unique bucket
+
+|`min_threshold` |SizeTieredCompactionStrategy |4 |Minimum number of
+SSTables needed to start a minor compaction.
+
+|`max_threshold` |SizeTieredCompactionStrategy |32 |Maximum number of
+SSTables processed by one minor compaction.
+
+|`bucket_low` |SizeTieredCompactionStrategy |0.5 |Size tiered consider
+sstables to be within the same bucket if their size is within
+[average_size * `bucket_low`, average_size * `bucket_high` ] (i.e the
+default groups sstable whose sizes diverges by at most 50%)
+
+|`bucket_high` |SizeTieredCompactionStrategy |1.5 |Size tiered consider
+sstables to be within the same bucket if their size is within
+[average_size * `bucket_low`, average_size * `bucket_high` ] (i.e the
+default groups sstable whose sizes diverges by at most 50%).
+
+|`sstable_size_in_mb` |LeveledCompactionStrategy |5MB |The target size
+(in MB) for sstables in the leveled strategy. Note that while sstable
+sizes should stay less or equal to `sstable_size_in_mb`, it is possible
+to exceptionally have a larger sstable as during compaction, data for a
+given partition key are never split into 2 sstables
+
+|`timestamp_resolution` |TimeWindowCompactionStrategy |MICROSECONDS |The
+timestamp resolution used when inserting data, could be MILLISECONDS,
+MICROSECONDS etc (should be understandable by Java TimeUnit) - don’t
+change this unless you do mutations with USING TIMESTAMP (or equivalent
+directly in the client)
+
+|`compaction_window_unit` |TimeWindowCompactionStrategy |DAYS |The Java
+TimeUnit used for the window size, set in conjunction with
+`compaction_window_size`. Must be one of DAYS, HOURS, MINUTES
+
+|`compaction_window_size` |TimeWindowCompactionStrategy |1 |The number
+of `compaction_window_unit` units that make up a time window.
+
+|`unsafe_aggressive_sstable_expiration` |TimeWindowCompactionStrategy
+|false |Expired sstables will be dropped without checking its data is
+shadowing other sstables. This is a potentially risky option that can
+lead to data loss or deleted data re-appearing, going beyond what
+`unchecked_tombstone_compaction` does for single sstable compaction. Due
+to the risk the jvm must also be started with
+`-Dcassandra.unsafe_aggressive_sstable_expiration=true`.
+|===
+
+[[compressionOptions]]
+===== Compression options
+
+For the `compression` property, the following sub-options are available:
+
+[cols=",,,,,",options="header",]
+|===
+|option |default |description | | |
+|`class` |LZ4Compressor |The compression algorithm to use. Default
+compressor are: LZ4Compressor, SnappyCompressor and DeflateCompressor.
+Use `'enabled' : false` to disable compression. Custom compressor can be
+provided by specifying the full class name as a link:#constants[string
+constant]. | | |
+
+|`enabled` |true |By default compression is enabled. To disable it, set
+`enabled` to `false` |`chunk_length_in_kb` |64KB |On disk SSTables are
+compressed by block (to allow random reads). This defines the size (in
+KB) of said block. Bigger values may improve the compression rate, but
+increases the minimum size of data to be read from disk for a read
+
+|`crc_check_chance` |1.0 |When compression is enabled, each compressed
+block includes a checksum of that block for the purpose of detecting
+disk bitrot and avoiding the propagation of corruption to other replica.
+This option defines the probability with which those checksums are
+checked during read. By default they are always checked. Set to 0 to
+disable checksum checking and to 0.5 for instance to check them every
+other read | | |
+|===
+
+[[cachingOptions]]
+===== Caching options
+
+For the `caching` property, the following sub-options are available:
+
+[cols=",,",options="header",]
+|===
+|option |default |description
+|`keys` |ALL |Whether to cache keys (``key cache'') for this table.
+Valid values are: `ALL` and `NONE`.
+
+|`rows_per_partition` |NONE |The amount of rows to cache per partition
+(``row cache''). If an integer `n` is specified, the first `n` queried
+rows of a partition will be cached. Other possible options are `ALL`, to
+cache all rows of a queried partition, or `NONE` to disable row caching.
+|===
+
+===== Other considerations:
+
+* When link:#insertStmt[inserting] / link:#updateStmt[updating] a given
+row, not all columns needs to be defined (except for those part of the
+key), and missing columns occupy no space on disk. Furthermore, adding
+new columns (see `ALTER TABLE`) is a constant time operation. There is
+thus no need to try to anticipate future usage (or to cry when you
+haven’t) when creating a table.
+
+[[alterTableStmt]]
+==== ALTER TABLE
+
+_Syntax:_
+
+bc(syntax).. +
+::= ALTER (TABLE | COLUMNFAMILY) (IF EXISTS)?
+
+::= ADD (IF NOT EXISTS)? +
+| ADD (IF NOT EXISTS)? ( ( , )* ) +
+| DROP (IF EXISTS)? +
+| DROP (IF EXISTS)? ( ( , )* ) +
+| RENAME (IF EXISTS)? TO (AND TO)* +
+| WITH ( AND )* +
+p. +
+_Sample:_
+
+bc(sample).. +
+ALTER TABLE addamsFamily
+
+ALTER TABLE addamsFamily +
+ADD gravesite varchar;
+
+ALTER TABLE addamsFamily +
+WITH comment = `A most excellent and useful column family'; +
+p. +
+The `ALTER` statement is used to manipulate table definitions. It allows
+for adding new columns, dropping existing ones, or updating the table
+options. As with table creation, `ALTER COLUMNFAMILY` is allowed as an
+alias for `ALTER TABLE`.
+If the table does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+The `<tablename>` is the table name optionally preceded by the keyspace
+name. The `<instruction>` defines the alteration to perform:
+
+* `ADD`: Adds a new column to the table. The `<identifier>` for the new
+column must not conflict with an existing column. Moreover, columns
+cannot be added to tables defined with the `COMPACT STORAGE` option.
+If the new column already exists, the statement will return an error, unless `IF NOT EXISTS` is used in which case the operation is a no-op.
+* `DROP`: Removes a column from the table. Dropped columns will
+immediately become unavailable in the queries and will not be included
+in compacted sstables in the future. If a column is readded, queries
+won’t return values written before the column was last dropped. It is
+assumed that timestamps represent actual time, so if this is not your
+case, you should NOT read previously dropped columns. Columns can’t be
+dropped from tables defined with the `COMPACT STORAGE` option.
+If the dropped column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+* `RENAME` a primary key column of a table. Non primary key columns cannot be renamed.
+Furthermore, renaming a column to another name which already exists isn't allowed.
+It's important to keep in mind that renamed columns shouldn't have dependent seconday indexes.
+If the renamed column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+* `WITH`: Allows to update the options of the table. The
+link:#createTableOptions[supported `<option>`] (and syntax) are the same
+as for the `CREATE TABLE` statement except that `COMPACT STORAGE` is not
+supported. Note that setting any `compaction` sub-options has the effect
+of erasing all previous `compaction` options, so you need to re-specify
+all the sub-options if you want to keep them. The same note applies to
+the set of `compression` sub-options.
+
+===== CQL type compatibility:
+
+CQL data types may be converted only as the following table.
+
+[cols=",",options="header",]
+|===
+|Data type may be altered to: |Data type
+|timestamp |bigint
+
+|ascii, bigint, boolean, date, decimal, double, float, inet, int,
+smallint, text, time, timestamp, timeuuid, tinyint, uuid, varchar,
+varint |blob
+
+|int |date
+
+|ascii, varchar |text
+
+|bigint |time
+
+|bigint |timestamp
+
+|timeuuid |uuid
+
+|ascii, text |varchar
+
+|bigint, int, timestamp |varint
+|===
+
+Clustering columns have stricter requirements, only the below
+conversions are allowed.
+
+[cols=",",options="header",]
+|===
+|Data type may be altered to: |Data type
+|ascii, text, varchar |blob
+|ascii, varchar |text
+|ascii, text |varchar
+|===
+
+[[dropTableStmt]]
+==== DROP TABLE
+
+_Syntax:_
+
+bc(syntax). ::= DROP TABLE ( IF EXISTS )?
+
+_Sample:_
+
+bc(sample). DROP TABLE worldSeriesAttendees;
+
+The `DROP TABLE` statement results in the immediate, irreversible
+removal of a table, including all data contained in it. As for table
+creation, `DROP COLUMNFAMILY` is allowed as an alias for `DROP TABLE`.
+
+If the table does not exist, the statement will return an error, unless
+`IF EXISTS` is used in which case the operation is a no-op.
+
+[[truncateStmt]]
+==== TRUNCATE
+
+_Syntax:_
+
+bc(syntax). ::= TRUNCATE ( TABLE | COLUMNFAMILY )?
+
+_Sample:_
+
+bc(sample). TRUNCATE superImportantData;
+
+The `TRUNCATE` statement permanently removes all data from a table.
+
+[[createIndexStmt]]
+==== CREATE INDEX
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE ( CUSTOM )? INDEX ( IF NOT EXISTS )? ( )? +
+ON `(' `)' +
+( USING ( WITH OPTIONS = )? )?
+
+::= +
+| keys( ) +
+p. +
+_Sample:_
+
+bc(sample). +
+CREATE INDEX userIndex ON NerdMovies (user); +
+CREATE INDEX ON Mutants (abilityId); +
+CREATE INDEX ON users (keys(favs)); +
+CREATE CUSTOM INDEX ON users (email) USING `path.to.the.IndexClass'; +
+CREATE CUSTOM INDEX ON users (email) USING `path.to.the.IndexClass' WITH
+OPTIONS = \{’storage’: `/mnt/ssd/indexes/'};
+
+The `CREATE INDEX` statement is used to create a new (automatic)
+secondary index for a given (existing) column in a given table. A name
+for the index itself can be specified before the `ON` keyword, if
+desired. If data already exists for the column, it will be indexed
+asynchronously. After the index is created, new data for the column is
+indexed automatically at insertion time.
+
+Attempting to create an already existing index will return an error
+unless the `IF NOT EXISTS` option is used. If it is used, the statement
+will be a no-op if the index already exists.
+
+[[keysIndex]]
+===== Indexes on Map Keys
+
+When creating an index on a link:#map[map column], you may index either
+the keys or the values. If the column identifier is placed within the
+`keys()` function, the index will be on the map keys, allowing you to
+use `CONTAINS KEY` in `WHERE` clauses. Otherwise, the index will be on
+the map values.
+
+[[dropIndexStmt]]
+==== DROP INDEX
+
+_Syntax:_
+
+bc(syntax). ::= DROP INDEX ( IF EXISTS )? ( `.' )?
+
+_Sample:_
+
+bc(sample).. +
+DROP INDEX userIndex;
+
+DROP INDEX userkeyspace.address_index; +
+p. +
+The `DROP INDEX` statement is used to drop an existing secondary index.
+The argument of the statement is the index name, which may optionally
+specify the keyspace of the index.
+
+If the index does not exists, the statement will return an error, unless
+`IF EXISTS` is used in which case the operation is a no-op.
+
+[[createMVStmt]]
+==== CREATE MATERIALIZED VIEW
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE MATERIALIZED VIEW ( IF NOT EXISTS )? AS +
+SELECT ( `(' ( `,' ) * `)' | `*' ) +
+FROM +
+( WHERE )? +
+PRIMARY KEY `(' ( `,' )* `)' +
+( WITH ( AND )* )? +
+p. +
+_Sample:_
+
+bc(sample).. +
+CREATE MATERIALIZED VIEW monkeySpecies_by_population AS +
+SELECT * +
+FROM monkeySpecies +
+WHERE population IS NOT NULL AND species IS NOT NULL +
+PRIMARY KEY (population, species) +
+WITH comment=`Allow query by population instead of species'; +
+p. +
+The `CREATE MATERIALIZED VIEW` statement creates a new materialized
+view. Each such view is a set of _rows_ which corresponds to rows which
+are present in the underlying, or base, table specified in the `SELECT`
+statement. A materialized view cannot be directly updated, but updates
+to the base table will cause corresponding updates in the view.
+
+Attempting to create an already existing materialized view will return
+an error unless the `IF NOT EXISTS` option is used. If it is used, the
+statement will be a no-op if the materialized view already exists.
+
+[[createMVWhere]]
+===== `WHERE` Clause
+
+The `<where-clause>` is similar to the link:#selectWhere[where clause of
+a `SELECT` statement], with a few differences. First, the where clause
+must contain an expression that disallows `NULL` values in columns in
+the view’s primary key. If no other restriction is desired, this can be
+accomplished with an `IS NOT NULL` expression. Second, only columns
+which are in the base table’s primary key may be restricted with
+expressions other than `IS NOT NULL`. (Note that this second restriction
+may be lifted in the future.)
+
+[[alterMVStmt]]
+==== ALTER MATERIALIZED VIEW
+
+_Syntax:_
+
+bc(syntax). ::= ALTER MATERIALIZED VIEW +
+WITH ( AND )*
+
+The `ALTER MATERIALIZED VIEW` statement allows options to be update;
+these options are the same as `CREATE TABLE`’s options.
+
+[[dropMVStmt]]
+==== DROP MATERIALIZED VIEW
+
+_Syntax:_
+
+bc(syntax). ::= DROP MATERIALIZED VIEW ( IF EXISTS )?
+
+_Sample:_
+
+bc(sample). DROP MATERIALIZED VIEW monkeySpecies_by_population;
+
+The `DROP MATERIALIZED VIEW` statement is used to drop an existing
+materialized view.
+
+If the materialized view does not exists, the statement will return an
+error, unless `IF EXISTS` is used in which case the operation is a
+no-op.
+
+[[createTypeStmt]]
+==== CREATE TYPE
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE TYPE ( IF NOT EXISTS )? +
+`(' ( `,' )* `)'
+
+::= ( `.' )?
+
+::=
+
+_Sample:_
+
+bc(sample).. +
+CREATE TYPE address ( +
+street_name text, +
+street_number int, +
+city text, +
+state text, +
+zip int +
+)
+
+CREATE TYPE work_and_home_addresses ( +
+home_address address, +
+work_address address +
+) +
+p. +
+The `CREATE TYPE` statement creates a new user-defined type. Each type
+is a set of named, typed fields. Field types may be any valid type,
+including collections and other existing user-defined types.
+
+Attempting to create an already existing type will result in an error
+unless the `IF NOT EXISTS` option is used. If it is used, the statement
+will be a no-op if the type already exists.
+
+[[createTypeName]]
+===== `<typename>`
+
+Valid type names are identifiers. The names of existing CQL types and
+link:#appendixB[reserved type names] may not be used.
+
+If the type name is provided alone, the type is created with the current
+keyspace (see `USE`). If it is prefixed by an existing keyspace name,
+the type is created within the specified keyspace instead of the current
+keyspace.
+
+[[alterTypeStmt]]
+==== ALTER TYPE
+
+_Syntax:_
+
+bc(syntax).. +
+::= ALTER TYPE (IF EXISTS)?
+
+::= ADD (IF NOT EXISTS)? +
+| RENAME (IF EXISTS)? TO ( AND TO )* +
+p. +
+_Sample:_
+
+bc(sample).. +
+ALTER TYPE address ADD country text
+
+ALTER TYPE address RENAME zip TO zipcode AND street_name TO street +
+p. +
+The `ALTER TYPE` statement is used to manipulate type definitions. It
+allows for adding new fields, renaming existing fields, or changing the
+type of existing fields. If the type does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+[[dropTypeStmt]]
+==== DROP TYPE
+
+_Syntax:_
+
+bc(syntax).. +
+::= DROP TYPE ( IF EXISTS )? +
+p. +
+The `DROP TYPE` statement results in the immediate, irreversible removal
+of a type. Attempting to drop a type that is still in use by another
+type or a table will result in an error.
+
+If the type does not exist, an error will be returned unless `IF EXISTS`
+is used, in which case the operation is a no-op.
+
+[[createTriggerStmt]]
+==== CREATE TRIGGER
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE TRIGGER ( IF NOT EXISTS )? ( )? +
+ON +
+USING
+
+_Sample:_
+
+bc(sample). +
+CREATE TRIGGER myTrigger ON myTable USING
+`org.apache.cassandra.triggers.InvertedIndex';
+
+The actual logic that makes up the trigger can be written in any Java
+(JVM) language and exists outside the database. You place the trigger
+code in a `lib/triggers` subdirectory of the Cassandra installation
+directory, it loads during cluster startup, and exists on every node
+that participates in a cluster. The trigger defined on a table fires
+before a requested DML statement occurs, which ensures the atomicity of
+the transaction.
+
+[[dropTriggerStmt]]
+==== DROP TRIGGER
+
+_Syntax:_
+
+bc(syntax).. +
+::= DROP TRIGGER ( IF EXISTS )? ( )? +
+ON +
+p. +
+_Sample:_
+
+bc(sample). +
+DROP TRIGGER myTrigger ON myTable;
+
+`DROP TRIGGER` statement removes the registration of a trigger created
+using `CREATE TRIGGER`.
+
+[[createFunctionStmt]]
+==== CREATE FUNCTION
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE ( OR REPLACE )? +
+FUNCTION ( IF NOT EXISTS )? +
+( `.' )? +
+`(' ( `,' )* `)' +
+( CALLED | RETURNS NULL ) ON NULL INPUT +
+RETURNS +
+LANGUAGE +
+AS
+
+_Sample:_
+
+bc(sample). +
+CREATE OR REPLACE FUNCTION somefunction +
+( somearg int, anotherarg text, complexarg frozen, listarg list ) +
+RETURNS NULL ON NULL INPUT +
+RETURNS text +
+LANGUAGE java +
+AS $$ +
+// some Java code +
+$$; +
+CREATE FUNCTION akeyspace.fname IF NOT EXISTS +
+( someArg int ) +
+CALLED ON NULL INPUT +
+RETURNS text +
+LANGUAGE java +
+AS $$ +
+// some Java code +
+$$;
+
+`CREATE FUNCTION` creates or replaces a user-defined function.
+
+[[functionSignature]]
+===== Function Signature
+
+Signatures are used to distinguish individual functions. The signature
+consists of:
+
+. The fully qualified function name - i.e _keyspace_ plus
+_function-name_
+. The concatenated list of all argument types
+
+Note that keyspace names, function names and argument types are subject
+to the default naming conventions and case-sensitivity rules.
+
+`CREATE FUNCTION` with the optional `OR REPLACE` keywords either creates
+a function or replaces an existing one with the same signature. A
+`CREATE FUNCTION` without `OR REPLACE` fails if a function with the same
+signature already exists.
+
+Behavior on invocation with `null` values must be defined for each
+function. There are two options:
+
+. `RETURNS NULL ON NULL INPUT` declares that the function will always
+return `null` if any of the input arguments is `null`.
+. `CALLED ON NULL INPUT` declares that the function will always be
+executed.
+
+If the optional `IF NOT EXISTS` keywords are used, the function will
+only be created if another function with the same signature does not
+exist.
+
+`OR REPLACE` and `IF NOT EXIST` cannot be used together.
+
+Functions belong to a keyspace. If no keyspace is specified in
+`<function-name>`, the current keyspace is used (i.e. the keyspace
+specified using the link:#useStmt[`USE`] statement). It is not possible
+to create a user-defined function in one of the system keyspaces.
+
+See the section on link:#udfs[user-defined functions] for more
+information.
+
+[[dropFunctionStmt]]
+==== DROP FUNCTION
+
+_Syntax:_
+
+bc(syntax).. +
+::= DROP FUNCTION ( IF EXISTS )? +
+( `.' )? +
+( `(' ( `,' )* `)' )?
+
+_Sample:_
+
+bc(sample). +
+DROP FUNCTION myfunction; +
+DROP FUNCTION mykeyspace.afunction; +
+DROP FUNCTION afunction ( int ); +
+DROP FUNCTION afunction ( text );
+
+`DROP FUNCTION` statement removes a function created using
+`CREATE FUNCTION`. +
+You must specify the argument types (link:#functionSignature[signature]
+) of the function to drop if there are multiple functions with the same
+name but a different signature (overloaded functions).
+
+`DROP FUNCTION` with the optional `IF EXISTS` keywords drops a function
+if it exists.
+
+[[createAggregateStmt]]
+==== CREATE AGGREGATE
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE ( OR REPLACE )? +
+AGGREGATE ( IF NOT EXISTS )? +
+( `.' )? +
+`(' ( `,' )* `)' +
+SFUNC +
+STYPE +
+( FINALFUNC )? +
+( INITCOND )? +
+p. +
+_Sample:_
+
+bc(sample). +
+CREATE AGGREGATE myaggregate ( val text ) +
+SFUNC myaggregate_state +
+STYPE text +
+FINALFUNC myaggregate_final +
+INITCOND `foo';
+
+See the section on link:#udas[user-defined aggregates] for a complete
+example.
+
+`CREATE AGGREGATE` creates or replaces a user-defined aggregate.
+
+`CREATE AGGREGATE` with the optional `OR REPLACE` keywords either
+creates an aggregate or replaces an existing one with the same
+signature. A `CREATE AGGREGATE` without `OR REPLACE` fails if an
+aggregate with the same signature already exists.
+
+`CREATE AGGREGATE` with the optional `IF NOT EXISTS` keywords either
+creates an aggregate if it does not already exist.
+
+`OR REPLACE` and `IF NOT EXIST` cannot be used together.
+
+Aggregates belong to a keyspace. If no keyspace is specified in
+`<aggregate-name>`, the current keyspace is used (i.e. the keyspace
+specified using the link:#useStmt[`USE`] statement). It is not possible
+to create a user-defined aggregate in one of the system keyspaces.
+
+Signatures for user-defined aggregates follow the
+link:#functionSignature[same rules] as for user-defined functions.
+
+`STYPE` defines the type of the state value and must be specified.
+
+The optional `INITCOND` defines the initial state value for the
+aggregate. It defaults to `null`. A non-`null` `INITCOND` must be
+specified for state functions that are declared with
+`RETURNS NULL ON NULL INPUT`.
+
+`SFUNC` references an existing function to be used as the state
+modifying function. The type of first argument of the state function
+must match `STYPE`. The remaining argument types of the state function
+must match the argument types of the aggregate function. State is not
+updated for state functions declared with `RETURNS NULL ON NULL INPUT`
+and called with `null`.
+
+The optional `FINALFUNC` is called just before the aggregate result is
+returned. It must take only one argument with type `STYPE`. The return
+type of the `FINALFUNC` may be a different type. A final function
+declared with `RETURNS NULL ON NULL INPUT` means that the aggregate’s
+return value will be `null`, if the last state is `null`.
+
+If no `FINALFUNC` is defined, the overall return type of the aggregate
+function is `STYPE`. If a `FINALFUNC` is defined, it is the return type
+of that function.
+
+See the section on link:#udas[user-defined aggregates] for more
+information.
+
+[[dropAggregateStmt]]
+==== DROP AGGREGATE
+
+_Syntax:_
+
+bc(syntax).. +
+::= DROP AGGREGATE ( IF EXISTS )? +
+( `.' )? +
+( `(' ( `,' )* `)' )? +
+p.
+
+_Sample:_
+
+bc(sample). +
+DROP AGGREGATE myAggregate; +
+DROP AGGREGATE myKeyspace.anAggregate; +
+DROP AGGREGATE someAggregate ( int ); +
+DROP AGGREGATE someAggregate ( text );
+
+The `DROP AGGREGATE` statement removes an aggregate created using
+`CREATE AGGREGATE`. You must specify the argument types of the aggregate
+to drop if there are multiple aggregates with the same name but a
+different signature (overloaded aggregates).
+
+`DROP AGGREGATE` with the optional `IF EXISTS` keywords drops an
+aggregate if it exists, and does nothing if a function with the
+signature does not exist.
+
+Signatures for user-defined aggregates follow the
+link:#functionSignature[same rules] as for user-defined functions.
+
+[[dataManipulation]]
+=== Data Manipulation
+
+[[insertStmt]]
+==== INSERT
+
+_Syntax:_
+
+bc(syntax).. +
+::= INSERT INTO +
+( ( VALUES ) +
+| ( JSON )) +
+( IF NOT EXISTS )? +
+( USING ( AND )* )?
+
+::= `(' ( `,' )* `)'
+
+::= `(' ( `,' )* `)'
+
+::= TIMESTAMP +
+| TTL +
+p. +
+_Sample:_
+
+bc(sample).. +
+INSERT INTO NerdMovies (movie, director, main_actor, year) +
+VALUES (`Serenity', `Joss Whedon', `Nathan Fillion', 2005) +
+USING TTL 86400;
+
+INSERT INTO NerdMovies JSON `\{``movie'': ``Serenity'', ``director'':
+``Joss Whedon'', ``year'': 2005}' +
+p. +
+The `INSERT` statement writes one or more columns for a given row in a
+table. Note that since a row is identified by its `PRIMARY KEY`, at
+least the columns composing it must be specified. The list of columns to
+insert to must be supplied when using the `VALUES` syntax. When using
+the `JSON` syntax, they are optional. See the section on
+link:#insertJson[`INSERT JSON`] for more details.
+
+Note that unlike in SQL, `INSERT` does not check the prior existence of
+the row by default: the row is created if none existed before, and
+updated otherwise. Furthermore, there is no mean to know which of
+creation or update happened.
+
+It is however possible to use the `IF NOT EXISTS` condition to only
+insert if the row does not exist prior to the insertion. But please note
+that using `IF NOT EXISTS` will incur a non negligible performance cost
+(internally, Paxos will be used) so this should be used sparingly.
+
+All updates for an `INSERT` are applied atomically and in isolation.
+
+Please refer to the link:#updateOptions[`UPDATE`] section for
+information on the `<option>` available and to the
+link:#collections[collections] section for use of
+`<collection-literal>`. Also note that `INSERT` does not support
+counters, while `UPDATE` does.
+
+[[updateStmt]]
+==== UPDATE
+
+_Syntax:_
+
+bc(syntax).. +
+::= UPDATE +
+( USING ( AND )* )? +
+SET ( `,' )* +
+WHERE +
+( IF ( AND condition )* )?
+
+::= `=' +
+| `=' (`+' | `-') ( | | ) +
+| `=' `+' +
+| `[' `]' `=' +
+| `.' `='
+
+::= +
+| CONTAINS (KEY)? +
+| IN +
+| `[' `]' +
+| `[' `]' IN +
+| `.' +
+| `.' IN
+
+::= `<' | `<=' | `=' | `!=' | `>=' | `>' +
+::= ( | `(' ( ( `,' )* )? `)')
+
+::= ( AND )*
+
+::= `=' +
+| `(' (`,' )* `)' `=' +
+| IN `(' ( ( `,' )* )? `)' +
+| IN +
+| `(' (`,' )* `)' IN `(' ( ( `,' )* )? `)' +
+| `(' (`,' )* `)' IN
+
+::= TIMESTAMP +
+| TTL +
+p. +
+_Sample:_
+
+bc(sample).. +
+UPDATE NerdMovies USING TTL 400 +
+SET director = `Joss Whedon', +
+main_actor = `Nathan Fillion', +
+year = 2005 +
+WHERE movie = `Serenity';
+
+UPDATE UserActions SET total = total + 2 WHERE user =
+B70DE1D0-9908-4AE3-BE34-5573E5B09F14 AND action = `click'; +
+p. +
+The `UPDATE` statement writes one or more columns for a given row in a
+table. The `<where-clause>` is used to select the row to update and must
+include all columns composing the `PRIMARY KEY`. Other columns values
+are specified through `<assignment>` after the `SET` keyword.
+
+Note that unlike in SQL, `UPDATE` does not check the prior existence of
+the row by default (except through the use of `<condition>`, see below):
+the row is created if none existed before, and updated otherwise.
+Furthermore, there are no means to know whether a creation or update
+occurred.
+
+It is however possible to use the conditions on some columns through
+`IF`, in which case the row will not be updated unless the conditions
+are met. But, please note that using `IF` conditions will incur a
+non-negligible performance cost (internally, Paxos will be used) so this
+should be used sparingly.
+
+In an `UPDATE` statement, all updates within the same partition key are
+applied atomically and in isolation.
+
+The `c = c + 3` form of `<assignment>` is used to increment/decrement
+counters. The identifier after the `=' sign *must* be the same than the
+one before the `=' sign (Only increment/decrement is supported on
+counters, not the assignment of a specific value).
+
+The `id = id + <collection-literal>` and `id[value1] = value2` forms of
+`<assignment>` are for collections. Please refer to the
+link:#collections[relevant section] for more details.
+
+The `id.field = <term>` form of `<assignemt>` is for setting the value
+of a single field on a non-frozen user-defined types.
+
+[[updateOptions]]
+===== `<options>`
+
+The `UPDATE` and `INSERT` statements support the following options:
+
+* `TIMESTAMP`: sets the timestamp for the operation. If not specified,
+the coordinator will use the current time (in microseconds) at the start
+of statement execution as the timestamp. This is usually a suitable
+default.
+* `TTL`: specifies an optional Time To Live (in seconds) for the
+inserted values. If set, the inserted values are automatically removed
+from the database after the specified time. Note that the TTL concerns
+the inserted values, not the columns themselves. This means that any
+subsequent update of the column will also reset the TTL (to whatever TTL
+is specified in that update). By default, values never expire. A TTL of
+0 is equivalent to no TTL. If the table has a default_time_to_live, a
+TTL of 0 will remove the TTL for the inserted or updated values.
+
+[[deleteStmt]]
+==== DELETE
+
+_Syntax:_
+
+bc(syntax).. +
+::= DELETE ( ( `,' )* )? +
+FROM +
+( USING TIMESTAMP )? +
+WHERE +
+( IF ( EXISTS | ( ( AND )*) ) )?
+
+::= +
+| `[' `]' +
+| `.'
+
+::= ( AND )*
+
+::= +
+| `(' (`,' )* `)' +
+| IN `(' ( ( `,' )* )? `)' +
+| IN +
+| `(' (`,' )* `)' IN `(' ( ( `,' )* )? `)' +
+| `(' (`,' )* `)' IN
+
+::= `=' | `<' | `>' | `<=' | `>=' +
+::= ( | `(' ( ( `,' )* )? `)')
+
+::= ( | `!=') +
+| CONTAINS (KEY)? +
+| IN +
+| `[' `]' ( | `!=') +
+| `[' `]' IN +
+| `.' ( | `!=') +
+| `.' IN
+
+_Sample:_
+
+bc(sample).. +
+DELETE FROM NerdMovies USING TIMESTAMP 1240003134 WHERE movie =
+`Serenity';
+
+DELETE phone FROM Users WHERE userid IN
+(C73DE1D3-AF08-40F3-B124-3FF3E5109F22,
+B70DE1D0-9908-4AE3-BE34-5573E5B09F14); +
+p. +
+The `DELETE` statement deletes columns and rows. If column names are
+provided directly after the `DELETE` keyword, only those columns are
+deleted from the row indicated by the `<where-clause>`. The `id[value]`
+syntax in `<selection>` is for non-frozen collections (please refer to
+the link:#collections[collection section] for more details). The
+`id.field` syntax is for the deletion of non-frozen user-defined types.
+Otherwise, whole rows are removed. The `<where-clause>` specifies which
+rows are to be deleted. Multiple rows may be deleted with one statement
+by using an `IN` clause. A range of rows may be deleted using an
+inequality operator (such as `>=`).
+
+`DELETE` supports the `TIMESTAMP` option with the same semantics as the
+link:#updateStmt[`UPDATE`] statement.
+
+In a `DELETE` statement, all deletions within the same partition key are
+applied atomically and in isolation.
+
+A `DELETE` operation can be conditional through the use of an `IF`
+clause, similar to `UPDATE` and `INSERT` statements. However, as with
+`INSERT` and `UPDATE` statements, this will incur a non-negligible
+performance cost (internally, Paxos will be used) and so should be used
+sparingly.
+
+[[batchStmt]]
+==== BATCH
+
+_Syntax:_
+
+bc(syntax).. +
+::= BEGIN ( UNLOGGED | COUNTER ) BATCH +
+( USING ( AND )* )? +
+( `;' )* +
+APPLY BATCH
+
+::= +
+| +
+|
+
+::= TIMESTAMP +
+p. +
+_Sample:_
+
+bc(sample). +
+BEGIN BATCH +
+INSERT INTO users (userid, password, name) VALUES (`user2', `ch@ngem3b',
+`second user'); +
+UPDATE users SET password = `ps22dhds' WHERE userid = `user3'; +
+INSERT INTO users (userid, password) VALUES (`user4', `ch@ngem3c'); +
+DELETE name FROM users WHERE userid = `user1'; +
+APPLY BATCH;
+
+The `BATCH` statement group multiple modification statements
+(insertions/updates and deletions) into a single statement. It serves
+several purposes:
+
+. It saves network round-trips between the client and the server (and
+sometimes between the server coordinator and the replicas) when batching
+multiple updates.
+. All updates in a `BATCH` belonging to a given partition key are
+performed in isolation.
+. By default, all operations in the batch are performed as `LOGGED`, to
+ensure all mutations eventually complete (or none will). See the notes
+on link:#unloggedBatch[`UNLOGGED`] for more details.
+
+Note that:
+
+* `BATCH` statements may only contain `UPDATE`, `INSERT` and `DELETE`
+statements.
+* Batches are _not_ a full analogue for SQL transactions.
+* If a timestamp is not specified for each operation, then all
+operations will be applied with the same timestamp. Due to Cassandra’s
+conflict resolution procedure in the case of
+http://wiki.apache.org/cassandra/FAQ#clocktie[timestamp ties],
+operations may be applied in an order that is different from the order
+they are listed in the `BATCH` statement. To force a particular
+operation ordering, you must specify per-operation timestamps.
+
+[[unloggedBatch]]
+===== `UNLOGGED`
+
+By default, Cassandra uses a batch log to ensure all operations in a
+batch eventually complete or none will (note however that operations are
+only isolated within a single partition).
+
+There is a performance penalty for batch atomicity when a batch spans
+multiple partitions. If you do not want to incur this penalty, you can
+tell Cassandra to skip the batchlog with the `UNLOGGED` option. If the
+`UNLOGGED` option is used, a failed batch might leave the patch only
+partly applied.
+
+[[counterBatch]]
+===== `COUNTER`
+
+Use the `COUNTER` option for batched counter updates. Unlike other
+updates in Cassandra, counter updates are not idempotent.
+
+[[batchOptions]]
+===== `<option>`
+
+`BATCH` supports both the `TIMESTAMP` option, with similar semantic to
+the one described in the link:#updateOptions[`UPDATE`] statement (the
+timestamp applies to all the statement inside the batch). However, if
+used, `TIMESTAMP` *must not* be used in the statements within the batch.
+
+=== Queries
+
+[[selectStmt]]
+==== SELECT
+
+_Syntax:_
+
+bc(syntax).. +
+::= SELECT ( JSON )? +
+FROM +
+( WHERE )? +
+( GROUP BY )? +
+( ORDER BY )? +
+( PER PARTITION LIMIT )? +
+( LIMIT )? +
+( ALLOW FILTERING )?
+
+::= DISTINCT?
+
+::= (AS )? ( `,' (AS )? )* +
+| `*'
+
+::= +
+| +
+| WRITETIME `(' `)' +
+| MAXWRITETIME `(' `)' +
+| COUNT `(' `*' `)' +
+| TTL `(' `)' +
+| CAST `(' AS `)' +
+| `(' ( (`,' )*)? `)' +
+| `.' +
+| `[' `]' +
+| `[' ? .. ? `]'
+
+::= ( AND )*
+
+::= +
+| `(' (`,' )* `)' +
+| IN `(' ( ( `,' )* )? `)' +
+| `(' (`,' )* `)' IN `(' ( ( `,' )* )? `)' +
+| TOKEN `(' ( `,' )* `)'
+
+::= `=' | `<' | `>' | `<=' | `>=' | CONTAINS | CONTAINS KEY +
+::= (`,' )* +
+::= ( `,' )* +
+::= ( ASC | DESC )? +
+::= `(' (`,' )* `)' +
+p. +
+_Sample:_
+
+bc(sample).. +
+SELECT name, occupation FROM users WHERE userid IN (199, 200, 207);
+
+SELECT JSON name, occupation FROM users WHERE userid = 199;
+
+SELECT name AS user_name, occupation AS user_occupation FROM users;
+
+SELECT time, value +
+FROM events +
+WHERE event_type = `myEvent' +
+AND time > `2011-02-03' +
+AND time <= `2012-01-01'
+
+SELECT COUNT (*) FROM users;
+
+SELECT COUNT (*) AS user_count FROM users;
+
+The `SELECT` statements reads one or more columns for one or more rows
+in a table. It returns a result-set of rows, where each row contains the
+collection of columns corresponding to the query. If the `JSON` keyword
+is used, the results for each row will contain only a single column
+named ``json''. See the section on link:#selectJson[`SELECT JSON`] for
+more details.
+
+[[selectSelection]]
+===== `<select-clause>`
+
+The `<select-clause>` determines which columns needs to be queried and
+returned in the result-set. It consists of either the comma-separated
+list of or the wildcard character (`*`) to select all the columns
+defined for the table. Please note that for wildcard `SELECT` queries
+the order of columns returned is not specified and is not guaranteed to
+be stable between Cassandra versions.
+
+A `<selector>` is either a column name to retrieve or a `<function>` of
+one or more `<term>`s. The function allowed are the same as for `<term>`
+and are described in the link:#functions[function section]. In addition
+to these generic functions, the `WRITETIME` and `MAXWRITETIME` (resp. `TTL`)
+function allows to select the timestamp of when the column was inserted (resp.
+the time to live (in seconds) for the column (or null if the column has
+no expiration set)) and the link:#castFun[`CAST`] function can be used
+to convert one data type to another. The `WRITETIME` and `TTL` functions
+can't be used on multi-cell columns such as non-frozen collections or
+non-frozen user-defined types.
+
+Additionally, individual values of maps and sets can be selected using
+`[ <term> ]`. For maps, this will return the value corresponding to the
+key, if such entry exists. For sets, this will return the key that is
+selected if it exists and is thus mainly a way to check element
+existence. It is also possible to select a slice of a set or map with
+`[ <term> ... <term> `], where both bound can be omitted.
+
+Any `<selector>` can be aliased using `AS` keyword (see examples).
+Please note that `<where-clause>` and `<order-by>` clause should refer
+to the columns by their original names and not by their aliases.
+
+The `COUNT` keyword can be used with parenthesis enclosing `*`. If so,
+the query will return a single result: the number of rows matching the
+query. Note that `COUNT(1)` is supported as an alias.
+
+[[selectWhere]]
+===== `<where-clause>`
+
+The `<where-clause>` specifies which rows must be queried. It is
+composed of relations on the columns that are part of the `PRIMARY KEY`
+and/or have a link:#createIndexStmt[secondary index] defined on them.
+
+Not all relations are allowed in a query. For instance, non-equal
+relations (where `IN` is considered as an equal relation) on a partition
+key are not supported (but see the use of the `TOKEN` method below to do
+non-equal queries on the partition key). Moreover, for a given partition
+key, the clustering columns induce an ordering of rows and relations on
+them is restricted to the relations that allow to select a *contiguous*
+(for the ordering) set of rows. For instance, given
+
+bc(sample). +
+CREATE TABLE posts ( +
+userid text, +
+blog_title text, +
+posted_at timestamp, +
+entry_title text, +
+content text, +
+category int, +
+PRIMARY KEY (userid, blog_title, posted_at) +
+)
+
+The following query is allowed:
+
+bc(sample). +
+SELECT entry_title, content FROM posts WHERE userid=`john doe' AND
+blog_title=`John'`s Blog' AND posted_at >= `2012-01-01' AND posted_at <
+`2012-01-31'
+
+But the following one is not, as it does not select a contiguous set of
+rows (and we suppose no secondary indexes are set):
+
+bc(sample). +
+// Needs a blog_title to be set to select ranges of posted_at +
+SELECT entry_title, content FROM posts WHERE userid=`john doe' AND
+posted_at >= `2012-01-01' AND posted_at < `2012-01-31'
+
+When specifying relations, the `TOKEN` function can be used on the
+`PARTITION KEY` column to query. In that case, rows will be selected
+based on the token of their `PARTITION_KEY` rather than on the value.
+Note that the token of a key depends on the partitioner in use, and that
+in particular the RandomPartitioner won’t yield a meaningful order. Also
+note that ordering partitioners always order token values by bytes (so
+even if the partition key is of type int, `token(-1) > token(0)` in
+particular). Example:
+
+bc(sample). +
+SELECT * FROM posts WHERE token(userid) > token(`tom') AND token(userid)
+< token(`bob')
+
+Moreover, the `IN` relation is only allowed on the last column of the
+partition key and on the last column of the full primary key.
+
+It is also possible to ``group'' `CLUSTERING COLUMNS` together in a
+relation using the tuple notation. For instance:
+
+bc(sample). +
+SELECT * FROM posts WHERE userid=`john doe' AND (blog_title, posted_at)
+> (`John'`s Blog', `2012-01-01')
+
+will request all rows that sorts after the one having ``John’s Blog'' as
+`blog_tile` and `2012-01-01' for `posted_at` in the clustering order. In
+particular, rows having a `post_at <= '2012-01-01'` will be returned as
+long as their `blog_title > 'John''s Blog'`, which wouldn’t be the case
+for:
+
+bc(sample). +
+SELECT * FROM posts WHERE userid=`john doe' AND blog_title > `John'`s
+Blog' AND posted_at > `2012-01-01'
+
+The tuple notation may also be used for `IN` clauses on
+`CLUSTERING COLUMNS`:
+
+bc(sample). +
+SELECT * FROM posts WHERE userid=`john doe' AND (blog_title, posted_at)
+IN ((`John'`s Blog', `2012-01-01), (’Extreme Chess', `2014-06-01'))
+
+The `CONTAINS` operator may only be used on collection columns (lists,
+sets, and maps). In the case of maps, `CONTAINS` applies to the map
+values. The `CONTAINS KEY` operator may only be used on map columns and
+applies to the map keys.
+
+[[selectOrderBy]]
+===== `<order-by>`
+
+The `ORDER BY` option allows to select the order of the returned
+results. It takes as argument a list of column names along with the
+order for the column (`ASC` for ascendant and `DESC` for descendant,
+omitting the order being equivalent to `ASC`). Currently the possible
+orderings are limited (which depends on the table
+link:#createTableOptions[`CLUSTERING ORDER`] ):
+
+* if the table has been defined without any specific `CLUSTERING ORDER`,
+then then allowed orderings are the order induced by the clustering
+columns and the reverse of that one.
+* otherwise, the orderings allowed are the order of the
+`CLUSTERING ORDER` option and the reversed one.
+
+[[selectGroupBy]]
+===== `<group-by>`
+
+The `GROUP BY` option allows to condense into a single row all selected
+rows that share the same values for a set of columns.
+
+Using the `GROUP BY` option, it is only possible to group rows at the
+partition key level or at a clustering column level. By consequence, the
+`GROUP BY` option only accept as arguments primary key column names in
+the primary key order. If a primary key column is restricted by an
+equality restriction it is not required to be present in the `GROUP BY`
+clause.
+
+Aggregate functions will produce a separate value for each group. If no
+`GROUP BY` clause is specified, aggregates functions will produce a
+single value for all the rows.
+
+If a column is selected without an aggregate function, in a statement
+with a `GROUP BY`, the first value encounter in each group will be
+returned.
+
+[[selectLimit]]
+===== `LIMIT` and `PER PARTITION LIMIT`
+
+The `LIMIT` option to a `SELECT` statement limits the number of rows
+returned by a query, while the `PER PARTITION LIMIT` option limits the
+number of rows returned for a given partition by the query. Note that
+both type of limit can used in the same statement.
+
+[[selectAllowFiltering]]
+===== `ALLOW FILTERING`
+
+By default, CQL only allows select queries that don’t involve
+``filtering'' server side, i.e. queries where we know that all (live)
+record read will be returned (maybe partly) in the result set. The
+reasoning is that those ``non filtering'' queries have predictable
+performance in the sense that they will execute in a time that is
+proportional to the amount of data *returned* by the query (which can be
+controlled through `LIMIT`).
+
+The `ALLOW FILTERING` option allows to explicitly allow (some) queries
+that require filtering. Please note that a query using `ALLOW FILTERING`
+may thus have unpredictable performance (for the definition above), i.e.
+even a query that selects a handful of records *may* exhibit performance
+that depends on the total amount of data stored in the cluster.
+
+For instance, considering the following table holding user profiles with
+their year of birth (with a secondary index on it) and country of
+residence:
+
+bc(sample).. +
+CREATE TABLE users ( +
+username text PRIMARY KEY, +
+firstname text, +
+lastname text, +
+birth_year int, +
+country text +
+)
+
+CREATE INDEX ON users(birth_year); +
+p.
+
+Then the following queries are valid:
+
+bc(sample). +
+SELECT * FROM users; +
+SELECT firstname, lastname FROM users WHERE birth_year = 1981;
+
+because in both case, Cassandra guarantees that these queries
+performance will be proportional to the amount of data returned. In
+particular, if no users are born in 1981, then the second query
+performance will not depend of the number of user profile stored in the
+database (not directly at least: due to secondary index implementation
+consideration, this query may still depend on the number of node in the
+cluster, which indirectly depends on the amount of data stored.
+Nevertheless, the number of nodes will always be multiple number of
+magnitude lower than the number of user profile stored). Of course, both
+query may return very large result set in practice, but the amount of
+data returned can always be controlled by adding a `LIMIT`.
+
+However, the following query will be rejected:
+
+bc(sample). +
+SELECT firstname, lastname FROM users WHERE birth_year = 1981 AND
+country = `FR';
+
+because Cassandra cannot guarantee that it won’t have to scan large
+amount of data even if the result to those query is small. Typically, it
+will scan all the index entries for users born in 1981 even if only a
+handful are actually from France. However, if you ``know what you are
+doing'', you can force the execution of this query by using
+`ALLOW FILTERING` and so the following query is valid:
+
+bc(sample). +
+SELECT firstname, lastname FROM users WHERE birth_year = 1981 AND
+country = `FR' ALLOW FILTERING;
+
+[[databaseRoles]]
+=== Database Roles
+
+[[createRoleStmt]]
+==== CREATE ROLE
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE ROLE ( IF NOT EXISTS )? ( WITH ( AND )* )?
+
+::= PASSWORD = +
+| LOGIN = +
+| SUPERUSER = +
+| OPTIONS = +
+p.
+
+_Sample:_
+
+bc(sample). +
+CREATE ROLE new_role; +
+CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true; +
+CREATE ROLE bob WITH PASSWORD = `password_b' AND LOGIN = true AND
+SUPERUSER = true; +
+CREATE ROLE carlos WITH OPTIONS = \{ `custom_option1' : `option1_value',
+`custom_option2' : 99 };
+
+By default roles do not possess `LOGIN` privileges or `SUPERUSER`
+status.
+
+link:#permissions[Permissions] on database resources are granted to
+roles; types of resources include keyspaces, tables, functions and roles
+themselves. Roles may be granted to other roles to create hierarchical
+permissions structures; in these hierarchies, permissions and
+`SUPERUSER` status are inherited, but the `LOGIN` privilege is not.
+
+If a role has the `LOGIN` privilege, clients may identify as that role
+when connecting. For the duration of that connection, the client will
+acquire any roles and privileges granted to that role.
+
+Only a client with with the `CREATE` permission on the database roles
+resource may issue `CREATE ROLE` requests (see the
+link:#permissions[relevant section] below), unless the client is a
+`SUPERUSER`. Role management in Cassandra is pluggable and custom
+implementations may support only a subset of the listed options.
+
+Role names should be quoted if they contain non-alphanumeric characters.
+
+[[createRolePwd]]
+===== Setting credentials for internal authentication
+
+Use the `WITH PASSWORD` clause to set a password for internal
+authentication, enclosing the password in single quotation marks. +
+If internal authentication has not been set up or the role does not have
+`LOGIN` privileges, the `WITH PASSWORD` clause is not necessary.
+
+[[createRoleConditional]]
+===== Creating a role conditionally
+
+Attempting to create an existing role results in an invalid query
+condition unless the `IF NOT EXISTS` option is used. If the option is
+used and the role exists, the statement is a no-op.
+
+bc(sample). +
+CREATE ROLE other_role; +
+CREATE ROLE IF NOT EXISTS other_role;
+
+[[alterRoleStmt]]
+==== ALTER ROLE
+
+_Syntax:_
+
+bc(syntax).. +
+::= ALTER ROLE (IF EXISTS)? ( WITH ( AND )* )?
+
+::= PASSWORD = +
+| LOGIN = +
+| SUPERUSER = +
+| OPTIONS = +
+p.
+
+_Sample:_
+
+bc(sample). +
+ALTER ROLE bob WITH PASSWORD = `PASSWORD_B' AND SUPERUSER = false;
+
+If the role does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+Conditions on executing `ALTER ROLE` statements:
+
+* A client must have `SUPERUSER` status to alter the `SUPERUSER` status
+of another role
+* A client cannot alter the `SUPERUSER` status of any role it currently
+holds
+* A client can only modify certain properties of the role with which it
+identified at login (e.g. `PASSWORD`)
+* To modify properties of a role, the client must be granted `ALTER`
+link:#permissions[permission] on that role
+
+[[dropRoleStmt]]
+==== DROP ROLE
+
+_Syntax:_
+
+bc(syntax).. +
+::= DROP ROLE ( IF EXISTS )? +
+p.
+
+_Sample:_
+
+bc(sample). +
+DROP ROLE alice; +
+DROP ROLE IF EXISTS bob;
+
+`DROP ROLE` requires the client to have `DROP`
+link:#permissions[permission] on the role in question. In addition,
+client may not `DROP` the role with which it identified at login.
+Finaly, only a client with `SUPERUSER` status may `DROP` another
+`SUPERUSER` role. +
+Attempting to drop a role which does not exist results in an invalid
+query condition unless the `IF EXISTS` option is used. If the option is
+used and the role does not exist the statement is a no-op.
+
+[[grantRoleStmt]]
+==== GRANT ROLE
+
+_Syntax:_
+
+bc(syntax). +
+::= GRANT TO
+
+_Sample:_
+
+bc(sample). +
+GRANT report_writer TO alice;
+
+This statement grants the `report_writer` role to `alice`. Any
+permissions granted to `report_writer` are also acquired by `alice`. +
+Roles are modelled as a directed acyclic graph, so circular grants are
+not permitted. The following examples result in error conditions:
+
+bc(sample). +
+GRANT role_a TO role_b; +
+GRANT role_b TO role_a;
+
+bc(sample). +
+GRANT role_a TO role_b; +
+GRANT role_b TO role_c; +
+GRANT role_c TO role_a;
+
+[[revokeRoleStmt]]
+==== REVOKE ROLE
+
+_Syntax:_
+
+bc(syntax). +
+::= REVOKE FROM
+
+_Sample:_
+
+bc(sample). +
+REVOKE report_writer FROM alice;
+
+This statement revokes the `report_writer` role from `alice`. Any
+permissions that `alice` has acquired via the `report_writer` role are
+also revoked.
+
+[[listRolesStmt]]
+===== LIST ROLES
+
+_Syntax:_
+
+bc(syntax). +
+::= LIST ROLES ( OF )? ( NORECURSIVE )?
+
+_Sample:_
+
+bc(sample). +
+LIST ROLES;
+
+Return all known roles in the system, this requires `DESCRIBE`
+permission on the database roles resource.
+
+bc(sample). +
+LIST ROLES OF `alice`;
+
+Enumerate all roles granted to `alice`, including those transitively
+aquired.
+
+bc(sample). +
+LIST ROLES OF `bob` NORECURSIVE
+
+List all roles directly granted to `bob`.
+
+[[createUserStmt]]
+==== CREATE USER
+
+Prior to the introduction of roles in Cassandra 2.2, authentication and
+authorization were based around the concept of a `USER`. For backward
+compatibility, the legacy syntax has been preserved with `USER` centric
+statments becoming synonyms for the `ROLE` based equivalents.
+
+_Syntax:_
+
+bc(syntax).. +
+::= CREATE USER ( IF NOT EXISTS )? ( WITH PASSWORD )? ()?
+
+::= SUPERUSER +
+| NOSUPERUSER +
+p.
+
+_Sample:_
+
+bc(sample). +
+CREATE USER alice WITH PASSWORD `password_a' SUPERUSER; +
+CREATE USER bob WITH PASSWORD `password_b' NOSUPERUSER;
+
+`CREATE USER` is equivalent to `CREATE ROLE` where the `LOGIN` option is
+`true`. So, the following pairs of statements are equivalent:
+
+bc(sample).. +
+CREATE USER alice WITH PASSWORD `password_a' SUPERUSER; +
+CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true AND
+SUPERUSER = true;
+
+CREATE USER IF NOT EXISTS alice WITH PASSWORD `password_a' SUPERUSER; +
+CREATE ROLE IF NOT EXISTS alice WITH PASSWORD = `password_a' AND LOGIN =
+true AND SUPERUSER = true;
+
+CREATE USER alice WITH PASSWORD `password_a' NOSUPERUSER; +
+CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true AND
+SUPERUSER = false;
+
+CREATE USER alice WITH PASSWORD `password_a' NOSUPERUSER; +
+CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true;
+
+CREATE USER alice WITH PASSWORD `password_a'; +
+CREATE ROLE alice WITH PASSWORD = `password_a' AND LOGIN = true; +
+p.
+
+[[alterUserStmt]]
+==== ALTER USER
+
+_Syntax:_
+
+bc(syntax).. +
+::= ALTER USER (IF EXISTS)? ( WITH PASSWORD )? ( )?
+
+::= SUPERUSER +
+| NOSUPERUSER +
+p.
+
+bc(sample). +
+ALTER USER alice WITH PASSWORD `PASSWORD_A'; +
+ALTER USER bob SUPERUSER;
+
+If the user does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+[[dropUserStmt]]
+==== DROP USER
+
+_Syntax:_
+
+bc(syntax).. +
+::= DROP USER ( IF EXISTS )? +
+p.
+
+_Sample:_
+
+bc(sample). +
+DROP USER alice; +
+DROP USER IF EXISTS bob;
+
+[[listUsersStmt]]
+==== LIST USERS
+
+_Syntax:_
+
+bc(syntax). +
+::= LIST USERS;
+
+_Sample:_
+
+bc(sample). +
+LIST USERS;
+
+This statement is equivalent to
+
+bc(sample). +
+LIST ROLES;
+
+but only roles with the `LOGIN` privilege are included in the output.
+
+[[dataControl]]
+=== Data Control
+
+==== Permissions
+
+Permissions on resources are granted to roles; there are several
+different types of resources in Cassandra and each type is modelled
+hierarchically:
+
+* The hierarchy of Data resources, Keyspaces and Tables has the
+structure `ALL KEYSPACES` -> `KEYSPACE` -> `TABLE`
+* Function resources have the structure `ALL FUNCTIONS` -> `KEYSPACE` ->
+`FUNCTION`
+* Resources representing roles have the structure `ALL ROLES` -> `ROLE`
+* Resources representing JMX ObjectNames, which map to sets of
+MBeans/MXBeans, have the structure `ALL MBEANS` -> `MBEAN`
+
+Permissions can be granted at any level of these hierarchies and they
+flow downwards. So granting a permission on a resource higher up the
+chain automatically grants that same permission on all resources lower
+down. For example, granting `SELECT` on a `KEYSPACE` automatically
+grants it on all `TABLES` in that `KEYSPACE`. Likewise, granting a
+permission on `ALL FUNCTIONS` grants it on every defined function,
+regardless of which keyspace it is scoped in. It is also possible to
+grant permissions on all functions scoped to a particular keyspace.
+
+Modifications to permissions are visible to existing client sessions;
+that is, connections need not be re-established following permissions
+changes.
+
+The full set of available permissions is:
+
+* `CREATE`
+* `ALTER`
+* `DROP`
+* `SELECT`
+* `MODIFY`
+* `AUTHORIZE`
+* `DESCRIBE`
+* `EXECUTE`
+* `UNMASK`
+* `SELECT_MASKED`
+
+Not all permissions are applicable to every type of resource. For
+instance, `EXECUTE` is only relevant in the context of functions or
+mbeans; granting `EXECUTE` on a resource representing a table is
+nonsensical. Attempting to `GRANT` a permission on resource to which it
+cannot be applied results in an error response. The following
+illustrates which permissions can be granted on which types of resource,
+and which statements are enabled by that permission.
+
+[cols=",,,,,",options="header",]
+|===
+|permission |resource |operations | | |
+|`CREATE` |`ALL KEYSPACES` |`CREATE KEYSPACE` <br> `CREATE TABLE` in any
+keyspace | | |
+
+|`CREATE` |`KEYSPACE` |`CREATE TABLE` in specified keyspace | | |
+
+|`CREATE` |`ALL FUNCTIONS` |`CREATE FUNCTION` in any keyspace <br>
+`CREATE AGGREGATE` in any keyspace | | |
+
+|`CREATE` |`ALL FUNCTIONS IN KEYSPACE` |`CREATE FUNCTION` in keyspace
+<br> `CREATE AGGREGATE` in keyspace | | |
+
+|`CREATE` |`ALL ROLES` |`CREATE ROLE` | | |
+
+|`ALTER` |`ALL KEYSPACES` |`ALTER KEYSPACE` <br> `ALTER TABLE` in any
+keyspace | | |
+
+|`ALTER` |`KEYSPACE` |`ALTER KEYSPACE` <br> `ALTER TABLE` in keyspace |
+| |
+
+|`ALTER` |`TABLE` |`ALTER TABLE` | | |
+
+|`ALTER` |`ALL FUNCTIONS` |`CREATE FUNCTION` replacing any existing <br>
+`CREATE AGGREGATE` replacing any existing | | |
+
+|`ALTER` |`ALL FUNCTIONS IN KEYSPACE` |`CREATE FUNCTION` replacing
+existing in keyspace <br> `CREATE AGGREGATE` replacing any existing in
+keyspace | | |
+
+|`ALTER` |`FUNCTION` |`CREATE FUNCTION` replacing existing <br>
+`CREATE AGGREGATE` replacing existing | | |
+
+|`ALTER` |`ALL ROLES` |`ALTER ROLE` on any role | | |
+
+|`ALTER` |`ROLE` |`ALTER ROLE` | | |
+
+|`DROP` |`ALL KEYSPACES` |`DROP KEYSPACE` <br> `DROP TABLE` in any
+keyspace | | |
+
+|`DROP` |`KEYSPACE` |`DROP TABLE` in specified keyspace | | |
+
+|`DROP` |`TABLE` |`DROP TABLE` | | |
+
+|`DROP` |`ALL FUNCTIONS` |`DROP FUNCTION` in any keyspace <br>
+`DROP AGGREGATE` in any existing | | |
+
+|`DROP` |`ALL FUNCTIONS IN KEYSPACE` |`DROP FUNCTION` in keyspace <br>
+`DROP AGGREGATE` in existing | | |
+
+|`DROP` |`FUNCTION` |`DROP FUNCTION` | | |
+
+|`DROP` |`ALL ROLES` |`DROP ROLE` on any role | | |
+
+|`DROP` |`ROLE` |`DROP ROLE` | | |
+
+|`SELECT` |`ALL KEYSPACES` |`SELECT` on any table | | |
+
+|`SELECT` |`KEYSPACE` |`SELECT` on any table in keyspace | | |
+
+|`SELECT` |`TABLE` |`SELECT` on specified table | | |
+
+|`SELECT` |`ALL MBEANS` |Call getter methods on any mbean | | |
+
+|`SELECT` |`MBEANS` |Call getter methods on any mbean matching a
+wildcard pattern | | |
+
+|`SELECT` |`MBEAN` |Call getter methods on named mbean | | |
+
+|`MODIFY` |`ALL KEYSPACES` |`INSERT` on any table <br> `UPDATE` on any
+table <br> `DELETE` on any table <br> `TRUNCATE` on any table | | |
+
+|`MODIFY` |`KEYSPACE` |`INSERT` on any table in keyspace <br> `UPDATE`
+on any table in keyspace <br> `DELETE` on any table in keyspace <br>
+`TRUNCATE` on any table in keyspace |`MODIFY` |`TABLE` |`INSERT` <br>
+`UPDATE` <br> `DELETE` <br> `TRUNCATE`
+
+|`MODIFY` |`ALL MBEANS` |Call setter methods on any mbean | | |
+
+|`MODIFY` |`MBEANS` |Call setter methods on any mbean matching a
+wildcard pattern | | |
+
+|`MODIFY` |`MBEAN` |Call setter methods on named mbean | | |
+
+|`AUTHORIZE` |`ALL KEYSPACES` |`GRANT PERMISSION` on any table <br>
+`REVOKE PERMISSION` on any table | | |
+
+|`AUTHORIZE` |`KEYSPACE` |`GRANT PERMISSION` on table in keyspace <br>
+`REVOKE PERMISSION` on table in keyspace | | |
+
+|`AUTHORIZE` |`TABLE` |`GRANT PERMISSION` <br> `REVOKE PERMISSION` | | |
+
+|`AUTHORIZE` |`ALL FUNCTIONS` |`GRANT PERMISSION` on any function <br>
+`REVOKE PERMISSION` on any function | | |
+
+|`AUTHORIZE` |`ALL FUNCTIONS IN KEYSPACE` |`GRANT PERMISSION` in
+keyspace <br> `REVOKE PERMISSION` in keyspace | | |
+
+|`AUTHORIZE` |`ALL FUNCTIONS IN KEYSPACE` |`GRANT PERMISSION` in
+keyspace <br> `REVOKE PERMISSION` in keyspace | | |
+
+|`AUTHORIZE` |`FUNCTION` |`GRANT PERMISSION` <br> `REVOKE PERMISSION` |
+| |
+
+|`AUTHORIZE` |`ALL MBEANS` |`GRANT PERMISSION` on any mbean <br>
+`REVOKE PERMISSION` on any mbean | | |
+
+|`AUTHORIZE` |`MBEANS` |`GRANT PERMISSION` on any mbean matching a
+wildcard pattern <br> `REVOKE PERMISSION` on any mbean matching a
+wildcard pattern | | |
+
+|`AUTHORIZE` |`MBEAN` |`GRANT PERMISSION` on named mbean <br>
+`REVOKE PERMISSION` on named mbean | | |
+
+|`AUTHORIZE` |`ALL ROLES` |`GRANT ROLE` grant any role <br>
+`REVOKE ROLE` revoke any role | | |
+
+|`AUTHORIZE` |`ROLES` |`GRANT ROLE` grant role <br> `REVOKE ROLE` revoke
+role | | |
+
+|`DESCRIBE` |`ALL ROLES` |`LIST ROLES` all roles or only roles granted
+to another, specified role | | |
+
+|`DESCRIBE` |@ALL MBEANS |Retrieve metadata about any mbean from the
+platform’s MBeanServer | | |
+
+|`DESCRIBE` |@MBEANS |Retrieve metadata about any mbean matching a
+wildcard patter from the platform’s MBeanServer | | |
+
+|`DESCRIBE` |@MBEAN |Retrieve metadata about a named mbean from the
+platform’s MBeanServer | | |
+
+|`EXECUTE` |`ALL FUNCTIONS` |`SELECT`, `INSERT`, `UPDATE` using any
+function <br> use of any function in `CREATE AGGREGATE` | | |
+
+|`EXECUTE` |`ALL FUNCTIONS IN KEYSPACE` |`SELECT`, `INSERT`, `UPDATE`
+using any function in keyspace <br> use of any function in keyspace in
+`CREATE AGGREGATE` | | |
+
+|`EXECUTE` |`FUNCTION` |`SELECT`, `INSERT`, `UPDATE` using function <br>
+use of function in `CREATE AGGREGATE` | | |
+
+|`EXECUTE` |`ALL MBEANS` |Execute operations on any mbean | | |
+
+|`EXECUTE` |`MBEANS` |Execute operations on any mbean matching a
+wildcard pattern | | |
+
+|`EXECUTE` |`MBEAN` |Execute operations on named mbean | | |
+
+|`UNMASK` |`ALL KEYSPACES` |See the clear contents of masked columns on any table | | |
+
+|`UNMASK` |`KEYSPACE` |See the clear contents of masked columns on any table in keyspace | | |
+
+|`UNMASK` |`TABLE` |See the clear contents of masked columns on the specified table | | |
+
+|`SELECT_MASKED` | `ALL KEYSPACES` | `SELECT` restricting masked columns on any table | | |
+
+|`SELECT_MASKED` | `KEYSPACE` | `SELECT` restricting masked columns on any table in specified keyspace | | |
+
+|`SELECT_MASKED` | `TABLE` | `SELECT` restricting masked columns on the specified table | | |
+|===
+
+[[grantPermissionsStmt]]
+==== GRANT PERMISSION
+
+_Syntax:_
+
+bc(syntax).. +
+::= GRANT ( ALL ( PERMISSIONS )? | ( PERMISSION )? ) ON TO
+
+::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | UNMASK | SELECT_MASKED
+EXECUTE
+
+::= ALL KEYSPACES +
+| KEYSPACE +
+| ( TABLE )? +
+| ALL ROLES +
+| ROLE +
+| ALL FUNCTIONS ( IN KEYSPACE )? +
+| FUNCTION +
+| ALL MBEANS +
+| ( MBEAN | MBEANS ) +
+p.
+
+_Sample:_
+
+bc(sample). +
+GRANT SELECT ON ALL KEYSPACES TO data_reader;
+
+This gives any user with the role `data_reader` permission to execute
+`SELECT` statements on any table across all keyspaces
+
+bc(sample). +
+GRANT MODIFY ON KEYSPACE keyspace1 TO data_writer;
+
+This give any user with the role `data_writer` permission to perform
+`UPDATE`, `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` queries on all
+tables in the `keyspace1` keyspace
+
+bc(sample). +
+GRANT DROP ON keyspace1.table1 TO schema_owner;
+
+This gives any user with the `schema_owner` role permissions to `DROP`
+`keyspace1.table1`.
+
+bc(sample). +
+GRANT EXECUTE ON FUNCTION keyspace1.user_function( int ) TO
+report_writer;
+
+This grants any user with the `report_writer` role permission to execute
+`SELECT`, `INSERT` and `UPDATE` queries which use the function
+`keyspace1.user_function( int )`
+
+bc(sample). +
+GRANT DESCRIBE ON ALL ROLES TO role_admin;
+
+This grants any user with the `role_admin` role permission to view any
+and all roles in the system with a `LIST ROLES` statement
+
+[[grantAll]]
+===== GRANT ALL
+
+When the `GRANT ALL` form is used, the appropriate set of permissions is
+determined automatically based on the target resource.
+
+[[autoGrantPermissions]]
+===== Automatic Granting
+
+When a resource is created, via a `CREATE KEYSPACE`, `CREATE TABLE`,
+`CREATE FUNCTION`, `CREATE AGGREGATE` or `CREATE ROLE` statement, the
+creator (the role the database user who issues the statement is
+identified as), is automatically granted all applicable permissions on
+the new resource.
+
+[[revokePermissionsStmt]]
+==== REVOKE PERMISSION
+
+_Syntax:_
+
+bc(syntax).. +
+::= REVOKE ( ALL ( PERMISSIONS )? | ( PERMISSION )? ) ON FROM
+
+::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | UNMASK | SELECT_MASKED
+EXECUTE
+
+::= ALL KEYSPACES +
+| KEYSPACE +
+| ( TABLE )? +
+| ALL ROLES +
+| ROLE +
+| ALL FUNCTIONS ( IN KEYSPACE )? +
+| FUNCTION +
+| ALL MBEANS +
+| ( MBEAN | MBEANS ) +
+p.
+
+_Sample:_
+
+bc(sample).. +
+REVOKE SELECT ON ALL KEYSPACES FROM data_reader; +
+REVOKE MODIFY ON KEYSPACE keyspace1 FROM data_writer; +
+REVOKE DROP ON keyspace1.table1 FROM schema_owner; +
+REVOKE EXECUTE ON FUNCTION keyspace1.user_function( int ) FROM
+report_writer; +
+REVOKE DESCRIBE ON ALL ROLES FROM role_admin; +
+p.
+
+[[listPermissionsStmt]]
+===== LIST PERMISSIONS
+
+_Syntax:_
+
+bc(syntax).. +
+::= LIST ( ALL ( PERMISSIONS )? | ) +
+( ON )? +
+( OF ( NORECURSIVE )? )?
+
+::= ALL KEYSPACES +
+| KEYSPACE +
+| ( TABLE )? +
+| ALL ROLES +
+| ROLE +
+| ALL FUNCTIONS ( IN KEYSPACE )? +
+| FUNCTION +
+| ALL MBEANS +
+| ( MBEAN | MBEANS ) +
+p.
+
+_Sample:_
+
+bc(sample). +
+LIST ALL PERMISSIONS OF alice;
+
+Show all permissions granted to `alice`, including those acquired
+transitively from any other roles.
+
+bc(sample). +
+LIST ALL PERMISSIONS ON keyspace1.table1 OF bob;
+
+Show all permissions on `keyspace1.table1` granted to `bob`, including
+those acquired transitively from any other roles. This also includes any
+permissions higher up the resource hierarchy which can be applied to
+`keyspace1.table1`. For example, should `bob` have `ALTER` permission on
+`keyspace1`, that would be included in the results of this query. Adding
+the `NORECURSIVE` switch restricts the results to only those permissions
+which were directly granted to `bob` or one of `bob`’s roles.
+
+bc(sample). +
+LIST SELECT PERMISSIONS OF carlos;
+
+Show any permissions granted to `carlos` or any of `carlos`’s roles,
+limited to `SELECT` permissions on any resource.
+
+[[types]]
+=== Data Types
+
+CQL supports a rich set of data types for columns defined in a table,
+including collection types. On top of those native +
+and collection types, users can also provide custom types (through a
+JAVA class extending `AbstractType` loadable by +
+Cassandra). The syntax of types is thus:
+
+bc(syntax).. +
+::= +
+| +
+| +
+| // Used for custom types. The fully-qualified name of a JAVA class
+
+::= ascii +
+| bigint +
+| blob +
+| boolean +
+| counter +
+| date +
+| decimal +
+| double +
+| float +
+| inet +
+| int +
+| smallint +
+| text +
+| time +
+| timestamp +
+| timeuuid +
+| tinyint +
+| uuid +
+| varchar +
+| varint
+
+::= list `<' `>' +
+| set `<' `>' +
+| map `<' `,' `>' +
+::= tuple `<' (`,' )* `>' +
+p. Note that the native types are keywords and as such are
+case-insensitive. They are however not reserved ones.
+
+The following table gives additional informations on the native data
+types, and on which kind of link:#constants[constants] each type
+supports:
+
+[cols=",,",options="header",]
+|===
+|type |constants supported |description
+|`ascii` |strings |ASCII character string
+
+|`bigint` |integers |64-bit signed long
+
+|`blob` |blobs |Arbitrary bytes (no validation)
+
+|`boolean` |booleans |true or false
+
+|`counter` |integers |Counter column (64-bit signed value). See
+link:#counters[Counters] for details
+
+|`date` |integers, strings |A date (with no corresponding time value).
+See link:#usingdates[Working with dates] below for more information.
+
+|`decimal` |integers, floats |Variable-precision decimal
+
+|`double` |integers |64-bit IEEE-754 floating point
+
+|`float` |integers, floats |32-bit IEEE-754 floating point
+
+|`inet` |strings |An IP address. It can be either 4 bytes long (IPv4) or
+16 bytes long (IPv6). There is no `inet` constant, IP address should be
+inputed as strings
+
+|`int` |integers |32-bit signed int
+
+|`smallint` |integers |16-bit signed int
+
+|`text` |strings |UTF8 encoded string
+
+|`time` |integers, strings |A time with nanosecond precision. See
+link:#usingtime[Working with time] below for more information.
+
+|`timestamp` |integers, strings |A timestamp. Strings constant are allow
+to input timestamps as dates, see link:#usingtimestamps[Working with
+timestamps] below for more information.
+
+|`timeuuid` |uuids |Type 1 UUID. This is generally used as a
+``conflict-free'' timestamp. Also see the link:#timeuuidFun[functions on
+Timeuuid]
+
+|`tinyint` |integers |8-bit signed int
+
+|`uuid` |uuids |Type 1 or type 4 UUID
+
+|`varchar` |strings |UTF8 encoded string
+
+|`varint` |integers |Arbitrary-precision integer
+|===
+
+For more information on how to use the collection types, see the
+link:#collections[Working with collections] section below.
+
+[[usingtimestamps]]
+==== Working with timestamps
+
+Values of the `timestamp` type are encoded as 64-bit signed integers
+representing a number of milliseconds since the standard base time known
+as ``the epoch'': January 1 1970 at 00:00:00 GMT.
+
+Timestamp can be input in CQL as simple long integers, giving the number
+of milliseconds since the epoch, as defined above.
+
+They can also be input as string literals in any of the following ISO
+8601 formats, each representing the time and date Mar 2, 2011, at
+04:05:00 AM, GMT.:
+
+* `2011-02-03 04:05+0000`
+* `2011-02-03 04:05:00+0000`
+* `2011-02-03 04:05:00.000+0000`
+* `2011-02-03T04:05+0000`
+* `2011-02-03T04:05:00+0000`
+* `2011-02-03T04:05:00.000+0000`
+
+The `+0000` above is an RFC 822 4-digit time zone specification; `+0000`
+refers to GMT. US Pacific Standard Time is `-0800`. The time zone may be
+omitted if desired— the date will be interpreted as being in the time
+zone under which the coordinating Cassandra node is configured.
+
+* `2011-02-03 04:05`
+* `2011-02-03 04:05:00`
+* `2011-02-03 04:05:00.000`
+* `2011-02-03T04:05`
+* `2011-02-03T04:05:00`
+* `2011-02-03T04:05:00.000`
+
+There are clear difficulties inherent in relying on the time zone
+configuration being as expected, though, so it is recommended that the
+time zone always be specified for timestamps when feasible.
+
+The time of day may also be omitted, if the date is the only piece that
+matters:
+
+* `2011-02-03`
+* `2011-02-03+0000`
+
+In that case, the time of day will default to 00:00:00, in the specified
+or default time zone.
+
+[[usingdates]]
+==== Working with dates
+
+Values of the `date` type are encoded as 32-bit unsigned integers
+representing a number of days with ``the epoch'' at the center of the
+range (2^31). Epoch is January 1st, 1970
+
+A date can be input in CQL as an unsigned integer as defined above.
+
+They can also be input as string literals in the following format:
+
+* `2014-01-01`
+
+[[usingtime]]
+==== Working with time
+
+Values of the `time` type are encoded as 64-bit signed integers
+representing the number of nanoseconds since midnight.
+
+A time can be input in CQL as simple long integers, giving the number of
+nanoseconds since midnight.
+
+They can also be input as string literals in any of the following
+formats:
+
+* `08:12:54`
+* `08:12:54.123`
+* `08:12:54.123456`
+* `08:12:54.123456789`
+
+==== Counters
+
+The `counter` type is used to define _counter columns_. A counter column
+is a column whose value is a 64-bit signed integer and on which 2
+operations are supported: incrementation and decrementation (see
+link:#updateStmt[`UPDATE`] for syntax). Note the value of a counter
+cannot be set. A counter doesn’t exist until first
+incremented/decremented, and the first incrementation/decrementation is
+made as if the previous value was 0. Deletion of counter columns is
+supported but have some limitations (see the
+http://wiki.apache.org/cassandra/Counters[Cassandra Wiki] for more
+information).
+
+The use of the counter type is limited in the following way:
+
+* It cannot be used for column that is part of the `PRIMARY KEY` of a
+table.
+* A table that contains a counter can only contain counters. In other
+words, either all the columns of a table outside the `PRIMARY KEY` have
+the counter type, or none of them have it.
+
+[[collections]]
+==== Working with collections
+
+===== Noteworthy characteristics
+
+Collections are meant for storing/denormalizing relatively small amount
+of data. They work well for things like ``the phone numbers of a given
+user'', ``labels applied to an email'', etc. But when items are expected
+to grow unbounded (``all the messages sent by a given user'', ``events
+registered by a sensor'', …), then collections are not appropriate
+anymore and a specific table (with clustering columns) should be used.
+Concretely, collections have the following limitations:
+
+* Collections are always read in their entirety (and reading one is not
+paged internally).
+* Collections cannot have more than 65535 elements. More precisely,
+while it may be possible to insert more than 65535 elements, it is not
+possible to read more than the 65535 first elements (see
+https://issues.apache.org/jira/browse/CASSANDRA-5428[CASSANDRA-5428] for
+details).
+* While insertion operations on sets and maps never incur a
+read-before-write internally, some operations on lists do (see the
+section on lists below for details). It is thus advised to prefer sets
+over lists when possible.
+
+Please note that while some of those limitations may or may not be
+loosen in the future, the general rule that collections are for
+denormalizing small amount of data is meant to stay.
+
+[[map]]
+===== Maps
+
+A `map` is a link:#types[typed] set of key-value pairs, where keys are
+unique. Furthermore, note that the map are internally sorted by their
+keys and will thus always be returned in that order. To create a column
+of type `map`, use the `map` keyword suffixed with comma-separated key
+and value types, enclosed in angle brackets. For example:
+
+bc(sample). +
+CREATE TABLE users ( +
+id text PRIMARY KEY, +
+given text, +
+surname text, +
+favs map<text, text> // A map of text keys, and text values +
+)
+
+Writing `map` data is accomplished with a JSON-inspired syntax. To write
+a record using `INSERT`, specify the entire map as a JSON-style
+associative array. _Note: This form will always replace the entire map._
+
+bc(sample). +
+// Inserting (or Updating) +
+INSERT INTO users (id, given, surname, favs) +
+VALUES (`jsmith', `John', `Smith', \{ `fruit' : `apple', `band' :
+`Beatles' })
+
+Adding or updating key-values of a (potentially) existing map can be
+accomplished either by subscripting the map column in an `UPDATE`
+statement or by adding a new map literal:
+
+bc(sample). +
+// Updating (or inserting) +
+UPDATE users SET favs[`author'] = `Ed Poe' WHERE id = `jsmith' +
+UPDATE users SET favs = favs + \{ `movie' : `Cassablanca' } WHERE id =
+`jsmith'
+
+Note that TTLs are allowed for both `INSERT` and `UPDATE`, but in both
+case the TTL set only apply to the newly inserted/updated _values_. In
+other words,
+
+bc(sample). +
+// Updating (or inserting) +
+UPDATE users USING TTL 10 SET favs[`color'] = `green' WHERE id =
+`jsmith'
+
+will only apply the TTL to the `{ 'color' : 'green' }` record, the rest
+of the map remaining unaffected.
+
+Deleting a map record is done with:
+
+bc(sample). +
+DELETE favs[`author'] FROM users WHERE id = `jsmith'
+
+[[set]]
+===== Sets
+
+A `set` is a link:#types[typed] collection of unique values. Sets are
+ordered by their values. To create a column of type `set`, use the `set`
+keyword suffixed with the value type enclosed in angle brackets. For
+example:
+
+bc(sample). +
+CREATE TABLE images ( +
+name text PRIMARY KEY, +
+owner text, +
+date timestamp, +
+tags set +
+);
+
+Writing a `set` is accomplished by comma separating the set values, and
+enclosing them in curly braces. _Note: An `INSERT` will always replace
+the entire set._
+
+bc(sample). +
+INSERT INTO images (name, owner, date, tags) +
+VALUES (`cat.jpg', `jsmith', `now', \{ `kitten', `cat', `pet' });
+
+Adding and removing values of a set can be accomplished with an `UPDATE`
+by adding/removing new set values to an existing `set` column.
+
+bc(sample). +
+UPDATE images SET tags = tags + \{ `cute', `cuddly' } WHERE name =
+`cat.jpg'; +
+UPDATE images SET tags = tags - \{ `lame' } WHERE name = `cat.jpg';
+
+As with link:#map[maps], TTLs if used only apply to the newly
+inserted/updated _values_.
+
+[[list]]
+===== Lists
+
+A `list` is a link:#types[typed] collection of non-unique values where
+elements are ordered by there position in the list. To create a column
+of type `list`, use the `list` keyword suffixed with the value type
+enclosed in angle brackets. For example:
+
+bc(sample). +
+CREATE TABLE plays ( +
+id text PRIMARY KEY, +
+game text, +
+players int, +
+scores list +
+)
+
+Do note that as explained below, lists have some limitations and
+performance considerations to take into account, and it is advised to
+prefer link:#set[sets] over lists when this is possible.
+
+Writing `list` data is accomplished with a JSON-style syntax. To write a
+record using `INSERT`, specify the entire list as a JSON array. _Note:
+An `INSERT` will always replace the entire list._
+
+bc(sample). +
+INSERT INTO plays (id, game, players, scores) +
+VALUES (`123-afde', `quake', 3, [17, 4, 2]);
+
+Adding (appending or prepending) values to a list can be accomplished by
+adding a new JSON-style array to an existing `list` column.
+
+bc(sample). +
+UPDATE plays SET players = 5, scores = scores + [ 14, 21 ] WHERE id =
+`123-afde'; +
+UPDATE plays SET players = 5, scores = [ 12 ] + scores WHERE id =
+`123-afde';
+
+It should be noted that append and prepend are not idempotent
+operations. This means that if during an append or a prepend the
+operation timeout, it is not always safe to retry the operation (as this
+could result in the record appended or prepended twice).
+
+Lists also provides the following operation: setting an element by its
+position in the list, removing an element by its position in the list
+and remove all the occurrence of a given value in the list. _However,
+and contrarily to all the other collection operations, these three
+operations induce an internal read before the update, and will thus
+typically have slower performance characteristics_. Those operations
+have the following syntax:
+
+bc(sample). +
+UPDATE plays SET scores[1] = 7 WHERE id = `123-afde'; // sets the 2nd
+element of scores to 7 (raises an error is scores has less than 2
+elements) +
+DELETE scores[1] FROM plays WHERE id = `123-afde'; // deletes the 2nd
+element of scores (raises an error is scores has less than 2 elements) +
+UPDATE plays SET scores = scores - [ 12, 21 ] WHERE id = `123-afde'; //
+removes all occurrences of 12 and 21 from scores
+
+As with link:#map[maps], TTLs if used only apply to the newly
+inserted/updated _values_.
+
+=== Functions
+
+CQL3 distinguishes between built-in functions (so called `native
+functions') and link:#udfs[user-defined functions]. CQL3 includes
+several native functions, described below:
+
+[[castFun]]
+==== Cast
+
+The `cast` function can be used to converts one native datatype to
+another.
+
+The following table describes the conversions supported by the `cast`
+function. Cassandra will silently ignore any cast converting a datatype
+into its own datatype.
+
+[cols=",",options="header",]
+|===
+|from |to
+|`ascii` |`text`, `varchar`
+
+|`bigint` |`tinyint`, `smallint`, `int`, `float`, `double`, `decimal`,
+`varint`, `text`, `varchar`
+
+|`boolean` |`text`, `varchar`
+
+|`counter` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
+`decimal`, `varint`, `text`, `varchar`
+
+|`date` |`timestamp`
+
+|`decimal` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
+`varint`, `text`, `varchar`
+
+|`double` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `decimal`,
+`varint`, `text`, `varchar`
+
+|`float` |`tinyint`, `smallint`, `int`, `bigint`, `double`, `decimal`,
+`varint`, `text`, `varchar`
+
+|`inet` |`text`, `varchar`
+
+|`int` |`tinyint`, `smallint`, `bigint`, `float`, `double`, `decimal`,
+`varint`, `text`, `varchar`
+
+|`smallint` |`tinyint`, `int`, `bigint`, `float`, `double`, `decimal`,
+`varint`, `text`, `varchar`
+
+|`time` |`text`, `varchar`
+
+|`timestamp` |`date`, `text`, `varchar`
+
+|`timeuuid` |`timestamp`, `date`, `text`, `varchar`
+
+|`tinyint` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
+`decimal`, `varint`, `text`, `varchar`
+
+|`uuid` |`text`, `varchar`
+
+|`varint` |`tinyint`, `smallint`, `int`, `bigint`, `float`, `double`,
+`decimal`, `text`, `varchar`
+|===
+
+The conversions rely strictly on Java’s semantics. For example, the
+double value 1 will be converted to the text value `1.0'.
+
+bc(sample). +
+SELECT avg(cast(count as double)) FROM myTable
+
+[[tokenFun]]
+==== Token
+
+The `token` function allows to compute the token for a given partition
+key. The exact signature of the token function depends on the table
+concerned and of the partitioner used by the cluster.
+
+The type of the arguments of the `token` depend on the type of the
+partition key columns. The return type depend on the partitioner in use:
+
+* For Murmur3Partitioner, the return type is `bigint`.
+* For RandomPartitioner, the return type is `varint`.
+* For ByteOrderedPartitioner, the return type is `blob`.
+
+For instance, in a cluster using the default Murmur3Partitioner, if a
+table is defined by
+
+bc(sample). +
+CREATE TABLE users ( +
+userid text PRIMARY KEY, +
+username text, +
+… +
+)
+
+then the `token` function will take a single argument of type `text` (in
+that case, the partition key is `userid` (there is no clustering columns
+so the partition key is the same than the primary key)), and the return
+type will be `bigint`.
+
+[[uuidFun]]
+==== Uuid
+
+The `uuid` function takes no parameters and generates a random type 4
+uuid suitable for use in INSERT or SET statements.
+
+[[timeuuidFun]]
+==== Timeuuid functions
+
+===== `now`
+
+The `now` function takes no arguments and generates, on the coordinator
+node, a new unique timeuuid (at the time where the statement using it is
+executed). Note that this method is useful for insertion but is largely
+non-sensical in `WHERE` clauses. For instance, a query of the form
+
+bc(sample). +
+SELECT * FROM myTable WHERE t = now()
+
+will never return any result by design, since the value returned by
+`now()` is guaranteed to be unique.
+
+===== `min_timeuuid` and `max_timeuuid`
+
+The `min_timeuuid` (resp. `max_timeuuid`) function takes a `timestamp`
+value `t` (which can be link:#usingtimestamps[either a timestamp or a
+date string] ) and return a _fake_ `timeuuid` corresponding to the
+_smallest_ (resp. _biggest_) possible `timeuuid` having for timestamp
+`t`. So for instance:
+
+bc(sample). +
+SELECT * FROM myTable WHERE t > max_timeuuid(`2013-01-01 00:05+0000') AND
+t < min_timeuuid(`2013-02-02 10:00+0000')
+
+will select all rows where the `timeuuid` column `t` is strictly older
+than `2013-01-01 00:05+0000' but strictly younger than `2013-02-02
+10:00+0000'. Please note that
+`t >= max_timeuuid('2013-01-01 00:05+0000')` would still _not_ select a
+`timeuuid` generated exactly at `2013-01-01 00:05+0000' and is
+essentially equivalent to `t > max_timeuuid('2013-01-01 00:05+0000')`.
+
+_Warning_: We called the values generated by `min_timeuuid` and
+`max_timeuuid` _fake_ UUID because they do no respect the Time-Based UUID
+generation process specified by the
+http://www.ietf.org/rfc/rfc4122.txt[RFC 4122]. In particular, the value
+returned by these 2 methods will not be unique. This means you should
+only use those methods for querying (as in the example above). Inserting
+the result of those methods is almost certainly _a bad idea_.
+
+[[timeFun]]
+==== Time conversion functions
+
+A number of functions are provided to ``convert'' a `timeuuid`, a
+`timestamp` or a `date` into another `native` type.
+
+[cols=",,",options="header",]
+|===
+|function name |input type |description
+|`to_date` |`timeuuid` |Converts the `timeuuid` argument into a `date`
+type
+
+|`to_date` |`timestamp` |Converts the `timestamp` argument into a `date`
+type
+
+|`to_timestamp` |`timeuuid` |Converts the `timeuuid` argument into a
+`timestamp` type
+
+|`to_timestamp` |`date` |Converts the `date` argument into a `timestamp`
+type
+
+|`to_unix_timestamp` |`timeuuid` |Converts the `timeuuid` argument into a
+`bigInt` raw value
+
+|`to_unix_timestamp` |`timestamp` |Converts the `timestamp` argument into
+a `bigInt` raw value
+
+|`to_unix_timestamp` |`date` |Converts the `date` argument into a `bigInt`
+raw value
+|===
+
+[[blobFun]]
+==== Blob conversion functions
+
+A number of functions are provided to ``convert'' the native types into
+binary data (`blob`). For every `<native-type>` `type` supported by CQL3
+(a notable exceptions is `blob`, for obvious reasons), the function
+`type_as_blob` takes a argument of type `type` and return it as a `blob`.
+Conversely, the function `blob_as_type` takes a 64-bit `blob` argument and
+convert it to a `bigint` value. And so for instance, `bigint_as_blob(3)`
+is `0x0000000000000003` and `blob_as_bigint(0x0000000000000003)` is `3`.
+
+=== Aggregates
+
+Aggregate functions work on a set of rows. They receive values for each
+row and returns one value for the whole set. +
+If `normal` columns, `scalar functions`, `UDT` fields, `writetime`, `maxwritetime`
+or `ttl` are selected together with aggregate functions, the values
+returned for them will be the ones of the first row matching the query.
+
+CQL3 distinguishes between built-in aggregates (so called `native
+aggregates') and link:#udas[user-defined aggregates]. CQL3 includes
+several native aggregates, described below:
+
+[[countFct]]
+==== Count
+
+The `count` function can be used to count the rows returned by a query.
+Example:
+
+bc(sample). +
+SELECT COUNT (*) FROM plays; +
+SELECT COUNT (1) FROM plays;
+
+It also can be used to count the non null value of a given column.
+Example:
+
+bc(sample). +
+SELECT COUNT (scores) FROM plays;
+
+[[maxMinFcts]]
+==== Max and Min
+
+The `max` and `min` functions can be used to compute the maximum and the
+minimum value returned by a query for a given column.
+
+bc(sample). +
+SELECT MIN (players), MAX (players) FROM plays WHERE game = `quake';
+
+[[sumFct]]
+==== Sum
+
+The `sum` function can be used to sum up all the values returned by a
+query for a given column.
+
+bc(sample). +
+SELECT SUM (players) FROM plays;
+
+[[avgFct]]
+==== Avg
+
+The `avg` function can be used to compute the average of all the values
+returned by a query for a given column.
+
+bc(sample). +
+SELECT AVG (players) FROM plays;
+
+[[udfs]]
+=== User-Defined Functions
+
+User-defined functions allow execution of user-provided code in
+Cassandra. By default, Cassandra supports defining functions in _Java_
+and _JavaScript_. Support for other JSR 223 compliant scripting
+languages (such as Python, Ruby, and Scala) has been removed in 3.0.11.
+
+UDFs are part of the Cassandra schema. As such, they are automatically
+propagated to all nodes in the cluster.
+
+UDFs can be _overloaded_ - i.e. multiple UDFs with different argument
+types but the same function name. Example:
+
+bc(sample). +
+CREATE FUNCTION sample ( arg int ) …; +
+CREATE FUNCTION sample ( arg text ) …;
+
+User-defined functions are susceptible to all of the normal problems
+with the chosen programming language. Accordingly, implementations
+should be safe against null pointer exceptions, illegal arguments, or
+any other potential source of exceptions. An exception during function
+execution will result in the entire statement failing.
+
+It is valid to use _complex_ types like collections, tuple types and
+user-defined types as argument and return types. Tuple types and
+user-defined types are handled by the conversion functions of the
+DataStax Java Driver. Please see the documentation of the Java Driver
+for details on handling tuple types and user-defined types.
+
+Arguments for functions can be literals or terms. Prepared statement
+placeholders can be used, too.
+
+Note that you can use the double-quoted string syntax to enclose the UDF
+source code. For example:
+
+bc(sample).. +
+CREATE FUNCTION some_function ( arg int ) +
+RETURNS NULL ON NULL INPUT +
+RETURNS int +
+LANGUAGE java +
+AS $$ return arg; $$;
+
+SELECT some_function(column) FROM atable …; +
+UPDATE atable SET col = some_function(?) …; +
+p.
+
+bc(sample). +
+CREATE TYPE custom_type (txt text, i int); +
+CREATE FUNCTION fct_using_udt ( udtarg frozen ) +
+RETURNS NULL ON NULL INPUT +
+RETURNS text +
+LANGUAGE java +
+AS $$ return udtarg.getString(``txt''); $$;
+
+User-defined functions can be used in link:#selectStmt[`SELECT`],
+link:#insertStmt[`INSERT`] and link:#updateStmt[`UPDATE`] statements.
+
+The implicitly available `udfContext` field (or binding for script UDFs)
+provides the neccessary functionality to create new UDT and tuple
+values.
+
+bc(sample). +
+CREATE TYPE custom_type (txt text, i int); +
+CREATE FUNCTION fct_using_udt ( somearg int ) +
+RETURNS NULL ON NULL INPUT +
+RETURNS custom_type +
+LANGUAGE java +
+AS $$ +
+UDTValue udt = udfContext.newReturnUDTValue(); +
+udt.setString(``txt'', ``some string''); +
+udt.setInt(``i'', 42); +
+return udt; +
+$$;
+
+The definition of the `UDFContext` interface can be found in the Apache
+Cassandra source code for
+`org.apache.cassandra.cql3.functions.UDFContext`.
+
+bc(sample). +
+public interface UDFContext +
+\{ +
+UDTValue newArgUDTValue(String argName); +
+UDTValue newArgUDTValue(int argNum); +
+UDTValue newReturnUDTValue(); +
+UDTValue newUDTValue(String udtName); +
+TupleValue newArgTupleValue(String argName); +
+TupleValue newArgTupleValue(int argNum); +
+TupleValue newReturnTupleValue(); +
+TupleValue newTupleValue(String cqlDefinition); +
+}
+
+Java UDFs already have some imports for common interfaces and classes
+defined. These imports are: +
+Please note, that these convenience imports are not available for script
+UDFs.
+
+bc(sample). +
+import java.nio.ByteBuffer; +
+import java.util.List; +
+import java.util.Map; +
+import java.util.Set; +
+import org.apache.cassandra.cql3.functions.UDFContext; +
+import com.datastax.driver.core.TypeCodec; +
+import com.datastax.driver.core.TupleValue; +
+import com.datastax.driver.core.UDTValue;
+
+See link:#createFunctionStmt[`CREATE FUNCTION`] and
+link:#dropFunctionStmt[`DROP FUNCTION`].
+
+[[udas]]
+=== User-Defined Aggregates
+
+User-defined aggregates allow creation of custom aggregate functions
+using link:#udfs[UDFs]. Common examples of aggregate functions are
+_count_, _min_, and _max_.
+
+Each aggregate requires an _initial state_ (`INITCOND`, which defaults
+to `null`) of type `STYPE`. The first argument of the state function
+must have type `STYPE`. The remaining arguments of the state function
+must match the types of the user-defined aggregate arguments. The state
+function is called once for each row, and the value returned by the
+state function becomes the new state. After all rows are processed, the
+optional `FINALFUNC` is executed with last state value as its argument.
+
+`STYPE` is mandatory in order to be able to distinguish possibly
+overloaded versions of the state and/or final function (since the
+overload can appear after creation of the aggregate).
+
+User-defined aggregates can be used in link:#selectStmt[`SELECT`]
+statement.
+
+A complete working example for user-defined aggregates (assuming that a
+keyspace has been selected using the link:#useStmt[`USE`] statement):
+
+bc(sample).. +
+CREATE OR REPLACE FUNCTION averageState ( state tuple<int,bigint>, val
+int ) +
+CALLED ON NULL INPUT +
+RETURNS tuple<int,bigint> +
+LANGUAGE java +
+AS ’ +
+if (val != null) \{ +
+state.setInt(0, state.getInt(0)+1); +
+state.setLong(1, state.getLong(1)+val.intValue()); +
+} +
+return state; +
+’;
+
+CREATE OR REPLACE FUNCTION averageFinal ( state tuple<int,bigint> ) +
+CALLED ON NULL INPUT +
+RETURNS double +
+LANGUAGE java +
+AS ’ +
+double r = 0; +
+if (state.getInt(0) == 0) return null; +
+r = state.getLong(1); +
+r /= state.getInt(0); +
+return Double.valueOf®; +
+’;
+
+CREATE OR REPLACE AGGREGATE average ( int ) +
+SFUNC averageState +
+STYPE tuple<int,bigint> +
+FINALFUNC averageFinal +
+INITCOND (0, 0);
+
+CREATE TABLE atable ( +
+pk int PRIMARY KEY, +
+val int); +
+INSERT INTO atable (pk, val) VALUES (1,1); +
+INSERT INTO atable (pk, val) VALUES (2,2); +
+INSERT INTO atable (pk, val) VALUES (3,3); +
+INSERT INTO atable (pk, val) VALUES (4,4); +
+SELECT average(val) FROM atable; +
+p.
+
+See link:#createAggregateStmt[`CREATE AGGREGATE`] and
+link:#dropAggregateStmt[`DROP AGGREGATE`].
+
+[[json]]
+=== JSON Support
+
+Cassandra 2.2 introduces JSON support to link:#selectStmt[`SELECT`] and
+link:#insertStmt[`INSERT`] statements. This support does not
+fundamentally alter the CQL API (for example, the schema is still
+enforced), it simply provides a convenient way to work with JSON
+documents.
+
+[[selectJson]]
+==== SELECT JSON
+
+With `SELECT` statements, the new `JSON` keyword can be used to return
+each row as a single `JSON` encoded map. The remainder of the `SELECT`
+statment behavior is the same.
+
+The result map keys are the same as the column names in a normal result
+set. For example, a statement like ```SELECT JSON a, ttl(b) FROM ...`''
+would result in a map with keys `"a"` and `"ttl(b)"`. However, this is
+one notable exception: for symmetry with `INSERT JSON` behavior,
+case-sensitive column names with upper-case letters will be surrounded
+with double quotes. For example, ```SELECT JSON myColumn FROM ...`''
+would result in a map key `"\"myColumn\""` (note the escaped quotes).
+
+The map values will `JSON`-encoded representations (as described below)
+of the result set values.
+
+[[insertJson]]
+==== INSERT JSON
+
+With `INSERT` statements, the new `JSON` keyword can be used to enable
+inserting a `JSON` encoded map as a single row. The format of the `JSON`
+map should generally match that returned by a `SELECT JSON` statement on
+the same table. In particular, case-sensitive column names should be
+surrounded with double quotes. For example, to insert into a table with
+two columns named ``myKey'' and ``value'', you would do the following:
+
+bc(sample). +
+INSERT INTO mytable JSON `\{``\''myKey\``'': 0, ``value'': 0}'
+
+Any columns which are ommitted from the `JSON` map will be defaulted to
+a `NULL` value (which will result in a tombstone being created).
+
+[[jsonEncoding]]
+==== JSON Encoding of Cassandra Data Types
+
+Where possible, Cassandra will represent and accept data types in their
+native `JSON` representation. Cassandra will also accept string
+representations matching the CQL literal format for all single-field
+types. For example, floats, ints, UUIDs, and dates can be represented by
+CQL literal strings. However, compound types, such as collections,
+tuples, and user-defined types must be represented by native `JSON`
+collections (maps and lists) or a JSON-encoded string representation of
+the collection.
+
+The following table describes the encodings that Cassandra will accept
+in `INSERT JSON` values (and `from_json()` arguments) as well as the
+format Cassandra will use when returning data for `SELECT JSON`
+statements (and `from_json()`):
+
+[cols=",,,",options="header",]
+|===
+|type |formats accepted |return format |notes
+|`ascii` |string |string |Uses JSON’s `\u` character escape
+
+|`bigint` |integer, string |integer |String must be valid 64 bit integer
+
+|`blob` |string |string |String should be 0x followed by an even number
+of hex digits
+
+|`boolean` |boolean, string |boolean |String must be ``true'' or
+``false''
+
+|`date` |string |string |Date in format `YYYY-MM-DD`, timezone UTC
+
+|`decimal` |integer, float, string |float |May exceed 32 or 64-bit
+IEEE-754 floating point precision in client-side decoder
+
+|`double` |integer, float, string |float |String must be valid integer
+or float
+
+|`float` |integer, float, string |float |String must be valid integer or
+float
+
+|`inet` |string |string |IPv4 or IPv6 address
+
+|`int` |integer, string |integer |String must be valid 32 bit integer
+
+|`list` |list, string |list |Uses JSON’s native list representation
+
+|`map` |map, string |map |Uses JSON’s native map representation
+
+|`smallint` |integer, string |integer |String must be valid 16 bit
+integer
+
+|`set` |list, string |list |Uses JSON’s native list representation
+
+|`text` |string |string |Uses JSON’s `\u` character escape
+
+|`time` |string |string |Time of day in format `HH-MM-SS[.fffffffff]`
+
+|`timestamp` |integer, string |string |A timestamp. Strings constant are
+allow to input timestamps as dates, see link:#usingdates[Working with
+dates] below for more information. Datestamps with format
+`YYYY-MM-DD HH:MM:SS.SSS` are returned.
+
+|`timeuuid` |string |string |Type 1 UUID. See link:#constants[Constants]
+for the UUID format
+
+|`tinyint` |integer, string |integer |String must be valid 8 bit integer
+
+|`tuple` |list, string |list |Uses JSON’s native list representation
+
+|`UDT` |map, string |map |Uses JSON’s native map representation with
+field names as keys
+
+|`uuid` |string |string |See link:#constants[Constants] for the UUID
+format
+
+|`varchar` |string |string |Uses JSON’s `\u` character escape
+
+|`varint` |integer, string |integer |Variable length; may overflow 32 or
+64 bit integers in client-side decoder
+|===
+
+[[from_json]]
+==== The from_json() Function
+
+The `from_json()` function may be used similarly to `INSERT JSON`, but
+for a single column value. It may only be used in the `VALUES` clause of
+an `INSERT` statement or as one of the column values in an `UPDATE`,
+`DELETE`, or `SELECT` statement. For example, it cannot be used in the
+selection clause of a `SELECT` statement.
+
+[[to_json]]
+==== The to_json() Function
+
+The `to_json()` function may be used similarly to `SELECT JSON`, but for
+a single column value. It may only be used in the selection clause of a
+`SELECT` statement.
+
+[[appendixA]]
+=== Appendix A: CQL Keywords
+
+CQL distinguishes between _reserved_ and _non-reserved_ keywords.
+Reserved keywords cannot be used as identifier, they are truly reserved
+for the language (but one can enclose a reserved keyword by
+double-quotes to use it as an identifier). Non-reserved keywords however
+only have a specific meaning in certain context but can used as
+identifer otherwise. The only _raison d’être_ of these non-reserved
+keywords is convenience: some keyword are non-reserved when it was
+always easy for the parser to decide whether they were used as keywords
+or not.
+
+[cols=",",options="header",]
+|===
+|Keyword |Reserved?
+|`ADD` |yes
+|`AGGREGATE` |no
+|`ALL` |no
+|`ALLOW` |yes
+|`ALTER` |yes
+|`AND` |yes
+|`APPLY` |yes
+|`AS` |no
+|`ASC` |yes
+|`ASCII` |no
+|`AUTHORIZE` |yes
+|`BATCH` |yes
+|`BEGIN` |yes
+|`BIGINT` |no
+|`BLOB` |no
+|`BOOLEAN` |no
+|`BY` |yes
+|`CALLED` |no
+|`CAST` |no
+|`CLUSTERING` |no
+|`COLUMNFAMILY` |yes
+|`COMPACT` |no
+|`CONTAINS` |no
+|`COUNT` |no
+|`COUNTER` |no
+|`CREATE` |yes
+|`CUSTOM` |no
+|`DATE` |no
+|`DECIMAL` |no
+|`DEFAULT` |yes
+|`DELETE` |yes
+|`DESC` |yes
+|`DESCRIBE` |yes
+|`DISTINCT` |no
+|`DOUBLE` |no
+|`DROP` |yes
+|`DURATION` |no
+|`ENTRIES` |yes
+|`EXECUTE` |yes
+|`EXISTS` |no
+|`FILTERING` |no
+|`FINALFUNC` |no
+|`FLOAT` |no
+|`FROM` |yes
+|`FROZEN` |no
+|`FULL` |yes
+|`FUNCTION` |no
+|`FUNCTIONS` |no
+|`GRANT` |yes
+|`GROUP` |no
+|`IF` |yes
+|`IN` |yes
+|`INDEX` |yes
+|`INET` |no
+|`INFINITY` |yes
+|`INITCOND` |no
+|`INPUT` |no
+|`INSERT` |yes
+|`INT` |no
+|`INTO` |yes
+|`IS` |yes
+|`JSON` |no
+|`KEY` |no
+|`KEYS` |no
+|`KEYSPACE` |yes
+|`KEYSPACES` |no
+|`LANGUAGE` |no
+|`LIKE` |no
+|`LIMIT` |yes
+|`LIST` |no
+|`LOGIN` |no
+|`MAP` |no
+|`MASKED` |no
+|`MATERIALIZED` |yes
+|`MBEAN` |yes
+|`MBEANS` |yes
+|`MODIFY` |yes
+|`NAN` |yes
+|`NOLOGIN` |no
+|`NORECURSIVE` |yes
+|`NOSUPERUSER` |no
+|`NOT` |yes
+|`NULL` |yes
+|`OF` |yes
+|`ON` |yes
+|`OPTIONS` |no
+|`OR` |yes
+|`ORDER` |yes
+|`PARTITION` |no
+|`PASSWORD` |no
+|`PER` |no
+|`PERMISSION` |no
+|`PERMISSIONS` |no
+|`PRIMARY` |yes
+|`RENAME` |yes
+|`REPLACE` |yes
+|`RETURNS` |no
+|`REVOKE` |yes
+|`ROLE` |no
+|`ROLES` |no
+|`SCHEMA` |yes
+|`SELECT` |yes
+|`SELECT_MASKED` |no
+|`SET` |yes
+|`SFUNC` |no
+|`SMALLINT` |no
+|`STATIC` |no
+|`STORAGE` |no
+|`STYPE` |no
+|`SUPERUSER` |no
+|`TABLE` |yes
+|`TEXT` |no
+|`TIME` |no
+|`TIMESTAMP` |no
+|`TIMEUUID` |no
+|`TINYINT` |no
+|`TO` |yes
+|`TOKEN` |yes
+|`TRIGGER` |no
+|`TRUNCATE` |yes
+|`TTL` |no
+|`TUPLE` |no
+|`TYPE` |no
+|`UNLOGGED` |yes
+|`UNMASK` |no
+|`UNSET` |yes
+|`UPDATE` |yes
+|`USE` |yes
+|`USER` |no
+|`USERS` |no
+|`USING` |yes
+|`UUID` |no
+|`VALUES` |no
+|`VARCHAR` |no
+|`VARINT` |no
+|`VIEW` |yes
+|`WHERE` |yes
+|`WITH` |yes
+|`WRITETIME` |no
+|===
+
+[[appendixB]]
+=== Appendix B: CQL Reserved Types
+
+The following type names are not currently used by CQL, but are reserved
+for potential future use. User-defined types may not use reserved type
+names as their name.
+
+[cols="",options="header",]
+|===
+|type
+|`bitstring`
+|`byte`
+|`complex`
+|`date`
+|`enum`
+|`interval`
+|`macaddr`
+|===
+
+=== Changes
+
+The following describes the changes in each version of CQL.
+
+==== 3.4.3
+
+* Support for `GROUP BY`. See link:#selectGroupBy[`<group-by>`] (see
+https://issues.apache.org/jira/browse/CASSANDRA-10707)[CASSANDRA-10707].
+
+==== 3.4.2
+
+* Support for selecting elements and slices of a collection
+(https://issues.apache.org/jira/browse/CASSANDRA-7396)[CASSANDRA-7396].
+
+==== 3.4.2
+
+* link:#updateOptions[`INSERT/UPDATE options`] for tables having a
+default_time_to_live specifying a TTL of 0 will remove the TTL from the
+inserted or updated values
+* link:#alterTableStmt[`ALTER TABLE`] `ADD` and `DROP` now allow mutiple
+columns to be added/removed
+* New link:#selectLimit[`PER PARTITION LIMIT`] option (see
+https://issues.apache.org/jira/browse/CASSANDRA-7017)[CASSANDRA-7017].
+* link:#udfs[User-defined functions] can now instantiate `UDTValue` and
+`TupleValue` instances via the new `UDFContext` interface (see
+https://issues.apache.org/jira/browse/CASSANDRA-10818)[CASSANDRA-10818].
+* ``User-defined types''#createTypeStmt may now be stored in a
+non-frozen form, allowing individual fields to be updated and deleted in
+link:#updateStmt[`UPDATE` statements] and link:#deleteStmt[`DELETE`
+statements], respectively.
+(https://issues.apache.org/jira/browse/CASSANDRA-7423)[CASSANDRA-7423]
+
+==== 3.4.1
+
+* Adds `CAST` functions. See link:#castFun[`Cast`].
+
+==== 3.4.0
+
+* Support for link:#createMVStmt[materialized views]
+* link:#deleteStmt[`DELETE`] support for inequality expressions and `IN`
+restrictions on any primary key columns
+* link:#updateStmt[`UPDATE`] support for `IN` restrictions on any
+primary key columns
+
+==== 3.3.1
+
+* The syntax `TRUNCATE TABLE X` is now accepted as an alias for
+`TRUNCATE X`
+
+==== 3.3.0
+
+* Adds new link:#aggregates[aggregates]
+* User-defined functions are now supported through
+link:#createFunctionStmt[`CREATE FUNCTION`] and
+link:#dropFunctionStmt[`DROP FUNCTION`].
+* User-defined aggregates are now supported through
+link:#createAggregateStmt[`CREATE AGGREGATE`] and
+link:#dropAggregateStmt[`DROP AGGREGATE`].
+* Allows double-dollar enclosed strings literals as an alternative to
+single-quote enclosed strings.
+* Introduces Roles to supercede user based authentication and access
+control
+* link:#usingdates[`Date`] and link:usingtime[`Time`] data types have
+been added
+* link:#json[`JSON`] support has been added
+* `Tinyint` and `Smallint` data types have been added
+* Adds new time conversion functions and deprecate `dateOf` and
+`unixTimestampOf`. See link:#timeFun[`Time conversion functions`]
+
+==== 3.2.0
+
+* User-defined types are now supported through
+link:#createTypeStmt[`CREATE TYPE`], link:#alterTypeStmt[`ALTER TYPE`],
+and link:#dropTypeStmt[`DROP TYPE`]
+* link:#createIndexStmt[`CREATE INDEX`] now supports indexing collection
+columns, including indexing the keys of map collections through the
+`keys()` function
+* Indexes on collections may be queried using the new `CONTAINS` and
+`CONTAINS KEY` operators
+* Tuple types were added to hold fixed-length sets of typed positional
+fields (see the section on link:#types[types] )
+* link:#dropIndexStmt[`DROP INDEX`] now supports optionally specifying a
+keyspace
+
+==== 3.1.7
+
+* `SELECT` statements now support selecting multiple rows in a single
+partition using an `IN` clause on combinations of clustering columns.
+See link:#selectWhere[SELECT WHERE] clauses.
+* `IF NOT EXISTS` and `IF EXISTS` syntax is now supported by
+`CREATE USER` and `DROP USER` statmenets, respectively.
+
+==== 3.1.6
+
+* A new link:#uuidFun[`uuid` method] has been added.
+* Support for `DELETE ... IF EXISTS` syntax.
+
+==== 3.1.5
+
+* It is now possible to group clustering columns in a relatiion, see
+link:#selectWhere[SELECT WHERE] clauses.
+* Added support for `STATIC` columns, see link:#createTableStatic[static
+in CREATE TABLE].
+
+==== 3.1.4
+
+* `CREATE INDEX` now allows specifying options when creating CUSTOM
+indexes (see link:#createIndexStmt[CREATE INDEX reference] ).
+
+==== 3.1.3
+
+* Millisecond precision formats have been added to the timestamp parser
+(see link:#usingtimestamps[working with dates] ).
+
+==== 3.1.2
+
+* `NaN` and `Infinity` has been added as valid float contants. They are
+now reserved keywords. In the unlikely case you we using them as a
+column identifier (or keyspace/table one), you will noew need to double
+quote them (see link:#identifiers[quote identifiers] ).
+
+==== 3.1.1
+
+* `SELECT` statement now allows listing the partition keys (using the
+`DISTINCT` modifier). See
+https://issues.apache.org/jira/browse/CASSANDRA-4536[CASSANDRA-4536].
+* The syntax `c IN ?` is now supported in `WHERE` clauses. In that case,
+the value expected for the bind variable will be a list of whatever type
+`c` is.
+* It is now possible to use named bind variables (using `:name` instead
+of `?`).
+
+==== 3.1.0
+
+* link:#alterTableStmt[ALTER TABLE] `DROP` option has been reenabled for
+CQL3 tables and has new semantics now: the space formerly used by
+dropped columns will now be eventually reclaimed (post-compaction). You
+should not readd previously dropped columns unless you use timestamps
+with microsecond precision (see
+https://issues.apache.org/jira/browse/CASSANDRA-3919[CASSANDRA-3919] for
+more details).
+* `SELECT` statement now supports aliases in select clause. Aliases in
+WHERE and ORDER BY clauses are not supported. See the
+link:#selectStmt[section on select] for details.
+* `CREATE` statements for `KEYSPACE`, `TABLE` and `INDEX` now supports
+an `IF NOT EXISTS` condition. Similarly, `DROP` statements support a
+`IF EXISTS` condition.
+* `INSERT` statements optionally supports a `IF NOT EXISTS` condition
+and `UPDATE` supports `IF` conditions.
+
+==== 3.0.5
+
+* `SELECT`, `UPDATE`, and `DELETE` statements now allow empty `IN`
+relations (see
+https://issues.apache.org/jira/browse/CASSANDRA-5626)[CASSANDRA-5626].
+
+==== 3.0.4
+
+* Updated the syntax for custom link:#createIndexStmt[secondary
+indexes].
+* Non-equal condition on the partition key are now never supported, even
+for ordering partitioner as this was not correct (the order was *not*
+the one of the type of the partition key). Instead, the `token` method
+should always be used for range queries on the partition key (see
+link:#selectWhere[WHERE clauses] ).
+
+==== 3.0.3
+
+* Support for custom link:#createIndexStmt[secondary indexes] has been
+added.
+
+==== 3.0.2
+
+* Type validation for the link:#constants[constants] has been fixed. For
+instance, the implementation used to allow `'2'` as a valid value for an
+`int` column (interpreting it has the equivalent of `2`), or `42` as a
+valid `blob` value (in which case `42` was interpreted as an hexadecimal
+representation of the blob). This is no longer the case, type validation
+of constants is now more strict. See the link:#types[data types] section
+for details on which constant is allowed for which type.
+* The type validation fixed of the previous point has lead to the
+introduction of link:#constants[blobs constants] to allow inputing
+blobs. Do note that while inputing blobs as strings constant is still
+supported by this version (to allow smoother transition to blob
+constant), it is now deprecated (in particular the link:#types[data
+types] section does not list strings constants as valid blobs) and will
+be removed by a future version. If you were using strings as blobs, you
+should thus update your client code ASAP to switch blob constants.
+* A number of functions to convert native types to blobs have also been
+introduced. Furthermore the token function is now also allowed in select
+clauses. See the link:#functions[section on functions] for details.
+
+==== 3.0.1
+
+* link:#usingtimestamps[Date strings] (and timestamps) are no longer
+accepted as valid `timeuuid` values. Doing so was a bug in the sense
+that date string are not valid `timeuuid`, and it was thus resulting in
+https://issues.apache.org/jira/browse/CASSANDRA-4936[confusing
+behaviors]. However, the following new methods have been added to help
+working with `timeuuid`: `now`, `minTimeuuid`, `maxTimeuuid` , `dateOf`
+and `unixTimestampOf`. See the link:#timeuuidFun[section dedicated to
+these methods] for more detail.
+* ``Float constants''#constants now support the exponent notation. In
+other words, `4.2E10` is now a valid floating point value.
+
+=== Versioning
+
+Versioning of the CQL language adheres to the http://semver.org[Semantic
+Versioning] guidelines. Versions take the form X.Y.Z where X, Y, and Z
+are integer values representing major, minor, and patch level
+respectively. There is no correlation between Cassandra release versions
+and the CQL language version.
+
+[cols=",",options="header",]
+|===
+|version |description
+|Major |The major version _must_ be bumped when backward incompatible
+changes are introduced. This should rarely occur.
+
+|Minor |Minor version increments occur when new, but backward
+compatible, functionality is introduced.
+
+|Patch |The patch version is incremented when bugs are fixed.
+|===
diff --git a/doc/modules/cassandra/pages/developing/cql/ddl.adoc b/doc/modules/cassandra/pages/developing/cql/ddl.adoc
new file mode 100644
index 0000000..cb905f5
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/ddl.adoc
@@ -0,0 +1,806 @@
+= Data Definition
+:tabs:
+
+CQL stores data in _tables_, whose schema defines the layout of the
+data in the table. Tables are located in _keyspaces_.
+A keyspace defines options that apply to all the keyspace's tables.
+The xref:cql/ddl.adoc#replication-strategy[replication strategy]
+is an important keyspace option, as is the replication factor.
+A good general rule is one keyspace per application.
+It is common for a cluster to define only one keyspace for an active application.
+
+This section describes the statements used to create, modify, and remove
+those keyspace and tables.
+
+== Common definitions
+
+The names of the keyspaces and tables are defined by the following
+grammar:
+
+[source,bnf]
+----
+include::example$BNF/ks_table.bnf[]
+----
+
+Both keyspace and table name should be comprised of only alphanumeric
+characters, cannot be empty and are limited in size to 48 characters
+(that limit exists mostly to avoid filenames (which may include the
+keyspace and table name) to go over the limits of certain file systems).
+By default, keyspace and table names are case-insensitive (`myTable` is
+equivalent to `mytable`) but case sensitivity can be forced by using
+double-quotes (`"myTable"` is different from `mytable`).
+
+Further, a table is always part of a keyspace and a table name can be
+provided fully-qualified by the keyspace it is part of. If is is not
+fully-qualified, the table is assumed to be in the _current_ keyspace
+(see xref:cql/ddl.adoc#use-statement[USE] statement).
+
+Further, the valid names for columns are defined as:
+
+[source,bnf]
+----
+include::example$BNF/column.bnf[]
+----
+
+We also define the notion of statement options for use in the following
+section:
+
+[source,bnf]
+----
+include::example$BNF/options.bnf[]
+----
+
+[[create-keyspace-statement]]
+== CREATE KEYSPACE
+
+A keyspace is created with a `CREATE KEYSPACE` statement:
+
+[source,bnf]
+----
+include::example$BNF/create_ks.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/create_ks.cql[]
+----
+
+Attempting to create a keyspace that already exists will return an error
+unless the `IF NOT EXISTS` option is used. If it is used, the statement
+will be a no-op if the keyspace already exists.
+
+The supported `options` are:
+
+[cols=",,,,",options="header",]
+|===
+|name | kind | mandatory | default | description
+|`replication` | _map_ | yes | n/a | The replication strategy and options to use for the keyspace (see
+details below).
+|`durable_writes` | _simple_ | no | true | Whether to use the commit log for updates on this keyspace (disable this
+option at your own risk!).
+|===
+
+The `replication` property is mandatory and must contain the `'class'` sub-option that defines the desired
+xref:cql/ddl.adoc#replication-strategy[replication strategy] class.
+The rest of the sub-options depend on which replication strategy is used.
+By default, Cassandra supports the following `'class'` values:
+
+[[replication-strategy]]
+=== `SimpleStrategy`
+
+A simple strategy that defines a replication factor for data to be
+spread across the entire cluster. This is generally not a wise choice
+for production, as it does not respect datacenter layouts and can
+lead to wildly varying query latency. For production, use
+`NetworkTopologyStrategy`. `SimpleStrategy` supports a single
+mandatory argument:
+
+[cols=",,,",options="header",]
+|===
+|sub-option |type |since |description
+|`'replication_factor'` | int | all | The number of replicas to store per range
+|===
+
+=== `NetworkTopologyStrategy`
+
+A production-ready replication strategy that sets the
+replication factor independently for each data-center. The rest of the
+sub-options are key-value pairs, with a key set to a data-center name and
+its value set to the associated replication factor. Options:
+
+[cols=",,,",options="header",]
+|===
+|sub-option |type |description
+|`'<datacenter>'` | int | The number of replicas to store per range in the provided datacenter.
+|`'replication_factor'` | int | The number of replicas to use as a default per datacenter if not
+specifically provided. Note that this always defers to existing
+definitions or explicit datacenter settings. For example, to have three
+replicas per datacenter, set a value of 3.
+|===
+
+When later altering keyspaces and changing the `replication_factor`,
+auto-expansion will only _add_ new datacenters for safety, it will not
+alter existing datacenters or remove any, even if they are no longer in
+the cluster. If you want to remove datacenters while setting the
+`replication_factor`, explicitly zero out the datacenter you want to
+have zero replicas.
+
+An example of auto-expanding datacenters with two datacenters: `DC1` and
+`DC2`:
+
+[source,cql]
+----
+include::example$CQL/autoexpand_ks.cql[]
+----
+will result in:
+[source,plaintext]
+----
+include::example$RESULTS/autoexpand_ks.result[]
+----
+
+An example of auto-expanding and overriding a datacenter:
+
+[source,cql]
+----
+include::example$CQL/autoexpand_ks_override.cql[]
+----
+will result in:
+[source,plaintext]
+----
+include::example$RESULTS/autoexpand_ks_override.result[]
+----
+
+An example that excludes a datacenter while using `replication_factor`:
+
+[source,cql]
+----
+include::example$CQL/autoexpand_exclude_dc.cql[]
+----
+will result in:
+[source,plaintext]
+----
+include::example$RESULTS/autoexpand_exclude_dc.result[]
+----
+
+If xref:new/transientreplication.adoc[transient replication] has been enabled, transient replicas can be
+configured for both `SimpleStrategy` and `NetworkTopologyStrategy` by
+defining replication factors in the format
+`'<total_replicas>/<transient_replicas>'`
+
+For instance, this keyspace will have 3 replicas in DC1, 1 of which is
+transient, and 5 replicas in DC2, 2 of which are transient:
+
+[source,cql]
+----
+include::example$CQL/create_ks_trans_repl.cql[]
+----
+
+[[use-statement]]
+== USE
+
+The `USE` statement changes the _current_ keyspace to the specified keyspace.
+A number of objects in CQL are bound to a keyspace (tables, user-defined types, functions, etc.) and the
+current keyspace is the default keyspace used when those objects are
+referred to in a query without a fully-qualified name (without a prefixed keyspace name).
+A `USE` statement specifies the keyspace to use as an argument:
+
+[source,bnf]
+----
+include::example$BNF/use_ks.bnf[]
+----
+Using CQL:
+[source,cql]
+----
+include::example$CQL/use_ks.cql[]
+----
+
+[[alter-keyspace-statement]]
+== ALTER KEYSPACE
+
+An `ALTER KEYSPACE` statement modifies the options of a keyspace:
+
+[source,bnf]
+----
+include::example$BNF/alter_ks.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/alter_ks.cql[]
+----
+If the keyspace does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+The supported options are the same as for xref:cql/ddl.adoc#create-keyspace-statement[creating a keyspace].
+
+[[drop-keyspace-statement]]
+== DROP KEYSPACE
+
+Dropping a keyspace is done with the `DROP KEYSPACE` statement:
+
+[source,bnf]
+----
+include::example$BNF/drop_ks.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/drop_ks.cql[]
+----
+
+Dropping a keyspace results in the immediate, irreversible removal of
+that keyspace, including all the tables, user-defined types, user-defined functions, and
+all the data contained in those tables.
+
+If the keyspace does not exists, the statement will return an error,
+unless `IF EXISTS` is used in which case the operation is a no-op.
+
+[[create-table-statement]]
+== CREATE TABLE
+
+Creating a new table uses the `CREATE TABLE` statement:
+
+[source,bnf]
+----
+include::example$BNF/create_table.bnf[]
+----
+
+For example, here are some CQL statements to create tables:
+
+[source,cql]
+----
+include::example$CQL/create_table.cql[]
+----
+
+A CQL table has a name and is composed of a set of _rows_.
+Creating a table amounts to defining which xref:cql/ddl.adoc#column-definition[columns] each rows will have,
+which of those columns comprise the xref:cql/ddl.adoc#primary-key[primary key], as well as defined
+xref:cql/ddl.adoc#create-table-options[options] for the table.
+
+Attempting to create an already existing table will return an error
+unless the `IF NOT EXISTS` directive is used. If it is used, the
+statement will be a no-op if the table already exists.
+
+[[column-definition]]
+=== Column definitions
+
+Every row in a CQL table will have the predefined columns defined at table creation.
+Columns can be added later using an xref:cql/ddl.adoc#alter-table-statement[alter statement].
+
+A `column_definition` is comprised of the name of the column and its xref:cql/ddl.adoc#data-type[type],
+restricting the values that are accepted for that column. Additionally, a column definition can have the
+following modifiers:
+
+* `STATIC`: declares the column as a xref:cql/ddl.adoc#static-column[static column]
+* `PRIMARY KEY`: declares the column as the sole component of the xref:cql/ddl.adoc#primary-key[primary key] of the table
+
+[[static-column]]
+==== Static columns
+
+Some columns can be declared as `STATIC` in a table definition. A column
+that is static will be “shared” by all the rows belonging to the same
+partition (having the same xref:cql/ddl.adoc#partition-key[partition key].
+
+For example:
+
+[{tabs}]
+====
+Code::
++
+--
+[source,cql]
+----
+include::example$CQL/create_static_column.cql[]
+include::example$CQL/insert_static_data.cql[]
+include::example$CQL/select_static_data.cql[]
+----
+--
+
+Results::
++
+--
+[source,cql]
+----
+include::example$RESULTS/select_static_data.result[]
+----
+--
+====
+
+As can be seen, the `s` value is the same (`static1`) for both of the
+rows in the partition (the partition key being `pk`, and both
+rows are in the same partition): the second insertion overrides the
+value for `s`.
+
+The use of static columns has the following restrictions:
+
+* A table without clustering columns cannot have static columns.
+In a table without clustering columns, every partition has only one row, and
+so every column is inherently static)
+* Only non-primary key columns can be static.
+
+[[primary-key]]
+=== The Primary key
+
+Within a table, a row is uniquely identified by its `PRIMARY KEY`, and
+hence all tables *must* define a single PRIMARY KEY.
+A `PRIMARY KEY` is composed of one or more of the defined columns in the table.
+Syntactically, the primary key is defined with the phrase `PRIMARY KEY`
+followed by a comma-separated list of the column names within parenthesis.
+If the primary key has only one column, you can alternatively add the `PRIMARY KEY` phrase to
+that column in the table definition.
+The order of the columns in the primary key definition defines the partition key and
+clustering columns.
+
+A CQL primary key is composed of two parts:
+
+xref:cql/ddl.adoc#partition-key[partition key]::
+* It is the first component of the primary key definition.
+It can be a single column or, using an additional set of parenthesis, can be multiple columns.
+A table must have at least one partition key, the smallest possible table definition is:
++
+[source,cql]
+----
+include::example$CQL/create_table_single_pk.cql[]
+----
+xref:cql/ddl.adoc#clustering-columns[clustering columns]::
+* The columns are the columns that follow the partition key in the primary key definition.
+The order of those columns define the _clustering order_.
+
+Some examples of primary key definition are:
+
+* `PRIMARY KEY (a)`: `a` is the single partition key and there are no clustering columns
+* `PRIMARY KEY (a, b, c)` : `a` is the single partition key and `b` and `c` are the clustering columns
+* `PRIMARY KEY ((a, b), c)` : `a` and `b` compose the _composite_ partition key and `c` is the clustering column
+
+[IMPORTANT]
+====
+The primary key uniquely identifies a row in the table, as described above.
+A consequence of this uniqueness is that if another row is inserted using the same primary key,
+then an `UPSERT` occurs and an existing row with the same primary key is replaced.
+Columns that are not part of the primary key cannot define uniqueness.
+====
+
+[[partition-key]]
+==== Partition key
+
+Within a table, CQL defines the notion of a _partition_ that defines the location of data within a Cassandra cluster.
+A partition is the set of rows that share the same value for their partition key.
+
+Note that if the partition key is composed of multiple columns, then rows belong to the same partition
+when they have the same values for all those partition key columns.
+A hash is computed from the partition key columns and that hash value defines the partition location.
+So, for instance, given the following table definition and content:
+
+[source,cql]
+----
+include::example$CQL/create_table_compound_pk.cql[]
+include::example$CQL/insert_table_compound_pk.cql[]
+include::example$CQL/select_table_compound_pk.cql[]
+----
+
+will result in
+[source,cql]
+----
+include::example$RESULTS/select_table_compound_pk.result[]
+----
+<1> Rows 1 and 2 are in the same partition, because both columns `a` and `b` are zero.
+<2> Rows 3 and 4 are in the same partition, but a different one, because column `a` is zero and column `b` is 1 in both rows.
+<3> Row 5 is in a third partition by itself, because both columns `a` and `b` are 1.
+
+Note that a table always has a partition key, and that if the table has
+no `clustering columns`, then every partition of that table has a single row.
+because the partition key, compound or otherwise, identifies a single location.
+
+The most important property of partition is that all the rows belonging
+to the same partition are guaranteed to be stored on the same set of
+replica nodes.
+In other words, the partition key of a table defines which rows will be localized on the same
+node in the cluster.
+The localization of data is important to the efficient retrieval of data, requiring the Cassandra coordinator
+to contact as few nodes as possible.
+However, there is a flip-side to this guarantee, and all rows sharing a partition key will be stored on the same
+node, creating a hotspot for both reading and writing.
+While selecting a primary key that groups table rows assists batch updates and can ensure that the updates are
+_atomic_ and done in _isolation_, the partitions must be sized "just right, not too big nor too small".
+
+Data modeling that considers the querying patterns and assigns primary keys based on the queries will have the lowest
+latency in fetching data.
+
+[[clustering-columns]]
+==== Clustering columns
+
+The clustering columns of a table define the clustering order for the partition of that table.
+For a given `partition`, all rows are ordered by that clustering order. Clustering columns also add uniqueness to
+a row in a table.
+
+For instance, given:
+
+[source,cql]
+----
+include::example$CQL/create_table_clustercolumn.cql[]
+include::example$CQL/insert_table_clustercolumn.cql[]
+include::example$CQL/select_table_clustercolumn.cql[]
+----
+
+will result in
+[source,cql]
+----
+include::example$RESULTS/select_table_clustercolumn.result[]
+----
+<1> Row 1 is in one partition, and Rows 2-5 are in a different one. The display order is also different.
+
+Looking more closely at the four rows in the same partition, the `b` clustering column defines the order in which those rows
+are displayed.
+Whereas the partition key of the table groups rows on the same node, the clustering columns control
+how those rows are stored on the node.
+
+That sorting allows the very efficient retrieval of a range of rows within a partition:
+
+[source,cql]
+----
+include::example$CQL/select_range.cql[]
+----
+
+will result in
+[source,cql]
+----
+include::example$RESULTS/select_range.result[]
+----
+
+[[create-table-options]]
+=== Table options
+
+A CQL table has a number of options that can be set at creation (and,
+for most of them, altered later). These options are specified after the
+`WITH` keyword.
+
+One important option that cannot be changed after creation, `CLUSTERING ORDER BY`, influences how queries can be done against the table. It is worth discussing in more detail here.
+
+[[clustering-order]]
+==== Clustering order
+
+The clustering order of a table is defined by the clustering columns.
+By default, the clustering order is ascending for the clustering column's data types.
+For example, integers order from 1, 2, ... n, while text orders from A to Z.
+
+The `CLUSTERING ORDER BY` table option uses a comma-separated list of the
+clustering columns, each set for either `ASC` (for _ascending_ order) or `DESC` (for _descending order).
+The default is ascending for all clustering columns if the `CLUSTERING ORDER BY` option is not set.
+
+This option is basically a hint for the storage engine that changes the order in which it stores the row.
+Beware of the consequences of setting this option:
+
+* It changes the default ascending order of results when queried with a `SELECT` statement with no `ORDER BY` clause.
+
+* It limits how the `ORDER BY` clause is used in `SELECT` statements on that table.
+Results can only be ordered with either the original clustering order or the reverse clustering order.
+Suppose you create a table with two clustering columns `a` and `b`, defined `WITH CLUSTERING ORDER BY (a DESC, b ASC)`.
+Queries on the table can use `ORDER BY (a DESC, b ASC)` or `ORDER BY (a ASC, b DESC)`.
+Mixed order, such as `ORDER BY (a ASC, b ASC)` or `ORDER BY (a DESC, b DESC)` will not return expected order.
+
+* It has a performance impact on queries. Queries in reverse clustering order are slower than the default ascending order.
+If you plan to query mostly in descending order, declare the clustering order in the table schema using `WITH CLUSTERING ORDER BY ()`.
+This optimization is common for time series, to retrieve the data from newest to oldest.
+
+[[create-table-general-options]]
+==== Other table options
+
+A table supports the following options:
+
+[width="100%",cols="30%,9%,11%,50%",options="header",]
+|===
+|option | kind | default | description
+
+| `comment` | _simple_ | none | A free-form, human-readable comment
+| xref:cql/ddl.adoc#spec_retry[`speculative_retry`] | _simple_ | 99PERCENTILE | Speculative retry options
+| `cdc` |_boolean_ |false |Create a Change Data Capture (CDC) log on the table
+| `additional_write_policy` |_simple_ |99PERCENTILE | Same as `speculative_retry`
+| `gc_grace_seconds` |_simple_ |864000 |Time to wait before garbage collecting tombstones (deletion markers)
+| `bloom_filter_fp_chance` |_simple_ |0.00075 |The target probability of
+false positive of the sstable bloom filters. Said bloom filters will be
+sized to provide the provided probability, thus lowering this value
+impacts the size of bloom filters in-memory and on-disk.
+| `default_time_to_live` |_simple_ |0 |Default expiration time (“TTL”) in seconds for a table
+| `compaction` |_map_ |_see below_ | xref:operating/compaction/index.adoc#cql-compaction-options[Compaction options]
+| `compression` |_map_ |_see below_ | xref:operating/compression/index.adoc#cql-compression-options[Compression options]
+| `caching` |_map_ |_see below_ |Caching options
+| `memtable_flush_period_in_ms` |_simple_ |0 |Time (in ms) before Cassandra flushes memtables to disk
+| `read_repair` |_simple_ |BLOCKING |Sets read repair behavior (see below)
+|===
+
+[[spec_retry]]
+===== Speculative retry options
+
+By default, Cassandra read coordinators only query as many replicas as
+necessary to satisfy consistency levels: one for consistency level
+`ONE`, a quorum for `QUORUM`, and so on. `speculative_retry` determines
+when coordinators may query additional replicas, a useful action when
+replicas are slow or unresponsive. Speculative retries reduce the latency.
+The speculative_retry option configures rapid read protection, where a coordinator sends more
+requests than needed to satisfy the consistency level.
+
+[IMPORTANT]
+====
+Frequently reading from additional replicas can hurt cluster
+performance. When in doubt, keep the default `99PERCENTILE`.
+====
+
+Pre-Cassandra 4.0 speculative retry policy takes a single string as a parameter:
+
+* `NONE`
+* `ALWAYS`
+* `99PERCENTILE` (PERCENTILE)
+* `50MS` (CUSTOM)
+
+An example of setting speculative retry sets a custom value:
+
+[source,cql]
+----
+include::example$CQL/alter_table_spec_retry.cql[]
+----
+
+This example uses a percentile for the setting:
+
+[source,cql]
+----
+include::example$CQL/alter_table_spec_retry_percent.cql[]
+----
+
+A percentile setting can backfire. If a single host becomes unavailable, it can
+force up the percentiles. A value of `p99` will not speculate as intended because the
+value at the specified percentile has increased too much. If the consistency level is set to `ALL`, all
+replicas are queried regardless of the speculative retry setting.
+
+Cassandra 4.0 supports case-insensitivity for speculative retry values (https://issues.apache.org/jira/browse/CASSANDRA-14293[CASSANDRA-14293]). For example, assigning the value as `none`, `None`, or `NONE` has the same effect.
+
+Additionally, the following values are added:
+
+[cols=",,",options="header",]
+|===
+|Format |Example |Description
+| `XPERCENTILE` | 90.5PERCENTILE | Coordinators record average per-table response times
+for all replicas. If a replica takes longer than `X` percent of this
+table's average response time, the coordinator queries an additional
+replica. `X` must be between 0 and 100.
+| `XP` | 90.5P | Same as `XPERCENTILE`
+| `Yms` | 25ms | If a replica takes more than `Y` milliseconds to respond, the
+coordinator queries an additional replica.
+| `MIN(XPERCENTILE,YMS)` | MIN(99PERCENTILE,35MS) | A hybrid policy that uses either the
+specified percentile or fixed milliseconds depending on which value is
+lower at the time of calculation. Parameters are `XPERCENTILE`, `XP`, or
+`Yms`. This setting helps protect against a single slow instance.
+
+| `MAX(XPERCENTILE,YMS)` `ALWAYS` `NEVER` | MAX(90.5P,25ms) | A hybrid policy that uses either the specified
+percentile or fixed milliseconds depending on which value is higher at
+the time of calculation.
+|===
+
+Cassandra 4.0 adds support for hybrid `MIN()` and `MAX()` speculative retry policies, with a mix and match of either `MIN(), MAX()`, `MIN(), MIN()`, or `MAX(), MAX()` (https://issues.apache.org/jira/browse/CASSANDRA-14293[CASSANDRA-14293]).
+The hybrid mode will still speculate if the normal `p99` for the table is < 50ms, the minimum value.
+But if the `p99` level goes higher than the maximum value, then that value can be used.
+In a hybrid value, one value must be a fixed time (ms) value and the other a percentile value.
+
+To illustrate variations, the following examples are all valid:
+
+[source,cql]
+----
+include::example$CQL/spec_retry_values.cql[]
+----
+
+The `additional_write_policy` setting specifies the threshold at which a cheap
+quorum write will be upgraded to include transient replicas.
+
+[[cql-compaction-options]]
+===== Compaction options
+
+The `compaction` options must minimally define the `'class'` sub-option,
+to specify the compaction strategy class to use.
+The supported classes are:
+
+* `'SizeTieredCompactionStrategy'`, xref:operating/compaction/stcs.adoc#stcs[STCS] (Default)
+* `'LeveledCompactionStrategy'`, xref:operating/compaction/lcs.adoc#lcs[LCS]
+* `'TimeWindowCompactionStrategy'`, xref:operating/compaction/twcs.adoc#twcs[TWCS]
+
+If a custom strategies is required, specify the full class name as a xref:cql/definitions.adoc#constants[string constant].
+
+All default strategies support a number of xref:operating/compaction/index.adoc#compaction-options[common options], as well as options specific to the strategy chosen. See the section corresponding to your strategy for details: xref:operating/compaction/stcs.adoc#stcs_options[STCS], xref:operating/compaction/lcs.adoc#lcs_options[LCS], xref:operating/compaction/twcs.adoc#twcs_options[TWCS].
+
+[[cql-compression-options]]
+===== Compression options
+
+The `compression` options define if and how the SSTables of the table
+are compressed. Compression is configured on a per-table basis as an
+optional argument to `CREATE TABLE` or `ALTER TABLE`. The following
+sub-options are available:
+
+[cols=",,",options="header",]
+|===
+|Option |Default |Description
+| `class` | LZ4Compressor | The compression algorithm to use. Default compressor are: LZ4Compressor,
+SnappyCompressor, DeflateCompressor and ZstdCompressor.
+Use `'enabled' : false` to disable compression.
+Custom compressor can be provided by specifying the full class name as a xref:cql/definitions.adoc#constants[string constant].
+
+| `enabled` | true | Enable/disable sstable compression.
+If the `enabled` option is set to `false`, no other options must be specified.
+
+| `chunk_length_in_kb` | 64 | On disk SSTables are compressed by block (to allow random reads).
+This option defines the size (in KB) of said block. See xref:cql/ddl.adoc#chunk_note[note] for further information.
+
+| `crc_check_chance` | 1.0 | Determines how likely Cassandra is to verify the checksum on each
+compression chunk during reads.
+
+| `compression_level` | 3 | Compression level. Only applicable for `ZstdCompressor`.
+Accepts values between `-131072` and `22`.
+|===
+
+[[chunk_note]]
+[NOTE]
+====
+Bigger values may improve the compression rate, but will increase the minimum size of data to be read from
+disk for a read.
+The default value is an optimal value for compressing tables.
+Chunk length must be a power of 2 when computing the chunk number from an uncompressed file offset.
+Block size may be adjusted based on read/write access patterns such as:
+
+* How much data is typically requested at once
+* Average size of rows in the table
+====
+
+For instance, to create a table with LZ4Compressor and a `chunk_length_in_kb` of 4 KB:
+
+[source,cql]
+----
+include::example$CQL/chunk_length.cql[]
+----
+
+[[cql-caching-options]]
+===== Caching options
+
+Caching optimizes the use of cache memory of a table. The cached data is
+weighed by size and access frequency.
+The `caching` options can configure both the `key cache` and the `row cache` for the table.
+The following sub-options are available:
+
+[cols=",,",options="header",]
+|===
+|Option |Default |Description
+| `keys` | ALL | Whether to cache keys (key cache) for this table. Valid values are: `ALL` and `NONE`.
+
+| `rows_per_partition` | NONE | The amount of rows to cache per partition (row cache).
+If an integer `n` is specified, the first `n` queried rows of a partition will be cached.
+Valid values are: `ALL`, to cache all rows of a queried partition, or `NONE` to disable row caching.
+|===
+
+For instance, to create a table with both a key cache and 10 rows cached per partition:
+
+[source,cql]
+----
+include::example$CQL/caching_option.cql[]
+----
+
+[[read-repair-options]]
+===== Read Repair options
+
+The `read_repair` options configure the read repair behavior, tuning for various performance and consistency behaviors.
+
+The values are:
+[cols=",,",options="header",]
+|===
+|Option |Default |Description
+|`BLOCKING` | yes | If a read repair is triggered, the read blocks writes sent to other replicas until the consistency level is reached by the writes.
+
+|`NONE` | no | If set, the coordinator reconciles any differences between replicas, but doesn't attempt to repair them.
+|===
+
+Two consistency properties are affected by read repair behavior.
+
+* Monotonic quorum reads: Monotonic quorum reads
+prevents reads from appearing to go back in time in some circumstances.
+When monotonic quorum reads are not provided and a write fails to reach
+a quorum of replicas, the read values may be visible in one read, and then disappear
+in a subsequent read. `BLOCKING` provides this behavior.
+* Write atomicity: Write atomicity prevents reads
+from returning partially-applied writes. Cassandra attempts to provide
+partition-level write atomicity, but since only the data covered by a
+SELECT statement is repaired by a read repair, read repair can break
+write atomicity when data is read at a more granular level than it is
+written. For example, read repair can break write atomicity if you write
+multiple rows to a clustered partition in a batch, but then select a
+single row by specifying the clustering column in a SELECT statement.
+`NONE` provides this behavior.
+
+===== Other considerations:
+
+* Adding new columns (see `ALTER TABLE` below) is a constant time
+operation. Thus, there is no need to anticipate future usage while initially creating a table.
+
+[[alter-table-statement]]
+== ALTER TABLE
+
+Altering an existing table uses the `ALTER TABLE` statement:
+
+[source,bnf]
+----
+include::example$BNF/alter_table.bnf[]
+----
+If the table does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/alter_table_add_column.cql[]
+include::example$CQL/alter_table_with_comment.cql[]
+----
+
+The `ALTER TABLE` statement can:
+
+* `ADD` a new column to a table. The primary key of a table cannot ever be altered.
+A new column, thus, cannot be part of the primary key.
+Adding a column is a constant-time operation based on the amount of data in the table.
+If the new column already exists, the statement will return an error, unless `IF NOT EXISTS` is used in which case the operation is a no-op.
+* `DROP` a column from a table. This command drops both the column and all
+its content. Be aware that, while the column becomes immediately
+unavailable, its content are removed lazily during compaction. Because of this lazy removal,
+the command is a constant-time operation based on the amount of data in the table.
+Also, it is important to know that once a column is dropped, a column with the same name can be re-added,
+unless the dropped column was a non-frozen column like a collection.
+If the dropped column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+[WARNING]
+.Warning
+====
+Dropping a column assumes that the timestamps used for the value of this
+column are "real" timestamp in microseconds. Using "real" timestamps in
+microseconds is the default is and is *strongly* recommended but as
+Cassandra allows the client to provide any timestamp on any table, it is
+theoretically possible to use another convention. Please be aware that
+if you do so, dropping a column will not correctly execute.
+====
+
+* `RENAME` a primary key column of a table. Non primary key columns cannot be renamed.
+Furthermore, renaming a column to another name which already exists isn't allowed.
+It's important to keep in mind that renamed columns shouldn't have dependent seconday indexes.
+If the renamed column does not already exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+* Use `WITH` to change a table option. The xref:CQL/ddl.adoc#create-table-options[supported options]
+are the same as those used when creating a table, with the exception of `CLUSTERING ORDER`.
+However, setting any `compaction` sub-options will erase *ALL* previous `compaction` options, so you need to re-specify
+all the sub-options you wish to keep. The same is true for `compression` sub-options.
+
+[[drop-table-statement]]
+== DROP TABLE
+
+Dropping a table uses the `DROP TABLE` statement:
+
+[source,bnf]
+----
+include::example$BNF/drop_table.bnf[]
+----
+
+Dropping a table results in the immediate, irreversible removal of the
+table, including all data it contains.
+
+If the table does not exist, the statement will return an error, unless
+`IF EXISTS` is used, when the operation is a no-op.
+
+[[truncate-statement]]
+== TRUNCATE
+
+A table can be truncated using the `TRUNCATE` statement:
+
+[source,bnf]
+----
+include::example$BNF/truncate_table.bnf[]
+----
+
+`TRUNCATE TABLE foo` is the preferred syntax for consistency with other DDL
+statements.
+However, tables are the only object that can be truncated currently, and the `TABLE` keyword can be omitted.
+
+Truncating a table permanently removes all existing data from the table, but without removing the table itself.
diff --git a/doc/modules/cassandra/pages/developing/cql/definitions.adoc b/doc/modules/cassandra/pages/developing/cql/definitions.adoc
new file mode 100644
index 0000000..14e6048
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/definitions.adoc
@@ -0,0 +1,187 @@
+= Definitions
+
+== Conventions
+
+To aid in specifying the CQL syntax, we will use the following
+conventions in this document:
+
+* Language rules will be given in an informal
+http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form#Variants[BNF
+variant] notation. In particular, we'll use square brakets (`[ item ]`)
+for optional items, `*` and `+` for repeated items (where `+` imply at
+least one).
+* The grammar will also use the following convention for convenience:
+non-terminal term will be lowercase (and link to their definition) while
+terminal keywords will be provided "all caps". Note however that
+keywords are `identifiers` and are thus case insensitive in practice. We
+will also define some early construction using regexp, which we'll
+indicate with `re(<some regular expression>)`.
+* The grammar is provided for documentation purposes and leave some
+minor details out. For instance, the comma on the last column definition
+in a `CREATE TABLE` statement is optional but supported if present even
+though the grammar in this document suggests otherwise. Also, not
+everything accepted by the grammar is necessarily valid CQL.
+* References to keywords or pieces of CQL code in running text will be
+shown in a `fixed-width font`.
+
+[[identifiers]]
+== Identifiers and keywords
+
+The CQL language uses _identifiers_ (or _names_) to identify tables,
+columns and other objects. An identifier is a token matching the regular
+expression `[a-zA-Z][a-zA-Z0-9_]*`.
+
+A number of such identifiers, like `SELECT` or `WITH`, are _keywords_.
+They have a fixed meaning for the language and most are reserved. The
+list of those keywords can be found in xref:cql/appendices.adoc#appendix-A[Appendix A].
+
+Identifiers and (unquoted) keywords are case insensitive. Thus `SELECT`
+is the same than `select` or `sElEcT`, and `myId` is the same than
+`myid` or `MYID`. A convention often used (in particular by the samples
+of this documentation) is to use uppercase for keywords and lowercase
+for other identifiers.
+
+There is a second kind of identifier called a _quoted identifier_
+defined by enclosing an arbitrary sequence of characters (non-empty) in
+double-quotes(`"`). Quoted identifiers are never keywords. Thus
+`"select"` is not a reserved keyword and can be used to refer to a
+column (note that using this is particularly ill-advised), while `select`
+would raise a parsing error. Also, unlike unquoted identifiers
+and keywords, quoted identifiers are case sensitive (`"My Quoted Id"` is
+_different_ from `"my quoted id"`). A fully lowercase quoted identifier
+that matches `[a-zA-Z][a-zA-Z0-9_]*` is however _equivalent_ to the
+unquoted identifier obtained by removing the double-quote (so `"myid"`
+is equivalent to `myid` and to `myId` but different from `"myId"`).
+Inside a quoted identifier, the double-quote character can be repeated
+to escape it, so `"foo "" bar"` is a valid identifier.
+
+[NOTE]
+.Note
+====
+The _quoted identifier_ can declare columns with arbitrary names, and
+these can sometime clash with specific names used by the server. For
+instance, when using conditional update, the server will respond with a
+result set containing a special result named `"[applied]"`. If you’ve
+declared a column with such a name, this could potentially confuse some
+tools and should be avoided. In general, unquoted identifiers should be
+preferred but if you use quoted identifiers, it is strongly advised that you
+avoid any name enclosed by squared brackets (like `"[applied]"`) and any
+name that looks like a function call (like `"f(x)"`).
+====
+
+More formally, we have:
+
+[source, bnf]
+----
+include::example$BNF/identifier.bnf[]
+----
+
+[[constants]]
+== Constants
+
+CQL defines the following _constants_:
+
+[source, bnf]
+----
+include::example$BNF/constant.bnf[]
+----
+
+In other words:
+
+* A string constant is an arbitrary sequence of characters enclosed by
+single-quote(`'`). A single-quote can be included by repeating it, e.g.
+`'It''s raining today'`. Those are not to be confused with quoted
+`identifiers` that use double-quotes. Alternatively, a string can be
+defined by enclosing the arbitrary sequence of characters by two dollar
+characters, in which case single-quote can be used without escaping
+(`$$It's raining today$$`). That latter form is often used when defining
+xref:cql/functions.adoc#udfs[user-defined functions] to avoid having to escape single-quote
+characters in function body (as they are more likely to occur than
+`$$`).
+* Integer, float and boolean constant are defined as expected. Note
+however than float allows the special `NaN` and `Infinity` constants.
+* CQL supports
+https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID]
+constants.
+* The content for blobs is provided in hexadecimal and prefixed by `0x`.
+* The special `NULL` constant denotes the absence of value.
+
+For how these constants are typed, see the xref:cql/types.adoc[Data types] section.
+
+== Terms
+
+CQL has the notion of a _term_, which denotes the kind of values that
+CQL support. Terms are defined by:
+
+[source, bnf]
+----
+include::example$BNF/term.bnf[]
+----
+
+A term is thus one of:
+
+* A xref:cql/defintions.adoc#constants[constant]
+* A literal for either a xref:cql/types.adoc#collections[collection],
+a xref:cql/types.adoc#udts[user-defined type] or a xref:cql/types.adoc#tuples[tuple]
+* A xref:cql/functions.adoc#cql-functions[function] call, either a xref:cql/functions.adoc#scalar-native-functions[native function]
+or a xref:cql/functions.adoc#user-defined-scalar-functions[user-defined function]
+* An xref:cql/operators.adoc#arithmetic_operators[arithmetic operation] between terms
+* A type hint
+* A bind marker, which denotes a variable to be bound at execution time.
+See the section on `prepared-statements` for details. A bind marker can
+be either anonymous (`?`) or named (`:some_name`). The latter form
+provides a more convenient way to refer to the variable for binding it
+and should generally be preferred.
+
+== Comments
+
+A comment in CQL is a line beginning by either double dashes (`--`) or
+double slash (`//`).
+
+Multi-line comments are also supported through enclosure within `/*` and
+`*/` (but nesting is not supported).
+
+[source,cql]
+----
+-- This is a comment
+// This is a comment too
+/* This is
+ a multi-line comment */
+----
+
+== Statements
+
+CQL consists of statements that can be divided in the following
+categories:
+
+* `data-definition` statements, to define and change how the data is
+stored (keyspaces and tables).
+* `data-manipulation` statements, for selecting, inserting and deleting
+data.
+* `secondary-indexes` statements.
+* `materialized-views` statements.
+* `cql-roles` statements.
+* `cql-permissions` statements.
+* `User-Defined Functions (UDFs)` statements.
+* `udts` statements.
+* `cql-triggers` statements.
+
+All the statements are listed below and are described in the rest of
+this documentation (see links above):
+
+[source, bnf]
+----
+include::example$BNF/cql_statement.bnf[]
+----
+
+== Prepared Statements
+
+CQL supports _prepared statements_. Prepared statements are an
+optimization that allows to parse a query only once but execute it
+multiple times with different concrete values.
+
+Any statement that uses at least one bind marker (see `bind_marker`)
+will need to be _prepared_. After which the statement can be _executed_
+by provided concrete values for each of its marker. The exact details of
+how a statement is prepared and then executed depends on the CQL driver
+used and you should refer to your driver documentation.
diff --git a/doc/modules/cassandra/pages/developing/cql/dml.adoc b/doc/modules/cassandra/pages/developing/cql/dml.adoc
new file mode 100644
index 0000000..60b9374
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/dml.adoc
@@ -0,0 +1,465 @@
+= Data Manipulation
+
+This section describes the statements supported by CQL to insert,
+update, delete and query data.
+
+[[select-statement]]
+== SELECT
+
+Querying data from data is done using a `SELECT` statement:
+
+[source,bnf]
+----
+include::example$BNF/select_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/select_statement.cql[]
+----
+
+The `SELECT` statements reads one or more columns for one or more rows
+in a table. It returns a result-set of the rows matching the request,
+where each row contains the values for the selection corresponding to
+the query. Additionally, xref:cql/functions.adoc#cql-functions[functions] including
+xref:cql/functions.adoc#aggregate-functions[aggregations] can be applied to the result.
+
+A `SELECT` statement contains at least a xref:cql/dml.adoc#selection-clause[selection clause] and the name of the table on which
+the selection is executed.
+CQL does *not* execute joins or sub-queries and a select statement only apply to a single table.
+A select statement can also have a xref:cql/dml.adoc#where-clause[where clause] that can further narrow the query results.
+Additional clauses can xref:cql/dml.adoc#ordering-clause[order] or xref:cql/dml.adoc#limit-clause[limit] the results.
+Lastly, xref:cql/dml.adoc#allow-filtering[queries that require full cluster filtering] can append `ALLOW FILTERING` to any query.
+For virtual tables, from https://issues.apache.org/jira/browse/CASSANDRA-18238[CASSANDRA-18238], it is not necessary to specify `ALLOW FILTERING` when a query would normally require that. Please consult the documentation for virtual tables to know more.
+
+[[selection-clause]]
+=== Selection clause
+
+The `select_clause` determines which columns will be queried and returned in the result set.
+This clause can also apply transformations to apply to the result before returning.
+The selection clause consists of a comma-separated list of specific _selectors_ or, alternatively, the wildcard character (`*`) to select all the columns defined in the table.
+
+==== Selectors
+
+A `selector` can be one of:
+
+* A column name of the table selected, to retrieve the values for that
+column.
+* A term, which is usually used nested inside other selectors like
+functions (if a term is selected directly, then the corresponding column
+of the result-set will simply have the value of this term for every row
+returned).
+* A casting, which allows to convert a nested selector to a (compatible)
+type.
+* A function call, where the arguments are selector themselves. See the
+section on xref:cql/functions.adoc#cql-functions[functions] for more details.
+* The special call `COUNT(*)` to the xref:cql/functions.adoc#count-function[COUNT function],
+which counts all non-null results.
+
+==== Aliases
+
+Every _top-level_ selector can also be aliased (using AS).
+If so, the name of the corresponding column in the result set will be
+that of the alias. For instance:
+
+[source,cql]
+----
+include::example$CQL/as.cql[]
+----
+
+[NOTE]
+====
+Currently, aliases aren't recognized in the `WHERE` or `ORDER BY` clauses in the statement.
+You must use the orignal column name instead.
+====
+
+[[writetime-and-ttl-function]]
+==== `WRITETIME`, `MAXWRITETIME` and `TTL` function
+
+Selection supports three special functions that aren't allowed anywhere
+else: `WRITETIME`, `MAXWRITETIME` and `TTL`.
+All functions take only one argument, a column name. If the column is a collection or UDT, it's possible to add element
+selectors, such as `WRITETTIME(phones[2..4])` or `WRITETTIME(user.name)`.
+These functions retrieve meta-information that is stored internally for each column:
+
+* `WRITETIME` stores the timestamp of the value of the column.
+* `MAXWRITETIME` stores the largest timestamp of the value of the column. For non-collection and non-UDT columns, `MAXWRITETIME`
+is equivalent to `WRITETIME`. In the other cases, it returns the largest timestamp of the values in the column.
+* `TTL` stores the remaining time to live (in seconds) for the value of the column if it is set to expire; otherwise the value is `null`.
+
+The `WRITETIME` and `TTL` functions can be used on multi-cell columns such as non-frozen collections or non-frozen
+user-defined types. In that case, the functions will return the list of timestamps or TTLs for each selected cell.
+
+[[where-clause]]
+=== The `WHERE` clause
+
+The `WHERE` clause specifies which rows are queried. It specifies
+a relationship for `PRIMARY KEY` columns or a column that has
+a xref:cql/indexes.adoc#create-index-statement[secondary index] defined, along with a set value.
+
+Not all relationships are allowed in a query. For instance, only an equality
+is allowed on a partition key. The `IN` clause is considered an equality for one or more values.
+The `TOKEN` clause can be used to query for partition key non-equalities.
+A partition key must be specified before clustering columns in the `WHERE` clause. The relationship
+for clustering columns must specify a *contiguous* set of rows to order.
+
+For instance, given:
+
+[source,cql]
+----
+include::example$CQL/table_for_where.cql[]
+----
+
+The following query is allowed:
+
+[source,cql]
+----
+include::example$CQL/where.cql[]
+----
+
+But the following one is not, as it does not select a contiguous set of
+rows (and we suppose no secondary indexes are set):
+
+[source,cql]
+----
+include::example$CQL/where_fail.cql[]
+----
+
+When specifying relationships, the `TOKEN` function can be applied to the `PARTITION KEY` column to query.
+Rows will be selected based on the token of the `PARTITION_KEY` rather than on the value.
+[IMPORTANT]
+====
+The token of a key depends on the partitioner in use, and that
+in particular the `RandomPartitioner` won't yield a meaningful order.
+Also note that ordering partitioners always order token values by bytes (so
+even if the partition key is of type int, `token(-1) > token(0)` in
+particular).
+====
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/token.cql[]
+----
+
+The `IN` relationship is only allowed on the last column of the
+partition key or on the last column of the full primary key.
+
+It is also possible to “group” `CLUSTERING COLUMNS` together in a
+relation using the tuple notation.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/where_group_cluster_columns.cql[]
+----
+
+This query will return all rows that sort after the one having “John's Blog” as
+`blog_tile` and '2012-01-01' for `posted_at` in the clustering order. In
+particular, rows having a `post_at <= '2012-01-01'` will be returned, as
+long as their `blog_title > 'John''s Blog'`.
+
+That would not be the case for this example:
+
+[source,cql]
+----
+include::example$CQL/where_no_group_cluster_columns.cql[]
+----
+
+The tuple notation may also be used for `IN` clauses on clustering columns:
+
+[source,cql]
+----
+include::example$CQL/where_in_tuple.cql[]
+----
+
+The `CONTAINS` operator may only be used for collection columns (lists,
+sets, and maps). In the case of maps, `CONTAINS` applies to the map
+values. The `CONTAINS KEY` operator may only be used on map columns and
+applies to the map keys.
+
+[[group-by-clause]]
+=== Grouping results
+
+The `GROUP BY` option can condense all selected
+rows that share the same values for a set of columns into a single row.
+
+Using the `GROUP BY` option, rows can be grouped at the partition key or clustering column level.
+Consequently, the `GROUP BY` option only accepts primary key columns in defined order as arguments.
+If a primary key column is restricted by an equality restriction, it is not included in the `GROUP BY` clause.
+
+Aggregate functions will produce a separate value for each group.
+If no `GROUP BY` clause is specified, aggregates functions will produce a single value for all the rows.
+
+If a column is selected without an aggregate function, in a statement
+with a `GROUP BY`, the first value encounter in each group will be
+returned.
+
+[[ordering-clause]]
+=== Ordering results
+
+The `ORDER BY` clause selects the order of the returned results.
+The argument is a list of column names and each column's order
+(`ASC` for ascendant and `DESC` for descendant,
+The possible orderings are limited by the xref:cql/ddl.adoc#clustering-order[clustering order] defined on the table:
+
+* if the table has been defined without any specific `CLUSTERING ORDER`, then the order is as defined by the clustering columns
+or the reverse
+* otherwise, the order is defined by the `CLUSTERING ORDER` option and the reversed one.
+
+[[limit-clause]]
+=== Limiting results
+
+The `LIMIT` option to a `SELECT` statement limits the number of rows
+returned by a query. The `PER PARTITION LIMIT` option limits the
+number of rows returned for a given partition by the query. Both types of limits can used in the same statement.
+
+[[allow-filtering]]
+=== Allowing filtering
+
+By default, CQL only allows select queries that don't involve a full scan of all partitions.
+If all partitions are scanned, then returning the results may experience a significant latency proportional to the
+amount of data in the table. The `ALLOW FILTERING` option explicitly executes a full scan. Thus, the performance of
+the query can be unpredictable.
+
+For example, consider the following table of user profiles with birth year and country of residence.
+The birth year has a secondary index defined.
+
+[source,cql]
+----
+include::example$CQL/allow_filtering.cql[]
+----
+
+The following queries are valid:
+
+[source,cql]
+----
+include::example$CQL/query_allow_filtering.cql[]
+----
+
+In both cases, the query performance is proportional to the amount of data returned.
+The first query returns all rows, because all users are selected.
+The second query returns only the rows defined by the secondary index, a per-node implementation; the results will
+depend on the number of nodes in the cluster, and is indirectly proportional to the amount of data stored.
+The number of nodes will always be multiple number of magnitude lower than the number of user profiles stored.
+Both queries may return very large result sets, but the addition of a `LIMIT` clause can reduced the latency.
+
+The following query will be rejected:
+
+[source,cql]
+----
+include::example$CQL/query_fail_allow_filtering.cql[]
+----
+
+Cassandra cannot guarantee that large amounts of data won't have to scanned amount of data, even if the result is small.
+If you know that the dataset is small, and the performance will be reasonable, add `ALLOW FILTERING` to allow the query to
+execute:
+
+[source,cql]
+----
+include::example$CQL/query_nofail_allow_filtering.cql[]
+----
+
+[[insert-statement]]
+== INSERT
+
+Inserting data for a row is done using an `INSERT` statement:
+
+[source,bnf]
+----
+include::example$BNF/insert_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/insert_statement.cql[]
+----
+
+The `INSERT` statement writes one or more columns for a given row in a
+table.
+Since a row is identified by its `PRIMARY KEY`, at least one columns must be specified.
+The list of columns to insert must be supplied with the `VALUES` syntax.
+When using the `JSON` syntax, `VALUES` are optional.
+See the section on xref:cql/dml.adoc#cql-json[JSON support] for more detail.
+All updates for an `INSERT` are applied atomically and in isolation.
+
+Unlike in SQL, `INSERT` does not check the prior existence of the row by default.
+The row is created if none existed before, and updated otherwise.
+Furthermore, there is no means of knowing which action occurred.
+
+The `IF NOT EXISTS` condition can restrict the insertion if the row does not exist.
+However, note that using `IF NOT EXISTS` will incur a non-negligible performance cost, because Paxos is used,
+so this should be used sparingly.
+
+Please refer to the xref:cql/dml.adoc#update-parameters[UPDATE] section for informations on the `update_parameter`.
+Also note that `INSERT` does not support counters, while `UPDATE` does.
+
+[[update-statement]]
+== UPDATE
+
+Updating a row is done using an `UPDATE` statement:
+
+[source, bnf]
+----
+include::example$BNF/update_statement.bnf[]
+----
+
+For instance:
+
+[source,cql]
+----
+include::example$CQL/update_statement.cql[]
+----
+
+The `UPDATE` statement writes one or more columns for a given row in a
+table.
+The `WHERE` clause is used to select the row to update and must include all columns of the `PRIMARY KEY`.
+Non-primary key columns are set using the `SET` keyword.
+In an `UPDATE` statement, all updates within the same partition key are applied atomically and in isolation.
+
+Unlike in SQL, `UPDATE` does not check the prior existence of the row by default.
+The row is created if none existed before, and updated otherwise.
+Furthermore, there is no means of knowing which action occurred.
+
+The `IF` condition can be used to choose whether the row is updated or not if a particular condition is met.
+However, like the `IF NOT EXISTS` condition, a non-negligible performance cost can be incurred.
+
+Regarding the `SET` assignment:
+
+* `c = c + 3` will increment/decrement counters, the only operation allowed.
+The column name after the '=' sign *must* be the same than the one before the '=' sign.
+Increment/decrement is only allowed on counters.
+See the section on xref:cql/dml.adoc#counters[counters] for details.
+* `id = id + <some-collection>` and `id[value1] = value2` are for collections.
+See the xref:cql/types.adoc#collections[collections] for details.
+* `id.field = 3` is for setting the value of a field on a non-frozen user-defined types.
+See the xref:cql/types.adoc#udts[UDTs] for details.
+
+=== Update parameters
+
+`UPDATE` and `INSERT` statements support the following parameters:
+
+* `TTL`: specifies an optional Time To Live (in seconds) for the
+inserted values. If set, the inserted values are automatically removed
+from the database after the specified time. Note that the TTL concerns
+the inserted values, not the columns themselves. This means that any
+subsequent update of the column will also reset the TTL (to whatever TTL
+is specified in that update). By default, values never expire. A TTL of
+0 is equivalent to no TTL. If the table has a default_time_to_live, a
+TTL of 0 will remove the TTL for the inserted or updated values. A TTL
+of `null` is equivalent to inserting with a TTL of 0.
+
+`UPDATE`, `INSERT`, `DELETE` and `BATCH` statements support the following parameters:
+
+* `TIMESTAMP`: sets the timestamp for the operation. If not specified,
+the coordinator will use the current time (in microseconds) at the start
+of statement execution as the timestamp. This is usually a suitable
+default.
+
+[[delete_statement]]
+== DELETE
+
+Deleting rows or parts of rows uses the `DELETE` statement:
+
+[source,bnf]
+----
+include::example$BNF/delete_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/delete_statement.cql[]
+----
+
+The `DELETE` statement deletes columns and rows. If column names are
+provided directly after the `DELETE` keyword, only those columns are
+deleted from the row indicated by the `WHERE` clause. Otherwise, whole
+rows are removed.
+
+The `WHERE` clause specifies which rows are to be deleted. Multiple rows
+may be deleted with one statement by using an `IN` operator. A range of
+rows may be deleted using an inequality operator (such as `>=`).
+
+`DELETE` supports the `TIMESTAMP` option with the same semantics as in
+xref:cql/dml.adoc#update-parameters[updates].
+
+In a `DELETE` statement, all deletions within the same partition key are
+applied atomically and in isolation.
+
+A `DELETE` operation can be conditional through the use of an `IF`
+clause, similar to `UPDATE` and `INSERT` statements. However, as with
+`INSERT` and `UPDATE` statements, this will incur a non-negligible
+performance cost because Paxos is used, and should be used sparingly.
+
+[[batch_statement]]
+== BATCH
+
+Multiple `INSERT`, `UPDATE` and `DELETE` can be executed in a single
+statement by grouping them through a `BATCH` statement:
+
+[source, bnf]
+----
+include::example$BNF/batch_statement.bnf[]
+----
+
+For instance:
+
+[source,cql]
+----
+include::example$CQL/batch_statement.cql[]
+----
+
+The `BATCH` statement group multiple modification statements
+(insertions/updates and deletions) into a single statement. It serves
+several purposes:
+
+* It saves network round-trips between the client and the server (and
+sometimes between the server coordinator and the replicas) when batching
+multiple updates.
+* All updates in a `BATCH` belonging to a given partition key are
+performed in isolation.
+* By default, all operations in the batch are performed as _logged_, to
+ensure all mutations eventually complete (or none will). See the notes
+on xref:cql/dml.adoc#unlogged-batches[UNLOGGED batches] for more details.
+
+Note that:
+
+* `BATCH` statements may only contain `UPDATE`, `INSERT` and `DELETE`
+statements (not other batches for instance).
+* Batches are _not_ a full analogue for SQL transactions.
+* If a timestamp is not specified for each operation, then all
+operations will be applied with the same timestamp (either one generated
+automatically, or the timestamp provided at the batch level). Due to
+Cassandra's conflict resolution procedure in the case of
+http://wiki.apache.org/cassandra/FAQ#clocktie[timestamp ties],
+operations may be applied in an order that is different from the order
+they are listed in the `BATCH` statement. To force a particular
+operation ordering, you must specify per-operation timestamps.
+* A LOGGED batch to a single partition will be converted to an UNLOGGED
+batch as an optimization.
+
+[[unlogged-batches]]
+=== `UNLOGGED` batches
+
+By default, Cassandra uses a batch log to ensure all operations in a
+batch eventually complete or none will (note however that operations are
+only isolated within a single partition).
+
+There is a performance penalty for batch atomicity when a batch spans
+multiple partitions. If you do not want to incur this penalty, you can
+tell Cassandra to skip the batchlog with the `UNLOGGED` option. If the
+`UNLOGGED` option is used, a failed batch might leave the patch only
+partly applied.
+
+=== `COUNTER` batches
+
+Use the `COUNTER` option for batched counter updates. Unlike other
+updates in Cassandra, counter updates are not idempotent.
diff --git a/doc/modules/cassandra/pages/developing/cql/functions.adoc b/doc/modules/cassandra/pages/developing/cql/functions.adoc
new file mode 100644
index 0000000..6474b72
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/functions.adoc
@@ -0,0 +1,579 @@
+// Need some intro for UDF and native functions in general and point those to it.
+// [[cql-functions]][[native-functions]]
+
+== Functions
+
+CQL supports 2 main categories of functions:
+
+* xref:cql/functions.adoc#scalar-functions[scalar functions] that take a number of values and produce an output
+* xref:cql/functions.adoc#aggregate-functions[aggregate functions] that aggregate multiple rows resulting from a `SELECT` statement
+
+In both cases, CQL provides a number of native "hard-coded" functions as
+well as the ability to create new user-defined functions.
+
+[NOTE]
+.Note
+====
+By default, the use of user-defined functions is disabled by default for
+security concerns (even when enabled, the execution of user-defined
+functions is sandboxed and a "rogue" function should not be allowed to
+do evil, but no sandbox is perfect so using user-defined functions is
+opt-in). See the `user_defined_functions_enabled` in `cassandra.yaml` to
+enable them.
+====
+
+A function is identifier by its name:
+
+[source, bnf]
+----
+include::example$BNF/function.bnf[]
+----
+
+=== Scalar functions
+
+[[scalar-native-functions]]
+==== Native functions
+
+===== Cast
+
+The `cast` function can be used to converts one native datatype to
+another.
+
+The following table describes the conversions supported by the `cast`
+function. Cassandra will silently ignore any cast converting a datatype
+into its own datatype.
+
+[cols=",",options="header",]
+|===
+|From |To
+
+| `ascii` | `text`, `varchar`
+
+| `bigint` | `tinyint`, `smallint`, `int`, `float`, `double`, `decimal`, `varint`,
+`text`, `varchar`
+
+| `boolean` | `text`, `varchar`
+
+| `counter` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `decimal`,
+`varint`, `text`, `varchar`
+
+| `date` | `timestamp`
+
+| `decimal` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `varint`,
+`text`, `varchar`
+
+| `double` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `decimal`, `varint`,
+`text`, `varchar`
+
+| `float` | `tinyint`, `smallint`, `int`, `bigint`, `double`, `decimal`, `varint`,
+`text`, `varchar`
+
+| `inet` | `text`, `varchar`
+
+| `int` | `tinyint`, `smallint`, `bigint`, `float`, `double`, `decimal`, `varint`,
+`text`, `varchar`
+
+| `smallint` | `tinyint`, `int`, `bigint`, `float`, `double`, `decimal`, `varint`,
+`text`, `varchar`
+
+| `time` | `text`, `varchar`
+
+| `timestamp` | `date`, `text`, `varchar`
+
+| `timeuuid` | `timestamp`, `date`, `text`, `varchar`
+
+| `tinyint` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `decimal`,
+`varint`, `text`, `varchar`
+
+| `uuid` | `text`, `varchar`
+
+| `varint` | `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `decimal`,
+`text`, `varchar`
+|===
+
+The conversions rely strictly on Java's semantics. For example, the
+double value 1 will be converted to the text value '1.0'. For instance:
+
+[source,cql]
+----
+SELECT avg(cast(count as double)) FROM myTable
+----
+
+===== Token
+
+The `token` function computes the token for a given partition key.
+The exact signature of the token function depends on the table concerned and the partitioner used by the cluster.
+
+The type of the arguments of the `token` depend on the partition key column type. The returned type depends on the defined partitioner:
+
+[cols=",",options="header",]
+|===
+|Partitioner | Returned type
+| Murmur3Partitioner | `bigint`
+| RandomPartitioner | `varint`
+| ByteOrderedPartitioner | `blob`
+|===
+
+For example, consider the following table:
+
+[source,cql]
+----
+include::example$CQL/create_table_simple.cql[]
+----
+
+The table uses the default Murmur3Partitioner.
+The `token` function uses the single argument `text`, because the partition key is `userid` of text type.
+The returned type will be `bigint`.
+
+===== Uuid
+
+The `uuid` function takes no parameters and generates a random type 4
+uuid suitable for use in `INSERT` or `UPDATE` statements.
+
+===== Timeuuid functions
+
+====== `now`
+
+The `now` function takes no arguments and generates, on the coordinator
+node, a new unique timeuuid at the time the function is invoked. Note
+that this method is useful for insertion but is largely non-sensical in
+`WHERE` clauses.
+
+For example, a query of the form:
+
+[source,cql]
+----
+include::example$CQL/timeuuid_now.cql[]
+----
+
+will not return a result, by design, since the value returned by
+`now()` is guaranteed to be unique.
+
+`current_timeuuid` is an alias of `now`.
+
+====== `min_timeuuid` and `max_timeuuid`
+
+The `min_timeuuid` function takes a `timestamp` value `t`, either a timestamp or a date string.
+It returns a _fake_ `timeuuid` corresponding to the _smallest_ possible `timeuuid` for timestamp `t`.
+The `max_timeuuid` works similarly, but returns the _largest_ possible `timeuuid`.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/timeuuid_min_max.cql[]
+----
+
+will select all rows where the `timeuuid` column `t` is later than `'2013-01-01 00:05+0000'` and earlier than `'2013-02-02 10:00+0000'`.
+The clause `t >= maxTimeuuid('2013-01-01 00:05+0000')` would still _not_ select a `timeuuid` generated exactly at '2013-01-01 00:05+0000', and is essentially equivalent to `t > maxTimeuuid('2013-01-01 00:05+0000')`.
+
+[NOTE]
+.Note
+====
+The values generated by `min_timeuuid` and `max_timeuuid` are called _fake_ UUID because they do no respect the time-based UUID generation process
+specified by the http://www.ietf.org/rfc/rfc4122.txt[IETF RFC 4122].
+In particular, the value returned by these two methods will not be unique.
+Thus, only use these methods for *querying*, not for *insertion*, to prevent possible data overwriting.
+====
+
+===== Datetime functions
+
+====== Retrieving the current date/time
+
+The following functions can be used to retrieve the date/time at the
+time where the function is invoked:
+
+[cols=",",options="header",]
+|===
+|Function name |Output type
+
+| `current_timestamp` | `timestamp`
+
+| `current_date` | `date`
+
+| `current_time` | `time`
+
+| `current_timeuuid` | `timeUUID`
+|===
+
+For example the last two days of data can be retrieved using:
+
+[source,cql]
+----
+include::example$CQL/current_date.cql[]
+----
+
+====== Time conversion functions
+
+A number of functions are provided to convert a `timeuuid`, a `timestamp` or a `date` into another `native` type.
+
+[cols=",,",options="header",]
+|===
+|Function name |Input type |Description
+
+| `to_date` | `timeuuid` | Converts the `timeuuid` argument into a `date` type
+
+| `to_date` | `timestamp` | Converts the `timestamp` argument into a `date` type
+
+| `to_timestamp` | `timeuuid` | Converts the `timeuuid` argument into a `timestamp` type
+
+| `to_timestamp` | `date` | Converts the `date` argument into a `timestamp` type
+
+| `to_unix_timestamp` | `timeuuid` | Converts the `timeuuid` argument into a `bigInt` raw value
+
+| `to_unix_timestamp` | `timestamp` | Converts the `timestamp` argument into a `bigInt` raw value
+
+| `to_unix_timestamp` | `date` | Converts the `date` argument into a `bigInt` raw value
+|===
+
+===== Blob conversion functions
+
+A number of functions are provided to convert the native types into
+binary data, or a `blob`.
+For every xref:cql/types.adoc#native-types[type] supported by CQL, the function `type_as_blob` takes a argument of type `type` and returns it as a `blob`.
+Conversely, the function `blob_as_type` takes a 64-bit `blob` argument and converts it to a `bigint` value.
+For example, `bigint_as_blob(3)` returns `0x0000000000000003` and `blob_as_bigint(0x0000000000000003)` returns `3`.
+
+===== Math Functions
+
+Cql provides the following math functions: `abs`, `exp`, `log`, `log10`, and `round`.
+The return type for these functions is always the same as the input type.
+
+[cols=",",options="header",]
+|===
+|Function name |Description
+
+|`abs` | Returns the absolute value of the input.
+
+|`exp` | Returns the number e to the power of the input.
+
+|`log` | Returns the natural log of the input.
+
+|`log10` | Returns the log base 10 of the input.
+
+|`round` | Rounds the input to the nearest whole number using rounding mode `HALF_UP`.
+|===
+
+===== Collection functions
+
+A number of functions are provided to operate on collection columns.
+
+[cols=",,",options="header",]
+|===
+|Function name |Input type |Description
+
+| `map_keys` | `map` | Gets the keys of the `map` argument, returned as a `set`.
+
+| `map_values` | `map` | Gets the values of the `map` argument, returned as a `list`.
+
+| `collection_count` | `map`, `set` or `list` | Gets the number of elements in the collection argument.
+
+| `collection_min` | `set` or `list` | Gets the minimum element in the collection argument.
+
+| `collection_max` | `set` or `list` | Gets the maximum element in the collection argument.
+
+| `collection_sum` | numeric `set` or `list` | Computes the sum of the elements in the collection argument. The returned value is of the same type as the input collection elements, so there is a risk of overflowing the data type if the sum of the values exceeds the maximum value that the type can represent.
+
+| `collection_avg` | numeric `set` or `list` | Computes the average of the elements in the collection argument. The average of an empty collection returns zero. The returned value is of the same type as the input collection elements, which might include rounding and truncations. For example `collection_avg([1, 2])` returns `1` instead of `1.5`.
+|===
+
+[[data-masking-functions]]
+===== Data masking functions
+
+A number of functions allow to obscure the real contents of a column containing sensitive data.
+
+include::partial$masking_functions.adoc[]
+
+[[user-defined-scalar-functions]]
+==== User-defined functions
+
+User-defined functions (UDFs) execute user-provided code in Cassandra.
+By default, Cassandra supports defining functions in _Java_.
+
+UDFs are part of the Cassandra schema, and are automatically propagated to all nodes in the cluster.
+UDFs can be _overloaded_, so that multiple UDFs with different argument types can have the same function name.
+
+
+[NOTE]
+.Note
+====
+_JavaScript_ user-defined functions have been deprecated in Cassandra 4.1. In preparation for Cassandra 5.0, their removal is
+already in progress. For more information - CASSANDRA-17281, CASSANDRA-18252.
+====
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/function_overload.cql[]
+----
+
+UDFs are susceptible to all of the normal problems with the chosen programming language.
+Accordingly, implementations should be safe against null pointer exceptions, illegal arguments, or any other potential source of exceptions.
+An exception during function execution will result in the entire statement failing.
+Valid queries for UDF use are `SELECT`, `INSERT` and `UPDATE` statements.
+
+_Complex_ types like collections, tuple types and user-defined types are valid argument and return types in UDFs.
+Tuple types and user-defined types use the DataStax Java Driver conversion functions.
+Please see the Java Driver documentation for details on handling tuple types and user-defined types.
+
+Arguments for functions can be literals or terms.
+Prepared statement placeholders can be used, too.
+
+Note the use the double dollar-sign syntax to enclose the UDF source code.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/function_dollarsign.cql[]
+----
+
+The implicitly available `udfContext` field (or binding for script UDFs) provides the necessary functionality to create new UDT and tuple values:
+
+[source,cql]
+----
+include::example$CQL/function_udfcontext.cql[]
+----
+
+The definition of the `UDFContext` interface can be found in the Apache Cassandra source code for `org.apache.cassandra.cql3.functions.UDFContext`.
+
+[source,java]
+----
+include::example$JAVA/udfcontext.java[]
+----
+
+Java UDFs already have some imports for common interfaces and classes defined. These imports are:
+
+[source,java]
+----
+include::example$JAVA/udf_imports.java[]
+----
+
+Please note, that these convenience imports are not available for script UDFs.
+
+[[create-function-statement]]
+==== CREATE FUNCTION statement
+
+Creating a new user-defined function uses the `CREATE FUNCTION` statement:
+
+[source,bnf]
+----
+include::example$BNF/create_function_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/create_function.cql[]
+----
+
+`CREATE FUNCTION` with the optional `OR REPLACE` keywords creates either a function or replaces an existing one with the same signature.
+A `CREATE FUNCTION` without `OR REPLACE` fails if a function with the same signature already exists.
+If the optional `IF NOT EXISTS` keywords are used, the function will only be created only if another function with the same signature does not
+exist.
+`OR REPLACE` and `IF NOT EXISTS` cannot be used together.
+
+Behavior for `null` input values must be defined for each function:
+
+* `RETURNS NULL ON NULL INPUT` declares that the function will always return `null` if any of the input arguments is `null`.
+* `CALLED ON NULL INPUT` declares that the function will always be executed.
+
+===== Function Signature
+
+Signatures are used to distinguish individual functions. The signature consists of a fully-qualified function name of the <keyspace>.<function_name> and a concatenated list of all the argument types.
+
+Note that keyspace names, function names and argument types are subject to the default naming conventions and case-sensitivity rules.
+
+Functions belong to a keyspace; if no keyspace is specified, the current keyspace is used.
+User-defined functions are not allowed in the system keyspaces.
+
+[[drop-function-statement]]
+==== DROP FUNCTION statement
+
+Dropping a function uses the `DROP FUNCTION` statement:
+
+[source, bnf]
+----
+include::example$BNF/drop_function_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/drop_function.cql[]
+----
+
+You must specify the argument types of the function, the arguments_signature, in the drop command if there are multiple overloaded functions with the same name but different signatures.
+`DROP FUNCTION` with the optional `IF EXISTS` keywords drops a function if it exists, but does not throw an error if it doesn't.
+
+[[aggregate-functions]]
+=== Aggregate functions
+
+Aggregate functions work on a set of rows.
+Values for each row are input, to return a single value for the set of rows aggregated.
+
+If `normal` columns, `scalar functions`, `UDT` fields, `writetime`, or `ttl` are selected together with aggregate functions, the values
+returned for them will be the ones of the first row matching the query.
+
+==== Native aggregates
+
+[[count-function]]
+===== Count
+
+The `count` function can be used to count the rows returned by a query.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/count.cql[]
+----
+
+It also can count the non-null values of a given column:
+
+[source,cql]
+----
+include::example$CQL/count_nonnull.cql[]
+----
+
+===== Max and Min
+
+The `max` and `min` functions compute the maximum and the minimum value returned by a query for a given column.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/min_max.cql[]
+----
+
+===== Sum
+
+The `sum` function sums up all the values returned by a query for a given column.
+
+The returned value is of the same type as the input collection elements, so there is a risk of overflowing if the sum of the values exceeds the maximum value that the type can represent.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/sum.cql[]
+----
+
+The returned value is of the same type as the input values, so there is a risk of overflowing the type if the sum of the
+values exceeds the maximum value that the type can represent. You can use type casting to cast the input values as a
+type large enough to contain the type. For example:
+
+[source,cql]
+----
+include::example$CQL/sum_with_cast.cql[]
+----
+
+===== Avg
+
+The `avg` function computes the average of all the values returned by a query for a given column.
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/avg.cql[]
+----
+
+The average of an empty collection returns zero.
+
+The returned value is of the same type as the input values, which might include rounding and truncations.
+For example `collection_avg([1, 2])` returns `1` instead of `1.5`.
+You can use type casting to cast to a type with the desired decimal precision. For example:
+
+[source,cql]
+----
+include::example$CQL/avg_with_cast.cql[]
+----
+
+[[user-defined-aggregates-functions]]
+==== User-Defined Aggregates (UDAs)
+
+User-defined aggregates allow the creation of custom aggregate functions.
+User-defined aggregates can be used in `SELECT` statement.
+
+Each aggregate requires an _initial state_ of type `STYPE` defined with the `INITCOND`value (default value: `null`).
+The first argument of the state function must have type `STYPE`.
+The remaining arguments of the state function must match the types of the user-defined aggregate arguments.
+The state function is called once for each row, and the value returned by the state function becomes the new state.
+After all rows are processed, the optional `FINALFUNC` is executed with last state value as its argument.
+
+The `STYPE` value is mandatory in order to distinguish possibly overloaded versions of the state and/or final function, since the
+overload can appear after creation of the aggregate.
+
+
+A complete working example for user-defined aggregates (assuming that a
+keyspace has been selected using the `USE` statement):
+
+[source,cql]
+----
+include::example$CQL/uda.cql[]
+----
+
+[[create-aggregate-statement]]
+==== CREATE AGGREGATE statement
+
+Creating (or replacing) a user-defined aggregate function uses the
+`CREATE AGGREGATE` statement:
+
+[source, bnf]
+----
+include::example$BNF/create_aggregate_statement.bnf[]
+----
+
+See above for a complete example.
+
+The `CREATE AGGREGATE` command with the optional `OR REPLACE` keywords creates either an aggregate or replaces an existing one with the same
+signature.
+A `CREATE AGGREGATE` without `OR REPLACE` fails if an aggregate with the same signature already exists.
+The `CREATE AGGREGATE` command with the optional `IF NOT EXISTS` keywords creates an aggregate if it does not already exist.
+The `OR REPLACE` and `IF NOT EXISTS` phrases cannot be used together.
+
+The `STYPE` value defines the type of the state value and must be specified.
+The optional `INITCOND` defines the initial state value for the aggregate; the default value is `null`.
+A non-null `INITCOND` must be specified for state functions that are declared with `RETURNS NULL ON NULL INPUT`.
+
+The `SFUNC` value references an existing function to use as the state-modifying function.
+The first argument of the state function must have type `STYPE`.
+The remaining arguments of the state function must match the types of the user-defined aggregate arguments.
+The state function is called once for each row, and the value returned by the state function becomes the new state.
+State is not updated for state functions declared with `RETURNS NULL ON NULL INPUT` and called with `null`.
+After all rows are processed, the optional `FINALFUNC` is executed with last state value as its argument.
+It must take only one argument with type `STYPE`, but the return type of the `FINALFUNC` may be a different type.
+A final function declared with `RETURNS NULL ON NULL INPUT` means that the aggregate's return value will be `null`, if the last state is `null`.
+
+If no `FINALFUNC` is defined, the overall return type of the aggregate function is `STYPE`.
+If a `FINALFUNC` is defined, it is the return type of that function.
+
+[[drop-aggregate-statement]]
+==== DROP AGGREGATE statement
+
+Dropping an user-defined aggregate function uses the `DROP AGGREGATE`
+statement:
+
+[source, bnf]
+----
+include::example$BNF/drop_aggregate_statement.bnf[]
+----
+
+For instance:
+
+[source,cql]
+----
+include::example$CQL/drop_aggregate.cql[]
+----
+
+The `DROP AGGREGATE` statement removes an aggregate created using `CREATE AGGREGATE`.
+You must specify the argument types of the aggregate to drop if there are multiple overloaded aggregates with the same name but a
+different signature.
+
+The `DROP AGGREGATE` command with the optional `IF EXISTS` keywords drops an aggregate if it exists, and does nothing if a function with the
+signature does not exist.
diff --git a/doc/modules/cassandra/pages/developing/cql/index.adoc b/doc/modules/cassandra/pages/developing/cql/index.adoc
new file mode 100644
index 0000000..84e84a9
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/index.adoc
@@ -0,0 +1,25 @@
+= The Cassandra Query Language (CQL)
+
+This document describes the Cassandra Query Language
+(CQL) version 3.
+Note that this document describes the last version of the language.
+However, the xref:cql/changes.adoc[changes] section provides the differences between the versions of CQL since version 3.0.
+
+CQL offers a model similar to SQL.
+The data is stored in *tables* containing *rows* of *columns*.
+For that reason, when used in this document, these terms (tables, rows and columns) have the same definition that they have in SQL.
+
+* xref:cql/definitions.adoc[Definitions]
+* xref:cql/types.adoc[Data types]
+* xref:cql/ddl.adoc[Data definition language]
+* xref:cql/dml.adoc[Data manipulation language]
+* xref:cql/operators.adoc[Operators]
+* xref:cql/indexes.adoc[Secondary indexes]
+* xref:cql/mvs.adoc[Materialized views]
+* xref:cql/functions.adoc[Functions]
+* xref:cql/json.adoc[JSON]
+* xref:cql/security.adoc[CQL security]
+* xref:cql/dynamic_data_masking.adoc[Dynamic data masking]
+* xref:cql/triggers.adoc[Triggers]
+* xref:cql/appendices.adoc[Appendices]
+* xref:cql/changes.adoc[Changes]
diff --git a/doc/modules/cassandra/pages/cql/indexes.adoc b/doc/modules/cassandra/pages/developing/cql/indexes.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/cql/indexes.adoc
rename to doc/modules/cassandra/pages/developing/cql/indexes.adoc
diff --git a/doc/modules/cassandra/pages/developing/cql/json.adoc b/doc/modules/cassandra/pages/developing/cql/json.adoc
new file mode 100644
index 0000000..d3462e8
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/json.adoc
@@ -0,0 +1,125 @@
+= JSON Support
+
+Cassandra 2.2 introduces JSON support to `SELECT <select-statement>` and
+`INSERT <insert-statement>` statements.
+This support does not fundamentally alter the CQL API (for example, the schema is still
+enforced).
+It simply provides a convenient way to work with JSON documents.
+
+== SELECT JSON
+
+With `SELECT` statements, the `JSON` keyword is used to return each row as a single `JSON` encoded map.
+The remainder of the `SELECT` statement behavior is the same.
+
+The result map keys match the column names in a normal result set.
+For example, a statement like `SELECT JSON a, ttl(b) FROM ...` would result in a map with keys `"a"` and `"ttl(b)"`.
+However, there is one notable exception: for symmetry with `INSERT JSON` behavior, case-sensitive column names with upper-case letters will be surrounded with double quotes.
+For example, `SELECT JSON myColumn FROM ...` would result in a map key `"\"myColumn\""` with escaped quotes).
+
+The map values will JSON-encoded representations (as described below) of the result set values.
+
+== INSERT JSON
+
+With `INSERT` statements, the new `JSON` keyword can be used to enable
+inserting a `JSON` encoded map as a single row. The format of the `JSON`
+map should generally match that returned by a `SELECT JSON` statement on
+the same table. In particular, case-sensitive column names should be
+surrounded with double quotes. For example, to insert into a table with
+two columns named "myKey" and "value", you would do the following:
+
+[source,cql]
+----
+include::example$CQL/insert_json.cql[]
+----
+
+By default (or if `DEFAULT NULL` is explicitly used), a column omitted
+from the `JSON` map will be set to `NULL`, meaning that any pre-existing
+value for that column will be removed (resulting in a tombstone being
+created). Alternatively, if the `DEFAULT UNSET` directive is used after
+the value, omitted column values will be left unset, meaning that
+pre-existing values for those column will be preserved.
+
+== JSON Encoding of Cassandra Data Types
+
+Where possible, Cassandra will represent and accept data types in their
+native `JSON` representation. Cassandra will also accept string
+representations matching the CQL literal format for all single-field
+types. For example, floats, ints, UUIDs, and dates can be represented by
+CQL literal strings. However, compound types, such as collections,
+tuples, and user-defined types must be represented by native `JSON`
+collections (maps and lists) or a JSON-encoded string representation of
+the collection.
+
+The following table describes the encodings that Cassandra will accept
+in `INSERT JSON` values (and `from_json()` arguments) as well as the
+format Cassandra will use when returning data for `SELECT JSON`
+statements (and `from_json()`):
+
+[cols=",,,",options="header",]
+|===
+|Type |Formats accepted |Return format |Notes
+
+| `ascii` | string | string | Uses JSON's `\u` character escape
+
+| `bigint` | integer, string | integer | String must be valid 64 bit integer
+
+| `blob` | string | string | String should be 0x followed by an even number of hex digits
+
+| `boolean` | boolean, string | boolean | String must be "true" or "false"
+
+| `date` | string | string | Date in format `YYYY-MM-DD`, timezone UTC
+
+| `decimal` | integer, float, string | float | May exceed 32 or 64-bit IEEE-754 floating point precision in client-side decoder
+
+| `double` | integer, float, string | float | String must be valid integer or float
+
+| `float` | integer, float, string | float | String must be valid integer or float
+
+| `inet` | string | string | IPv4 or IPv6 address
+
+| `int` | integer, string | integer | String must be valid 32 bit integer
+
+| `list` | list, string | list | Uses JSON's native list representation
+
+| `map` | map, string | map | Uses JSON's native map representation
+
+| `smallint` | integer, string | integer | String must be valid 16 bit integer
+
+| `set` | list, string | list | Uses JSON's native list representation
+
+| `text` | string | string | Uses JSON's `\u` character escape
+
+| `time` | string | string | Time of day in format `HH-MM-SS[.fffffffff]`
+
+| `timestamp` | integer, string | string | A timestamp. Strings constant allows to input `timestamps
+as dates <timestamps>`. Datestamps with format `YYYY-MM-DD HH:MM:SS.SSS`
+are returned.
+
+| `timeuuid` | string | string | Type 1 UUID. See `constant` for the UUID format
+
+| `tinyint` | integer, string | integer | String must be valid 8 bit integer
+
+| `tuple` | list, string | list | Uses JSON's native list representation
+
+| `UDT` | map, string | map | Uses JSON's native map representation with field names as keys
+
+| `uuid` | string | string | See `constant` for the UUID format
+
+| `varchar` | string | string | Uses JSON's `\u` character escape
+
+| `varint` | integer, string | integer | Variable length; may overflow 32 or 64 bit integers in client-side decoder
+|===
+
+== The from_json() Function
+
+The `from_json()` function may be used similarly to `INSERT JSON`, but
+for a single column value. It may only be used in the `VALUES` clause of
+an `INSERT` statement or as one of the column values in an `UPDATE`,
+`DELETE`, or `SELECT` statement. For example, it cannot be used in the
+selection clause of a `SELECT` statement.
+
+== The to_json() Function
+
+The `to_json()` function may be used similarly to `SELECT JSON`, but for
+a single column value. It may only be used in the selection clause of a
+`SELECT` statement.
diff --git a/doc/modules/cassandra/pages/cql/mvs.adoc b/doc/modules/cassandra/pages/developing/cql/mvs.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/cql/mvs.adoc
rename to doc/modules/cassandra/pages/developing/cql/mvs.adoc
diff --git a/doc/modules/cassandra/pages/cql/operators.adoc b/doc/modules/cassandra/pages/developing/cql/operators.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/cql/operators.adoc
rename to doc/modules/cassandra/pages/developing/cql/operators.adoc
diff --git a/doc/modules/cassandra/pages/developing/cql/security.adoc b/doc/modules/cassandra/pages/developing/cql/security.adoc
new file mode 100644
index 0000000..166eb87
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/security.adoc
@@ -0,0 +1,630 @@
+role_name ::= identifier | string= Security
+
+[[cql-roles]]
+== Database Roles
+
+CQL uses database roles to represent users and group of users.
+Syntactically, a role is defined by:
+
+[source, bnf]
+----
+include::example$BNF/role_name.bnf[]
+----
+
+
+[[create-role-statement]]
+=== CREATE ROLE
+
+Creating a role uses the `CREATE ROLE` statement:
+
+[source, bnf]
+----
+include::example$BNF/create_role_statement.bnf[]
+----
+
+For instance:
+
+[source,cql]
+----
+include::example$CQL/create_role.cql[]
+----
+
+By default roles do not possess `LOGIN` privileges or `SUPERUSER`
+status.
+
+xref:cql/security.adoc#cql-permissions[Permissions] on database resources are granted to
+roles; types of resources include keyspaces, tables, functions and roles
+themselves. Roles may be granted to other roles to create hierarchical
+permissions structures; in these hierarchies, permissions and
+`SUPERUSER` status are inherited, but the `LOGIN` privilege is not.
+
+If a role has the `LOGIN` privilege, clients may identify as that role
+when connecting. For the duration of that connection, the client will
+acquire any roles and privileges granted to that role.
+
+Only a client with with the `CREATE` permission on the database roles
+resource may issue `CREATE ROLE` requests (see the
+xref:cql/security.adoc#cql-permissions[relevant section]), unless the client is a
+`SUPERUSER`. Role management in Cassandra is pluggable and custom
+implementations may support only a subset of the listed options.
+
+Role names should be quoted if they contain non-alphanumeric characters.
+
+==== Setting credentials for internal authentication
+
+Use the `WITH PASSWORD` clause to set a password for internal
+authentication, enclosing the password in single quotation marks.
+
+If internal authentication has not been set up or the role does not have
+`LOGIN` privileges, the `WITH PASSWORD` clause is not necessary.
+
+USE `WITH HASHED PASSWORD` to provide the jBcrypt hashed password directly. See the `hash_password` tool.
+
+==== Restricting connections to specific datacenters
+
+If a `network_authorizer` has been configured, you can restrict login
+roles to specific datacenters with the `ACCESS TO DATACENTERS` clause
+followed by a set literal of datacenters the user can access. Not
+specifiying datacenters implicitly grants access to all datacenters. The
+clause `ACCESS TO ALL DATACENTERS` can be used for explicitness, but
+there's no functional difference.
+
+==== Creating a role conditionally
+
+Attempting to create an existing role results in an invalid query
+condition unless the `IF NOT EXISTS` option is used. If the option is
+used and the role exists, the statement is a no-op:
+
+[source,cql]
+----
+include::example$CQL/create_role_ifnotexists.cql[]
+----
+
+[[alter-role-statement]]
+=== ALTER ROLE
+
+Altering a role options uses the `ALTER ROLE` statement:
+
+[source, bnf]
+----
+include::example$BNF/alter_role_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/alter_role.cql[]
+----
+If the role does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+USE `WITH HASHED PASSWORD` to provide the jBcrypt hashed password directly. See the `hash_password` tool.
+
+==== Restricting connections to specific datacenters
+
+If a `network_authorizer` has been configured, you can restrict login
+roles to specific datacenters with the `ACCESS TO DATACENTERS` clause
+followed by a set literal of datacenters the user can access. To remove
+any data center restrictions, use the `ACCESS TO ALL DATACENTERS`
+clause.
+
+Conditions on executing `ALTER ROLE` statements:
+
+* a client must have `SUPERUSER` status to alter the `SUPERUSER` status
+of another role
+* a client cannot alter the `SUPERUSER` status of any role it currently
+holds
+* a client can only modify certain properties of the role with which it
+identified at login (e.g. `PASSWORD`)
+* to modify properties of a role, the client must be granted `ALTER`
+`permission <cql-permissions>` on that role
+
+[[drop-role-statement]]
+=== DROP ROLE
+
+Dropping a role uses the `DROP ROLE` statement:
+
+[source, bnf]
+----
+include::example$BNF/drop_role_statement.bnf[]
+----
+
+`DROP ROLE` requires the client to have `DROP`
+`permission <cql-permissions>` on the role in question. In addition,
+client may not `DROP` the role with which it identified at login.
+Finally, only a client with `SUPERUSER` status may `DROP` another
+`SUPERUSER` role.
+
+Attempting to drop a role which does not exist results in an invalid
+query condition unless the `IF EXISTS` option is used. If the option is
+used and the role does not exist the statement is a no-op.
+
+[NOTE]
+.Note
+====
+DROP ROLE intentionally does not terminate any open user sessions.
+Currently connected sessions will remain connected and will retain the
+ability to perform any database actions which do not require
+xref:cql/security.adoc#authorization[authorization].
+However, if authorization is enabled, xref:cql/security.adoc#cql-permissions[permissions] of the dropped role are also revoked,
+subject to the xref:cql/security.adoc#auth-caching[caching options] configured in xref:cql/configuring.adoc#cassandra.yaml[cassandra-yaml] file.
+Should a dropped role be subsequently recreated and have new xref:security.adoc#grant-permission-statement[permissions] or
+xref:security.adoc#grant-role-statement[roles]` granted to it, any client sessions still
+connected will acquire the newly granted permissions and roles.
+====
+
+[[grant-role-statement]]
+=== GRANT ROLE
+
+Granting a role to another uses the `GRANT ROLE` statement:
+
+[source, bnf]
+----
+include::example$BNF/grant_role_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/grant_role.cql[]
+----
+
+This statement grants the `report_writer` role to `alice`. Any
+permissions granted to `report_writer` are also acquired by `alice`.
+
+Roles are modelled as a directed acyclic graph, so circular grants are
+not permitted. The following examples result in error conditions:
+
+[source,cql]
+----
+include::example$CQL/role_error.cql[]
+----
+
+[[revoke-role-statement]]
+=== REVOKE ROLE
+
+Revoking a role uses the `REVOKE ROLE` statement:
+
+[source, bnf]
+----
+include::example$BNF/revoke_role_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/revoke_role.cql[]
+----
+
+This statement revokes the `report_writer` role from `alice`. Any
+permissions that `alice` has acquired via the `report_writer` role are
+also revoked.
+
+[[list-roles-statement]]
+=== LIST ROLES
+
+All the known roles (in the system or granted to specific role) can be
+listed using the `LIST ROLES` statement:
+
+[source, bnf]
+----
+include::example$BNF/list_roles_statement.bnf[]
+----
+
+For instance:
+
+[source,cql]
+----
+include::example$CQL/list_roles.cql[]
+----
+
+returns all known roles in the system, this requires `DESCRIBE`
+permission on the database roles resource.
+
+This example enumerates all roles granted to `alice`, including those transitively
+acquired:
+
+[source,cql]
+----
+include::example$CQL/list_roles_of.cql[]
+----
+
+This example lists all roles directly granted to `bob` without including any of the
+transitively acquired ones:
+
+[source,cql]
+----
+include::example$CQL/list_roles_nonrecursive.cql[]
+----
+
+== Users
+
+Prior to the introduction of roles in Cassandra 2.2, authentication and
+authorization were based around the concept of a `USER`. For backward
+compatibility, the legacy syntax has been preserved with `USER` centric
+statements becoming synonyms for the `ROLE` based equivalents. In other
+words, creating/updating a user is just a different syntax for
+creating/updating a role.
+
+[[create-user-statement]]
+=== CREATE USER
+
+Creating a user uses the `CREATE USER` statement:
+
+[source, bnf]
+----
+include::example$BNF/create_user_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/create_user.cql[]
+----
+
+The `CREATE USER` command is equivalent to `CREATE ROLE` where the `LOGIN` option is `true`.
+So, the following pairs of statements are equivalent:
+
+[source,cql]
+----
+include::example$CQL/create_user_role.cql[]
+----
+
+[[alter-user-statement]]
+=== ALTER USER
+
+Altering the options of a user uses the `ALTER USER` statement:
+
+[source, bnf]
+----
+include::example$BNF/alter_user_statement.bnf[]
+----
+If the role does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+For example:
+
+[source,cql]
+----
+include::example$CQL/alter_user.cql[]
+----
+
+[[drop-user-statement]]
+=== DROP USER
+
+Dropping a user uses the `DROP USER` statement:
+
+[source, bnf]
+----
+include::example$BNF/drop_user_statement.bnf[]
+----
+
+[[list-users-statement]]
+=== LIST USERS
+
+Existing users can be listed using the `LIST USERS` statement:
+
+[source, bnf]
+----
+include::example$BNF/list_users_statement.bnf[]
+----
+
+Note that this statement is equivalent to xref:security.adoc#list-roles-statement[`LIST ROLES], but only roles with the `LOGIN` privilege are included in the output.
+
+== Data Control
+
+[[cql-permissions]]
+=== Permissions
+
+Permissions on resources are granted to roles; there are several
+different types of resources in Cassandra and each type is modelled
+hierarchically:
+
+* The hierarchy of Data resources, Keyspaces and Tables has the
+structure `ALL KEYSPACES` -> `KEYSPACE` -> `TABLE`.
+* Function resources have the structure `ALL FUNCTIONS` -> `KEYSPACE` ->
+`FUNCTION`
+* Resources representing roles have the structure `ALL ROLES` -> `ROLE`
+* Resources representing JMX ObjectNames, which map to sets of
+MBeans/MXBeans, have the structure `ALL MBEANS` -> `MBEAN`
+
+Permissions can be granted at any level of these hierarchies and they
+flow downwards. So granting a permission on a resource higher up the
+chain automatically grants that same permission on all resources lower
+down. For example, granting `SELECT` on a `KEYSPACE` automatically
+grants it on all `TABLES` in that `KEYSPACE`. Likewise, granting a
+permission on `ALL FUNCTIONS` grants it on every defined function,
+regardless of which keyspace it is scoped in. It is also possible to
+grant permissions on all functions scoped to a particular keyspace.
+
+Modifications to permissions are visible to existing client sessions;
+that is, connections need not be re-established following permissions
+changes.
+
+The full set of available permissions is:
+
+* `CREATE`
+* `ALTER`
+* `DROP`
+* `SELECT`
+* `MODIFY`
+* `AUTHORIZE`
+* `DESCRIBE`
+* `EXECUTE`
+* `UNMASK`
+* `SELECT_MASKED`
+
+Not all permissions are applicable to every type of resource. For
+instance, `EXECUTE` is only relevant in the context of functions or
+mbeans; granting `EXECUTE` on a resource representing a table is
+nonsensical. Attempting to `GRANT` a permission on resource to which it
+cannot be applied results in an error response. The following
+illustrates which permissions can be granted on which types of resource,
+and which statements are enabled by that permission.
+
+[cols=",,",options="header",]
+|===
+|Permission |Resource |Operations
+
+| `CREATE` | `ALL KEYSPACES` | `CREATE KEYSPACE` and `CREATE TABLE` in any keyspace
+
+| `CREATE` | `KEYSPACE` | `CREATE TABLE` in specified keyspace
+
+| `CREATE` | `ALL FUNCTIONS` | `CREATE FUNCTION` in any keyspace and `CREATE AGGREGATE` in any keyspace
+
+| `CREATE` | `ALL FUNCTIONS IN KEYSPACE` | `CREATE FUNCTION` and `CREATE AGGREGATE` in specified keyspace
+
+| `CREATE` | `ALL ROLES` | `CREATE ROLE`
+
+| `ALTER` | `ALL KEYSPACES` | `ALTER KEYSPACE` and `ALTER TABLE` in any keyspace
+
+| `ALTER` | `KEYSPACE` | `ALTER KEYSPACE` and `ALTER TABLE` in specified keyspace
+
+| `ALTER` | `TABLE` | `ALTER TABLE`
+
+| `ALTER` | `ALL FUNCTIONS` | `CREATE FUNCTION` and `CREATE AGGREGATE`: replacing any existing
+
+| `ALTER` | `ALL FUNCTIONS IN KEYSPACE` | `CREATE FUNCTION` and `CREATE AGGREGATE`: replacing existing in specified keyspace
+
+| `ALTER` | `FUNCTION` | `CREATE FUNCTION` and `CREATE AGGREGATE`: replacing existing
+
+| `ALTER` | `ALL ROLES` | `ALTER ROLE` on any role
+
+| `ALTER` | `ROLE` | `ALTER ROLE`
+
+| `DROP` | `ALL KEYSPACES` | `DROP KEYSPACE` and `DROP TABLE` in any keyspace
+
+| `DROP` | `KEYSPACE` | `DROP TABLE` in specified keyspace
+
+| `DROP` | `TABLE` | `DROP TABLE`
+
+| `DROP` | `ALL FUNCTIONS` | `DROP FUNCTION` and `DROP AGGREGATE` in any keyspace
+
+| `DROP` | `ALL FUNCTIONS IN KEYSPACE` | `DROP FUNCTION` and `DROP AGGREGATE` in specified keyspace
+
+| `DROP` | `FUNCTION` | `DROP FUNCTION`
+
+| `DROP` | `ALL ROLES` | `DROP ROLE` on any role
+
+| `DROP` | `ROLE` | `DROP ROLE`
+
+| `SELECT` | `ALL KEYSPACES` | `SELECT` on any table
+
+| `SELECT` | `KEYSPACE` | `SELECT` on any table in specified keyspace
+
+| `SELECT` | `TABLE` | `SELECT` on specified table
+
+| `SELECT` | `ALL MBEANS` | Call getter methods on any mbean
+
+| `SELECT` | `MBEANS` | Call getter methods on any mbean matching a wildcard pattern
+
+| `SELECT` | `MBEAN` | Call getter methods on named mbean
+
+| `MODIFY` | `ALL KEYSPACES` | `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` on any table
+
+| `MODIFY` | `KEYSPACE` | `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` on any table in specified
+keyspace
+
+| `MODIFY` | `TABLE` | `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` on specified table
+
+| `MODIFY` | `ALL MBEANS` | Call setter methods on any mbean
+
+| `MODIFY` | `MBEANS` | Call setter methods on any mbean matching a wildcard pattern
+
+| `MODIFY` | `MBEAN` | Call setter methods on named mbean
+
+| `AUTHORIZE` | `ALL KEYSPACES` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any table
+
+| `AUTHORIZE` | `KEYSPACE` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any table in specified keyspace
+
+| `AUTHORIZE` | `TABLE` | `GRANT PERMISSION` and `REVOKE PERMISSION` on specified table
+
+| `AUTHORIZE` | `ALL FUNCTIONS` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any function
+
+| `AUTHORIZE` | `ALL FUNCTIONS IN KEYSPACE` | `GRANT PERMISSION` and `REVOKE PERMISSION` in specified keyspace
+
+| `AUTHORIZE` | `FUNCTION` | `GRANT PERMISSION` and `REVOKE PERMISSION` on specified function
+
+| `AUTHORIZE` | `ALL MBEANS` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any mbean
+
+| `AUTHORIZE` | `MBEANS` | `GRANT PERMISSION` and `REVOKE PERMISSION` on any mbean matching a wildcard pattern
+
+| `AUTHORIZE` | `MBEAN` | `GRANT PERMISSION` and `REVOKE PERMISSION` on named mbean
+
+| `AUTHORIZE` | `ALL ROLES` | `GRANT ROLE` and `REVOKE ROLE` on any role
+
+| `AUTHORIZE` | `ROLES` | `GRANT ROLE` and `REVOKE ROLE` on specified roles
+
+| `DESCRIBE` | `ALL ROLES` | `LIST ROLES` on all roles or only roles granted to another, specified role
+
+| `DESCRIBE` | `ALL MBEANS` | Retrieve metadata about any mbean from the platform's MBeanServer
+
+
+| `DESCRIBE` | `MBEANS` | Retrieve metadata about any mbean matching a wildcard patter from the
+platform's MBeanServer
+
+| `DESCRIBE` | `MBEAN` | Retrieve metadata about a named mbean from the platform's MBeanServer
+
+| `EXECUTE` | `ALL FUNCTIONS` | `SELECT`, `INSERT` and `UPDATE` using any function, and use of any
+function in `CREATE AGGREGATE`
+
+| `EXECUTE` | `ALL FUNCTIONS IN KEYSPACE` | `SELECT`, `INSERT` and `UPDATE` using any function in specified keyspace
+and use of any function in keyspace in `CREATE AGGREGATE`
+
+| `EXECUTE` | `FUNCTION` | `SELECT`, `INSERT` and `UPDATE` using specified function and use of the function in `CREATE AGGREGATE`
+
+| `EXECUTE` | `ALL MBEANS` | Execute operations on any mbean
+
+| `EXECUTE` | `MBEANS` | Execute operations on any mbean matching a wildcard pattern
+
+| `EXECUTE` | `MBEAN` | Execute operations on named mbean
+
+|`UNMASK` |`ALL KEYSPACES` | See the clear contents of masked columns on any table
+
+|`UNMASK` |`KEYSPACE` | See the clear contents of masked columns on any table in keyspace
+
+|`UNMASK` |`TABLE` | See the clear contents of masked columns on the specified table
+
+|`SELECT_MASKED` | `ALL KEYSPACES` | `SELECT` restricting masked columns on any table
+
+|`SELECT_MASKED` | `KEYSPACE` | `SELECT` restricting masked columns on any table in specified keyspace
+
+|`SELECT_MASKED` | `TABLE` | `SELECT` restricting masked columns on the specified table
+|===
+
+[[grant-permission-statement]]
+=== GRANT PERMISSION
+
+Granting a permission uses the `GRANT PERMISSION` statement:
+
+[source, bnf]
+----
+include::example$BNF/grant_permission_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/grant_perm.cql[]
+----
+
+This example gives any user with the role `data_reader` permission to execute
+`SELECT` statements on any table across all keyspaces:
+
+[source,cql]
+----
+include::example$CQL/grant_modify.cql[]
+----
+
+To give any user with the role `data_writer` permission to perform
+`UPDATE`, `INSERT`, `UPDATE`, `DELETE` and `TRUNCATE` queries on all
+tables in the `keyspace1` keyspace:
+
+[source,cql]
+----
+include::example$CQL/grant_drop.cql[]
+----
+
+To give any user with the `schema_owner` role permissions to `DROP` a specific
+`keyspace1.table1`:
+
+[source,cql]
+----
+include::example$CQL/grant_execute.cql[]
+----
+
+This command grants any user with the `report_writer` role permission to execute
+`SELECT`, `INSERT` and `UPDATE` queries which use the function
+`keyspace1.user_function( int )`:
+
+[source,cql]
+----
+include::example$CQL/grant_describe.cql[]
+----
+
+This grants any user with the `role_admin` role permission to view any
+and all roles in the system with a `LIST ROLES` statement.
+
+==== GRANT ALL
+
+When the `GRANT ALL` form is used, the appropriate set of permissions is
+determined automatically based on the target resource.
+
+==== Automatic Granting
+
+When a resource is created, via a `CREATE KEYSPACE`, `CREATE TABLE`,
+`CREATE FUNCTION`, `CREATE AGGREGATE` or `CREATE ROLE` statement, the
+creator (the role the database user who issues the statement is
+identified as), is automatically granted all applicable permissions on
+the new resource.
+
+[[revoke-permission-statement]]
+=== REVOKE PERMISSION
+
+Revoking a permission from a role uses the `REVOKE PERMISSION`
+statement:
+
+[source, bnf]
+----
+include::example$BNF/revoke_permission_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/revoke_perm.cql[]
+----
+
+Because of their function in normal driver operations, certain tables
+cannot have their `SELECT` permissions revoked. The
+following tables will be available to all authorized users regardless of
+their assigned role:
+
+[source,cql]
+----
+include::example$CQL/no_revoke.cql[]
+----
+
+[[list-permissions-statement]]
+=== LIST PERMISSIONS
+
+Listing granted permissions uses the `LIST PERMISSIONS` statement:
+
+[source, bnf]
+----
+include::example$BNF/list_permissions_statement.bnf[]
+----
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/list_perm.cql[]
+----
+
+Show all permissions granted to `alice`, including those acquired
+transitively from any other roles:
+
+[source,cql]
+----
+include::example$CQL/list_all_perm.cql[]
+----
+
+Show all permissions on `keyspace1.table1` granted to `bob`, including
+those acquired transitively from any other roles. This also includes any
+permissions higher up the resource hierarchy which can be applied to
+`keyspace1.table1`. For example, should `bob` have `ALTER` permission on
+`keyspace1`, that would be included in the results of this query. Adding
+the `NORECURSIVE` switch restricts the results to only those permissions
+which were directly granted to `bob` or one of `bob`'s roles:
+
+[source,cql]
+----
+include::example$CQL/list_select_perm.cql[]
+----
+
+Show any permissions granted to `carlos` or any of `carlos`'s roles,
+limited to `SELECT` permissions on any resource.
diff --git a/doc/modules/cassandra/pages/cql/triggers.adoc b/doc/modules/cassandra/pages/developing/cql/triggers.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/cql/triggers.adoc
rename to doc/modules/cassandra/pages/developing/cql/triggers.adoc
diff --git a/doc/modules/cassandra/pages/developing/cql/types.adoc b/doc/modules/cassandra/pages/developing/cql/types.adoc
new file mode 100644
index 0000000..c17dc38
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/cql/types.adoc
@@ -0,0 +1,538 @@
+= Data Types
+
+CQL is a typed language and supports a rich set of data types, including
+xref:cql/types.adoc#native-types[native types], xref:cql/types.adoc#collections[collection types],
+xref:cql/types.adoc#udts[user-defined types], xref:cql/types.adoc#tuples[tuple types], and xref:cql/types.adoc#custom-types[custom
+types]:
+
+[source, bnf]
+----
+include::example$BNF/cql_type.bnf[]
+----
+
+== Native types
+
+The native types supported by CQL are:
+
+[source, bnf]
+----
+include::example$BNF/native_type.bnf[]
+----
+
+The following table gives additional informations on the native data
+types, and on which kind of xref:cql/definitions.adoc#constants[constants] each type supports:
+
+[cols=",,",options="header",]
+|===
+| Type | Constants supported | Description
+
+| `ascii` | `string` | ASCII character string
+| `bigint` | `integer` | 64-bit signed long
+| `blob` | `blob` | Arbitrary bytes (no validation)
+| `boolean` | `boolean` | Either `true` or `false`
+| `counter` | `integer` | Counter column (64-bit signed value). See `counters` for details.
+| `date` | `integer`, `string` | A date (with no corresponding time value). See `dates` below for details.
+| `decimal` | `integer`, `float` | Variable-precision decimal
+| `double` | `integer` `float` | 64-bit IEEE-754 floating point
+| `duration` | `duration`, | A duration with nanosecond precision. See `durations` below for details.
+| `float` | `integer`, `float` | 32-bit IEEE-754 floating point
+| `inet` | `string` | An IP address, either IPv4 (4 bytes long) or IPv6 (16 bytes long). Note
+that there is no `inet` constant, IP address should be input as strings.
+| `int` | `integer` | 32-bit signed int
+| `smallint` | `integer` | 16-bit signed int
+| `text` | `string` | UTF8 encoded string
+| `time` | `integer`, `string` | A time (with no corresponding date value) with nanosecond precision. See
+`times` below for details.
+| `timestamp` | `integer`, `string` | A timestamp (date and time) with millisecond precision. See `timestamps`
+below for details.
+| `timeuuid` | `uuid` | Version 1 https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID],
+generally used as a “conflict-free” timestamp. Also see `timeuuid-functions`.
+| `tinyint` | `integer` | 8-bit signed int
+| `uuid` | `uuid` | A https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] (of any version)
+| `varchar` | `string` | UTF8 encoded string
+| `varint` | `integer` | Arbitrary-precision integer
+|===
+
+=== Counters
+
+The `counter` type is used to define _counter columns_. A counter column
+is a column whose value is a 64-bit signed integer and on which 2
+operations are supported: incrementing and decrementing (see the
+xref:cql/dml.adoc#update-statement[UPDATE] statement for syntax).
+Note that the value of a counter cannot
+be set: a counter does not exist until first incremented/decremented,
+and that first increment/decrement is made as if the prior value was 0.
+
+[[counter-limitations]]
+Counters have a number of important limitations:
+
+* They cannot be used for columns part of the `PRIMARY KEY` of a table.
+* A table that contains a counter can only contain counters. In other
+words, either all the columns of a table outside the `PRIMARY KEY` have
+the `counter` type, or none of them have it.
+* Counters do not support xref:cql/dml.adoc#writetime-and-ttl-function[expiration].
+* The deletion of counters is supported, but is only guaranteed to work
+the first time you delete a counter. In other words, you should not
+re-update a counter that you have deleted (if you do, proper behavior is
+not guaranteed).
+* Counter updates are, by nature, not
+https://en.wikipedia.org/wiki/Idempotence[idemptotent]. An important
+consequence is that if a counter update fails unexpectedly (timeout or
+loss of connection to the coordinator node), the client has no way to
+know if the update has been applied or not. In particular, replaying the
+update may or may not lead to an over count.
+
+[[timestamps]]
+== Working with timestamps
+
+Values of the `timestamp` type are encoded as 64-bit signed integers
+representing a number of milliseconds since the standard base time known
+as https://en.wikipedia.org/wiki/Unix_time[the epoch]: January 1 1970 at
+00:00:00 GMT.
+
+Timestamps can be input in CQL either using their value as an `integer`,
+or using a `string` that represents an
+https://en.wikipedia.org/wiki/ISO_8601[ISO 8601] date. For instance, all
+of the values below are valid `timestamp` values for Mar 2, 2011, at
+04:05:00 AM, GMT:
+
+* `1299038700000`
+* `'2011-02-03 04:05+0000'`
+* `'2011-02-03 04:05:00+0000'`
+* `'2011-02-03 04:05:00.000+0000'`
+* `'2011-02-03T04:05+0000'`
+* `'2011-02-03T04:05:00+0000'`
+* `'2011-02-03T04:05:00.000+0000'`
+
+The `+0000` above is an RFC 822 4-digit time zone specification; `+0000`
+refers to GMT. US Pacific Standard Time is `-0800`. The time zone may be
+omitted if desired (`'2011-02-03 04:05:00'`), and if so, the date will
+be interpreted as being in the time zone under which the coordinating
+Cassandra node is configured. There are however difficulties inherent in
+relying on the time zone configuration being as expected, so it is
+recommended that the time zone always be specified for timestamps when
+feasible.
+
+The time of day may also be omitted (`'2011-02-03'` or
+`'2011-02-03+0000'`), in which case the time of day will default to
+00:00:00 in the specified or default time zone. However, if only the
+date part is relevant, consider using the xref:cql/types.adoc#dates[date] type.
+
+[[dates]]
+== Date type
+
+Values of the `date` type are encoded as 32-bit unsigned integers
+representing a number of days with “the epoch” at the center of the
+range (2^31). Epoch is January 1st, 1970
+
+For xref:cql/types.adoc#timestamps[timestamps], a date can be input either as an
+`integer` or using a date `string`. In the later case, the format should
+be `yyyy-mm-dd` (so `'2011-02-03'` for instance).
+
+[[times]]
+== Time type
+
+Values of the `time` type are encoded as 64-bit signed integers
+representing the number of nanoseconds since midnight.
+
+For xref:cql/types.adoc#timestamps[timestamps], a time can be input either as an
+`integer` or using a `string` representing the time. In the later case,
+the format should be `hh:mm:ss[.fffffffff]` (where the sub-second
+precision is optional and if provided, can be less than the nanosecond).
+So for instance, the following are valid inputs for a time:
+
+* `'08:12:54'`
+* `'08:12:54.123'`
+* `'08:12:54.123456'`
+* `'08:12:54.123456789'`
+
+[[durations]]
+== Duration type
+
+Values of the `duration` type are encoded as 3 signed integer of
+variable lengths. The first integer represents the number of months, the
+second the number of days and the third the number of nanoseconds. This
+is due to the fact that the number of days in a month can change, and a
+day can have 23 or 25 hours depending on the daylight saving.
+Internally, the number of months and days are decoded as 32 bits
+integers whereas the number of nanoseconds is decoded as a 64 bits
+integer.
+
+A duration can be input as:
+
+* `(quantity unit)+` like `12h30m` where the unit can be:
+** `y`: years (12 months)
+** `mo`: months (1 month)
+** `w`: weeks (7 days)
+** `d`: days (1 day)
+** `h`: hours (3,600,000,000,000 nanoseconds)
+** `m`: minutes (60,000,000,000 nanoseconds)
+** `s`: seconds (1,000,000,000 nanoseconds)
+** `ms`: milliseconds (1,000,000 nanoseconds)
+** `us` or `µs` : microseconds (1000 nanoseconds)
+** `ns`: nanoseconds (1 nanosecond)
+* ISO 8601 format: `P[n]Y[n]M[n]DT[n]H[n]M[n]S or P[n]W`
+* ISO 8601 alternative format: `P[YYYY]-[MM]-[DD]T[hh]:[mm]:[ss]`
+
+For example:
+
+[source,cql]
+----
+include::example$CQL/insert_duration.cql[]
+----
+
+[[duration-limitation]]
+Duration columns cannot be used in a table's `PRIMARY KEY`. This
+limitation is due to the fact that durations cannot be ordered. It is
+effectively not possible to know if `1mo` is greater than `29d` without
+a date context.
+
+A `1d` duration is not equal to a `24h` one as the duration type has
+been created to be able to support daylight saving.
+
+== Collections
+
+CQL supports three kinds of collections: `maps`, `sets` and `lists`. The
+types of those collections is defined by:
+
+[source,bnf]
+----
+include::example$BNF/collection_type.bnf[]
+----
+
+and their values can be inputd using collection literals:
+
+[source,bnf]
+----
+include::example$BNF/collection_literal.bnf[]
+----
+
+Note however that neither `bind_marker` nor `NULL` are supported inside
+collection literals.
+
+=== Noteworthy characteristics
+
+Collections are meant for storing/denormalizing relatively small amount
+of data. They work well for things like “the phone numbers of a given
+user”, “labels applied to an email”, etc. But when items are expected to
+grow unbounded (“all messages sent by a user”, “events registered by a
+sensor”...), then collections are not appropriate and a specific table
+(with clustering columns) should be used. Concretely, (non-frozen)
+collections have the following noteworthy characteristics and
+limitations:
+
+* Individual collections are not indexed internally. Which means that
+even to access a single element of a collection, the whole collection
+has to be read (and reading one is not paged internally).
+* While insertion operations on sets and maps never incur a
+read-before-write internally, some operations on lists do. Further, some
+lists operations are not idempotent by nature (see the section on
+xref:cql/types.adoc#lists[lists] below for details), making their retry in case of
+timeout problematic. It is thus advised to prefer sets over lists when
+possible.
+
+Please note that while some of those limitations may or may not be
+removed/improved upon in the future, it is a anti-pattern to use a
+(single) collection to store large amounts of data.
+
+=== Maps
+
+A `map` is a (sorted) set of key-value pairs, where keys are unique and
+the map is sorted by its keys. You can define and insert a map with:
+
+[source,cql]
+----
+include::example$CQL/map.cql[]
+----
+
+Further, maps support:
+
+* Updating or inserting one or more elements:
++
+[source,cql]
+----
+include::example$CQL/update_map.cql[]
+----
+* Removing one or more element (if an element doesn't exist, removing it
+is a no-op but no error is thrown):
++
+[source,cql]
+----
+include::example$CQL/delete_map.cql[]
+----
++
+Note that for removing multiple elements in a `map`, you remove from it
+a `set` of keys.
+
+Lastly, TTLs are allowed for both `INSERT` and `UPDATE`, but in both
+cases the TTL set only apply to the newly inserted/updated elements. In
+other words:
+
+[source,cql]
+----
+include::example$CQL/update_ttl_map.cql[]
+----
+
+will only apply the TTL to the `{ 'color' : 'green' }` record, the rest
+of the map remaining unaffected.
+
+=== Sets
+
+A `set` is a (sorted) collection of unique values. You can define and
+insert a set with:
+
+[source,cql]
+----
+include::example$CQL/set.cql[]
+----
+
+Further, sets support:
+
+* Adding one or multiple elements (as this is a set, inserting an
+already existing element is a no-op):
++
+[source,cql]
+----
+include::example$CQL/update_set.cql[]
+----
+* Removing one or multiple elements (if an element doesn't exist,
+removing it is a no-op but no error is thrown):
++
+[source,cql]
+----
+include::example$CQL/delete_set.cql[]
+----
+
+Lastly, for xref:cql/types.adoc#sets[sets], TTLs are only applied to newly inserted values.
+
+=== Lists
+
+[NOTE]
+.Note
+====
+As mentioned above and further discussed at the end of this section,
+lists have limitations and specific performance considerations that you
+should take into account before using them. In general, if you can use a
+xref:cql/types.adoc#sets[set] instead of list, always prefer a set.
+====
+
+A `list` is a (sorted) collection of non-unique values where
+elements are ordered by their position in the list. You can define and
+insert a list with:
+
+[source,cql]
+----
+include::example$CQL/list.cql[]
+----
+
+Further, lists support:
+
+* Appending and prepending values to a list:
++
+[source,cql]
+----
+include::example$CQL/update_list.cql[]
+----
+
+[WARNING]
+.Warning
+====
+The append and prepend operations are not idempotent by nature. So in
+particular, if one of these operations times out, then retrying the
+operation is not safe and it may (or may not) lead to
+appending/prepending the value twice.
+====
+
+* Setting the value at a particular position in a list that has a pre-existing element for that position. An error
+will be thrown if the list does not have the position:
++
+[source,cql]
+----
+include::example$CQL/update_particular_list_element.cql[]
+----
+* Removing an element by its position in the list that has a pre-existing element for that position. An error
+will be thrown if the list does not have the position. Further, as the operation removes an
+element from the list, the list size will decrease by one element, shifting
+the position of all the following elements one forward:
++
+[source,cql]
+----
+include::example$CQL/delete_element_list.cql[]
+----
+
+* Deleting _all_ the occurrences of particular values in the list (if a
+particular element doesn't occur at all in the list, it is simply
+ignored and no error is thrown):
++
+[source,cql]
+----
+include::example$CQL/delete_all_elements_list.cql[]
+----
+
+[WARNING]
+.Warning
+====
+Setting and removing an element by position and removing occurences of
+particular values incur an internal _read-before-write_. These operations will
+run slowly and use more resources than usual updates (with the
+exclusion of conditional write that have their own cost).
+====
+
+Lastly, for xref:cql/types.adoc#lists[lists], TTLs only apply to newly inserted values.
+
+[[udts]]
+== User-Defined Types (UDTs)
+
+CQL support the definition of user-defined types (UDTs). Such a
+type can be created, modified and removed using the
+`create_type_statement`, `alter_type_statement` and
+`drop_type_statement` described below. But once created, a UDT is simply
+referred to by its name:
+
+[source, bnf]
+----
+include::example$BNF/udt.bnf[]
+----
+
+=== Creating a UDT
+
+Creating a new user-defined type is done using a `CREATE TYPE` statement
+defined by:
+
+[source, bnf]
+----
+include::example$BNF/create_type.bnf[]
+----
+
+A UDT has a name (used to declared columns of that type) and is a set of
+named and typed fields. Fields name can be any type, including
+collections or other UDT. For instance:
+
+[source,cql]
+----
+include::example$CQL/udt.cql[]
+----
+
+Things to keep in mind about UDTs:
+
+* Attempting to create an already existing type will result in an error
+unless the `IF NOT EXISTS` option is used. If it is used, the statement
+will be a no-op if the type already exists.
+* A type is intrinsically bound to the keyspace in which it is created,
+and can only be used in that keyspace. At creation, if the type name is
+prefixed by a keyspace name, it is created in that keyspace. Otherwise,
+it is created in the current keyspace.
+* As of Cassandra , UDT have to be frozen in most cases, hence the
+`frozen<address>` in the table definition above.
+
+=== UDT literals
+
+Once a user-defined type has been created, value can be input using a
+UDT literal:
+
+[source,bnf]
+----
+include::example$BNF/udt_literal.bnf[]
+----
+
+In other words, a UDT literal is like a xref:cql/types.adoc#maps[map]` literal but its
+keys are the names of the fields of the type. For instance, one could
+insert into the table define in the previous section using:
+
+[source,cql]
+----
+include::example$CQL/insert_udt.cql[]
+----
+
+To be valid, a UDT literal can only include fields defined by the
+type it is a literal of, but it can omit some fields (these will be set to `NULL`).
+
+=== Altering a UDT
+
+An existing user-defined type can be modified using an `ALTER TYPE`
+statement:
+
+[source,bnf]
+----
+include::example$BNF/alter_udt_statement.bnf[]
+----
+If the type does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+You can:
+
+* Add a new field to the type (`ALTER TYPE address ADD country text`).
+That new field will be `NULL` for any values of the type created before
+the addition. If the new field exists, the statement will return an error, unless `IF NOT EXISTS` is used in which case the operation is a no-op.
+* Rename the fields of the type. If the field(s) does not exist, the statement will return an error, unless `IF EXISTS` is used in which case the operation is a no-op.
+
+[source,cql]
+----
+include::example$CQL/rename_udt_field.cql[]
+----
+
+=== Dropping a UDT
+
+You can drop an existing user-defined type using a `DROP TYPE`
+statement:
+
+[source,bnf]
+----
+include::example$BNF/drop_udt_statement.bnf[]
+----
+
+Dropping a type results in the immediate, irreversible removal of that
+type. However, attempting to drop a type that is still in use by another
+type, table or function will result in an error.
+
+If the type dropped does not exist, an error will be returned unless
+`IF EXISTS` is used, in which case the operation is a no-op.
+
+== Tuples
+
+CQL also support tuples and tuple types (where the elements can be of
+different types). Functionally, tuples can be though as anonymous UDT
+with anonymous fields. Tuple types and tuple literals are defined by:
+
+[source,bnf]
+----
+include::example$BNF/tuple.bnf[]
+----
+
+and can be created:
+
+[source,cql]
+----
+include::example$CQL/tuple.cql[]
+----
+
+Unlike other composed types, like collections and UDTs, a tuple is always
+`frozen <frozen>` (without the need of the `frozen` keyword)
+and it is not possible to update only some elements of a tuple (without
+updating the whole tuple). Also, a tuple literal should always have the
+same number of value than declared in the type it is a tuple of (some of
+those values can be null but they need to be explicitly declared as so).
+
+== Custom Types
+
+[NOTE]
+.Note
+====
+Custom types exists mostly for backward compatibility purposes and their
+usage is discouraged. Their usage is complex, not user friendly and the
+other provided types, particularly xref:cql/types.adoc#udts[user-defined types], should
+almost always be enough.
+====
+
+A custom type is defined by:
+
+[source,bnf]
+----
+include::example$BNF/custom_type.bnf[]
+----
+
+A custom type is a `string` that contains the name of Java class that
+extends the server side `AbstractType` class and that can be loaded by
+Cassandra (it should thus be in the `CLASSPATH` of every node running
+Cassandra). That class will define what values are valid for the type
+and how the time sorts when used for a clustering column. For any other
+purpose, a value of a custom type is the same than that of a `blob`, and
+can in particular be input using the `blob` literal syntax.
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_conceptual.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_conceptual.adoc
new file mode 100644
index 0000000..86fdd1d
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_conceptual.adoc
@@ -0,0 +1,44 @@
+= Conceptual Data Modeling
+
+First, let’s create a simple domain model that is easy to understand in
+the relational world, and then see how you might map it from a
+relational to a distributed hashtable model in Cassandra.
+
+Let's use an example that is complex enough to show the various data
+structures and design patterns, but not something that will bog you down
+with details. Also, a domain that’s familiar to everyone will allow you
+to concentrate on how to work with Cassandra, not on what the
+application domain is all about.
+
+For example, let's use a domain that is easily understood and that
+everyone can relate to: making hotel reservations.
+
+The conceptual domain includes hotels, guests that stay in the hotels, a
+collection of rooms for each hotel, the rates and availability of those
+rooms, and a record of reservations booked for guests. Hotels typically
+also maintain a collection of “points of interest,” which are parks,
+museums, shopping galleries, monuments, or other places near the hotel
+that guests might want to visit during their stay. Both hotels and
+points of interest need to maintain geolocation data so that they can be
+found on maps for mashups, and to calculate distances.
+
+The conceptual domain is depicted below using the entity–relationship
+model popularized by Peter Chen. This simple diagram represents the
+entities in the domain with rectangles, and attributes of those entities
+with ovals. Attributes that represent unique identifiers for items are
+underlined. Relationships between entities are represented as diamonds,
+and the connectors between the relationship and each entity show the
+multiplicity of the connection.
+
+image::data-modeling_hotel_erd.png[image]
+
+Obviously, in the real world, there would be many more considerations
+and much more complexity. For example, hotel rates are notoriously
+dynamic, and calculating them involves a wide array of factors. Here
+you’re defining something complex enough to be interesting and touch on
+the important points, but simple enough to maintain the focus on
+learning Cassandra.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc
new file mode 100644
index 0000000..16c69ef
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc
@@ -0,0 +1,195 @@
+= Logical Data Modeling
+
+Now that you have defined your queries, you’re ready to begin designing
+Cassandra tables. First, create a logical model containing a table for
+each query, capturing entities and relationships from the conceptual
+model.
+
+To name each table, you’ll identify the primary entity type for which
+you are querying and use that to start the entity name. If you are
+querying by attributes of other related entities, append those to the
+table name, separated with `_by_`. For example, `hotels_by_poi`.
+
+Next, you identify the primary key for the table, adding partition key
+columns based on the required query attributes, and clustering columns
+in order to guarantee uniqueness and support desired sort ordering.
+
+The design of the primary key is extremely important, as it will
+determine how much data will be stored in each partition and how that
+data is organized on disk, which in turn will affect how quickly
+Cassandra processes reads.
+
+Complete each table by adding any additional attributes identified by
+the query. If any of these additional attributes are the same for every
+instance of the partition key, mark the column as static.
+
+Now that was a pretty quick description of a fairly involved process, so
+it will be worthwhile to work through a detailed example. First, let’s
+introduce a notation that you can use to represent logical models.
+
+Several individuals within the Cassandra community have proposed
+notations for capturing data models in diagrammatic form. This document
+uses a notation popularized by Artem Chebotko which provides a simple,
+informative way to visualize the relationships between queries and
+tables in your designs. This figure shows the Chebotko notation for a
+logical data model.
+
+image::data-modeling_chebotko_logical.png[image]
+
+Each table is shown with its title and a list of columns. Primary key
+columns are identified via symbols such as *K* for partition key columns
+and **C**↑ or **C**↓ to represent clustering columns. Lines are shown
+entering tables or between tables to indicate the queries that each
+table is designed to support.
+
+== Hotel Logical Data Model
+
+The figure below shows a Chebotko logical data model for the queries
+involving hotels, points of interest, rooms, and amenities. One thing
+you'll notice immediately is that the Cassandra design doesn’t include
+dedicated tables for rooms or amenities, as you had in the relational
+design. This is because the workflow didn’t identify any queries
+requiring this direct access.
+
+image::data-modeling_hotel_logical.png[image]
+
+Let’s explore the details of each of these tables.
+
+The first query Q1 is to find hotels near a point of interest, so you’ll
+call this table `hotels_by_poi`. Searching by a named point of interest
+is a clue that the point of interest should be a part of the primary
+key. Let’s reference the point of interest by name, because according to
+the workflow that is how users will start their search.
+
+You’ll note that you certainly could have more than one hotel near a
+given point of interest, so you’ll need another component in the primary
+key in order to make sure you have a unique partition for each hotel. So
+you add the hotel key as a clustering column.
+
+An important consideration in designing your table’s primary key is
+making sure that it defines a unique data element. Otherwise you run the
+risk of accidentally overwriting data.
+
+Now for the second query (Q2), you’ll need a table to get information
+about a specific hotel. One approach would have been to put all of the
+attributes of a hotel in the `hotels_by_poi` table, but you added only
+those attributes that were required by the application workflow.
+
+From the workflow diagram, you know that the `hotels_by_poi` table is
+used to display a list of hotels with basic information on each hotel,
+and the application knows the unique identifiers of the hotels returned.
+When the user selects a hotel to view details, you can then use Q2,
+which is used to obtain details about the hotel. Because you already
+have the `hotel_id` from Q1, you use that as a reference to the hotel
+you’re looking for. Therefore the second table is just called `hotels`.
+
+Another option would have been to store a set of `poi_names` in the
+hotels table. This is an equally valid approach. You’ll learn through
+experience which approach is best for your application.
+
+Q3 is just a reverse of Q1—looking for points of interest near a hotel,
+rather than hotels near a point of interest. This time, however, you
+need to access the details of each point of interest, as represented by
+the `pois_by_hotel` table. As previously, you add the point of interest
+name as a clustering key to guarantee uniqueness.
+
+At this point, let’s now consider how to support query Q4 to help the
+user find available rooms at a selected hotel for the nights they are
+interested in staying. Note that this query involves both a start date
+and an end date. Because you’re querying over a range instead of a
+single date, you know that you’ll need to use the date as a clustering
+key. Use the `hotel_id` as a primary key to group room data for each
+hotel on a single partition, which should help searches be super fast.
+Let’s call this the `available_rooms_by_hotel_date` table.
+
+To support searching over a range, use `clustering columns
+<clustering-columns>` to store attributes that you need to access in a
+range query. Remember that the order of the clustering columns is
+important.
+
+The design of the `available_rooms_by_hotel_date` table is an instance
+of the *wide partition* pattern. This pattern is sometimes called the
+*wide row* pattern when discussing databases that support similar
+models, but wide partition is a more accurate description from a
+Cassandra perspective. The essence of the pattern is to group multiple
+related rows in a partition in order to support fast access to multiple
+rows within the partition in a single query.
+
+In order to round out the shopping portion of the data model, add the
+`amenities_by_room` table to support Q5. This will allow users to view
+the amenities of one of the rooms that is available for the desired stay
+dates.
+
+== Reservation Logical Data Model
+
+Now let's switch gears to look at the reservation queries. The figure
+shows a logical data model for reservations. You’ll notice that these
+tables represent a denormalized design; the same data appears in
+multiple tables, with differing keys.
+
+image::data-modeling_reservation_logical.png[image]
+
+In order to satisfy Q6, the `reservations_by_guest` table can be used to
+look up the reservation by guest name. You could envision query Q7 being
+used on behalf of a guest on a self-serve website or a call center agent
+trying to assist the guest. Because the guest name might not be unique,
+you include the guest ID here as a clustering column as well.
+
+Q8 and Q9 in particular help to remind you to create queries that
+support various stakeholders of the application, not just customers but
+staff as well, and perhaps even the analytics team, suppliers, and so
+on.
+
+The hotel staff might wish to see a record of upcoming reservations by
+date in order to get insight into how the hotel is performing, such as
+what dates the hotel is sold out or undersold. Q8 supports the retrieval
+of reservations for a given hotel by date.
+
+Finally, you create a `guests` table. This provides a single location
+that used to store guest information. In this case, you specify a
+separate unique identifier for guest records, as it is not uncommon for
+guests to have the same name. In many organizations, a customer database
+such as the `guests` table would be part of a separate customer
+management application, which is why other guest access patterns were
+omitted from the example.
+
+== Patterns and Anti-Patterns
+
+As with other types of software design, there are some well-known
+patterns and anti-patterns for data modeling in Cassandra. You’ve
+already used one of the most common patterns in this hotel model—the
+wide partition pattern.
+
+The *time series* pattern is an extension of the wide partition pattern.
+In this pattern, a series of measurements at specific time intervals are
+stored in a wide partition, where the measurement time is used as part
+of the partition key. This pattern is frequently used in domains
+including business analysis, sensor data management, and scientific
+experiments.
+
+The time series pattern is also useful for data other than measurements.
+Consider the example of a banking application. You could store each
+customer’s balance in a row, but that might lead to a lot of read and
+write contention as various customers check their balance or make
+transactions. You’d probably be tempted to wrap a transaction around
+writes just to protect the balance from being updated in error. In
+contrast, a time series–style design would store each transaction as a
+timestamped row and leave the work of calculating the current balance to
+the application.
+
+One design trap that many new users fall into is attempting to use
+Cassandra as a queue. Each item in the queue is stored with a timestamp
+in a wide partition. Items are appended to the end of the queue and read
+from the front, being deleted after they are read. This is a design that
+seems attractive, especially given its apparent similarity to the time
+series pattern. The problem with this approach is that the deleted items
+are now `tombstones <asynch-deletes>` that Cassandra must scan past in
+order to read from the front of the queue. Over time, a growing number
+of tombstones begins to degrade read performance.
+
+The queue anti-pattern serves as a reminder that any design that relies
+on the deletion of data is potentially a poorly performing design.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_physical.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_physical.adoc
new file mode 100644
index 0000000..2066970
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_physical.adoc
@@ -0,0 +1,96 @@
+= Physical Data Modeling
+
+Once you have a logical data model defined, creating the physical model
+is a relatively simple process.
+
+You walk through each of the logical model tables, assigning types to
+each item. You can use any valid `CQL data type <data-types>`, including
+the basic types, collections, and user-defined types. You may identify
+additional user-defined types that can be created to simplify your
+design.
+
+After you’ve assigned data types, you analyze the model by performing
+size calculations and testing out how the model works. You may make some
+adjustments based on your findings. Once again let's cover the data
+modeling process in more detail by working through an example.
+
+Before getting started, let’s look at a few additions to the Chebotko
+notation for physical data models. To draw physical models, you need to
+be able to add the typing information for each column. This figure shows
+the addition of a type for each column in a sample table.
+
+image::data-modeling_chebotko_physical.png[image]
+
+The figure includes a designation of the keyspace containing each table
+and visual cues for columns represented using collections and
+user-defined types. Note the designation of static columns and secondary
+index columns. There is no restriction on assigning these as part of a
+logical model, but they are typically more of a physical data modeling
+concern.
+
+== Hotel Physical Data Model
+
+Now let’s get to work on the physical model. First, you need keyspaces
+to contain the tables. To keep the design relatively simple, create a
+`hotel` keyspace to contain tables for hotel and availability data, and
+a `reservation` keyspace to contain tables for reservation and guest
+data. In a real system, you might divide the tables across even more
+keyspaces in order to separate concerns.
+
+For the `hotels` table, use Cassandra’s `text` type to represent the
+hotel’s `id`. For the address, create an `address` user defined type.
+Use the `text` type to represent the phone number, as there is
+considerable variance in the formatting of numbers between countries.
+
+While it would make sense to use the `uuid` type for attributes such as
+the `hotel_id`, this document uses mostly `text` attributes as
+identifiers, to keep the samples simple and readable. For example, a
+common convention in the hospitality industry is to reference properties
+by short codes like "AZ123" or "NY229". This example uses these values
+for `hotel_ids`, while acknowledging they are not necessarily globally
+unique.
+
+You’ll find that it’s often helpful to use unique IDs to uniquely
+reference elements, and to use these `uuids` as references in tables
+representing other entities. This helps to minimize coupling between
+different entity types. This may prove especially effective if you are
+using a microservice architectural style for your application, in which
+there are separate services responsible for each entity type.
+
+As you work to create physical representations of various tables in the
+logical hotel data model, you use the same approach. The resulting
+design is shown in this figure:
+
+image::data-modeling_hotel_physical.png[image]
+
+Note that the `address` type is also included in the design. It is
+designated with an asterisk to denote that it is a user-defined type,
+and has no primary key columns identified. This type is used in the
+`hotels` and `hotels_by_poi` tables.
+
+User-defined types are frequently used to help reduce duplication of
+non-primary key columns, as was done with the `address` user-defined
+type. This can reduce complexity in the design.
+
+Remember that the scope of a UDT is the keyspace in which it is defined.
+To use `address` in the `reservation` keyspace defined below design,
+you’ll have to declare it again. This is just one of the many trade-offs
+you have to make in data model design.
+
+== Reservation Physical Data Model
+
+Now, let’s examine reservation tables in the design. Remember that the
+logical model contained three denormalized tables to support queries for
+reservations by confirmation number, guest, and hotel and date. For the
+first iteration of your physical data model design, assume you're going
+to manage this denormalization manually. Note that this design could be
+revised to use Cassandra’s (experimental) materialized view feature.
+
+image::data-modeling_reservation_physical.png[image]
+
+Note that the `address` type is reproduced in this keyspace and
+`guest_id` is modeled as a `uuid` type in all of the tables.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
new file mode 100644
index 0000000..7378006
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
@@ -0,0 +1,60 @@
+= Defining Application Queries
+
+Let’s try the query-first approach to start designing the data model for
+a hotel application. The user interface design for the application is
+often a great artifact to use to begin identifying queries. Let’s assume
+that you’ve talked with the project stakeholders and your UX designers
+have produced user interface designs or wireframes for the key use
+cases. You’ll likely have a list of shopping queries like the following:
+
+* Q1. Find hotels near a given point of interest.
+* Q2. Find information about a given hotel, such as its name and
+location.
+* Q3. Find points of interest near a given hotel.
+* Q4. Find an available room in a given date range.
+* Q5. Find the rate and amenities for a room.
+
+It is often helpful to be able to refer to queries by a shorthand number
+rather that explaining them in full. The queries listed here are
+numbered Q1, Q2, and so on, which is how they are referenced in diagrams
+throughout the example.
+
+Now if the application is to be a success, you’ll certainly want
+customers to be able to book reservations at hotels. This includes steps
+such as selecting an available room and entering their guest
+information. So clearly you will also need some queries that address the
+reservation and guest entities from the conceptual data model. Even
+here, however, you’ll want to think not only from the customer
+perspective in terms of how the data is written, but also in terms of
+how the data will be queried by downstream use cases.
+
+You natural tendency as might be to focus first on designing the tables
+to store reservation and guest records, and only then start thinking
+about the queries that would access them. You may have felt a similar
+tension already when discussing the shopping queries before, thinking
+“but where did the hotel and point of interest data come from?” Don’t
+worry, you will see soon enough. Here are some queries that describe how
+users will access reservations:
+
+* Q6. Lookup a reservation by confirmation number.
+* Q7. Lookup a reservation by hotel, date, and guest name.
+* Q8. Lookup all reservations by guest name.
+* Q9. View guest details.
+
+All of the queries are shown in the context of the workflow of the
+application in the figure below. Each box on the diagram represents a
+step in the application workflow, with arrows indicating the flows
+between steps and the associated query. If you’ve modeled the
+application well, each step of the workflow accomplishes a task that
+“unlocks” subsequent steps. For example, the “View hotels near POI” task
+helps the application learn about several hotels, including their unique
+keys. The key for a selected hotel may be used as part of Q2, in order
+to obtain detailed description of the hotel. The act of booking a room
+creates a reservation record that may be accessed by the guest and hotel
+staff at a later time through various additional queries.
+
+image::data-modeling_hotel_queries.png[image]
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_rdbms.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_rdbms.adoc
new file mode 100644
index 0000000..50007d4
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_rdbms.adoc
@@ -0,0 +1,144 @@
+= RDBMS Design
+
+When you set out to build a new data-driven application that will use a
+relational database, you might start by modeling the domain as a set of
+properly normalized tables and use foreign keys to reference related
+data in other tables.
+
+The figure below shows how you might represent the data storage for your
+application using a relational database model. The relational model
+includes a couple of “join” tables in order to realize the many-to-many
+relationships from the conceptual model of hotels-to-points of interest,
+rooms-to-amenities, rooms-to-availability, and guests-to-rooms (via a
+reservation).
+
+image::data-modeling_hotel_relational.png[image]
+
+== Design Differences Between RDBMS and Cassandra
+
+Let’s take a minute to highlight some of the key differences in doing
+data modeling for Cassandra versus a relational database.
+
+=== No joins
+
+You cannot perform joins in Cassandra. If you have designed a data model
+and find that you need something like a join, you’ll have to either do
+the work on the client side, or create a denormalized second table that
+represents the join results for you. This latter option is preferred in
+Cassandra data modeling. Performing joins on the client should be a very
+rare case; you really want to duplicate (denormalize) the data instead.
+
+=== No referential integrity
+
+Although Cassandra supports features such as lightweight transactions
+and batches, Cassandra itself has no concept of referential integrity
+across tables. In a relational database, you could specify foreign keys
+in a table to reference the primary key of a record in another table.
+But Cassandra does not enforce this. It is still a common design
+requirement to store IDs related to other entities in your tables, but
+operations such as cascading deletes are not available.
+
+=== Denormalization
+
+In relational database design, you are often taught the importance of
+normalization. This is not an advantage when working with Cassandra
+because it performs best when the data model is denormalized. It is
+often the case that companies end up denormalizing data in relational
+databases as well. There are two common reasons for this. One is
+performance. Companies simply can’t get the performance they need when
+they have to do so many joins on years’ worth of data, so they
+denormalize along the lines of known queries. This ends up working, but
+goes against the grain of how relational databases are intended to be
+designed, and ultimately makes one question whether using a relational
+database is the best approach in these circumstances.
+
+A second reason that relational databases get denormalized on purpose is
+a business document structure that requires retention. That is, you have
+an enclosing table that refers to a lot of external tables whose data
+could change over time, but you need to preserve the enclosing document
+as a snapshot in history. The common example here is with invoices. You
+already have customer and product tables, and you’d think that you could
+just make an invoice that refers to those tables. But this should never
+be done in practice. Customer or price information could change, and
+then you would lose the integrity of the invoice document as it was on
+the invoice date, which could violate audits, reports, or laws, and
+cause other problems.
+
+In the relational world, denormalization violates Codd’s normal forms,
+and you try to avoid it. But in Cassandra, denormalization is, well,
+perfectly normal. It’s not required if your data model is simple. But
+don’t be afraid of it.
+
+Historically, denormalization in Cassandra has required designing and
+managing multiple tables using techniques described in this
+documentation. Beginning with the 3.0 release, Cassandra provides a
+feature known as `materialized views <materialized-views>` which allows
+you to create multiple denormalized views of data based on a base table
+design. Cassandra manages materialized views on the server, including
+the work of keeping the views in sync with the table.
+
+=== Query-first design
+
+Relational modeling, in simple terms, means that you start from the
+conceptual domain and then represent the nouns in the domain in tables.
+You then assign primary keys and foreign keys to model relationships.
+When you have a many-to-many relationship, you create the join tables
+that represent just those keys. The join tables don’t exist in the real
+world, and are a necessary side effect of the way relational models
+work. After you have all your tables laid out, you can start writing
+queries that pull together disparate data using the relationships
+defined by the keys. The queries in the relational world are very much
+secondary. It is assumed that you can always get the data you want as
+long as you have your tables modeled properly. Even if you have to use
+several complex subqueries or join statements, this is usually true.
+
+By contrast, in Cassandra you don’t start with the data model; you start
+with the query model. Instead of modeling the data first and then
+writing queries, with Cassandra you model the queries and let the data
+be organized around them. Think of the most common query paths your
+application will use, and then create the tables that you need to
+support them.
+
+Detractors have suggested that designing the queries first is overly
+constraining on application design, not to mention database modeling.
+But it is perfectly reasonable to expect that you should think hard
+about the queries in your application, just as you would, presumably,
+think hard about your relational domain. You may get it wrong, and then
+you’ll have problems in either world. Or your query needs might change
+over time, and then you’ll have to work to update your data set. But
+this is no different from defining the wrong tables, or needing
+additional tables, in an RDBMS.
+
+=== Designing for optimal storage
+
+In a relational database, it is frequently transparent to the user how
+tables are stored on disk, and it is rare to hear of recommendations
+about data modeling based on how the RDBMS might store tables on disk.
+However, that is an important consideration in Cassandra. Because
+Cassandra tables are each stored in separate files on disk, it’s
+important to keep related columns defined together in the same table.
+
+A key goal that you will see as you begin creating data models in
+Cassandra is to minimize the number of partitions that must be searched
+in order to satisfy a given query. Because the partition is a unit of
+storage that does not get divided across nodes, a query that searches a
+single partition will typically yield the best performance.
+
+=== Sorting is a design decision
+
+In an RDBMS, you can easily change the order in which records are
+returned to you by using `ORDER BY` in your query. The default sort
+order is not configurable; by default, records are returned in the order
+in which they are written. If you want to change the order, you just
+modify your query, and you can sort by any list of columns.
+
+In Cassandra, however, sorting is treated differently; it is a design
+decision. The sort order available on queries is fixed, and is
+determined entirely by the selection of clustering columns you supply in
+the `CREATE TABLE` command. The CQL `SELECT` statement does support
+`ORDER BY` semantics, but only in the order specified by the clustering
+columns.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_refining.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_refining.adoc
new file mode 100644
index 0000000..6dd8ffa
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_refining.adoc
@@ -0,0 +1,202 @@
+= Evaluating and Refining Data Models
+:stem: latexmath
+
+Once you’ve created a physical model, there are some steps you’ll want
+to take to evaluate and refine table designs to help ensure optimal
+performance.
+
+== Calculating Partition Size
+
+The first thing that you want to look for is whether your tables will
+have partitions that will be overly large, or to put it another way, too
+wide. Partition size is measured by the number of cells (values) that
+are stored in the partition. Cassandra’s hard limit is 2 billion cells
+per partition, but you’ll likely run into performance issues before
+reaching that limit.
+
+In order to calculate the size of partitions, use the following formula:
+
+[latexmath]
+++++
+\[N_v = N_r (N_c - N_{pk} - N_s) + N_s\]
+++++
+
+The number of values (or cells) in the partition (N~v~) is equal to the
+number of static columns (N~s~) plus the product of the number of rows
+(N~r~) and the number of of values per row. The number of values per row
+is defined as the number of columns (N~c~) minus the number of primary
+key columns (N~pk~) and static columns (N~s~).
+
+The number of columns tends to be relatively static, although it is
+possible to alter tables at runtime. For this reason, a primary driver
+of partition size is the number of rows in the partition. This is a key
+factor that you must consider in determining whether a partition has the
+potential to get too large. Two billion values sounds like a lot, but in
+a sensor system where tens or hundreds of values are measured every
+millisecond, the number of values starts to add up pretty fast.
+
+Let’s take a look at one of the tables to analyze the partition size.
+Because it has a wide partition design with one partition per hotel,
+look at the `available_rooms_by_hotel_date` table. The table has four
+columns total (N~c~ = 4), including three primary key columns (N~pk~ =
+3) and no static columns (N~s~ = 0). Plugging these values into the
+formula, the result is:
+
+[latexmath]
+++++
+\[N_v = N_r (4 - 3 - 0) + 0 = 1N_r\]
+++++
+
+Therefore the number of values for this table is equal to the number of
+rows. You still need to determine a number of rows. To do this, make
+estimates based on the application design. The table is storing a record
+for each room, in each of hotel, for every night. Let's assume the
+system will be used to store two years of inventory at a time, and there
+are 5,000 hotels in the system, with an average of 100 rooms in each
+hotel.
+
+Since there is a partition for each hotel, the estimated number of rows
+per partition is as follows:
+
+[latexmath]
+++++
+\[N_r = 100 rooms/hotel \times 730 days = 73,000 rows\]
+++++
+
+This relatively small number of rows per partition is not going to get
+you in too much trouble, but if you start storing more dates of
+inventory, or don’t manage the size of the inventory well using TTL, you
+could start having issues. You still might want to look at breaking up
+this large partition, which you'll see how to do shortly.
+
+When performing sizing calculations, it is tempting to assume the
+nominal or average case for variables such as the number of rows.
+Consider calculating the worst case as well, as these sorts of
+predictions have a way of coming true in successful systems.
+
+== Calculating Size on Disk
+
+In addition to calculating the size of a partition, it is also an
+excellent idea to estimate the amount of disk space that will be
+required for each table you plan to store in the cluster. In order to
+determine the size, use the following formula to determine the size S~t~
+of a partition:
+
+[latexmath]
+++++
+\[S_t = \displaystyle\sum_i sizeOf\big (c_{k_i}\big) + \displaystyle\sum_j sizeOf\big(c_{s_j}\big) + N_r\times \bigg(\displaystyle\sum_k sizeOf\big(c_{r_k}\big) + \displaystyle\sum_l sizeOf\big(c_{c_l}\big)\bigg) +\]
+++++
+
+[latexmath]
+++++
+\[N_v\times sizeOf\big(t_{avg}\big)\]
+++++
+
+This is a bit more complex than the previous formula, but let's break it
+down a bit at a time. Let’s take a look at the notation first:
+
+* In this formula, c~k~ refers to partition key columns, c~s~ to static
+columns, c~r~ to regular columns, and c~c~ to clustering columns.
+* The term t~avg~ refers to the average number of bytes of metadata
+stored per cell, such as timestamps. It is typical to use an estimate of
+8 bytes for this value.
+* You'll recognize the number of rows N~r~ and number of values N~v~
+from previous calculations.
+* The *sizeOf()* function refers to the size in bytes of the CQL data
+type of each referenced column.
+
+The first term asks you to sum the size of the partition key columns.
+For this example, the `available_rooms_by_hotel_date` table has a single
+partition key column, the `hotel_id`, which is of type `text`. Assuming
+that hotel identifiers are simple 5-character codes, you have a 5-byte
+value, so the sum of the partition key column sizes is 5 bytes.
+
+The second term asks you to sum the size of the static columns. This
+table has no static columns, so the size is 0 bytes.
+
+The third term is the most involved, and for good reason—it is
+calculating the size of the cells in the partition. Sum the size of the
+clustering columns and regular columns. The two clustering columns are
+the `date`, which is 4 bytes, and the `room_number`, which is a 2-byte
+short integer, giving a sum of 6 bytes. There is only a single regular
+column, the boolean `is_available`, which is 1 byte in size. Summing the
+regular column size (1 byte) plus the clustering column size (6 bytes)
+gives a total of 7 bytes. To finish up the term, multiply this value by
+the number of rows (73,000), giving a result of 511,000 bytes (0.51 MB).
+
+The fourth term is simply counting the metadata that that Cassandra
+stores for each cell. In the storage format used by Cassandra 3.0 and
+later, the amount of metadata for a given cell varies based on the type
+of data being stored, and whether or not custom timestamp or TTL values
+are specified for individual cells. For this table, reuse the number of
+values from the previous calculation (73,000) and multiply by 8, which
+gives 0.58 MB.
+
+Adding these terms together, you get a final estimate:
+
+[latexmath]
+++++
+\[Partition size = 16 bytes + 0 bytes + 0.51 MB + 0.58 MB = 1.1 MB\]
+++++
+
+This formula is an approximation of the actual size of a partition on
+disk, but is accurate enough to be quite useful. Remembering that the
+partition must be able to fit on a single node, it looks like the table
+design will not put a lot of strain on disk storage.
+
+Cassandra’s storage engine was re-implemented for the 3.0 release,
+including a new format for SSTable files. The previous format stored a
+separate copy of the clustering columns as part of the record for each
+cell. The newer format eliminates this duplication, which reduces the
+size of stored data and simplifies the formula for computing that size.
+
+Keep in mind also that this estimate only counts a single replica of
+data. You will need to multiply the value obtained here by the number of
+partitions and the number of replicas specified by the keyspace’s
+replication strategy in order to determine the total required total
+capacity for each table. This will come in handy when you plan your
+cluster.
+
+== Breaking Up Large Partitions
+
+As discussed previously, the goal is to design tables that can provide
+the data you need with queries that touch a single partition, or failing
+that, the minimum possible number of partitions. However, as shown in
+the examples, it is quite possible to design wide partition-style tables
+that approach Cassandra’s built-in limits. Performing sizing analysis on
+tables may reveal partitions that are potentially too large, either in
+number of values, size on disk, or both.
+
+The technique for splitting a large partition is straightforward: add an
+additional column to the partition key. In most cases, moving one of the
+existing columns into the partition key will be sufficient. Another
+option is to introduce an additional column to the table to act as a
+sharding key, but this requires additional application logic.
+
+Continuing to examine the available rooms example, if you add the `date`
+column to the partition key for the `available_rooms_by_hotel_date`
+table, each partition would then represent the availability of rooms at
+a specific hotel on a specific date. This will certainly yield
+partitions that are significantly smaller, perhaps too small, as the
+data for consecutive days will likely be on separate nodes.
+
+Another technique known as *bucketing* is often used to break the data
+into moderate-size partitions. For example, you could bucketize the
+`available_rooms_by_hotel_date` table by adding a `month` column to the
+partition key, perhaps represented as an integer. The comparision with
+the original design is shown in the figure below. While the `month`
+column is partially duplicative of the `date`, it provides a nice way of
+grouping related data in a partition that will not get too large.
+
+image::data-modeling_hotel_bucketing.png[image]
+
+If you really felt strongly about preserving a wide partition design,
+you could instead add the `room_id` to the partition key, so that each
+partition would represent the availability of the room across all dates.
+Because there was no query identified that involves searching
+availability of a specific room, the first or second design approach is
+most suitable to the application needs.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc
new file mode 100644
index 0000000..04a0434
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc
@@ -0,0 +1,130 @@
+= Defining Database Schema
+
+Once you have finished evaluating and refining the physical model,
+you’re ready to implement the schema in CQL. Here is the schema for the
+`hotel` keyspace, using CQL’s comment feature to document the query
+pattern supported by each table:
+
+[source,cql]
+----
+CREATE KEYSPACE hotel WITH replication =
+ {‘class’: ‘SimpleStrategy’, ‘replication_factor’ : 3};
+
+CREATE TYPE hotel.address (
+ street text,
+ city text,
+ state_or_province text,
+ postal_code text,
+ country text );
+
+CREATE TABLE hotel.hotels_by_poi (
+ poi_name text,
+ hotel_id text,
+ name text,
+ phone text,
+ address frozen<address>,
+ PRIMARY KEY ((poi_name), hotel_id) )
+ WITH comment = ‘Q1. Find hotels near given poi’
+ AND CLUSTERING ORDER BY (hotel_id ASC) ;
+
+CREATE TABLE hotel.hotels (
+ id text PRIMARY KEY,
+ name text,
+ phone text,
+ address frozen<address>,
+ pois set<text> )
+ WITH comment = ‘Q2. Find information about a hotel’;
+
+CREATE TABLE hotel.pois_by_hotel (
+ poi_name text,
+ hotel_id text,
+ description text,
+ PRIMARY KEY ((hotel_id), poi_name) )
+ WITH comment = Q3. Find pois near a hotel’;
+
+CREATE TABLE hotel.available_rooms_by_hotel_date (
+ hotel_id text,
+ date date,
+ room_number smallint,
+ is_available boolean,
+ PRIMARY KEY ((hotel_id), date, room_number) )
+ WITH comment = ‘Q4. Find available rooms by hotel date’;
+
+CREATE TABLE hotel.amenities_by_room (
+ hotel_id text,
+ room_number smallint,
+ amenity_name text,
+ description text,
+ PRIMARY KEY ((hotel_id, room_number), amenity_name) )
+ WITH comment = ‘Q5. Find amenities for a room’;
+----
+
+Notice that the elements of the partition key are surrounded with
+parentheses, even though the partition key consists of the single column
+`poi_name`. This is a best practice that makes the selection of
+partition key more explicit to others reading your CQL.
+
+Similarly, here is the schema for the `reservation` keyspace:
+
+[source,cql]
+----
+CREATE KEYSPACE reservation WITH replication = {‘class’:
+ ‘SimpleStrategy’, ‘replication_factor’ : 3};
+
+CREATE TYPE reservation.address (
+ street text,
+ city text,
+ state_or_province text,
+ postal_code text,
+ country text );
+
+CREATE TABLE reservation.reservations_by_confirmation (
+ confirm_number text,
+ hotel_id text,
+ start_date date,
+ end_date date,
+ room_number smallint,
+ guest_id uuid,
+ PRIMARY KEY (confirm_number) )
+ WITH comment = ‘Q6. Find reservations by confirmation number’;
+
+CREATE TABLE reservation.reservations_by_hotel_date (
+ hotel_id text,
+ start_date date,
+ end_date date,
+ room_number smallint,
+ confirm_number text,
+ guest_id uuid,
+ PRIMARY KEY ((hotel_id, start_date), room_number) )
+ WITH comment = ‘Q7. Find reservations by hotel and date’;
+
+CREATE TABLE reservation.reservations_by_guest (
+ guest_last_name text,
+ hotel_id text,
+ start_date date,
+ end_date date,
+ room_number smallint,
+ confirm_number text,
+ guest_id uuid,
+ PRIMARY KEY ((guest_last_name), hotel_id) )
+ WITH comment = ‘Q8. Find reservations by guest name’;
+
+CREATE TABLE reservation.guests (
+ guest_id uuid PRIMARY KEY,
+ first_name text,
+ last_name text,
+ title text,
+ emails set,
+ phone_numbers list,
+ addresses map<text,
+ frozen<address>,
+ confirm_number text )
+ WITH comment = ‘Q9. Find guest by ID’;
+----
+
+You now have a complete Cassandra schema for storing data for a hotel
+application.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_tools.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_tools.adoc
new file mode 100644
index 0000000..608caaa
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_tools.adoc
@@ -0,0 +1,44 @@
+= Cassandra Data Modeling Tools
+
+There are several tools available to help you design and manage your
+Cassandra schema and build queries.
+
+* https://hackolade.com/nosqldb.html#cassandra[Hackolade] is a data
+modeling tool that supports schema design for Cassandra and many other
+NoSQL databases. Hackolade supports the unique concepts of CQL such as
+partition keys and clustering columns, as well as data types including
+collections and UDTs. It also provides the ability to create Chebotko
+diagrams.
+* http://kdm.dataview.org/[Kashlev Data Modeler] is a Cassandra data
+modeling tool that automates the data modeling methodology described in
+this documentation, including identifying access patterns, conceptual,
+logical, and physical data modeling, and schema generation. It also
+includes model patterns that you can optionally leverage as a starting
+point for your designs.
+* DataStax DevCenter is a tool for managing schema, executing queries
+and viewing results. While the tool is no longer actively supported, it
+is still popular with many developers and is available as a
+https://academy.datastax.com/downloads[free download]. DevCenter
+features syntax highlighting for CQL commands, types, and name literals.
+DevCenter provides command completion as you type out CQL commands and
+interprets the commands you type, highlighting any errors you make. The
+tool provides panes for managing multiple CQL scripts and connections to
+multiple clusters. The connections are used to run CQL commands against
+live clusters and view the results. The tool also has a query trace
+feature that is useful for gaining insight into the performance of your
+queries.
+* IDE Plugins - There are CQL plugins available for several Integrated
+Development Environments (IDEs), such as IntelliJ IDEA and Apache
+NetBeans. These plugins typically provide features such as schema
+management and query execution.
+
+Some IDEs and tools that claim to support Cassandra do not actually
+support CQL natively, but instead access Cassandra using a JDBC/ODBC
+driver and interact with Cassandra as if it were a relational database
+with SQL support. When selecting tools for working with Cassandra you’ll
+want to make sure they support CQL and reinforce Cassandra best
+practices for data modeling as presented in this documentation.
+
+_Material adapted from Cassandra, The Definitive Guide. Published by
+O'Reilly Media, Inc. Copyright © 2020 Jeff Carpenter, Eben Hewitt. All
+rights reserved. Used with permission._
diff --git a/doc/modules/cassandra/pages/data_modeling/images/Figure_1_data_model.jpg b/doc/modules/cassandra/pages/developing/data-modeling/images/Figure_1_data_model.jpg
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/Figure_1_data_model.jpg
rename to doc/modules/cassandra/pages/developing/data-modeling/images/Figure_1_data_model.jpg
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/Figure_2_data_model.jpg b/doc/modules/cassandra/pages/developing/data-modeling/images/Figure_2_data_model.jpg
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/Figure_2_data_model.jpg
rename to doc/modules/cassandra/pages/developing/data-modeling/images/Figure_2_data_model.jpg
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_chebotko_logical.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_chebotko_logical.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_chebotko_logical.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_chebotko_logical.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_chebotko_physical.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_chebotko_physical.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_chebotko_physical.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_chebotko_physical.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_bucketing.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_bucketing.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_bucketing.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_bucketing.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_erd.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_erd.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_erd.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_erd.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_logical.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_logical.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_logical.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_logical.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_physical.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_physical.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_physical.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_physical.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_queries.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_queries.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_queries.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_queries.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_relational.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_relational.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_hotel_relational.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_hotel_relational.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_reservation_logical.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_reservation_logical.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_reservation_logical.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_reservation_logical.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/data_modeling/images/data_modeling_reservation_physical.png b/doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_reservation_physical.png
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/images/data_modeling_reservation_physical.png
rename to doc/modules/cassandra/pages/developing/data-modeling/images/data_modeling_reservation_physical.png
Binary files differ
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/index.adoc b/doc/modules/cassandra/pages/developing/data-modeling/index.adoc
new file mode 100644
index 0000000..653e43a
--- /dev/null
+++ b/doc/modules/cassandra/pages/developing/data-modeling/index.adoc
@@ -0,0 +1,11 @@
+= Data Modeling
+
+* xref:data-modeling/intro.adoc[Introduction]
+* xref:data-modeling/data-modeling_rdbms.adoc[RDBMS]
+* xref:data-modeling/data-modeling_conceptual.adoc[Conceptual]
+* xref:data-modeling/data-modeling_logical.adoc[Logical]
+* xref:data-modeling/data-modeling_physical.adoc[Physical]
+* xref:data-modeling/data-modeling_schema.adoc[Schema]
+* xref:data-modeling/data-modeling_queries.adoc[Queries]
+* xref:data-modeling/data-modeling_refining.adoc[Refining]
+* xref:data-modeling/data-modeling_tools.adoc[Tools]
diff --git a/doc/modules/cassandra/pages/data_modeling/intro.adoc b/doc/modules/cassandra/pages/developing/data-modeling/intro.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/data_modeling/intro.adoc
rename to doc/modules/cassandra/pages/developing/data-modeling/intro.adoc
diff --git a/doc/modules/cassandra/pages/faq/index.adoc b/doc/modules/cassandra/pages/faq/index.adoc
deleted file mode 100644
index df74db9..0000000
--- a/doc/modules/cassandra/pages/faq/index.adoc
+++ /dev/null
@@ -1,272 +0,0 @@
-= Frequently Asked Questions
-
-[[why-cant-list-all]]
-== Why can't I set `listen_address` to listen on 0.0.0.0 (all my addresses)?
-
-Cassandra is a gossip-based distributed system and `listen_address` is
-the address a node tells other nodes to reach it at. Telling other nodes
-"contact me on any of my addresses" is a bad idea; if different nodes in
-the cluster pick different addresses for you, Bad Things happen.
-
-If you don't want to manually specify an IP to `listen_address` for each
-node in your cluster (understandable!), leave it blank and Cassandra
-will use `InetAddress.getLocalHost()` to pick an address. Then it's up
-to you or your ops team to make things resolve correctly (`/etc/hosts/`,
-dns, etc).
-
-One exception to this process is JMX, which by default binds to 0.0.0.0
-(Java bug 6425769).
-
-See `256` and `43` for more gory details.
-
-[[what-ports]]
-== What ports does Cassandra use?
-
-By default, Cassandra uses 7000 for cluster communication (7001 if SSL
-is enabled), 9042 for native protocol clients, and 7199 for JMX. The
-internode communication and native protocol ports are configurable in
-the `cassandra-yaml`. The JMX port is configurable in `cassandra-env.sh`
-(through JVM options). All ports are TCP.
-
-[[what-happens-on-joins]]
-== What happens to existing data in my cluster when I add new nodes?
-
-When a new nodes joins a cluster, it will automatically contact the
-other nodes in the cluster and copy the right data to itself. See
-`topology-changes`.
-
-[[asynch-deletes]]
-== I delete data from Cassandra, but disk usage stays the same. What gives?
-
-Data you write to Cassandra gets persisted to SSTables. Since SSTables
-are immutable, the data can't actually be removed when you perform a
-delete, instead, a marker (also called a "tombstone") is written to
-indicate the value's new status. Never fear though, on the first
-compaction that occurs between the data and the tombstone, the data will
-be expunged completely and the corresponding disk space recovered. See
-`compaction` for more detail.
-
-[[one-entry-ring]]
-== Why does nodetool ring only show one entry, even though my nodes logged that they see each other joining the ring?
-
-This happens when you have the same token assigned to each node. Don't
-do that.
-
-Most often this bites people who deploy by installing Cassandra on a VM
-(especially when using the Debian package, which auto-starts Cassandra
-after installation, thus generating and saving a token), then cloning
-that VM to other nodes.
-
-The easiest fix is to wipe the data and commitlog directories, thus
-making sure that each node will generate a random token on the next
-restart.
-
-[[change-replication-factor]]
-== Can I change the replication factor (a a keyspace) on a live cluster?
-
-Yes, but it will require running a full repair (or cleanup) to change
-the replica count of existing data:
-
-* `Alter <alter-keyspace-statement>` the replication factor for desired
-keyspace (using cqlsh for instance).
-* If you're reducing the replication factor, run `nodetool cleanup` on
-the cluster to remove surplus replicated data. Cleanup runs on a
-per-node basis.
-* If you're increasing the replication factor, run
-`nodetool repair -full` to ensure data is replicated according to the
-new configuration. Repair runs on a per-replica set basis. This is an
-intensive process that may result in adverse cluster performance. It's
-highly recommended to do rolling repairs, as an attempt to repair the
-entire cluster at once will most likely swamp it. Note that you will
-need to run a full repair (`-full`) to make sure that already repaired
-sstables are not skipped.
-
-[[can-large-blob]]
-== Can I Store (large) BLOBs in Cassandra?
-
-Cassandra isn't optimized for large file or BLOB storage and a single
-`blob` value is always read and send to the client entirely. As such,
-storing small blobs (less than single digit MB) should not be a problem,
-but it is advised to manually split large blobs into smaller chunks.
-
-Please note in particular that by default, any value greater than 16MiB
-will be rejected by Cassandra due the `max_mutation_size`
-configuration of the `cassandra-yaml` file (which default to half of
-`commitlog_segment_size`, which itself default to 32MiB).
-
-[[nodetool-connection-refused]]
-== Nodetool says "Connection refused to host: 127.0.1.1" for any remote host. What gives?
-
-Nodetool relies on JMX, which in turn relies on RMI, which in turn sets
-up its own listeners and connectors as needed on each end of the
-exchange. Normally all of this happens behind the scenes transparently,
-but incorrect name resolution for either the host connecting, or the one
-being connected to, can result in crossed wires and confusing
-exceptions.
-
-If you are not using DNS, then make sure that your `/etc/hosts` files
-are accurate on both ends. If that fails, try setting the
-`-Djava.rmi.server.hostname=<public name>` JVM option near the bottom of
-`cassandra-env.sh` to an interface that you can reach from the remote
-machine.
-
-[[to-batch-or-not-to-batch]]
-== Will batching my operations speed up my bulk load?
-
-No. Using batches to load data will generally just add "spikes" of
-latency. Use asynchronous INSERTs instead, or use true `bulk-loading`.
-
-An exception is batching updates to a single partition, which can be a
-Good Thing (as long as the size of a single batch stay reasonable). But
-never ever blindly batch everything!
-
-[[selinux]]
-== On RHEL nodes are unable to join the ring
-
-Check if https://en.wikipedia.org/wiki/Security-Enhanced_Linux[SELinux]
-is on; if it is, turn it off.
-
-[[how-to-unsubscribe]]
-== How do I unsubscribe from the email list?
-
-Send an email to `user-unsubscribe@cassandra.apache.org`.
-
-[[cassandra-eats-all-my-memory]]
-== Why does top report that Cassandra is using a lot more memory than the Java heap max?
-
-Cassandra uses https://en.wikipedia.org/wiki/Memory-mapped_file[Memory
-Mapped Files] (mmap) internally. That is, we use the operating system's
-virtual memory system to map a number of on-disk files into the
-Cassandra process' address space. This will "use" virtual memory; i.e.
-address space, and will be reported by tools like top accordingly, but
-on 64 bit systems virtual address space is effectively unlimited so you
-should not worry about that.
-
-What matters from the perspective of "memory use" in the sense as it is
-normally meant, is the amount of data allocated on brk() or mmap'd
-/dev/zero, which represent real memory used. The key issue is that for a
-mmap'd file, there is never a need to retain the data resident in
-physical memory. Thus, whatever you do keep resident in physical memory
-is essentially just there as a cache, in the same way as normal I/O will
-cause the kernel page cache to retain data that you read/write.
-
-The difference between normal I/O and mmap() is that in the mmap() case
-the memory is actually mapped to the process, thus affecting the virtual
-size as reported by top. The main argument for using mmap() instead of
-standard I/O is the fact that reading entails just touching memory - in
-the case of the memory being resident, you just read it - you don't even
-take a page fault (so no overhead in entering the kernel and doing a
-semi-context switch). This is covered in more detail
-http://www.varnish-cache.org/trac/wiki/ArchitectNotes[here].
-
-== What are seeds?
-
-Seeds are used during startup to discover the cluster.
-
-If you configure your nodes to refer some node as seed, nodes in your
-ring tend to send Gossip message to seeds more often (also see the
-`section on gossip <gossip>`) than to non-seeds. In other words, seeds
-are worked as hubs of Gossip network. With seeds, each node can detect
-status changes of other nodes quickly.
-
-Seeds are also referred by new nodes on bootstrap to learn other nodes
-in ring. When you add a new node to ring, you need to specify at least
-one live seed to contact. Once a node join the ring, it learns about the
-other nodes, so it doesn't need seed on subsequent boot.
-
-You can make a seed a node at any time. There is nothing special about
-seed nodes. If you list the node in seed list it is a seed
-
-Seeds do not auto bootstrap (i.e. if a node has itself in its seed list
-it will not automatically transfer data to itself) If you want a node to
-do that, bootstrap it first and then add it to seeds later. If you have
-no data (new install) you do not have to worry about bootstrap at all.
-
-Recommended usage of seeds:
-
-* pick two (or more) nodes per data center as seed nodes.
-* sync the seed list to all your nodes
-
-[[are-seeds-SPOF]]
-== Does single seed mean single point of failure?
-
-The ring can operate or boot without a seed; however, you will not be
-able to add new nodes to the cluster. It is recommended to configure
-multiple seeds in production system.
-
-[[cant-call-jmx-method]]
-== Why can't I call jmx method X on jconsole?
-
-Some of JMX operations use array argument and as jconsole doesn't
-support array argument, those operations can't be called with jconsole
-(the buttons are inactive for them). You need to write a JMX client to
-call such operations or need array-capable JMX monitoring tool.
-
-[[why-message-dropped]]
-== Why do I see "... messages dropped ..." in the logs?
-
-This is a symptom of load shedding -- Cassandra defending itself against
-more requests than it can handle.
-
-Internode messages which are received by a node, but do not get not to
-be processed within their proper timeout (see `read_request_timeout`,
-`write_request_timeout`, ... in the `cassandra-yaml`), are dropped
-rather than processed (since the as the coordinator node will no longer
-be waiting for a response).
-
-For writes, this means that the mutation was not applied to all replicas
-it was sent to. The inconsistency will be repaired by read repair, hints
-or a manual repair. The write operation may also have timeouted as a
-result.
-
-For reads, this means a read request may not have completed.
-
-Load shedding is part of the Cassandra architecture, if this is a
-persistent issue it is generally a sign of an overloaded node or
-cluster.
-
-[[oom-map-failed]]
-== Cassandra dies with `java.lang.OutOfMemoryError: Map failed`
-
-If Cassandra is dying *specifically* with the "Map failed" message, it
-means the OS is denying java the ability to lock more memory. In linux,
-this typically means memlock is limited. Check
-`/proc/<pid of cassandra>/limits` to verify this and raise it (eg, via
-ulimit in bash). You may also need to increase `vm.max_map_count.` Note
-that the debian package handles this for you automatically.
-
-[[what-on-same-timestamp-update]]
-== What happens if two updates are made with the same timestamp?
-
-Updates must be commutative, since they may arrive in different orders
-on different replicas. As long as Cassandra has a deterministic way to
-pick the winner (in a timestamp tie), the one selected is as valid as
-any other, and the specifics should be treated as an implementation
-detail. That said, in the case of a timestamp tie, Cassandra follows two
-rules: first, deletes take precedence over inserts/updates. Second, if
-there are two updates, the one with the lexically larger value is
-selected.
-
-[[why-bootstrapping-stream-error]]
-== Why bootstrapping a new node fails with a "Stream failed" error?
-
-Two main possibilities:
-
-. the GC may be creating long pauses disrupting the streaming process
-. compactions happening in the background hold streaming long enough
-that the TCP connection fails
-
-In the first case, regular GC tuning advices apply. In the second case,
-you need to set TCP keepalive to a lower value (default is very high on
-Linux). Try to just run the following:
-
-....
-$ sudo /sbin/sysctl -w net.ipv4.tcp_keepalive_time=60 net.ipv4.tcp_keepalive_intvl=60 net.ipv4.tcp_keepalive_probes=5
-....
-
-To make those settings permanent, add them to your `/etc/sysctl.conf`
-file.
-
-Note: https://cloud.google.com/compute/[GCE]'s firewall will always
-interrupt TCP connections that are inactive for more than 10 min.
-Running the above command is highly recommended in that environment.
diff --git a/doc/modules/cassandra/pages/getting_started/configuring.adoc b/doc/modules/cassandra/pages/getting-started/configuring.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/getting_started/configuring.adoc
rename to doc/modules/cassandra/pages/getting-started/configuring.adoc
diff --git a/doc/modules/cassandra/pages/getting-started/drivers.adoc b/doc/modules/cassandra/pages/getting-started/drivers.adoc
new file mode 100644
index 0000000..3deb613
--- /dev/null
+++ b/doc/modules/cassandra/pages/getting-started/drivers.adoc
@@ -0,0 +1,90 @@
+= Client drivers
+
+Here are known Cassandra client drivers organized by language. Before
+choosing a driver, you should verify the Cassandra version and
+functionality supported by a specific driver.
+
+== Java
+
+* http://achilles.archinnov.info/[Achilles]
+* https://github.com/Netflix/astyanax/wiki/Getting-Started[Astyanax]
+* https://github.com/noorq/casser[Casser]
+* https://github.com/datastax/java-driver[Datastax Java driver]
+* https://github.com/Impetus/kundera[Kundera]
+* https://github.com/deanhiller/playorm[PlayORM]
+
+== Python
+
+* https://github.com/datastax/python-driver[Datastax Python driver]
+
+== Ruby
+
+* https://github.com/datastax/ruby-driver[Datastax Ruby driver]
+
+== C# / .NET
+
+* https://github.com/pchalamet/cassandra-sharp[Cassandra Sharp]
+* https://github.com/datastax/csharp-driver[Datastax C# driver]
+* https://github.com/managedfusion/fluentcassandra[Fluent Cassandra]
+
+== Nodejs
+
+* https://github.com/datastax/nodejs-driver[Datastax Nodejs driver]
+
+== PHP
+
+* http://code.google.com/a/apache-extras.org/p/cassandra-pdo[CQL | PHP]
+* https://github.com/datastax/php-driver/[Datastax PHP driver]
+* https://github.com/aparkhomenko/php-cassandra[PHP-Cassandra]
+* https://github.com/duoshuo/php-cassandra[PHP Library for Cassandra]
+
+== C++
+
+* https://github.com/datastax/cpp-driver[Datastax C++ driver]
+* http://sourceforge.net/projects/libqtcassandra[libQTCassandra]
+
+== Scala
+
+* https://github.com/datastax/spark-cassandra-connector[Datastax Spark
+connector]
+* https://github.com/newzly/phantom[Phantom]
+* https://github.com/getquill/quill[Quill]
+
+== Clojure
+
+* https://github.com/mpenet/alia[Alia]
+* https://github.com/clojurewerkz/cassaforte[Cassaforte]
+* https://github.com/mpenet/hayt[Hayt]
+
+== Erlang
+
+* https://github.com/matehat/cqerl[CQerl]
+* https://github.com/silviucpp/erlcass[Erlcass]
+
+== Go
+
+* https://github.com/relops/cqlc[CQLc]
+* https://github.com/hailocab/gocassa[Gocassa]
+* https://github.com/gocql/gocql[GoCQL]
+
+== Haskell
+
+* https://github.com/ozataman/cassy[Cassy]
+
+== Rust
+
+* https://github.com/neich/rust-cql[Rust CQL]
+
+== Perl
+
+* https://github.com/tvdw/perl-dbd-cassandra[Cassandra::Client and
+DBD::Cassandra]
+
+== Elixir
+
+* https://github.com/lexhide/xandra[Xandra]
+* https://github.com/matehat/cqex[CQEx]
+
+== Dart
+
+* https://github.com/achilleasa/dart_cassandra_cql[dart_cassandra_cql]
diff --git a/doc/modules/cassandra/pages/getting-started/index.adoc b/doc/modules/cassandra/pages/getting-started/index.adoc
new file mode 100644
index 0000000..3d3c823
--- /dev/null
+++ b/doc/modules/cassandra/pages/getting-started/index.adoc
@@ -0,0 +1,30 @@
+= Getting Started
+
+This section covers how to get started using Apache Cassandra and should
+be the first thing to read if you are new to Cassandra.
+
+* xref:installing/installing.adoc[Installing Cassandra]: Installation instructions plus information on choosing a method.
+** [ xref:installing/installing.adoc#installing-the-docker-image[Docker] ]
+[ xref:installing/installing.adoc#installing-the-binary-tarball[tarball] ]
+[ xref:installing/installing.adoc#installing-the-debian-packages[Debian] ]
+[ xref:installing/installing.adoc#installing-the-rpm-packages[RPM] ]
+* xref:getting-started/configuring.adoc[Configuring Cassandra]
+* xref:getting-started/querying.adoc[Inserting and querying data]
+* xref:getting-started/drivers.adoc[Client drivers]: Drivers for various languages.
+** [ xref:getting-started/drivers.adoc#java[Java] ]
+ [ xref:getting-started/drivers.adoc#python[Python] ]
+ [ xref:getting-started/drivers.adoc#ruby[Ruby] ]
+ [ xref:getting-started/drivers.adoc#c-net[C# / .NET] ]
+ [ xref:getting-started/drivers.adoc#nodejs[Node.js] ]
+ [ xref:getting-started/drivers.adoc#php[PHP] ]
+ [ xref:getting-started/drivers.adoc#c[C++] ]
+ [ xref:getting-started/drivers.adoc#scala[Scala] ]
+ [ xref:getting-started/drivers.adoc#clojure[Clojure] ]
+ [ xref:getting-started/drivers.adoc#erlang[Erlang] ]
+ [ xref:getting-started/drivers.adoc#go[Go] ]
+ [ xref:getting-started/drivers.adoc#haskell[Haskell] ]
+ [ xref:getting-started/drivers.adoc#rust[Rust] ]
+ [ xref:getting-started/drivers.adoc#perl[Perl] ]
+ [ xref:getting-started/drivers.adoc#elixir[Elixir] ]
+ [ xref:getting-started/drivers.adoc#dart[Dart] ]
+* xref:getting-started/production.adoc[Production recommendations]
diff --git a/doc/modules/cassandra/pages/getting-started/production.adoc b/doc/modules/cassandra/pages/getting-started/production.adoc
new file mode 100644
index 0000000..ad28a35
--- /dev/null
+++ b/doc/modules/cassandra/pages/getting-started/production.adoc
@@ -0,0 +1,163 @@
+= Production recommendations
+
+The `cassandra.yaml` and `jvm.options` files have a number of notes and
+recommendations for production usage.
+This page expands on some of the information in the files.
+
+== Tokens
+
+Using more than one token-range per node is referred to as virtual nodes, or vnodes.
+`vnodes` facilitate flexible expansion with more streaming peers when a new node bootstraps
+into a cluster.
+Limiting the negative impact of streaming (I/O and CPU overhead) enables incremental cluster expansion.
+However, more tokens leads to sharing data with more peers, and results in decreased availability.
+These two factors must be balanced based on a cluster's characteristic reads and writes.
+To learn more,
+https://github.com/jolynch/python_performance_toolkit/raw/master/notebooks/cassandra_availability/whitepaper/cassandra-availability-virtual.pdf[Cassandra Availability in Virtual Nodes, Joseph Lynch and Josh Snyder] is recommended reading.
+
+Change the number of tokens using the setting in the `cassandra.yaml` file:
+
+`num_tokens: 16`
+
+Here are the most common token counts with a brief explanation of when
+and why you would use each one.
+
+[width="100%",cols="13%,87%",options="header",]
+|===
+|Token Count |Description
+|1 |Maximum availablility, maximum cluster size, fewest peers, but
+inflexible expansion. Must always double size of cluster to expand and
+remain balanced.
+
+|4 |A healthy mix of elasticity and availability. Recommended for
+clusters which will eventually reach over 30 nodes. Requires adding
+approximately 20% more nodes to remain balanced. Shrinking a cluster may
+result in cluster imbalance.
+
+|8 | Using 8 vnodes distributes the workload between systems with a ~10% variance
+and has minimal impact on performance.
+
+|16 |Best for heavily elastic clusters which expand and shrink
+regularly, but may have issues availability with larger clusters. Not
+recommended for clusters over 50 nodes.
+|===
+
+In addition to setting the token count, it's extremely important that
+`allocate_tokens_for_local_replication_factor` in `cassandra.yaml` is set to an
+appropriate number of replicates, to ensure even token allocation.
+
+== Read ahead
+
+Read ahead is an operating system feature that attempts to keep as much
+data as possible loaded in the page cache.
+Spinning disks can have long seek times causing high latency, so additional
+throughput on reads using page cache can improve performance.
+By leveraging read ahead, the OS can pull additional data into memory without
+the cost of additional seeks.
+This method works well when the available RAM is greater than the size of the
+hot dataset, but can be problematic when the reverse is true (dataset > RAM).
+The larger the hot dataset, the less read ahead is useful.
+
+Read ahead is definitely not useful in the following cases:
+
+* Small partitions, such as tables with a single partition key
+* Solid state drives (SSDs)
+
+
+Read ahead can actually increase disk usage, and in some cases result in as much
+as a 5x latency and throughput performance penalty.
+Read-heavy, key/value tables with small (under 1KB) rows are especially prone
+to this problem.
+
+The recommended read ahead settings are:
+
+[width="59%",cols="40%,60%",options="header",]
+|===
+|Hardware |Initial Recommendation
+|Spinning Disks |64KB
+|SSD |4KB
+|===
+
+Read ahead can be adjusted on Linux systems using the `blockdev` tool.
+
+For example, set the read ahead of the disk `/dev/sda1` to 4KB:
+
+[source, shell]
+----
+$ blockdev --setra 8 /dev/sda1
+----
+[NOTE]
+====
+The `blockdev` setting sets the number of 512 byte sectors to read ahead.
+The argument of 8 above is equivalent to 4KB, or 8 * 512 bytes.
+====
+
+All systems are different, so use these recommendations as a starting point and
+tune, based on your SLA and throughput requirements.
+To understand how read ahead impacts disk resource usage, we recommend carefully
+reading through the xref:troubleshooting/use_tools.adoc[Diving Deep, using external tools]
+section.
+
+== Compression
+
+Compressed data is stored by compressing fixed-size byte buffers and writing the
+data to disk.
+The buffer size is determined by the `chunk_length_in_kb` element in the compression
+map of a table's schema settings for `WITH COMPRESSION`.
+The default setting is 16KB starting with Cassandra {40_version}.
+
+Since the entire compressed buffer must be read off-disk, using a compression
+chunk length that is too large can lead to significant overhead when reading small records.
+Combined with the default read ahead setting, the result can be massive
+read amplification for certain workloads. Therefore, picking an appropriate
+value for this setting is important.
+
+LZ4Compressor is the default and recommended compression algorithm.
+If you need additional information on compression, read
+https://thelastpickle.com/blog/2018/08/08/compression_performance.html[The Last Pickle blogpost on compression performance].
+
+== Compaction
+
+There are different xref:compaction/index.adoc[compaction] strategies available
+for different workloads.
+We recommend reading about the different strategies to understand which is the
+best for your environment.
+Different tables may, and frequently do use different compaction strategies in
+the same cluster.
+
+== Encryption
+
+It is significantly better to set up peer-to-peer encryption and client server
+encryption when setting up your production cluster.
+Setting it up after the cluster is serving production traffic is challenging
+to do correctly.
+If you ever plan to use network encryption of any type, we recommend setting it
+up when initially configuring your cluster.
+Changing these configurations later is not impossible, but mistakes can
+result in downtime or data loss.
+
+== Ensure keyspaces are created with NetworkTopologyStrategy
+
+Production clusters should never use `SimpleStrategy`.
+Production keyspaces should use the `NetworkTopologyStrategy` (NTS).
+For example:
+
+[source, cql]
+----
+CREATE KEYSPACE mykeyspace WITH replication = {
+ 'class': 'NetworkTopologyStrategy',
+ 'datacenter1': 3
+};
+----
+
+Cassandra clusters initialized with `NetworkTopologyStrategy` can take advantage
+of the ability to configure multiple racks and data centers.
+
+== Configure racks and snitch
+
+**Correctly configuring or changing racks after a cluster has been provisioned is an unsupported process**.
+Migrating from a single rack to multiple racks is also unsupported and can
+result in data loss.
+Using `GossipingPropertyFileSnitch` is the most flexible solution for
+on-premise or mixed cloud environments.
+`Ec2Snitch` is reliable for AWS EC2 only environments.
diff --git a/doc/modules/cassandra/pages/getting-started/querying.adoc b/doc/modules/cassandra/pages/getting-started/querying.adoc
new file mode 100644
index 0000000..e0a6d6f
--- /dev/null
+++ b/doc/modules/cassandra/pages/getting-started/querying.adoc
@@ -0,0 +1,31 @@
+= Inserting and querying
+
+The API for Cassandra is xref:cql/ddl.adoc[`CQL`, the Cassandra Query Language]. To
+use CQL, you will need to connect to the cluster, using either:
+
+* `cqlsh`, a shell for CQL
+* a client driver for Cassandra
+* for the adventurous, check out https://zeppelin.apache.org/docs/0.7.0/interpreter/cassandra.html[Apache Zeppelin], a notebook-style tool
+
+== CQLSH
+
+`cqlsh` is a command-line shell for interacting with Cassandra using
+CQL. It is shipped with every Cassandra package, and can be found in the
+`bin` directory alongside the `cassandra` executable. It connects to the
+single node specified on the command line. For example:
+
+[source, shell]
+----
+include::example$BASH/cqlsh_localhost.sh[]
+----
+[source, cql]
+----
+include::example$RESULTS/cqlsh_localhost.result[]
+----
+If the command is used without specifying a node, `localhost` is the default. See the xref:tools/cqlsh.adoc[`cqlsh` section] for full documentation.
+
+== Client drivers
+
+A lot of xref:getting-started/drivers.adoc[client drivers] are provided by the Community and a list of
+known drivers is provided. You should refer to the documentation of each driver
+for more information.
diff --git a/doc/modules/cassandra/pages/getting-started/quickstart.adoc b/doc/modules/cassandra/pages/getting-started/quickstart.adoc
new file mode 100644
index 0000000..963d269
--- /dev/null
+++ b/doc/modules/cassandra/pages/getting-started/quickstart.adoc
@@ -0,0 +1,116 @@
+= Apache Cassandra Quickstart
+:tabs:
+
+_Interested in getting started with Cassandra? Follow these instructions._
+
+*STEP 1: GET CASSANDRA USING DOCKER*
+
+You'll need to have Docker Desktop for Mac, Docker Desktop for Windows, or
+similar software installed on your computer.
+
+[source, plaintext]
+----
+docker pull cassandra:latest
+----
+
+Apache Cassandra is also available as a https://cassandra.apache.org/download/[tarball or package download].
+
+*STEP 2: START CASSANDRA*
+
+[source, plaintext]
+----
+docker run --name cassandra cassandra
+----
+
+*STEP 3: CREATE FILES*
+
+In the directory where you plan to run the next step, create these two files
+so that some data can be automatically inserted in the next step.
+
+A _cqlshrc_ file will log into the Cassandra database with the default superuser:
+
+[source, plaintext]
+----
+[authentication]
+ username = cassandra
+ password = cassandra
+----
+
+Create a _scripts_ directory and change to that directory.
+The following _data.cql_ file will create a keyspace, the layer at which Cassandra
+replicates its data, a table to hold the data, and insert some data:
+
+[source, plaintext]
+----
+# Create a keyspace
+CREATE KEYSPACE IF NOT EXISTS store WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : '1' };
+
+# Create a table
+CREATE TABLE IF NOT EXISTS store.shopping_cart (
+ userid text PRIMARY KEY,
+ item_count int,
+ last_update_timestamp timestamp
+);
+
+# Insert some data
+INSERT INTO store.shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES ('9876', 2, to_timestamp(to_date(now))));
+INSERT INTO store.shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES (1234, 5, to_timestamp(toDate(now))));
+----
+
+You should now have a _cqlshrc_ file and _<currentdir>/scripts/data.cql_ file.
+
+*STEP 4: RUN CQLSH TO INTERACT*
+
+Cassandra is a distributed database that can read and write data across multiple
+nodes with peer-to-peer replication. The Cassandra Query Language (CQL) is
+similar to SQL but suited for the JOINless structure of Cassandra. The CQL
+shell, or `cqlsh`, is one tool to use in interacting with the database.
+
+[source, plaintext]
+----
+docker run --rm -it -v /<currentdir>/scripts:/scripts \
+-v /<currentdir/cqlshrc:/.cassandra/cqlshrc \
+--env CQLSH_HOST=host.docker.internal --env CQLSH_PORT=9042 nuvo/docker-cqlsh
+----
+
+For this quickstart, this cqlsh docker image also loads some data automatically,
+so you can start running queries.
+
+*STEP 5: READ SOME DATA*
+
+[source, plaintext]
+----
+SELECT * FROM store.shopping_cart;
+----
+
+*STEP 6: WRITE SOME MORE DATA*
+
+[source, plaintext]
+----
+INSERT (userid, item_count) VALUES (4567, 20) INTO store.shopping_cart;
+----
+
+*STEP 7: TERMINATE CASSANDRA*
+
+[source, plaintext]
+----
+docker rm cassandra
+----
+
+*CONGRATULATIONS!*
+
+Hey, that wasn't so hard, was it?
+
+To learn more, we suggest the following next steps:
+
+* Read through the *need link*[Overview] to learn main concepts and how Cassandra works at a
+high level.
+* To understand Cassandra in more detail, head over to the
+https://cassandra.apache.org/doc/latest/[Docs].
+* Browse through the https://cassandra.apache.org/case-studies/[Case Studies] to
+learn how other users in our worldwide community are getting value out of
+Cassandra.
diff --git a/doc/modules/cassandra/pages/getting_started/drivers.adoc b/doc/modules/cassandra/pages/getting_started/drivers.adoc
deleted file mode 100644
index eb15a55..0000000
--- a/doc/modules/cassandra/pages/getting_started/drivers.adoc
+++ /dev/null
@@ -1,90 +0,0 @@
-= Client drivers
-
-Here are known Cassandra client drivers organized by language. Before
-choosing a driver, you should verify the Cassandra version and
-functionality supported by a specific driver.
-
-== Java
-
-* http://achilles.archinnov.info/[Achilles]
-* https://github.com/Netflix/astyanax/wiki/Getting-Started[Astyanax]
-* https://github.com/noorq/casser[Casser]
-* https://github.com/datastax/java-driver[Datastax Java driver]
-* https://github.com/impetus-opensource/Kundera[Kundera]
-* https://github.com/deanhiller/playorm[PlayORM]
-
-== Python
-
-* https://github.com/datastax/python-driver[Datastax Python driver]
-
-== Ruby
-
-* https://github.com/datastax/ruby-driver[Datastax Ruby driver]
-
-== C# / .NET
-
-* https://github.com/pchalamet/cassandra-sharp[Cassandra Sharp]
-* https://github.com/datastax/csharp-driver[Datastax C# driver]
-* https://github.com/managedfusion/fluentcassandra[Fluent Cassandra]
-
-== Nodejs
-
-* https://github.com/datastax/nodejs-driver[Datastax Nodejs driver]
-
-== PHP
-
-* http://code.google.com/a/apache-extras.org/p/cassandra-pdo[CQL | PHP]
-* https://github.com/datastax/php-driver/[Datastax PHP driver]
-* https://github.com/aparkhomenko/php-cassandra[PHP-Cassandra]
-* https://github.com/duoshuo/php-cassandra[PHP Library for Cassandra]
-
-== C++
-
-* https://github.com/datastax/cpp-driver[Datastax C++ driver]
-* http://sourceforge.net/projects/libqtcassandra[libQTCassandra]
-
-== Scala
-
-* https://github.com/datastax/spark-cassandra-connector[Datastax Spark
-connector]
-* https://github.com/newzly/phantom[Phantom]
-* https://github.com/getquill/quill[Quill]
-
-== Clojure
-
-* https://github.com/mpenet/alia[Alia]
-* https://github.com/clojurewerkz/cassaforte[Cassaforte]
-* https://github.com/mpenet/hayt[Hayt]
-
-== Erlang
-
-* https://github.com/matehat/cqerl[CQerl]
-* https://github.com/silviucpp/erlcass[Erlcass]
-
-== Go
-
-* https://github.com/relops/cqlc[CQLc]
-* https://github.com/hailocab/gocassa[Gocassa]
-* https://github.com/gocql/gocql[GoCQL]
-
-== Haskell
-
-* https://github.com/ozataman/cassy[Cassy]
-
-== Rust
-
-* https://github.com/neich/rust-cql[Rust CQL]
-
-== Perl
-
-* https://github.com/tvdw/perl-dbd-cassandra[Cassandra::Client and
-DBD::Cassandra]
-
-== Elixir
-
-* https://github.com/lexhide/xandra[Xandra]
-* https://github.com/matehat/cqex[CQEx]
-
-== Dart
-
-* https://github.com/achilleasa/dart_cassandra_cql[dart_cassandra_cql]
diff --git a/doc/modules/cassandra/pages/getting_started/index.adoc b/doc/modules/cassandra/pages/getting_started/index.adoc
deleted file mode 100644
index af43c17..0000000
--- a/doc/modules/cassandra/pages/getting_started/index.adoc
+++ /dev/null
@@ -1,30 +0,0 @@
-= Getting Started
-
-This section covers how to get started using Apache Cassandra and should
-be the first thing to read if you are new to Cassandra.
-
-* xref:getting_started/installing.adoc[Installing Cassandra]: Installation instructions plus information on choosing a method.
-** [ xref:getting_started/installing.adoc#installing-the-docker-image[Docker] ]
-[ xref:getting_started/installing.adoc#installing-the-binary-tarball[tarball] ]
-[ xref:getting_started/installing.adoc#installing-the-debian-packages[Debian] ]
-[ xref:getting_started/installing.adoc#installing-the-rpm-packages[RPM] ]
-* xref:getting_started/configuring.adoc[Configuring Cassandra]
-* xref:getting_started/querying.adoc[Inserting and querying data]
-* xref:getting_started/drivers.adoc[Client drivers]: Drivers for various languages.
-** [ xref:getting_started/drivers.adoc#java[Java] ]
- [ xref:getting_started/drivers.adoc#python[Python] ]
- [ xref:getting_started/drivers.adoc#ruby[Ruby] ]
- [ xref:getting_started/drivers.adoc#c-net[C# / .NET] ]
- [ xref:getting_started/drivers.adoc#nodejs[Node.js] ]
- [ xref:getting_started/drivers.adoc#php[PHP] ]
- [ xref:getting_started/drivers.adoc#c[C++] ]
- [ xref:getting_started/drivers.adoc#scala[Scala] ]
- [ xref:getting_started/drivers.adoc#clojure[Clojure] ]
- [ xref:getting_started/drivers.adoc#erlang[Erlang] ]
- [ xref:getting_started/drivers.adoc#go[Go] ]
- [ xref:getting_started/drivers.adoc#haskell[Haskell] ]
- [ xref:getting_started/drivers.adoc#rust[Rust] ]
- [ xref:getting_started/drivers.adoc#perl[Perl] ]
- [ xref:getting_started/drivers.adoc#elixir[Elixir] ]
- [ xref:getting_started/drivers.adoc#dart[Dart] ]
-* xref:getting_started/production.adoc[Production recommendations]
diff --git a/doc/modules/cassandra/pages/getting_started/installing.adoc b/doc/modules/cassandra/pages/getting_started/installing.adoc
deleted file mode 100644
index c8ddcd0..0000000
--- a/doc/modules/cassandra/pages/getting_started/installing.adoc
+++ /dev/null
@@ -1,345 +0,0 @@
-= Installing Cassandra
-:tabs:
-
-These are the instructions for deploying the supported releases of
-Apache Cassandra on Linux servers.
-
-Cassandra runs on a wide array of Linux distributions including (but not
-limited to):
-
-* Ubuntu, most notably LTS releases 16.04 to 18.04
-* CentOS & RedHat Enterprise Linux (RHEL) including 6.6 to 7.7
-* Amazon Linux AMIs including 2016.09 through to Linux 2
-* Debian versions 8 & 9
-* SUSE Enterprise Linux 12
-
-This is not an exhaustive list of operating system platforms, nor is it
-prescriptive. However, users will be well-advised to conduct exhaustive
-tests of their own particularly for less-popular distributions of Linux.
-Deploying on older versions is not recommended unless you have previous
-experience with the older distribution in a production environment.
-
-== Prerequisites
-
-* Install the latest version of Java 8 or Java 11, either the
-http://www.oracle.com/technetwork/java/javase/downloads/index.html[Oracle
-Java Standard Edition 8] / http://www.oracle.com/technetwork/java/javase/downloads/index.html[Oracle Java Standard Edition 11 (Long Term Support)]
-or http://openjdk.java.net/[OpenJDK 8] / http://openjdk.java.net/[OpenJDK 11]. To
-verify that you have the correct version of java installed, type
-`java -version`.
-* *NOTE*: Experimental support for Java 11 was added in Cassandra {40_version}
-(https://issues.apache.org/jira/browse/CASSANDRA-9608[CASSANDRA-9608]).
-Full support is effective Cassandra 4.0.2 version (https://issues.apache.org/jira/browse/CASSANDRA-16894[CASSANDRA-16894])
-For more information, see
-https://github.com/apache/cassandra/blob/trunk/NEWS.txt[NEWS.txt].
-* For using cqlsh, the latest version of
-Python 3.6+ or Python 2.7 (support deprecated). To verify
-that you have the correct version of Python installed, type
-`python --version`.
-
-== Choosing an installation method
-
-There are three methods of installing Cassandra that are common:
-
-* Docker image
-* Tarball binary file
-* Package installation (RPM, YUM)
-
-If you are a current Docker user, installing a Docker image is simple.
-You'll need to install Docker Desktop for Mac, Docker Desktop for Windows,
-or have `docker` installed on Linux.
-Pull the appropriate image and then start Cassandra with a run command.
-
-For most users, installing the binary tarball is also a simple choice.
-The tarball unpacks all its contents into a single location with
-binaries and configuration files located in their own subdirectories.
-The most obvious attribute of the tarball installation is it does not
-require `root` permissions and can be installed on any Linux
-distribution.
-
-Packaged installations require `root` permissions, and are most appropriate for
-production installs.
-Install the RPM build on CentOS and RHEL-based distributions if you want to
-install Cassandra using YUM.
-Install the Debian build on Ubuntu and other Debian-based
-distributions if you want to install Cassandra using APT.
-Note that both the YUM and APT methods required `root` permissions and
-will install the binaries and configuration files as the `cassandra` OS user.
-
-== Installing the docker image
-
-[arabic, start=1]
-. Pull the docker image. For the latest image, use:
-
-[source, shell]
-----
-include::example$BASH/docker_pull.sh[]
-----
-
-This `docker pull` command will get the latest version of the 'Docker Official'
-Apache Cassandra image available from the https://hub.docker.com/_/cassandra[Dockerhub].
-
-[arabic, start=2]
-. Start Cassandra with a `docker run` command:
-
-[source, shell]
-----
-include::example$BASH/docker_run.sh[]
-----
-
-The `--name` option will be the name of the Cassandra cluster created.
-
-[arabic, start=3]
-. Start the CQL shell, `cqlsh` to interact with the Cassandra node created:
-
-[source, shell]
-----
-include::example$BASH/docker_cqlsh.sh[]
-----
-== Installing the binary tarball
-
-include::partial$java_version.adoc[]
-
-[arabic, start=2]
-. Download the binary tarball from one of the mirrors on the
-{cass_url}download/[Apache Cassandra Download] site.
-For example, to download Cassandra {40_version}:
-
-[source,shell]
-----
-include::example$BASH/curl_install.sh[]
-----
-
-NOTE: The mirrors only host the latest versions of each major supported
-release. To download an earlier version of Cassandra, visit the
-http://archive.apache.org/dist/cassandra/[Apache Archives].
-
-[arabic, start=3]
-. OPTIONAL: Verify the integrity of the downloaded tarball using one of
-the methods https://www.apache.org/dyn/closer.cgi#verify[here]. For
-example, to verify the hash of the downloaded file using GPG:
-
-[{tabs}]
-====
-Command::
-+
---
-[source,shell]
-----
-include::example$BASH/verify_gpg.sh[]
-----
---
-
-Result::
-+
---
-[source,plaintext]
-----
-include::example$RESULTS/verify_gpg.result[]
-----
---
-====
-
-Compare the signature with the SHA256 file from the Downloads site:
-
-[{tabs}]
-====
-Command::
-+
---
-[source,shell]
-----
-include::example$BASH/curl_verify_sha.sh[]
-----
---
-
-Result::
-+
---
-[source,plaintext]
-----
-28757dde589f70410f9a6a95c39ee7e6cde63440e2b06b91ae6b200614fa364d
-----
---
-====
-
-[arabic, start=4]
-. Unpack the tarball:
-
-[source,shell]
-----
-include::example$BASH/tarball.sh[]
-----
-
-The files will be extracted to the `apache-cassandra-4.0.0/` directory.
-This is the tarball installation location.
-
-[arabic, start=5]
-. Located in the tarball installation location are the directories for
-the scripts, binaries, utilities, configuration, data and log files:
-
-[source,plaintext]
-----
-include::example$TEXT/tarball_install_dirs.txt[]
-----
-<1> location of the commands to run cassandra, cqlsh, nodetool, and SSTable tools
-<2> location of cassandra.yaml and other configuration files
-<3> location of the commit logs, hints, and SSTables
-<4> location of system and debug logs
-<5>location of cassandra-stress tool
-
-For information on how to configure your installation, see
-{cass_url}doc/latest/getting_started/configuring.html[Configuring
-Cassandra].
-
-[arabic, start=6]
-. Start Cassandra:
-
-[source,shell]
-----
-include::example$BASH/start_tarball.sh[]
-----
-
-NOTE: This will run Cassandra as the authenticated Linux user.
-
-include::partial$tail_syslog.adoc[]
-You can monitor the progress of the startup with:
-
-[{tabs}]
-====
-Command::
-+
---
-[source,shell]
-----
-include::example$BASH/tail_syslog.sh[]
-----
---
-
-Result::
-+
---
-Cassandra is ready when you see an entry like this in the `system.log`:
-
-[source,plaintext]
-----
-include::example$RESULTS/tail_syslog.result[]
-----
---
-====
-
-include::partial$nodetool_and_cqlsh.adoc[]
-
-== Installing the Debian packages
-
-include::partial$java_version.adoc[]
-
-[arabic, start=2]
-. Add the Apache repository of Cassandra to the file
-`cassandra.sources.list`.
-include::partial$package_versions.adoc[]
-
-[source,shell]
-----
-include::example$BASH/get_deb_package.sh[]
-----
-
-[arabic, start=3]
-. Add the Apache Cassandra repository keys to the list of trusted keys
-on the server:
-
-[{tabs}]
-====
-Command::
-+
---
-[source,shell]
-----
-include::example$BASH/add_repo_keys.sh[]
-----
---
-
-Result::
-+
---
-[source,plaintext]
-----
-include::example$RESULTS/add_repo_keys.result[]
-----
---
-====
-
-[arabic, start=4]
-. Update the package index from sources:
-
-[source,shell]
-----
-include::example$BASH/apt-get_update.sh[]
-----
-
-[arabic, start=5]
-. Install Cassandra with APT:
-
-[source,shell]
-----
-include::example$BASH/apt-get_cass.sh[]
-----
-
-NOTE: For information on how to configure your installation, see
-{cass_url}doc/latest/getting_started/configuring.html[Configuring
-Cassandra].
-
-include::partial$tail_syslog.adoc[]
-
-include::partial$nodetool_and_cqlsh_nobin.adoc[]
-
-== Installing the RPM packages
-
-include::partial$java_version.adoc[]
-
-[arabic, start=2]
-. Add the Apache repository of Cassandra to the file
-`/etc/yum.repos.d/cassandra.repo` (as the `root` user).
-include::partial$package_versions.adoc[]
-
-[source,plaintext]
-----
-include::example$RESULTS/add_yum_repo.result[]
-----
-
-[arabic, start=3]
-. Update the package index from sources:
-
-[source,shell]
-----
-include::example$BASH/yum_update.sh[]
-----
-
-[arabic, start=4]
-. Install Cassandra with YUM:
-
-[source,shell]
-----
-include::example$BASH/yum_cass.sh[]
-----
-
-NOTE: A new Linux user `cassandra` will get created as part of the
-installation. The Cassandra service will also be run as this user.
-
-[arabic, start=5]
-. Start the Cassandra service:
-
-[source,shell]
-----
-include::example$BASH/yum_start.sh[]
-----
-
-include::partial$tail_syslog.adoc[]
-
-include::partial$nodetool_and_cqlsh_nobin.adoc[]
-
-== Further installation info
-
-For help with installation issues, see the
-{cass_url}doc/latest/troubleshooting/index.html[Troubleshooting]
-section.
diff --git a/doc/modules/cassandra/pages/getting_started/java11.adoc b/doc/modules/cassandra/pages/getting_started/java11.adoc
deleted file mode 100644
index a61a57f..0000000
--- a/doc/modules/cassandra/pages/getting_started/java11.adoc
+++ /dev/null
@@ -1,290 +0,0 @@
-= Support for Java 11
-
-In the new Java release cadence a new Java version is made available
-every six months. The more frequent release cycle is favored as it
-brings new Java features to the developers as and when they are
-developed without the wait that the earlier 3 year release model
-incurred. Not every Java version is a Long Term Support (LTS) version.
-After Java 8 the next LTS version is Java 11. Java 9, 10, 12 and 13 are
-all non-LTS versions.
-
-One of the objectives of the Apache Cassandra 4.0 version is to support
-the recent LTS Java versions 8 and 11
-(https://issues.apache.org/jira/browse/CASSANDRA-9608[CASSANDRA-9608]).
-Java 8 and Java 11 may be used to build and run Apache Cassandra 4.0. Effective Cassandra
-4.0.2 there is full Java 11 support, it is not experimental anymore.
-
-== Support Matrix
-
-The support matrix for the Java versions for compiling and running
-Apache Cassandra 4.0 is detailed in Table 1. The build version is along
-the vertical axis and the run version is along the horizontal axis.
-
-Table 1 : Support Matrix for Java
-
-[width="68%",cols="34%,30%,36%",]
-|===
-| |Java 8 (Run) |Java 11 (Run)
-|Java 8 (Build) |Supported |Supported
-|Java 11(Build) |Not Supported |Supported
-|===
-
-Essentially Apache 4.0 source code built with Java 11 cannot be run with
-Java 8. Next, we shall discuss using each of Java 8 and 11 to build and
-run Apache Cassandra 4.0.
-
-== Using Java 8 to Build
-
-To start with, install Java 8. As an example, for installing Java 8 on
-RedHat Linux the command is as follows:
-
-....
-$ sudo yum install java-1.8.0-openjdk-devel
-....
-
-Set `JAVA_HOME` and `JRE_HOME` environment variables in the shell bash
-script. First, open the bash script:
-
-....
-$ sudo vi ~/.bashrc
-....
-
-Set the environment variables including the `PATH`.
-
-....
-$ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
-$ export JRE_HOME=/usr/lib/jvm/java-1.8.0-openjdk/jre
-$ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
-....
-
-Download and install Apache Cassandra 4.0 source code from the Git along
-with the dependencies.
-
-....
-$ git clone https://github.com/apache/cassandra.git
-....
-
-If Cassandra is already running stop Cassandra with the following
-command.
-
-....
-[ec2-user@ip-172-30-3-146 bin]$ ./nodetool stopdaemon
-....
-
-Build the source code from the `cassandra` directory, which has the
-`build.xml` build script. The Apache Ant uses the Java version set in
-the `JAVA_HOME` environment variable.
-
-....
-$ cd ~/cassandra
-$ ant
-....
-
-Apache Cassandra 4.0 gets built with Java 8. Set the environment
-variable for `CASSANDRA_HOME` in the bash script. Also add the
-`CASSANDRA_HOME/bin` to the `PATH` variable.
-
-....
-$ export CASSANDRA_HOME=~/cassandra
-$ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$CASSANDRA_HOME/bin
-....
-
-To run Apache Cassandra 4.0 with either of Java 8 or Java 11 run the
-Cassandra application in the `CASSANDRA_HOME/bin` directory, which is in
-the `PATH` env variable.
-
-....
-$ cassandra
-....
-
-The Java version used to run Cassandra gets output as Cassandra is
-getting started. As an example if Java 11 is used, the run output should
-include similar to the following output snippet:
-
-....
-INFO [main] 2019-07-31 21:18:16,862 CassandraDaemon.java:480 - Hostname: ip-172-30-3-
-146.ec2.internal:7000:7001
-INFO [main] 2019-07-31 21:18:16,862 CassandraDaemon.java:487 - JVM vendor/version: OpenJDK
-64-Bit Server VM/11.0.3
-INFO [main] 2019-07-31 21:18:16,863 CassandraDaemon.java:488 - Heap size:
-1004.000MiB/1004.000MiB
-....
-
-The following output indicates a single node Cassandra 4.0 cluster has
-started.
-
-....
-INFO [main] 2019-07-31 21:18:19,687 InboundConnectionInitiator.java:130 - Listening on
-address: (127.0.0.1:7000), nic: lo, encryption: enabled (openssl)
-...
-...
-INFO [main] 2019-07-31 21:18:19,850 StorageService.java:512 - Unable to gossip with any
-peers but continuing anyway since node is in its own seed list
-INFO [main] 2019-07-31 21:18:19,864 StorageService.java:695 - Loading persisted ring state
-INFO [main] 2019-07-31 21:18:19,865 StorageService.java:814 - Starting up server gossip
-INFO [main] 2019-07-31 21:18:20,088 BufferPool.java:216 - Global buffer pool is enabled,
-when pool is exhausted (max is 251.000MiB) it will allocate on heap
-INFO [main] 2019-07-31 21:18:20,110 StorageService.java:875 - This node will not auto
-bootstrap because it is configured to be a seed node.
-...
-...
-INFO [main] 2019-07-31 21:18:20,809 StorageService.java:1507 - JOINING: Finish joining ring
-INFO [main] 2019-07-31 21:18:20,921 StorageService.java:2508 - Node 127.0.0.1:7000 state
-jump to NORMAL
-....
-
-== Using Java 11 to Build
-
-If Java 11 is used to build Apache Cassandra 4.0, first Java 11 must be
-installed and the environment variables set. As an example, to download
-and install Java 11 on RedHat Linux run the following command.
-
-....
-$ yum install java-11-openjdk-devel
-....
-
-Set the environment variables in the bash script for Java 11. The first
-command is to open the bash script.
-
-....
-$ sudo vi ~/.bashrc
-$ export JAVA_HOME=/usr/lib/jvm/java-11-openjdk
-$ export JRE_HOME=/usr/lib/jvm/java-11-openjdk/jre
-$ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
-....
-
-To build source code with Java 11 one of the following two options must
-be used.
-
-____
-[arabic]
-. {blank}
-+
-Include Apache Ant command-line option `-Duse.jdk=11` as follows:::
-....
-$ ant -Duse.jdk=11
-....
-. {blank}
-+
-Set environment variable `CASSANDRA_USE_JDK11` to `true`:::
-....
-$ export CASSANDRA_USE_JDK11=true
-....
-____
-
-As an example, set the environment variable `CASSANDRA_USE_JDK11` to
-`true`.
-
-....
-[ec2-user@ip-172-30-3-146 cassandra]$ export CASSANDRA_USE_JDK11=true
-[ec2-user@ip-172-30-3-146 cassandra]$ ant
-Buildfile: /home/ec2-user/cassandra/build.xml
-....
-
-Or, set the command-line option.
-
-....
-[ec2-user@ip-172-30-3-146 cassandra]$ ant -Duse.jdk11=true
-....
-
-The build output should include the following.
-
-....
-_build_java:
- [echo] Compiling for Java 11
-...
-...
-build:
-
-_main-jar:
- [copy] Copying 1 file to /home/ec2-user/cassandra/build/classes/main/META-INF
- [jar] Building jar: /home/ec2-user/cassandra/build/apache-cassandra-4.0-SNAPSHOT.jar
-...
-...
-_build-test:
- [javac] Compiling 739 source files to /home/ec2-user/cassandra/build/test/classes
- [copy] Copying 25 files to /home/ec2-user/cassandra/build/test/classes
-...
-...
-jar:
- [mkdir] Created dir: /home/ec2-user/cassandra/build/classes/stress/META-INF
- [mkdir] Created dir: /home/ec2-user/cassandra/build/tools/lib
- [jar] Building jar: /home/ec2-user/cassandra/build/tools/lib/stress.jar
- [mkdir] Created dir: /home/ec2-user/cassandra/build/classes/fqltool/META-INF
- [jar] Building jar: /home/ec2-user/cassandra/build/tools/lib/fqltool.jar
-
-BUILD SUCCESSFUL
-Total time: 1 minute 3 seconds
-[ec2-user@ip-172-30-3-146 cassandra]$
-....
-
-== Common Issues
-
-One of the two options mentioned must be used to compile with JDK 11 or
-the build fails and the following error message is output.
-
-....
-[ec2-user@ip-172-30-3-146 cassandra]$ ant
-Buildfile: /home/ec2-user/cassandra/build.xml
-validate-build-conf:
-
-BUILD FAILED
-/home/ec2-user/cassandra/build.xml:293: -Duse.jdk11=true or $CASSANDRA_USE_JDK11=true must
-be set when building from java 11
-Total time: 1 second
-[ec2-user@ip-172-30-3-146 cassandra]$
-....
-
-The Java 11 built Apache Cassandra 4.0 source code may be run with Java
-11 only. If a Java 11 built code is run with Java 8 the following error
-message gets output.
-
-....
-[root@localhost ~]# ssh -i cassandra.pem ec2-user@ec2-3-85-85-75.compute-1.amazonaws.com
-Last login: Wed Jul 31 20:47:26 2019 from 75.155.255.51
-[ec2-user@ip-172-30-3-146 ~]$ echo $JAVA_HOME
-/usr/lib/jvm/java-1.8.0-openjdk
-[ec2-user@ip-172-30-3-146 ~]$ cassandra
-...
-...
-Error: A JNI error has occurred, please check your installation and try again
-Exception in thread "main" java.lang.UnsupportedClassVersionError:
-org/apache/cassandra/service/CassandraDaemon has been compiled by a more recent version of
-the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes
-class file versions up to 52.0
- at java.lang.ClassLoader.defineClass1(Native Method)
- at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
- at ...
-...
-....
-
-The `CASSANDRA_USE_JDK11` variable or the command-line option
-`-Duse.jdk11` cannot be used to build with Java 8. To demonstrate set
-`JAVA_HOME` to version 8.
-
-....
-[root@localhost ~]# ssh -i cassandra.pem ec2-user@ec2-3-85-85-75.compute-1.amazonaws.com
-Last login: Wed Jul 31 21:41:50 2019 from 75.155.255.51
-[ec2-user@ip-172-30-3-146 ~]$ echo $JAVA_HOME
-/usr/lib/jvm/java-1.8.0-openjdk
-....
-
-Set the `CASSANDRA_USE_JDK11=true` or command-line option
-`-Duse.jdk11=true`. Subsequently, run Apache Ant to start the build. The
-build fails with error message listed.
-
-....
-[ec2-user@ip-172-30-3-146 ~]$ cd
-cassandra
-[ec2-user@ip-172-30-3-146 cassandra]$ export CASSANDRA_USE_JDK11=true
-[ec2-user@ip-172-30-3-146 cassandra]$ ant
-Buildfile: /home/ec2-user/cassandra/build.xml
-
-validate-build-conf:
-
-BUILD FAILED
-/home/ec2-user/cassandra/build.xml:285: -Duse.jdk11=true or $CASSANDRA_USE_JDK11=true cannot
-be set when building from java 8
-
-Total time: 0 seconds
-....
diff --git a/doc/modules/cassandra/pages/getting_started/production.adoc b/doc/modules/cassandra/pages/getting_started/production.adoc
deleted file mode 100644
index 93b2608..0000000
--- a/doc/modules/cassandra/pages/getting_started/production.adoc
+++ /dev/null
@@ -1,163 +0,0 @@
-= Production recommendations
-
-The `cassandra.yaml` and `jvm.options` files have a number of notes and
-recommendations for production usage.
-This page expands on some of the information in the files.
-
-== Tokens
-
-Using more than one token-range per node is referred to as virtual nodes, or vnodes.
-`vnodes` facilitate flexible expansion with more streaming peers when a new node bootstraps
-into a cluster.
-Limiting the negative impact of streaming (I/O and CPU overhead) enables incremental cluster expansion.
-However, more tokens leads to sharing data with more peers, and results in decreased availability.
-These two factors must be balanced based on a cluster's characteristic reads and writes.
-To learn more,
-https://github.com/jolynch/python_performance_toolkit/raw/master/notebooks/cassandra_availability/whitepaper/cassandra-availability-virtual.pdf[Cassandra Availability in Virtual Nodes, Joseph Lynch and Josh Snyder] is recommended reading.
-
-Change the number of tokens using the setting in the `cassandra.yaml` file:
-
-`num_tokens: 16`
-
-Here are the most common token counts with a brief explanation of when
-and why you would use each one.
-
-[width="100%",cols="13%,87%",options="header",]
-|===
-|Token Count |Description
-|1 |Maximum availablility, maximum cluster size, fewest peers, but
-inflexible expansion. Must always double size of cluster to expand and
-remain balanced.
-
-|4 |A healthy mix of elasticity and availability. Recommended for
-clusters which will eventually reach over 30 nodes. Requires adding
-approximately 20% more nodes to remain balanced. Shrinking a cluster may
-result in cluster imbalance.
-
-|8 | Using 8 vnodes distributes the workload between systems with a ~10% variance
-and has minimal impact on performance.
-
-|16 |Best for heavily elastic clusters which expand and shrink
-regularly, but may have issues availability with larger clusters. Not
-recommended for clusters over 50 nodes.
-|===
-
-In addition to setting the token count, it's extremely important that
-`allocate_tokens_for_local_replication_factor` in `cassandra.yaml` is set to an
-appropriate number of replicates, to ensure even token allocation.
-
-== Read ahead
-
-Read ahead is an operating system feature that attempts to keep as much
-data as possible loaded in the page cache.
-Spinning disks can have long seek times causing high latency, so additional
-throughput on reads using page cache can improve performance.
-By leveraging read ahead, the OS can pull additional data into memory without
-the cost of additional seeks.
-This method works well when the available RAM is greater than the size of the
-hot dataset, but can be problematic when the reverse is true (dataset > RAM).
-The larger the hot dataset, the less read ahead is useful.
-
-Read ahead is definitely not useful in the following cases:
-
-* Small partitions, such as tables with a single partition key
-* Solid state drives (SSDs)
-
-
-Read ahead can actually increase disk usage, and in some cases result in as much
-as a 5x latency and throughput performance penalty.
-Read-heavy, key/value tables with small (under 1KB) rows are especially prone
-to this problem.
-
-The recommended read ahead settings are:
-
-[width="59%",cols="40%,60%",options="header",]
-|===
-|Hardware |Initial Recommendation
-|Spinning Disks |64KB
-|SSD |4KB
-|===
-
-Read ahead can be adjusted on Linux systems using the `blockdev` tool.
-
-For example, set the read ahead of the disk `/dev/sda1\` to 4KB:
-
-[source, shell]
-----
-$ blockdev --setra 8 /dev/sda1
-----
-[NOTE]
-====
-The `blockdev` setting sets the number of 512 byte sectors to read ahead.
-The argument of 8 above is equivalent to 4KB, or 8 * 512 bytes.
-====
-
-All systems are different, so use these recommendations as a starting point and
-tune, based on your SLA and throughput requirements.
-To understand how read ahead impacts disk resource usage, we recommend carefully
-reading through the xref:troubleshooting/use_tools.adoc[Diving Deep, using external tools]
-section.
-
-== Compression
-
-Compressed data is stored by compressing fixed size byte buffers and writing the
-data to disk.
-The buffer size is determined by the `chunk_length_in_kb` element in the compression
-map of a table's schema settings for `WITH COMPRESSION`.
-The default setting is 16KB starting with Cassandra {40_version}.
-
-Since the entire compressed buffer must be read off-disk, using a compression
-chunk length that is too large can lead to significant overhead when reading small records.
-Combined with the default read ahead setting, the result can be massive
-read amplification for certain workloads. Therefore, picking an appropriate
-value for this setting is important.
-
-LZ4Compressor is the default and recommended compression algorithm.
-If you need additional information on compression, read
-https://thelastpickle.com/blog/2018/08/08/compression_performance.html[The Last Pickle blogpost on compression performance].
-
-== Compaction
-
-There are different xref:compaction/index.adoc[compaction] strategies available
-for different workloads.
-We recommend reading about the different strategies to understand which is the
-best for your environment.
-Different tables may, and frequently do use different compaction strategies in
-the same cluster.
-
-== Encryption
-
-It is significantly better to set up peer-to-peer encryption and client server
-encryption when setting up your production cluster.
-Setting it up after the cluster is serving production traffic is challenging
-to do correctly.
-If you ever plan to use network encryption of any type, we recommend setting it
-up when initially configuring your cluster.
-Changing these configurations later is not impossible, but mistakes can
-result in downtime or data loss.
-
-== Ensure keyspaces are created with NetworkTopologyStrategy
-
-Production clusters should never use `SimpleStrategy`.
-Production keyspaces should use the `NetworkTopologyStrategy` (NTS).
-For example:
-
-[source, cql]
-----
-CREATE KEYSPACE mykeyspace WITH replication = {
- 'class': 'NetworkTopologyStrategy',
- 'datacenter1': 3
-};
-----
-
-Cassandra clusters initialized with `NetworkTopologyStrategy` can take advantage
-of the ability to configure multiple racks and data centers.
-
-== Configure racks and snitch
-
-**Correctly configuring or changing racks after a cluster has been provisioned is an unsupported process**.
-Migrating from a single rack to multiple racks is also unsupported and can
-result in data loss.
-Using `GossipingPropertyFileSnitch` is the most flexible solution for on
-premise or mixed cloud environments.
-`Ec2Snitch` is reliable for AWS EC2 only environments.
diff --git a/doc/modules/cassandra/pages/getting_started/querying.adoc b/doc/modules/cassandra/pages/getting_started/querying.adoc
deleted file mode 100644
index a8b348a..0000000
--- a/doc/modules/cassandra/pages/getting_started/querying.adoc
+++ /dev/null
@@ -1,31 +0,0 @@
-= Inserting and querying
-
-The API for Cassandra is xref:cql/ddl.adoc[`CQL`, the Cassandra Query Language]. To
-use CQL, you will need to connect to the cluster, using either:
-
-* `cqlsh`, a shell for CQL
-* a client driver for Cassandra
-* for the adventurous, check out https://zeppelin.apache.org/docs/0.7.0/interpreter/cassandra.html[Apache Zeppelin], a notebook-style tool
-
-== CQLSH
-
-`cqlsh` is a command-line shell for interacting with Cassandra using
-CQL. It is shipped with every Cassandra package, and can be found in the
-`bin` directory alongside the `cassandra` executable. It connects to the
-single node specified on the command line. For example:
-
-[source, shell]
-----
-include::example$BASH/cqlsh_localhost.sh[]
-----
-[source, cql]
-----
-include::example$RESULTS/cqlsh_localhost.result[]
-----
-If the command is used without specifying a node, `localhost` is the default. See the xref:tools/cqlsh.adoc[`cqlsh` section] for full documentation.
-
-== Client drivers
-
-A lot of xref:getting_started/drivers.adoc[client drivers] are provided by the Community and a list of
-known drivers is provided. You should refer to the documentation of each driver
-for more information.
diff --git a/doc/modules/cassandra/pages/getting_started/quickstart.adoc b/doc/modules/cassandra/pages/getting_started/quickstart.adoc
deleted file mode 100644
index 69b55a6..0000000
--- a/doc/modules/cassandra/pages/getting_started/quickstart.adoc
+++ /dev/null
@@ -1,116 +0,0 @@
-= Apache Cassandra Quickstart
-:tabs:
-
-_Interested in getting started with Cassandra? Follow these instructions._
-
-*STEP 1: GET CASSANDRA USING DOCKER*
-
-You'll need to have Docker Desktop for Mac, Docker Desktop for Windows, or
-similar software installed on your computer.
-
-[source, plaintext]
-----
-docker pull cassandra:latest
-----
-
-Apache Cassandra is also available as a https://cassandra.apache.org/download/[tarball or package download].
-
-*STEP 2: START CASSANDRA*
-
-[source, plaintext]
-----
-docker run --name cassandra cassandra
-----
-
-*STEP 3: CREATE FILES*
-
-In the directory where you plan to run the next step, create these two files
-so that some data can be automatically inserted in the next step.
-
-A _cqlshrc_ file will log into the Cassandra database with the default superuser:
-
-[source, plaintext]
-----
-[authentication]
- username = cassandra
- password = cassandra
-----
-
-Create a _scripts_ directory and change to that directory.
-The following _data.cql_ file will create a keyspace, the layer at which Cassandra
-replicates its data, a table to hold the data, and insert some data:
-
-[source, plaintext]
-----
-# Create a keyspace
-CREATE KEYSPACE IF NOT EXISTS store WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : '1' };
-
-# Create a table
-CREATE TABLE IF NOT EXISTS store.shopping_cart (
- userid text PRIMARY KEY,
- item_count int,
- last_update_timestamp timestamp
-);
-
-# Insert some data
-INSERT INTO store.shopping_cart
-(userid, item_count, last_update_timestamp)
-VALUES ('9876', 2, toTimeStamp(toDate(now))));
-INSERT INTO store.shopping_cart
-(userid, item_count, last_update_timestamp)
-VALUES (1234, 5, toTimeStamp(toDate(now))));
-----
-
-You should now have a _cqlshrc_ file and _<currentdir>/scripts/data.cql_ file.
-
-*STEP 4: RUN CQLSH TO INTERACT*
-
-Cassandra is a distributed database that can read and write data across multiple
-nodes with peer-to-peer replication. The Cassandra Query Language (CQL) is
-similar to SQL but suited for the JOINless structure of Cassandra. The CQL
-shell, or `cqlsh`, is one tool to use in interacting with the database.
-
-[source, plaintext]
-----
-docker run --rm -it -v /<currentdir>/scripts:/scripts \
--v /<currentdir/cqlshrc:/.cassandra/cqlshrc \
---env CQLSH_HOST=host.docker.internal --env CQLSH_PORT=9042 nuvo/docker-cqlsh
-----
-
-For this quickstart, this cqlsh docker image also loads some data automatically,
-so you can start running queries.
-
-*STEP 5: READ SOME DATA*
-
-[source, plaintext]
-----
-SELECT * FROM store.shopping_cart;
-----
-
-*STEP 6: WRITE SOME MORE DATA*
-
-[source, plaintext]
-----
-INSERT (userid, item_count) VALUES (4567, 20) INTO store.shopping_cart;
-----
-
-*STEP 7: TERMINATE CASSANDRA*
-
-[source, plaintext]
-----
-docker rm cassandra
-----
-
-*CONGRATULATIONS!*
-
-Hey, that wasn't so hard, was it?
-
-To learn more, we suggest the following next steps:
-
-* Read through the *need link*[Overview] to learn main concepts and how Cassandra works at a
-high level.
-* To understand Cassandra in more detail, head over to the
-https://cassandra.apache.org/doc/latest/[Docs].
-* Browse through the https://cassandra.apache.org/case-studies/[Case Studies] to
-learn how other users in our worldwide community are getting value out of
-Cassandra.
diff --git a/doc/modules/cassandra/pages/installing/installing.adoc b/doc/modules/cassandra/pages/installing/installing.adoc
new file mode 100644
index 0000000..90c3089
--- /dev/null
+++ b/doc/modules/cassandra/pages/installing/installing.adoc
@@ -0,0 +1,345 @@
+= Installing Cassandra
+:tabs:
+
+These are the instructions for deploying the supported releases of
+Apache Cassandra on Linux servers.
+
+Cassandra runs on a wide array of Linux distributions including (but not
+limited to):
+
+* Ubuntu, most notably LTS releases 16.04 to 18.04
+* CentOS & RedHat Enterprise Linux (RHEL) including 6.6 to 7.7
+* Amazon Linux AMIs including 2016.09 through to Linux 2
+* Debian versions 8 & 9
+* SUSE Enterprise Linux 12
+
+This is not an exhaustive list of operating system platforms, nor is it
+prescriptive. However, users will be well-advised to conduct exhaustive
+tests of their own particularly for less-popular distributions of Linux.
+Deploying on older versions is not recommended unless you have previous
+experience with the older distribution in a production environment.
+
+== Prerequisites
+
+* Install the latest version of Java 8 or Java 11, either the
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[Oracle
+Java Standard Edition 8] / http://www.oracle.com/technetwork/java/javase/downloads/index.html[Oracle Java Standard Edition 11 (Long Term Support)]
+or http://openjdk.java.net/[OpenJDK 8] / http://openjdk.java.net/[OpenJDK 11]. To
+verify that you have the correct version of java installed, type
+`java -version`.
+* *NOTE*: Experimental support for Java 11 was added in Cassandra {40_version}
+(https://issues.apache.org/jira/browse/CASSANDRA-9608[CASSANDRA-9608]).
+Full support is effective Cassandra 4.0.2 version (https://issues.apache.org/jira/browse/CASSANDRA-16894[CASSANDRA-16894])
+For more information, see
+https://github.com/apache/cassandra/blob/trunk/NEWS.txt[NEWS.txt].
+* For using cqlsh, the latest version of
+Python 3.6+ or Python 2.7 (support deprecated). To verify
+that you have the correct version of Python installed, type
+`python --version`.
+
+== Choosing an installation method
+
+There are three methods of installing Cassandra that are common:
+
+* Docker image
+* Tarball binary file
+* Package installation (RPM, YUM)
+
+If you are a current Docker user, installing a Docker image is simple.
+You'll need to install Docker Desktop for Mac, Docker Desktop for Windows,
+or have `docker` installed on Linux.
+Pull the appropriate image and then start Cassandra with a run command.
+
+For most users, installing the binary tarball is also a simple choice.
+The tarball unpacks all its contents into a single location with
+binaries and configuration files located in their own subdirectories.
+The most obvious attribute of the tarball installation is it does not
+require `root` permissions and can be installed on any Linux
+distribution.
+
+Packaged installations require `root` permissions, and are most appropriate for
+production installs.
+Install the RPM build on CentOS and RHEL-based distributions if you want to
+install Cassandra using YUM.
+Install the Debian build on Ubuntu and other Debian-based
+distributions if you want to install Cassandra using APT.
+Note that both the YUM and APT methods required `root` permissions and
+will install the binaries and configuration files as the `cassandra` OS user.
+
+== Installing the docker image
+
+[arabic, start=1]
+. Pull the docker image. For the latest image, use:
+
+[source, shell]
+----
+include::example$BASH/docker_pull.sh[]
+----
+
+This `docker pull` command will get the latest version of the 'Docker Official'
+Apache Cassandra image available from the https://hub.docker.com/_/cassandra[Dockerhub].
+
+[arabic, start=2]
+. Start Cassandra with a `docker run` command:
+
+[source, shell]
+----
+include::example$BASH/docker_run.sh[]
+----
+
+The `--name` option will be the name of the Cassandra cluster created.
+
+[arabic, start=3]
+. Start the CQL shell, `cqlsh` to interact with the Cassandra node created:
+
+[source, shell]
+----
+include::example$BASH/docker_cqlsh.sh[]
+----
+== Installing the binary tarball
+
+include::partial$java_version.adoc[]
+
+[arabic, start=2]
+. Download the binary tarball from one of the mirrors on the
+{cass_url}download/[Apache Cassandra Download] site.
+For example, to download Cassandra {40_version}:
+
+[source,shell]
+----
+include::example$BASH/curl_install.sh[]
+----
+
+NOTE: The mirrors only host the latest versions of each major supported
+release. To download an earlier version of Cassandra, visit the
+http://archive.apache.org/dist/cassandra/[Apache Archives].
+
+[arabic, start=3]
+. OPTIONAL: Verify the integrity of the downloaded tarball using one of
+the methods https://www.apache.org/dyn/closer.cgi#verify[here]. For
+example, to verify the hash of the downloaded file using GPG:
+
+[{tabs}]
+====
+Command::
++
+--
+[source,shell]
+----
+include::example$BASH/verify_gpg.sh[]
+----
+--
+
+Result::
++
+--
+[source,plaintext]
+----
+include::example$RESULTS/verify_gpg.result[]
+----
+--
+====
+
+Compare the signature with the SHA256 file from the Downloads site:
+
+[{tabs}]
+====
+Command::
++
+--
+[source,shell]
+----
+include::example$BASH/curl_verify_sha.sh[]
+----
+--
+
+Result::
++
+--
+[source,plaintext]
+----
+28757dde589f70410f9a6a95c39ee7e6cde63440e2b06b91ae6b200614fa364d
+----
+--
+====
+
+[arabic, start=4]
+. Unpack the tarball:
+
+[source,shell]
+----
+include::example$BASH/tarball.sh[]
+----
+
+The files will be extracted to the `apache-cassandra-4.0.0/` directory.
+This is the tarball installation location.
+
+[arabic, start=5]
+. Located in the tarball installation location are the directories for
+the scripts, binaries, utilities, configuration, data and log files:
+
+[source,plaintext]
+----
+include::example$TEXT/tarball_install_dirs.txt[]
+----
+<1> location of the commands to run cassandra, cqlsh, nodetool, and SSTable tools
+<2> location of cassandra.yaml and other configuration files
+<3> location of the commit logs, hints, and SSTables
+<4> location of system and debug logs
+<5>location of cassandra-stress tool
+
+For information on how to configure your installation, see
+{cass_url}doc/latest/getting-started/configuring.html[Configuring
+Cassandra].
+
+[arabic, start=6]
+. Start Cassandra:
+
+[source,shell]
+----
+include::example$BASH/start_tarball.sh[]
+----
+
+NOTE: This will run Cassandra as the authenticated Linux user.
+
+include::partial$tail_syslog.adoc[]
+You can monitor the progress of the startup with:
+
+[{tabs}]
+====
+Command::
++
+--
+[source,shell]
+----
+include::example$BASH/tail_syslog.sh[]
+----
+--
+
+Result::
++
+--
+Cassandra is ready when you see an entry like this in the `system.log`:
+
+[source,plaintext]
+----
+include::example$RESULTS/tail_syslog.result[]
+----
+--
+====
+
+include::partial$nodetool_and_cqlsh.adoc[]
+
+== Installing the Debian packages
+
+include::partial$java_version.adoc[]
+
+[arabic, start=2]
+. Add the Apache repository of Cassandra to the file
+`cassandra.sources.list`.
+include::partial$package_versions.adoc[]
+
+[source,shell]
+----
+include::example$BASH/get_deb_package.sh[]
+----
+
+[arabic, start=3]
+. Add the Apache Cassandra repository keys to the list of trusted keys
+on the server:
+
+[{tabs}]
+====
+Command::
++
+--
+[source,shell]
+----
+include::example$BASH/add_repo_keys.sh[]
+----
+--
+
+Result::
++
+--
+[source,plaintext]
+----
+include::example$RESULTS/add_repo_keys.result[]
+----
+--
+====
+
+[arabic, start=4]
+. Update the package index from sources:
+
+[source,shell]
+----
+include::example$BASH/apt-get_update.sh[]
+----
+
+[arabic, start=5]
+. Install Cassandra with APT:
+
+[source,shell]
+----
+include::example$BASH/apt-get_cass.sh[]
+----
+
+NOTE: For information on how to configure your installation, see
+{cass_url}doc/latest/getting-started/configuring.html[Configuring
+Cassandra].
+
+include::partial$tail_syslog.adoc[]
+
+include::partial$nodetool_and_cqlsh_nobin.adoc[]
+
+== Installing the RPM packages
+
+include::partial$java_version.adoc[]
+
+[arabic, start=2]
+. Add the Apache repository of Cassandra to the file
+`/etc/yum.repos.d/cassandra.repo` (as the `root` user).
+include::partial$package_versions.adoc[]
+
+[source,plaintext]
+----
+include::example$RESULTS/add_yum_repo.result[]
+----
+
+[arabic, start=3]
+. Update the package index from sources:
+
+[source,shell]
+----
+include::example$BASH/yum_update.sh[]
+----
+
+[arabic, start=4]
+. Install Cassandra with YUM:
+
+[source,shell]
+----
+include::example$BASH/yum_cass.sh[]
+----
+
+NOTE: A new Linux user `cassandra` will get created as part of the
+installation. The Cassandra service will also be run as this user.
+
+[arabic, start=5]
+. Start the Cassandra service:
+
+[source,shell]
+----
+include::example$BASH/yum_start.sh[]
+----
+
+include::partial$tail_syslog.adoc[]
+
+include::partial$nodetool_and_cqlsh_nobin.adoc[]
+
+== Further installation info
+
+For help with installation issues, see the
+{cass_url}doc/latest/troubleshooting/index.html[Troubleshooting]
+section.
diff --git a/doc/modules/cassandra/pages/plugins/index.adoc b/doc/modules/cassandra/pages/integrating/plugins/index.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/plugins/index.adoc
rename to doc/modules/cassandra/pages/integrating/plugins/index.adoc
diff --git a/doc/modules/cassandra/pages/configuration/cass_cl_archive_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_cl_archive_file.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/cass_cl_archive_file.adoc
rename to doc/modules/cassandra/pages/managing/configuration/cass_cl_archive_file.adoc
diff --git a/doc/modules/cassandra/pages/configuration/cass_env_sh_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_env_sh_file.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/cass_env_sh_file.adoc
rename to doc/modules/cassandra/pages/managing/configuration/cass_env_sh_file.adoc
diff --git a/doc/modules/cassandra/pages/configuration/cass_jvm_options_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_jvm_options_file.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/cass_jvm_options_file.adoc
rename to doc/modules/cassandra/pages/managing/configuration/cass_jvm_options_file.adoc
diff --git a/doc/modules/cassandra/pages/managing/configuration/cass_logback_xml_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_logback_xml_file.adoc
new file mode 100644
index 0000000..7e64aea
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/configuration/cass_logback_xml_file.adoc
@@ -0,0 +1,197 @@
+= logback.xml file
+
+The `logback.xml` configuration file can optionally set logging levels
+for the logs written to `system.log` and `debug.log`. The logging levels
+can also be set using `nodetool setlogginglevels`.
+
+== Options
+
+=== `appender name="<appender_choice>"...</appender>`
+
+Specify log type and settings. Possible appender names are: `SYSTEMLOG`,
+`DEBUGLOG`, `ASYNCDEBUGLOG`, and `STDOUT`. `SYSTEMLOG` ensures that WARN
+and ERROR message are written synchronously to the specified file.
+`DEBUGLOG` and `ASYNCDEBUGLOG` ensure that DEBUG messages are written
+either synchronously or asynchronously, respectively, to the specified
+file. `STDOUT` writes all messages to the console in a human-readable
+format.
+
+*Example:* <appender name="SYSTEMLOG"
+class="ch.qos.logback.core.rolling.RollingFileAppender">
+
+=== `<file> <filename> </file>`
+
+Specify the filename for a log.
+
+*Example:* <file>$\{cassandra.logdir}/system.log</file>
+
+=== `<level> <log_level> </level>`
+
+Specify the level for a log. Part of the filter. Levels are: `ALL`,
+`TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `OFF`. `TRACE` creates the
+most verbose log, `ERROR` the least.
+
+[NOTE]
+.Note
+====
+Note: Increasing logging levels can generate heavy logging output on
+a moderately trafficked cluster. You can use the
+`nodetool getlogginglevels` command to see the current logging
+configuration.
+====
+
+*Default:* INFO
+
+*Example:* <level>INFO</level>
+
+=== `<rollingPolicy class="<rolling_policy_choice>" <fileNamePattern><pattern_info></fileNamePattern> ... </rollingPolicy>`
+
+Specify the policy for rolling logs over to an archive.
+
+*Example:* <rollingPolicy
+class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+
+=== `<fileNamePattern> <pattern_info> </fileNamePattern>`
+
+Specify the pattern information for rolling over the log to archive.
+Part of the rolling policy.
+
+*Example:*
+<fileNamePattern>$\{cassandra.logdir}/system.log.%d\{yyyy-MM-dd}.%i.zip</fileNamePattern>
+
+=== `<maxFileSize> <size> </maxFileSize>`
+
+Specify the maximum file size to trigger rolling a log. Part of the
+rolling policy.
+
+*Example:* <maxFileSize>50MB</maxFileSize>
+
+=== `<maxHistory> <number_of_days> </maxHistory>`
+
+Specify the maximum history in days to trigger rolling a log. Part of
+the rolling policy.
+
+*Example:* <maxHistory>7</maxHistory>
+
+=== `<encoder> <pattern>...</pattern> </encoder>`
+
+Specify the format of the message. Part of the rolling policy.
+
+*Example:* <maxHistory>7</maxHistory> *Example:* <encoder>
+<pattern>%-5level [%thread] %date\{ISO8601} %F:%L - %msg%n</pattern>
+</encoder>
+
+=== Logging to Cassandra virtual table
+
+It is possible to configure logback.xml in such a way that logs would appear in `system_views.system_log` table.
+This is achieved by appender implemented in class `VirtualTableAppender` which is called `CQLLOG` in the
+default configration. When the appender is commented out, no system logs are written to the virtual table.
+
+CQLLOG appender is special as the underlying structure it saves log messages into can not grow without any bound
+as a node would run out of memory. For this reason, `system_log` table is limited on its size.
+By default, it can hold at most 50 000 log messages, it can never hold more than 100 000 log messages.
+
+To specify how many rows you want that virtual table to hold at most, there is
+a system property called `cassandra.virtual.logs.max.rows` which takes an integer as value.
+
+You can execute CQL `truncate` query for `system_views.system_log` if you want to wipe out all the logs in virtual table
+to e.g. save some memory.
+
+It is recommended to set filter to at least `WARN` level so this table holds only important logging messages as
+each message will occupy memory.
+
+The appender to virtual table is commented out by default so logging to virtual table is not active.
+
+=== Contents of default `logback.xml`
+
+[source,XML]
+----
+<configuration scan="true" scanPeriod="60 seconds">
+ <jmxConfigurator />
+
+ <!-- No shutdown hook; we run it ourselves in StorageService after shutdown -->
+
+ <!-- SYSTEMLOG rolling file appender to system.log (INFO level) -->
+
+ <appender name="SYSTEMLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ <file>${cassandra.logdir}/system.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/system.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>7</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
+ </rollingPolicy>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- DEBUGLOG rolling file appender to debug.log (all levels) -->
+
+ <appender name="DEBUGLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${cassandra.logdir}/debug.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <!-- rollover daily -->
+ <fileNamePattern>${cassandra.logdir}/debug.log.%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
+ <!-- each file should be at most 50MB, keep 7 days worth of history, but at most 5GB -->
+ <maxFileSize>50MB</maxFileSize>
+ <maxHistory>7</maxHistory>
+ <totalSizeCap>5GB</totalSizeCap>
+ </rollingPolicy>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- ASYNCLOG assynchronous appender to debug.log (all levels) -->
+
+ <appender name="ASYNCDEBUGLOG" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>1024</queueSize>
+ <discardingThreshold>0</discardingThreshold>
+ <includeCallerData>true</includeCallerData>
+ <appender-ref ref="DEBUGLOG" />
+ </appender>
+
+ <!-- STDOUT console appender to stdout (INFO level) -->
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Uncomment bellow and corresponding appender-ref to activate logback metrics
+ <appender name="LogbackMetrics" class="com.codahale.metrics.logback.InstrumentedAppender" />
+ -->
+
+ <!-- Uncomment below configuration and corresponding appender-ref to activate
+ logging into system_views.system_logs virtual table. -->
+ <!-- <appender name="CQLLOG" class="org.apache.cassandra.utils.logging.VirtualTableAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>WARN</level>
+ </filter>
+ </appender> -->
+
+ <root level="INFO">
+ <appender-ref ref="SYSTEMLOG" />
+ <appender-ref ref="STDOUT" />
+ <appender-ref ref="ASYNCDEBUGLOG" /> <!-- Comment this line to disable debug.log -->
+ <!--
+ <appender-ref ref="LogbackMetrics" />
+ -->
+ <!--
+ <appender-ref ref="CQLLOG"/>
+ -->
+ </root>
+
+ <logger name="org.apache.cassandra" level="DEBUG"/>
+</configuration>
+----
diff --git a/doc/modules/cassandra/pages/configuration/cass_rackdc_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_rackdc_file.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/cass_rackdc_file.adoc
rename to doc/modules/cassandra/pages/managing/configuration/cass_rackdc_file.adoc
diff --git a/doc/modules/cassandra/pages/configuration/cass_topo_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_topo_file.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/cass_topo_file.adoc
rename to doc/modules/cassandra/pages/managing/configuration/cass_topo_file.adoc
diff --git a/doc/modules/cassandra/pages/configuration/configuration.adoc b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/configuration.adoc
rename to doc/modules/cassandra/pages/managing/configuration/configuration.adoc
diff --git a/doc/modules/cassandra/pages/configuration/index.adoc b/doc/modules/cassandra/pages/managing/configuration/index.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/configuration/index.adoc
rename to doc/modules/cassandra/pages/managing/configuration/index.adoc
diff --git a/doc/modules/cassandra/pages/operating/audit_logging.adoc b/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/audit_logging.adoc
rename to doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
diff --git a/doc/modules/cassandra/pages/managing/operating/auditlogging.adoc b/doc/modules/cassandra/pages/managing/operating/auditlogging.adoc
new file mode 100644
index 0000000..c83c5aa
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/auditlogging.adoc
@@ -0,0 +1,549 @@
+= Audit Logging
+
+Audit Logging is a new feature in Apache Cassandra 4.0 (https://issues.apache.org/jira/browse/CASSANDRA-12151[CASSANDRA-12151]).
+This new feature is safe for production use, with configurable limits to heap memory and disk space to prevent out-of-memory errors.
+All database activity is logged per-node as file-based records to a specified local filesystem directory.
+The audit log files are rolled periodically based on a configurable value.
+
+Some of the features of audit logging are:
+
+* No additional database capacity is needed to store audit logs.
+* No query tool is required to store the audit logs.
+* Latency of database operations is not affected, so there is no performance impact.
+* Heap memory usage is bounded by a weighted queue, with configurable maximum weight sitting in front of logging thread.
+* Disk utilization is bounded by a configurable size, deleting old log segments once the limit is reached.
+* Can be enabled or disabled at startup time using `cassandra.yaml` or at runtime using the JMX tool, ``nodetool``.
+* Can configure the settings in either the `cassandra.yaml` file or by using ``nodetool``.
+
+Audit logging includes all CQL requests, both successful and failed.
+It also captures all successful and failed authentication and authorization events, such as login attempts.
+The difference between Full Query Logging (FQL) and audit logging is that FQL captures only successful CQL requests, which allow replay or comparison of logs.
+Audit logs are useful for compliance and debugging, while FQL is useful for debugging, performance benchmarking, testing and auditing CQL queries.
+
+== Audit information logged
+
+The audit log contains:
+
+* all events in the configured keyspaces to include
+* all events in the configured categories to include
+* all events executed by the configured users to include
+
+The audit log does not contain:
+
+* configuration changes made in `cassandra.yaml` file
+* `nodetool` commands
+* Passwords mentioned as part of DCL statements: Passwords will be obfuscated as \*\*\*\*\*\*\*.
+ ** Statements that fail to parse will have everything after the appearance of the word password obfuscated as \*\*\*\*\*\*\*.
+ ** Statements with a mistyped word 'password' will be logged without obfuscation. Please make sure to use a different password on retries.
+
+The audit log is a series of log entries.
+An audit log entry contains:
+
+* keyspace (String) - Keyspace on which request is made
+* operation (String) - Database operation such as CQL command
+* user (String) - User name
+* scope (String) - Scope of request such as Table/Function/Aggregate name
+* type (AuditLogEntryType) - Type of request
+** CQL Audit Log Entry Type
+** Common Audit Log Entry Type
+* source (InetAddressAndPort) - Source IP Address from which request originated
+* timestamp (long ) - Timestamp of the request
+* batch (UUID) - Batch of request
+* options (QueryOptions) - CQL Query options
+* state (QueryState) - State related to a given query
+
+Each entry contains all applicable attributes for the given event, concatenated with a pipe (|).
+
+CQL audit log entry types are the following CQL commands. Each command is assigned to a particular specified category to log:
+
+[width="100%",cols="20%,80%",options="header",]
+|===
+| Category | CQL commands
+
+| DDL | ALTER_KEYSPACE, CREATE_KEYSPACE, DROP_KEYSPACE,
+ALTER_TABLE, CREATE_TABLE, DROP_TABLE,
+CREATE_FUNCTION, DROP_FUNCTION,
+CREATE_AGGREGATE, DROP_AGGREGATE,
+CREATE_INDEX, DROP_INDEX,
+ALTER_TYPE, CREATE_TYPE, DROP_TYPE,
+CREATE_TRIGGER, DROP_TRIGGER,
+ALTER_VIEW, CREATE_VIEW, DROP_VIEW,
+TRUNCATE
+| DML | BATCH, DELETE, UPDATE
+| DCL | GRANT, REVOKE,
+ALTER_ROLE, CREATE_ROLE, DROP_ROLE,
+LIST_ROLES, LIST_PERMISSIONS, LIST_USERS
+| OTHER | USE_KEYSPACE
+| QUERY | SELECT
+| PREPARE | PREPARE_STATEMENT
+|===
+
+Common audit log entry types are one of the following:
+
+[width="100%",cols="50%,50%",options="header",]
+|===
+| Category | CQL commands
+
+| AUTH | LOGIN_SUCCESS, LOGIN_ERROR, UNAUTHORIZED_ATTEMPT
+| ERROR | REQUEST_FAILURE
+|===
+
+== Availability and durability
+
+NOTE: Unlike data, audit log entries are not replicated
+
+For a given query, the corresponding audit entry is only stored on the coordinator node.
+For example, an ``INSERT`` in a keyspace with replication factor of 3 will produce an audit entry on one node, the coordinator who handled the request, and not on the two other nodes.
+For this reason, and depending on compliance requirements you must meet,
+make sure that audit logs are stored on a non-ephemeral storage.
+
+You can achieve custom needs with the <<archive_command>> option.
+
+== Configuring audit logging in cassandra.yaml
+
+The `cassandra.yaml` file can be used to configure and enable audit logging.
+Configuration and enablement may be the same or different on each node, depending on the `cassandra.yaml` file settings.
+
+Audit logging can also be configured using ``nodetool`` when enabling the feature, and will override any values set in the `cassandra.yaml` file, as discussed in <<enabling_audit_with_nodetool, Enabling Audit Logging with nodetool>>.
+
+Audit logs are generated on each enabled node, so logs on each node will have that node's queries.
+All options for audit logging can be set in the `cassandra.yaml` file under the ``audit_logging_options:``.
+
+The file includes the following options that can be uncommented for use:
+
+[source, yaml]
+----
+# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
+# on audit_logging for full details about the various configuration options.
+audit_logging_options:
+ enabled: false
+ logger:
+ - class_name: BinAuditLogger
+ # audit_logs_dir:
+ # included_keyspaces:
+ # excluded_keyspaces: system, system_schema, system_virtual_schema
+ # included_categories:
+ # excluded_categories:
+ # included_users:
+ # excluded_users:
+ # roll_cycle: HOURLY
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+----
+
+=== enabled
+
+Control whether audit logging is enabled or disabled (default).
+
+To enable audit logging set ``enabled: true``.
+
+If this option is enabled, audit logging will start when Cassandra is started.
+It can be disabled afterwards at runtime with <<enabling_audit_with_nodetool, nodetool>>.
+
+TIP: You can monitor whether audit logging is enabled with ``AuditLogEnabled`` attribute of the JMX MBean ``org.apache.cassandra.db:type=StorageService``.
+
+=== logger
+
+The type of audit logger is set with the `logger` option.
+Supported values are:
+
+- `BinAuditLogger` (default)
+- `FileAuditLogger`
+- `NoOpAuditLogger`
+
+`BinAuditLogger` logs events to a file in binary format.
+`FileAuditLogger` uses the standard logging mechanism, `slf4j` to log events to the `audit/audit.log` file. It is a synchronous, file-based audit logger. The roll_cycle will be set in the `logback.xml` file.
+`NoOpAuditLogger` is a no-op implementation of the audit logger that shoudl be specified when audit logging is disabled.
+
+For example:
+
+[source, yaml]
+----
+logger:
+ - class_name: FileAuditLogger
+----
+
+TIP: `BinAuditLogger` make use of open source https://github.com/OpenHFT/Chronicle-Queue[Chronicle Queue] under the hood. If you consider using audit logging for regulatory compliance purpose, it might be wise to be somewhat familiar with this library. See <<archive_command>> and <<roll_cycle>> for an example of the implications.
+
+=== audit_logs_dir
+
+To write audit logs, an existing directory must be set in ``audit_logs_dir``.
+
+The directory must have appropriate permissions set to allow reading, writing, and executing.
+Logging will recursively delete the directory contents as needed.
+Do not place links in this directory to other sections of the filesystem.
+For example, ``audit_logs_dir: /non_ephemeral_storage/audit/logs/hourly``.
+
+The audit log directory can also be configured using the system property `cassandra.logdir.audit`, which by default is set to `cassandra.logdir + /audit/`.
+
+=== included_keyspaces and excluded_keyspaces
+
+Set the keyspaces to include with the `included_keyspaces` option and
+the keyspaces to exclude with the `excluded_keyspaces` option.
+By default, `system`, `system_schema` and `system_virtual_schema` are excluded, and all other keyspaces are included.
+
+For example:
+[source, yaml]
+----
+included_keyspaces: test, demo
+excluded_keyspaces: system, system_schema, system_virtual_schema
+----
+
+=== included_categories and excluded_categories
+
+The categories of database operations to include are specified with the `included_categories` option as a comma-separated list.
+The categories of database operations to exclude are specified with `excluded_categories` option as a comma-separated list.
+The supported categories for audit log are: `AUTH`, `DCL`, `DDL`, `DML`, `ERROR`, `OTHER`, `PREPARE`, and `QUERY`.
+By default, all supported categories are included, and no category is excluded.
+
+[source, yaml]
+----
+included_categories: AUTH, ERROR, DCL
+excluded_categories: DDL, DML, QUERY, PREPARE
+----
+
+=== included_users and excluded_users
+
+Users to audit log are set with the `included_users` and `excluded_users` options.
+The `included_users` option specifies a comma-separated list of users to include explicitly.
+The `excluded_users` option specifies a comma-separated list of users to exclude explicitly.
+By default, all users are included, and no users are excluded.
+
+[source, yaml]
+----
+included_users:
+excluded_users: john, mary
+----
+
+[[roll_cycle]]
+=== roll_cycle
+
+The ``roll_cycle`` defines the frequency with which the audit log segments are rolled.
+Supported values are:
+
+- ``MINUTELY``
+- ``FIVE_MINUTELY``
+- ``TEN_MINUTELY``
+- ``TWENTY_MINUTELY``
+- ``HALF_HOURLY``
+- ``HOURLY`` (default)
+- ``TWO_HOURLY``
+- ``FOUR_HOURLY``
+- ``SIX_HOURLY``
+- ``DAILY``
+
+For example: ``roll_cycle: DAILY``
+
+WARNING: Read the following paragraph when changing ``roll_cycle`` on a production node.
+
+With the `BinLogger` implementation, any attempt to modify the roll cycle on a node where audit logging was previously enabled will fail silentely due to https://github.com/OpenHFT/Chronicle-Queue[Chronicle Queue] roll cycle inference mechanism (even if you delete the ``metadata.cq4t`` file).
+
+Here is an example of such an override visible in Cassandra logs:
+----
+INFO [main] <DATE TIME> BinLog.java:420 - Attempting to configure bin log: Path: /path/to/audit Roll cycle: TWO_HOURLY [...]
+WARN [main] <DATE TIME> SingleChronicleQueueBuilder.java:477 - Overriding roll cycle from TWO_HOURLY to FIVE_MINUTE
+----
+
+In order to change ``roll_cycle`` on a node, you have to:
+
+1. Stop Cassandra
+2. Move or offload all audit logs somewhere else (in a safe and durable location)
+3. Restart Cassandra.
+4. Check Cassandra logs
+5. Make sure that audit log filenames under ``audit_logs_dir`` correspond to the new roll cycle.
+
+=== block
+
+The ``block`` option specifies whether audit logging should block writing or drop log records if the audit logging falls behind. Supported boolean values are ``true`` (default) or ``false``.
+
+For example: ``block: false`` to drop records (e.g. if audit is used for troobleshooting)
+
+For regulatory compliance purposes, it's a good practice to explicitly set ``block: true`` to prevent any regression in case of future default value change.
+
+=== max_queue_weight
+
+The ``max_queue_weight`` option sets the maximum weight of in-memory queue for records waiting to be written to the file before blocking or dropping. The option must be set to a positive value. The default value is 268435456, or 256 MiB.
+
+For example, to change the default: ``max_queue_weight: 134217728 # 128 MiB``
+
+=== max_log_size
+
+The ``max_log_size`` option sets the maximum size of the rolled files to retain on disk before deleting the oldest file. The option must be set to a positive value. The default is 17179869184, or 16 GiB.
+For example, to change the default: ``max_log_size: 34359738368 # 32 GiB``
+
+WARNING: ``max_log_size`` is ignored if ``archive_command`` option is set.
+
+[[archive_command]]
+=== archive_command
+
+NOTE: If ``archive_command`` option is empty or unset (default), Cassandra uses a built-in DeletingArchiver that deletes the oldest files if ``max_log_size`` is reached.
+
+The ``archive_command`` option sets the user-defined archive script to execute on rolled log files.
+For example: ``archive_command: "/usr/local/bin/archiveit.sh %path"``
+
+``%path`` is replaced with the absolute file path of the file being rolled.
+
+When using a user-defined script, Cassandra does **not** use the DeletingArchiver, so it's the responsibility of the script to make any required cleanup.
+
+Cassandra will call the user-defined script as soon as the log file is rolled. It means that Chronicle Queue's QueueFileShrinkManager will not be able to shrink the sparse log file because it's done asynchronously. In other words, all log files will have at least the size of the default block size (80 MiB), even if there are only a few KB of real data. Consequently, some warnings will appear in Cassandra system.log:
+
+----
+WARN [main/queue~file~shrink~daemon] <DATE TIME> QueueFileShrinkManager.java:63 - Failed to shrink file as it exists no longer, file=/path/to/xxx.cq4
+----
+
+TIP: Because Cassandra does not make use of Pretoucher, you can configure Chronicle Queue to shrink files synchronously -- i.e. as soon as the file is rolled -- with ``chronicle.queue.synchronousFileShrinking`` JVM properties. For instance, you can add the following line at the end of ``cassandra-env.sh``: ``JVM_OPTS="$JVM_OPTS -Dchronicle.queue.synchronousFileShrinking=true"``
+
+=== max_archive_retries
+
+The ``max_archive_retries`` option sets the max number of retries of failed archive commands. The default is 10.
+
+For example: ``max_archive_retries: 10``
+
+Interval between each retry is hard coded to 5 minutes.
+
+[[enabling_audit_with_nodetool]]
+== Enabling Audit Logging with ``nodetool``
+
+Audit logging is enabled on a per-node basis using the ``nodetool enableauditlog`` command. The logging directory must be defined with ``audit_logs_dir`` in the `cassandra.yaml` file or uses the default value ``cassandra.logdir.audit``.
+
+The syntax of the ``nodetool enableauditlog`` command has all the same options that can be set in the ``cassandra.yaml`` file except ``audit_logs_dir``.
+In addition, ``nodetool`` has options to set which host and port to run the command on, and username and password if the command requires authentication.
+
+[source, plaintext]
+----
+ nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
+ [(-pp | --print-port)] [(-pw <password> | --password <password>)]
+ [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
+ [(-u <username> | --username <username>)] enableauditlog
+ [--excluded-categories <excluded_categories>]
+ [--excluded-keyspaces <excluded_keyspaces>]
+ [--excluded-users <excluded_users>]
+ [--included-categories <included_categories>]
+ [--included-keyspaces <included_keyspaces>]
+ [--included-users <included_users>] [--logger <logger>]
+
+OPTIONS
+ --excluded-categories <excluded_categories>
+ Comma separated list of Audit Log Categories to be excluded for
+ audit log. If not set the value from cassandra.yaml will be used
+
+ --excluded-keyspaces <excluded_keyspaces>
+ Comma separated list of keyspaces to be excluded for audit log. If
+ not set the value from cassandra.yaml will be used
+
+ --excluded-users <excluded_users>
+ Comma separated list of users to be excluded for audit log. If not
+ set the value from cassandra.yaml will be used
+
+ -h <host>, --host <host>
+ Node hostname or ip address
+
+ --included-categories <included_categories>
+ Comma separated list of Audit Log Categories to be included for
+ audit log. If not set the value from cassandra.yaml will be used
+
+ --included-keyspaces <included_keyspaces>
+ Comma separated list of keyspaces to be included for audit log. If
+ not set the value from cassandra.yaml will be used
+
+ --included-users <included_users>
+ Comma separated list of users to be included for audit log. If not
+ set the value from cassandra.yaml will be used
+
+ --logger <logger>
+ Logger name to be used for AuditLogging. Default BinAuditLogger. If
+ not set the value from cassandra.yaml will be used
+
+ -p <port>, --port <port>
+ Remote jmx agent port number
+
+ -pp, --print-port
+ Operate in 4.0 mode with hosts disambiguated by port number
+
+ -pw <password>, --password <password>
+ Remote jmx agent password
+
+ -pwf <passwordFilePath>, --password-file <passwordFilePath>
+ Path to the JMX password file
+
+ -u <username>, --username <username>
+ Remote jmx agent username
+----
+
+To enable audit logging, run following command on each node in the cluster on which you want to enable logging:
+
+[source, bash]
+----
+$ nodetool enableauditlog
+----
+
+== Disabling audit logging
+
+Use the `nodetool disableauditlog` command to disable audit logging.
+
+== Viewing audit logs
+
+The `auditlogviewer` tool is used to view (dump) audit logs if the logger was ``BinAuditLogger``..
+``auditlogviewer`` converts the binary log files into human-readable format; only the audit log directory must be supplied as a command-line option.
+If the logger ``FileAuditLogger`` was set, the log file are already in human-readable format and ``auditlogviewer`` is not needed to read files.
+
+
+The syntax of `auditlogviewer` is:
+
+[source, plaintext]
+----
+auditlogviewer
+
+Audit log files directory path is a required argument.
+usage: auditlogviewer <path1> [<path2>...<pathN>] [options]
+--
+View the audit log contents in human readable format
+--
+Options are:
+-f,--follow Upon reaching the end of the log continue indefinitely
+ waiting for more records
+-h,--help display this help message
+-r,--roll_cycle How often to roll the log file was rolled. May be
+ necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY,
+ DAILY). Default HOURLY.
+----
+
+== Example
+
+[arabic, start=1]
+. To demonstrate audit logging, first configure the ``cassandra.yaml`` file with the following settings:
+
+[source, yaml]
+----
+audit_logging_options:
+ enabled: true
+ logger: BinAuditLogger
+ audit_logs_dir: "/cassandra/audit/logs/hourly"
+ # included_keyspaces:
+ # excluded_keyspaces: system, system_schema, system_virtual_schema
+ # included_categories:
+ # excluded_categories:
+ # included_users:
+ # excluded_users:
+ roll_cycle: HOURLY
+ # block: true
+ # max_queue_weight: 268435456 # 256 MiB
+ # max_log_size: 17179869184 # 16 GiB
+ ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+ # archive_command:
+ # max_archive_retries: 10
+----
+
+[arabic, start=2]
+. Create the audit log directory `/cassandra/audit/logs/hourly` and set the directory permissions to read, write, and execute for all.
+
+[arabic, start=3]
+. Now create a demo keyspace and table and insert some data using ``cqlsh``:
+
+[source, cql]
+----
+ cqlsh> CREATE KEYSPACE auditlogkeyspace
+ ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
+ cqlsh> USE auditlogkeyspace;
+ cqlsh:auditlogkeyspace> CREATE TABLE t (
+ ...id int,
+ ...k int,
+ ...v text,
+ ...PRIMARY KEY (id)
+ ... );
+ cqlsh:auditlogkeyspace> INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
+ cqlsh:auditlogkeyspace> INSERT INTO t (id, k, v) VALUES (0, 1, 'val1');
+----
+
+All the supported CQL commands will be logged to the audit log directory.
+
+[arabic, start=4]
+. Change directory to the audit logs directory.
+
+[source, bash]
+----
+$ cd /cassandra/audit/logs/hourly
+----
+
+[arabic, start=5]
+. List the audit log files and directories.
+
+[source, bash]
+----
+$ ls -l
+----
+
+You should see results similar to:
+
+[source, plaintext]
+----
+total 28
+-rw-rw-r--. 1 ec2-user ec2-user 65536 Aug 2 03:01 directory-listing.cq4t
+-rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 03:01 20190802-02.cq4
+-rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 03:01 20190802-03.cq4
+----
+
+The audit log files will all be listed with a `.cq4` file type. The audit directory is of `.cq4t` type.
+
+[arabic, start=6]
+. Run `auditlogviewer` tool to view the audit logs.
+
+[source, bash]
+----
+$ auditlogviewer /cassandra/audit/logs/hourly
+----
+
+This command will return a readable version of the log. Here is a partial sample of the log for the commands in this demo:
+
+[source, plaintext]
+----
+WARN 03:12:11,124 Using Pauser.sleepy() as not enough processors, have 2, needs 8+
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711427328|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE AuditLogKeyspace;
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711427329|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE "auditlogkeyspace"
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711446279|type :SELECT|category:QUERY|ks:auditlogkeyspace|scope:t|operation:SELECT * FROM t;
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564713878834|type :DROP_TABLE|category:DDL|ks:auditlogkeyspace|scope:t|operation:DROP TABLE IF EXISTS
+AuditLogKeyspace.t;
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/3.91.56.164|port:42382|timestamp:1564714618360|ty
+pe:REQUEST_FAILURE|category:ERROR|operation:CREATE KEYSPACE AuditLogKeyspace
+WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};; Cannot add
+existing keyspace "auditlogkeyspace"
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564714690968|type :DROP_KEYSPACE|category:DDL|ks:auditlogkeyspace|operation:DROP KEYSPACE AuditLogKeyspace;
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/3.91.56.164|port:42406|timestamp:1564714708329|ty pe:CREATE_KEYSPACE|category:DDL|ks:auditlogkeyspace|operation:CREATE KEYSPACE
+AuditLogKeyspace
+WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
+Type: AuditLog
+LogMessage:
+user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564714870678|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE auditlogkeyspace;
+
+Password obfuscation examples:
+LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630496708|type:CREATE_ROLE|category:DCL|operation:CREATE ROLE role1 WITH PASSWORD = '*******';
+Type: audit
+LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630634552|type:ALTER_ROLE|category:DCL|operation:ATLER ROLE role1 WITH PASSWORD = '*******';
+Type: audit
+LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630698686|type:CREATE_ROLE|category:DCL|operation:CREATE USER user1 WITH PASSWORD '*******';
+Type: audit
+LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630747344|type:ALTER_ROLE|category:DCL|operation:ALTER USER user1 WITH PASSWORD '*******';
+----
+
+== Diagnostic events for user audit logging
+
+Any native transport-enabled client can subscribe to audit log events for diagnosing cluster issues.
+These events can be consumed by external tools to implement a Cassandra user auditing solution.
diff --git a/doc/modules/cassandra/pages/managing/operating/backups.adoc b/doc/modules/cassandra/pages/managing/operating/backups.adoc
new file mode 100644
index 0000000..78f9186
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/backups.adoc
@@ -0,0 +1,517 @@
+= Backups
+
+Apache Cassandra stores data in immutable SSTable files. Backups in
+Apache Cassandra database are backup copies of the database data that is
+stored as SSTable files. Backups are used for several purposes including
+the following:
+
+* To store a data copy for durability
+* To be able to restore a table if table data is lost due to
+node/partition/network failure
+* To be able to transfer the SSTable files to a different machine; for
+portability
+
+== Types of Backups
+
+Apache Cassandra supports two kinds of backup strategies.
+
+* Snapshots
+* Incremental Backups
+
+A _snapshot_ is a copy of a table’s SSTable files at a given time,
+created via hard links.
+The DDL to create the table is stored as well.
+Snapshots may be created by a user or created automatically.
+The setting `snapshot_before_compaction` in the `cassandra.yaml` file determines if
+snapshots are created before each compaction.
+By default, `snapshot_before_compaction` is set to false.
+Snapshots may be created automatically before keyspace truncation or dropping of a table by
+setting `auto_snapshot` to true (default) in `cassandra.yaml`.
+Truncates could be delayed due to the auto snapshots and another setting in
+`cassandra.yaml` determines how long the coordinator should wait for
+truncates to complete.
+By default Cassandra waits 60 seconds for auto snapshots to complete.
+
+An _incremental backup_ is a copy of a table’s SSTable files created by
+a hard link when memtables are flushed to disk as SSTables.
+Typically incremental backups are paired with snapshots to reduce the backup time
+as well as reduce disk space.
+Incremental backups are not enabled by default and must be enabled explicitly in `cassandra.yaml` (with
+`incremental_backups` setting) or with `nodetool`.
+Once enabled, Cassandra creates a hard link to each SSTable flushed or streamed
+locally in a `backups/` subdirectory of the keyspace data.
+Incremental backups of system tables are also created.
+
+== Data Directory Structure
+
+The directory structure of Cassandra data consists of different
+directories for keyspaces, and tables with the data files within the
+table directories.
+Directories backups and snapshots to store backups
+and snapshots respectively for a particular table are also stored within
+the table directory.
+The directory structure for Cassandra is illustrated in Figure 1.
+
+image::Figure_1_backups.jpg[Data directory structure for backups]
+
+Figure 1. Directory Structure for Cassandra Data
+
+=== Setting Up Example Tables for Backups and Snapshots
+
+In this section we shall create some example data that could be used to
+demonstrate incremental backups and snapshots.
+We have used a three node Cassandra cluster.
+First, the keyspaces are created.
+Then tables are created within a keyspace and table data is added.
+We have used two keyspaces `cqlkeyspace` and `catalogkeyspace` with two tables within
+each.
+
+Create the keyspace `cqlkeyspace`:
+
+[source,cql]
+----
+include::example$CQL/create_ks_backup.cql[]
+----
+
+Create two tables `t` and `t2` in the `cqlkeyspace` keyspace.
+
+[source,cql]
+----
+include::example$CQL/create_table_backup.cql[]
+----
+
+Add data to the tables:
+
+[source,cql]
+----
+include::example$CQL/insert_data_backup.cql[]
+----
+
+Query the table to list the data:
+
+[source,cql]
+----
+include::example$CQL/select_data_backup.cql[]
+----
+
+results in
+
+[source,cql]
+----
+include::example$RESULTS/select_data_backup.result[]
+----
+
+Create a second keyspace `catalogkeyspace`:
+
+[source,cql]
+----
+include::example$CQL/create_ks2_backup.cql[]
+----
+
+Create two tables `journal` and `magazine` in `catalogkeyspace`:
+
+[source,cql]
+----
+include::example$CQL/create_table2_backup.cql[]
+----
+
+Add data to the tables:
+
+[source,cql]
+----
+include::example$CQL/insert_data2_backup.cql[]
+----
+
+Query the tables to list the data:
+
+[source,cql]
+----
+include::example$CQL/select_data2_backup.cql[]
+----
+
+results in
+
+[source,cql]
+----
+include::example$RESULTS/select_data2_backup.result[]
+----
+
+== Snapshots
+
+In this section, we demonstrate creating snapshots.
+The command used to create a snapshot is `nodetool snapshot` with the usage:
+
+[source,bash]
+----
+include::example$BASH/nodetool_snapshot.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/nodetool_snapshot_help.result[]
+----
+
+=== Configuring for Snapshots
+
+To demonstrate creating snapshots with Nodetool on the commandline we
+have set `auto_snapshots` setting to `false` in the `cassandra.yaml` file:
+
+[source,yaml]
+----
+include::example$YAML/auto_snapshot.yaml[]
+----
+
+Also set `snapshot_before_compaction` to `false` to disable creating
+snapshots automatically before compaction:
+
+[source,yaml]
+----
+include::example$YAML/snapshot_before_compaction.yaml[]
+----
+
+=== Creating Snapshots
+
+Before creating any snapshots, search for snapshots and none will be listed:
+
+[source,bash]
+----
+include::example$BASH/find_snapshots.sh[]
+----
+
+We shall be using the example keyspaces and tables to create snapshots.
+
+==== Taking Snapshots of all Tables in a Keyspace
+
+Using the syntax above, create a snapshot called `catalog-ks` for all the tables
+in the `catalogkeyspace` keyspace:
+
+[source,bash]
+----
+include::example$BASH/snapshot_backup2.sh[]
+----
+
+results in
+
+[source,none]
+----
+include::example$RESULTS/snapshot_backup2.result[]
+----
+
+Using the `find` command above, the snapshots and `snapshots` directories
+are now found with listed files similar to:
+
+[source, plaintext]
+----
+include::example$RESULTS/snapshot_backup2_find.result[]
+----
+
+Snapshots of all tables in multiple keyspaces may be created similarly:
+
+[source,bash]
+----
+include::example$BASH/snapshot_both_backups.sh[]
+----
+
+==== Taking Snapshots of Single Table in a Keyspace
+
+To take a snapshot of a single table the `nodetool snapshot` command
+syntax becomes as follows:
+
+[source,bash]
+----
+include::example$BASH/snapshot_one_table.sh[]
+----
+
+Using the syntax above, create a snapshot for table `magazine` in keyspace `catalogkeyspace`:
+
+[source,bash]
+----
+include::example$BASH/snapshot_one_table2.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/snapshot_one_table2.result[]
+----
+
+==== Taking Snapshot of Multiple Tables from same Keyspace
+
+To take snapshots of multiple tables in a keyspace the list of
+_Keyspace.table_ must be specified with option `--kt-list`.
+For example, create snapshots for tables `t` and `t2` in the `cqlkeyspace` keyspace:
+
+[source,bash]
+----
+include::example$BASH/snapshot_mult_tables.sh[]
+----
+
+results in
+
+[source,plaintext]
+----
+include::example$RESULTS/snapshot_mult_tables.result[]
+----
+
+Multiple snapshots of the same set of tables may be created and tagged with a different name.
+As an example, create another snapshot for the same set of tables `t` and `t2` in the `cqlkeyspace`
+keyspace and tag the snapshots differently:
+
+[source,bash]
+----
+include::example$BASH/snapshot_mult_tables_again.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/snapshot_mult_tables_again.result[]
+----
+
+==== Taking Snapshot of Multiple Tables from Different Keyspaces
+
+To take snapshots of multiple tables that are in different keyspaces the
+command syntax is the same as when multiple tables are in the same
+keyspace.
+Each <keyspace>.<table> must be specified separately in the
+`--kt-list` option.
+
+For example, create a snapshot for table `t` in
+the `cqlkeyspace` and table `journal` in the catalogkeyspace and tag the
+snapshot `multi-ks`.
+
+[source,bash]
+----
+include::example$BASH/snapshot_mult_ks.sh[]
+----
+
+results in
+[source, plaintext]
+----
+include::example$RESULTS/snapshot_mult_ks.result[]
+----
+
+=== Listing Snapshots
+
+To list snapshots use the `nodetool listsnapshots` command. All the
+snapshots that we created in the preceding examples get listed:
+
+[source,bash]
+----
+include::example$BASH/nodetool_list_snapshots.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/nodetool_list_snapshots.result[]
+----
+
+=== Finding Snapshots Directories
+
+The `snapshots` directories may be listed with `find –name snapshots`
+command:
+
+[source,bash]
+----
+include::example$BASH/find_snapshots.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/snapshot_all.result[]
+----
+
+To list the snapshots for a particular table first change to the snapshots directory for that table.
+For example, list the snapshots for the `catalogkeyspace/journal` table:
+
+[source,bash]
+----
+include::example$BASH/find_two_snapshots.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/find_two_snapshots.result[]
+----
+
+A `snapshots` directory lists the SSTable files in the snapshot.
+A `schema.cql` file is also created in each snapshot that defines schema
+that can recreate the table with CQL when restoring from a snapshot:
+
+[source,bash]
+----
+include::example$BASH/snapshot_files.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/snapshot_files.result[]
+----
+
+=== Clearing Snapshots
+
+Snapshots may be cleared or deleted with the `nodetool clearsnapshot`
+command. Either a specific snapshot name must be specified or the `–all`
+option must be specified.
+
+For example, delete a snapshot called `magazine` from keyspace `cqlkeyspace`:
+
+[source,bash]
+----
+include::example$BASH/nodetool_clearsnapshot.sh[]
+----
+
+or delete all snapshots from `cqlkeyspace` with the –all option:
+
+[source,bash]
+----
+include::example$BASH/nodetool_clearsnapshot_all.sh[]
+----
+
+== Incremental Backups
+
+In the following sections, we shall discuss configuring and creating
+incremental backups.
+
+=== Configuring for Incremental Backups
+
+To create incremental backups set `incremental_backups` to `true` in
+`cassandra.yaml`.
+
+[source,yaml]
+----
+include::example$YAML/incremental_bups.yaml[]
+----
+
+This is the only setting needed to create incremental backups.
+By default `incremental_backups` setting is set to `false` because a new
+set of SSTable files is created for each data flush and if several CQL
+statements are to be run the `backups` directory could fill up quickly
+and use up storage that is needed to store table data.
+Incremental backups may also be enabled on the command line with the nodetool
+command `nodetool enablebackup`.
+Incremental backups may be disabled with `nodetool disablebackup` command.
+Status of incremental backups, whether they are enabled may be checked with `nodetool statusbackup`.
+
+=== Creating Incremental Backups
+
+After each table is created flush the table data with `nodetool flush`
+command. Incremental backups get created.
+
+[source,bash]
+----
+include::example$BASH/nodetool_flush.sh[]
+----
+
+=== Finding Incremental Backups
+
+Incremental backups are created within the Cassandra’s `data` directory
+within a table directory. Backups may be found with following command.
+
+[source,bash]
+----
+include::example$BASH/find_backups.sh[]
+----
+results in
+[source,none]
+----
+include::example$RESULTS/find_backups.result[]
+----
+
+=== Creating an Incremental Backup
+
+This section discusses how incremental backups are created in more
+detail using the keyspace and table previously created.
+
+Flush the keyspace and table:
+
+[source,bash]
+----
+include::example$BASH/nodetool_flush_table.sh[]
+----
+
+A search for backups and a `backups` directory will list a backup directory,
+even if we have added no table data yet.
+
+[source,bash]
+----
+include::example$BASH/find_backups.sh[]
+----
+
+results in
+
+[source,plaintext]
+----
+include::example$RESULTS/find_backups_table.result[]
+----
+
+Checking the `backups` directory will show that there are also no backup files:
+
+[source,bash]
+----
+include::example$BASH/check_backups.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/no_bups.result[]
+----
+
+If a row of data is added to the data, running the `nodetool flush` command will
+flush the table data and an incremental backup will be created:
+
+[source,bash]
+----
+include::example$BASH/flush_and_check.sh[]
+----
+
+results in
+
+[source, plaintext]
+----
+include::example$RESULTS/flush_and_check.result[]
+----
+
+[NOTE]
+.note
+====
+The `backups` directory for any table, such as `cqlkeyspace/t` is created in the
+`data` directory for that table.
+====
+
+Adding another row of data and flushing will result in another set of incremental backup files.
+The SSTable files are timestamped, which distinguishes the first incremental backup from the
+second:
+
+[source,none]
+----
+include::example$RESULTS/flush_and_check2.result[]
+----
+
+== Restoring from Incremental Backups and Snapshots
+
+The two main tools/commands for restoring a table after it has been
+dropped are:
+
+* sstableloader
+* nodetool refresh
+
+A snapshot contains essentially the same set of SSTable files as an
+incremental backup does with a few additional files. A snapshot includes
+a `schema.cql` file for the schema DDL to create a table in CQL. A table
+backup does not include DDL which must be obtained from a snapshot when
+restoring from an incremental backup.
diff --git a/doc/modules/cassandra/pages/operating/bloom_filters.adoc b/doc/modules/cassandra/pages/managing/operating/bloom_filters.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/bloom_filters.adoc
rename to doc/modules/cassandra/pages/managing/operating/bloom_filters.adoc
diff --git a/doc/modules/cassandra/pages/operating/bulk_loading.adoc b/doc/modules/cassandra/pages/managing/operating/bulk_loading.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/bulk_loading.adoc
rename to doc/modules/cassandra/pages/managing/operating/bulk_loading.adoc
diff --git a/doc/modules/cassandra/pages/operating/cdc.adoc b/doc/modules/cassandra/pages/managing/operating/cdc.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/cdc.adoc
rename to doc/modules/cassandra/pages/managing/operating/cdc.adoc
diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/index.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/index.adoc
new file mode 100644
index 0000000..1795f03
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/compaction/index.adoc
@@ -0,0 +1,336 @@
+= Compaction
+
+== Strategies
+
+Picking the right compaction strategy for your workload will ensure the
+best performance for both querying and for compaction itself.
+
+xref:operating/compaction/stcs.adoc[`Size Tiered Compaction Strategy (STCS)`]::
+ The default compaction strategy. Useful as a fallback when other
+ strategies don't fit the workload. Most useful for non pure time
+ series workloads with spinning disks, or when the I/O from `LCS`
+ is too high.
+xref:operating/compaction/lcs.adoc[`Leveled Compaction Strategy (LCS)`]::
+ Leveled Compaction Strategy (LCS) is optimized for read heavy
+ workloads, or workloads with lots of updates and deletes. It is not a
+ good choice for immutable time series data.
+xref:operating/compaction/twcs.adoc[`Time Window Compaction Strategy (TWCS)`]::
+ Time Window Compaction Strategy is designed for TTL'ed, mostly
+ immutable time series data.
+
+== Types of compaction
+
+The concept of compaction is used for different kinds of operations in
+Cassandra, the common thing about these operations is that it takes one
+or more SSTables and output new SSTables. The types of compactions are:
+
+Minor compaction::
+ triggered automatically in Cassandra.
+Major compaction::
+ a user executes a compaction over all SSTables on the node.
+User defined compaction::
+ a user triggers a compaction on a given set of SSTables.
+Scrub::
+ try to fix any broken SSTables. This can actually remove valid data if
+ that data is corrupted, if that happens you will need to run a full
+ repair on the node.
+UpgradeSSTables::
+ upgrade SSTables to the latest version. Run this after upgrading to a
+ new major version.
+Cleanup::
+ remove any ranges this node does not own anymore, typically triggered
+ on neighbouring nodes after a node has been bootstrapped since that
+ node will take ownership of some ranges from those nodes.
+Secondary index rebuild::
+ rebuild the secondary indexes on the node.
+Anticompaction::
+ after repair the ranges that were actually repaired are split out of
+ the SSTables that existed when repair started.
+Sub range compaction::
+ It is possible to only compact a given sub range - this could be
+ useful if you know a token that has been misbehaving - either
+ gathering many updates or many deletes.
+ (`nodetool compact -st x -et y`) will pick all SSTables containing the
+ range between x and y and issue a compaction for those SSTables. For
+ STCS this will most likely include all SSTables but with LCS it can
+ issue the compaction for a subset of the SSTables. With LCS the
+ resulting sstable will end up in L0.
+
+== When is a minor compaction triggered?
+
+* When an sstable is added to the node through flushing/streaming
+* When autocompaction is enabled after being disabled (`nodetool enableautocompaction`)
+* When compaction adds new SSTables
+* A check for new minor compactions every 5 minutes
+
+== Merging SSTables
+
+Compaction is about merging SSTables, since partitions in SSTables are
+sorted based on the hash of the partition key it is possible to
+efficiently merge separate SSTables. Content of each partition is also
+sorted so each partition can be merged efficiently.
+
+== Tombstones and Garbage Collection (GC) Grace
+
+=== Why Tombstones
+
+When a delete request is received by Cassandra it does not actually
+remove the data from the underlying store. Instead it writes a special
+piece of data known as a tombstone. The Tombstone represents the delete
+and causes all values which occurred before the tombstone to not appear
+in queries to the database. This approach is used instead of removing
+values because of the distributed nature of Cassandra.
+
+=== Deletes without tombstones
+
+Imagine a three node cluster which has the value [A] replicated to every
+node.:
+
+[source,none]
+----
+[A], [A], [A]
+----
+
+If one of the nodes fails and and our delete operation only removes
+existing values we can end up with a cluster that looks like:
+
+[source,none]
+----
+[], [], [A]
+----
+
+Then a repair operation would replace the value of [A] back onto the two
+nodes which are missing the value.:
+
+[source,none]
+----
+[A], [A], [A]
+----
+
+This would cause our data to be resurrected even though it had been
+deleted.
+
+=== Deletes with Tombstones
+
+Starting again with a three node cluster which has the value [A]
+replicated to every node.:
+
+[source,none]
+----
+[A], [A], [A]
+----
+
+If instead of removing data we add a tombstone record, our single node
+failure situation will look like this.:
+
+[source,none]
+----
+[A, Tombstone[A]], [A, Tombstone[A]], [A]
+----
+
+Now when we issue a repair the Tombstone will be copied to the replica,
+rather than the deleted data being resurrected.:
+
+[source,none]
+----
+[A, Tombstone[A]], [A, Tombstone[A]], [A, Tombstone[A]]
+----
+
+Our repair operation will correctly put the state of the system to what
+we expect with the record [A] marked as deleted on all nodes. This does
+mean we will end up accruing Tombstones which will permanently
+accumulate disk space. To avoid keeping tombstones forever we have a
+parameter known as `gc_grace_seconds` for every table in Cassandra.
+
+=== The gc_grace_seconds parameter and Tombstone Removal
+
+The table level `gc_grace_seconds` parameter controls how long Cassandra
+will retain tombstones through compaction events before finally removing
+them. This duration should directly reflect the amount of time a user
+expects to allow before recovering a failed node. After
+`gc_grace_seconds` has expired the tombstone may be removed (meaning
+there will no longer be any record that a certain piece of data was
+deleted), but as a tombstone can live in one sstable and the data it
+covers in another, a compaction must also include both sstable for a
+tombstone to be removed. More precisely, to be able to drop an actual
+tombstone the following needs to be true;
+
+* The tombstone must be older than `gc_grace_seconds`
+* If partition X contains the tombstone, the sstable containing the
+partition plus all SSTables containing data older than the tombstone
+containing X must be included in the same compaction. We don't need to
+care if the partition is in an sstable if we can guarantee that all data
+in that sstable is newer than the tombstone. If the tombstone is older
+than the data it cannot shadow that data.
+* If the option `only_purge_repaired_tombstones` is enabled, tombstones
+are only removed if the data has also been repaired.
+
+If a node remains down or disconnected for longer than
+`gc_grace_seconds` it's deleted data will be repaired back to the other
+nodes and re-appear in the cluster. This is basically the same as in the
+"Deletes without Tombstones" section. Note that tombstones will not be
+removed until a compaction event even if `gc_grace_seconds` has elapsed.
+
+The default value for `gc_grace_seconds` is 864000 which is equivalent
+to 10 days. This can be set when creating or altering a table using
+`WITH gc_grace_seconds`.
+
+== TTL
+
+Data in Cassandra can have an additional property called time to live -
+this is used to automatically drop data that has expired once the time
+is reached. Once the TTL has expired the data is converted to a
+tombstone which stays around for at least `gc_grace_seconds`. Note that
+if you mix data with TTL and data without TTL (or just different length
+of the TTL) Cassandra will have a hard time dropping the tombstones
+created since the partition might span many SSTables and not all are
+compacted at once.
+
+== Fully expired SSTables
+
+If an SSTable contains only tombstones and it is guaranteed that
+SSTable is not shadowing data in any other SSTable, then the compaction can drop
+that SSTable. If you see SSTables with only tombstones (note that TTL-ed
+data is considered tombstones once the time-to-live has expired), but it
+is not being dropped by compaction, it is likely that other SSTables
+contain older data. There is a tool called `sstableexpiredblockers` that
+will list which SSTables are droppable and which are blocking them from
+being dropped. With `TimeWindowCompactionStrategy` it
+is possible to remove the guarantee (not check for shadowing data) by
+enabling `unsafe_aggressive_sstable_expiration`.
+
+== Repaired/unrepaired data
+
+With incremental repairs Cassandra must keep track of what data is
+repaired and what data is unrepaired. With anticompaction repaired data
+is split out into repaired and unrepaired SSTables. To avoid mixing up
+the data again separate compaction strategy instances are run on the two
+sets of data, each instance only knowing about either the repaired or
+the unrepaired SSTables. This means that if you only run incremental
+repair once and then never again, you might have very old data in the
+repaired SSTables that block compaction from dropping tombstones in the
+unrepaired (probably newer) SSTables.
+
+== Data directories
+
+Since tombstones and data can live in different SSTables it is important
+to realize that losing an sstable might lead to data becoming live again
+- the most common way of losing SSTables is to have a hard drive break
+down. To avoid making data live tombstones and actual data are always in
+the same data directory. This way, if a disk is lost, all versions of a
+partition are lost and no data can get undeleted. To achieve this a
+compaction strategy instance per data directory is run in addition to
+the compaction strategy instances containing repaired/unrepaired data,
+this means that if you have 4 data directories there will be 8
+compaction strategy instances running. This has a few more benefits than
+just avoiding data getting undeleted:
+
+* It is possible to run more compactions in parallel - leveled
+compaction will have several totally separate levelings and each one can
+run compactions independently from the others.
+* Users can backup and restore a single data directory.
+* Note though that currently all data directories are considered equal,
+so if you have a tiny disk and a big disk backing two data directories,
+the big one will be limited the by the small one. One work around to
+this is to create more data directories backed by the big disk.
+
+== Single sstable tombstone compaction
+
+When an sstable is written a histogram with the tombstone expiry times
+is created and this is used to try to find SSTables with very many
+tombstones and run single sstable compaction on that sstable in hope of
+being able to drop tombstones in that sstable. Before starting this it
+is also checked how likely it is that any tombstones will actually will
+be able to be dropped how much this sstable overlaps with other
+SSTables. To avoid most of these checks the compaction option
+`unchecked_tombstone_compaction` can be enabled.
+
+[[compaction-options]]
+== Common options
+
+There is a number of common options for all the compaction strategies;
+
+`enabled` (default: true)::
+ Whether minor compactions should run. Note that you can have
+ 'enabled': true as a compaction option and then do 'nodetool
+ enableautocompaction' to start running compactions.
+`tombstone_threshold` (default: 0.2)::
+ How much of the sstable should be tombstones for us to consider doing
+ a single sstable compaction of that sstable.
+`tombstone_compaction_interval` (default: 86400s (1 day))::
+ Since it might not be possible to drop any tombstones when doing a
+ single sstable compaction we need to make sure that one sstable is not
+ constantly getting recompacted - this option states how often we
+ should try for a given sstable.
+`log_all` (default: false)::
+ New detailed compaction logging, see
+ `below <detailed-compaction-logging>`.
+`unchecked_tombstone_compaction` (default: false)::
+ The single sstable compaction has quite strict checks for whether it
+ should be started, this option disables those checks and for some
+ usecases this might be needed. Note that this does not change anything
+ for the actual compaction, tombstones are only dropped if it is safe
+ to do so - it might just rewrite an sstable without being able to drop
+ any tombstones.
+`only_purge_repaired_tombstone` (default: false)::
+ Option to enable the extra safety of making sure that tombstones are
+ only dropped if the data has been repaired.
+`min_threshold` (default: 4)::
+ Lower limit of number of SSTables before a compaction is triggered.
+ Not used for `LeveledCompactionStrategy`.
+`max_threshold` (default: 32)::
+ Upper limit of number of SSTables before a compaction is triggered.
+ Not used for `LeveledCompactionStrategy`.
+
+Further, see the section on each strategy for specific additional
+options.
+
+== Compaction nodetool commands
+
+The `nodetool <nodetool>` utility provides a number of commands related
+to compaction:
+
+`enableautocompaction`::
+ Enable compaction.
+`disableautocompaction`::
+ Disable compaction.
+`setcompactionthroughput`::
+ How fast compaction should run at most - defaults to 64MiB/s.
+`compactionstats`::
+ Statistics about current and pending compactions.
+`compactionhistory`::
+ List details about the last compactions.
+`setcompactionthreshold`::
+ Set the min/max sstable count for when to trigger compaction, defaults
+ to 4/32.
+
+== Switching the compaction strategy and options using JMX
+
+It is possible to switch compaction strategies and its options on just a
+single node using JMX, this is a great way to experiment with settings
+without affecting the whole cluster. The mbean is:
+
+[source,none]
+----
+org.apache.cassandra.db:type=ColumnFamilies,keyspace=<keyspace_name>,columnfamily=<table_name>
+----
+
+and the attribute to change is `CompactionParameters` or
+`CompactionParametersJson` if you use jconsole or jmc. The syntax for
+the json version is the same as you would use in an
+`ALTER TABLE <alter-table-statement>` statement -for example:
+
+[source,none]
+----
+{ 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 123, 'fanout_size': 10}
+----
+
+The setting is kept until someone executes an
+`ALTER TABLE <alter-table-statement>` that touches the compaction
+settings or restarts the node.
+
+[[detailed-compaction-logging]]
+== More detailed compaction logging
+
+Enable with the compaction option `log_all` and a more detailed
+compaction log file will be produced in your log directory.
diff --git a/doc/modules/cassandra/pages/operating/compaction/lcs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/lcs.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/compaction/lcs.adoc
rename to doc/modules/cassandra/pages/managing/operating/compaction/lcs.adoc
diff --git a/doc/modules/cassandra/pages/operating/compaction/stcs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/stcs.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/compaction/stcs.adoc
rename to doc/modules/cassandra/pages/managing/operating/compaction/stcs.adoc
diff --git a/doc/modules/cassandra/pages/operating/compaction/twcs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/twcs.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/compaction/twcs.adoc
rename to doc/modules/cassandra/pages/managing/operating/compaction/twcs.adoc
diff --git a/doc/modules/cassandra/pages/operating/compression.adoc b/doc/modules/cassandra/pages/managing/operating/compression.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/compression.adoc
rename to doc/modules/cassandra/pages/managing/operating/compression.adoc
diff --git a/doc/modules/cassandra/pages/managing/operating/denylisting_partitions.adoc b/doc/modules/cassandra/pages/managing/operating/denylisting_partitions.adoc
new file mode 100644
index 0000000..a81dc80
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/denylisting_partitions.adoc
@@ -0,0 +1,143 @@
+= Denylisting Partitions
+
+Due to access patterns and data modeling, sometimes there are specific partitions
+that are "hot" and can cause instability in a Cassandra cluster. This often occurs
+when your data model includes many update or insert operations on a single partition,
+causing the partition to grow very large over time and in turn making it very expensive
+to read and maintain.
+
+Cassandra supports "denylisting" these problematic partitions so that when clients
+issue point reads (`SELECT` statements with the partition key specified) or range
+reads (`SELECT *`, etc that pull a range of data) that intersect with a blocked
+partition key, the query will be immediately rejected with an `InvalidQueryException`.
+
+== How to denylist a partition key
+
+The ``system_distributed.denylisted_partitions`` table can be used to denylist partitions.
+There are a couple of ways to interact with and mutate this data. First: directly
+via CQL by inserting a record with the following details:
+
+- Keyspace name (ks_name)
+- Table name (table_name)
+- Partition Key (partition_key)
+
+The partition key format needs to be in the same form required by ``nodetool getendpoints``.
+
+Following are several examples for denylisting partition keys in keyspace `ks` and
+table `table1` for different data types on the primary key `Id`:
+
+- Id is a simple type - `INSERT INTO system_distributed.denylisted_partitions (ks_name, table_name, partition_key) VALUES ('ks','table1','1');`
+- Id is a blob - `INSERT INTO system_distributed.denylisted_partitions (ks_name, table_name, partition_key) VALUES ('ks','table1','12345f');`
+- Id has a colon - `INSERT INTO system_distributed.denylisted_partitions (ks_name, table_name, partition_key) VALUES ('ks','table1','1\:2');`
+
+In the case of composite column partition keys (Key1, Key2):
+
+- `INSERT INTO system_distributed.denylisted_partitions (ks_name, table_name, partition_key) VALUES ('ks', 'table1', 'k11:k21')`
+
+
+=== Special considerations
+
+The denylist has the property in that you want to keep your cache (see below) and
+CQL data on a replica set as close together as possible, so you don't have different
+nodes in your cluster denying or allowing different keys. To best achieve this,
+the workflow for a denylist change (addition or deletion) should always be as follows:
+
+JMX PATH (preferred for single changes):
+
+1. Call the JMX hook for ``denylistKey()`` with the desired key
+2. Double-check the cache reloaded with ``isKeyDenylisted()``
+3. Check for warnings about unrecognized keyspace/table combinations, limits, or
+consistency level. If you get a message about nodes being down and not hitting CL
+for denylist, recover the downed nodes and then trigger a re-load of the cache on each
+node with ``loadPartitionDenylist()``
+
+CQL PATH (preferred for bulk changes):
+
+1. Mutate the denylisted partition lists via CQL
+2. Trigger a re-load of the denylist cache on each node via JMX ``loadPartitionDenylist()`` (see below)
+3. Check for warnings about lack of availability for a denylist refresh. In the event nodes are down, recover them, then go to 2.
+
+Due to conditions on known unavailable range slices leading to alert storming on
+startup, the denylist cache will not load on node start unless it can achieve the
+configured consistency level in `cassandra.yaml` - `denylist_consistency_level`.
+The JMX call to `loadPartitionDenylist` will, however, load the cache regardless
+of the number of nodes available. This leaves the control for denylisting or not
+denylisting during degraded cluster states in the hands of the operator.
+
+== Denylisted Partitions Cache
+
+Cassandra internally maintains an on-heap cache of denylisted partitions loaded
+from ``system_distributed.denylisted_partitions``. The values for a table will be
+automatically repopulated every ``denylist_refresh`` as specified in the
+`conf/cassandra.yaml` file, defaulting to `600s`, or 10 minutes. Invalid records
+(unknown keyspaces, tables, or keys) will be ignored and not cached on load.
+
+The cache can be refreshed in the following ways:
+
+- During Cassandra node startup
+- Via the automatic on-heap cache refresh mechanisms. Note: this will occur asynchronously
+on query after the ``denylist_refresh`` time is hit.
+- Via the JMX command: ``loadPartitionDenylist`` in ``the org.apache.cassandra.service.
+StorageProxyMBean`` invocation point.
+
+The Cache size is bounded by the following two config properties
+
+- denylist_max_keys_per_table
+- denylist_max_keys_total
+
+On cache load, if a table exceeds the value allowed in `denylist_max_keys_per_table` (defaults to 1000),
+a warning will be printed to the logs and the remainder of the keys will not be cached.
+Similarly, if the total allowed size is exceeded, subsequent ks_name + table_name
+combinations (in clustering / lexicographical order) will be skipped as well, and a
+warning logged to the server logs.
+
+[NOTE]
+====
+Given the required workflow of 1) Mutate, 2) Reload cache, the auto-reload
+property seems superfluous. It exists to ensure that, should an operator make a
+mistake and denylist (or undenylist) a key but forget to reload the cache, that
+intent will be captured on the next cache reload.
+====
+
+== JMX Interface
+
+[cols="1,1"]
+|===
+| Command | Effect
+
+| loadPartitionDenylist()
+| Reloads cached denylist from CQL table
+
+| getPartitionDenylistLoadAttempts()
+| Gets the count of cache reload attempts
+
+| getPartitionDenylistLoadSuccesses()
+| Gets the count of cache reload successes
+
+| setEnablePartitionDenylist(boolean enabled)
+| Enables or disables the partition denylisting functionality
+
+| setEnableDenylistWrites(boolean enabled)
+| Enables or disables write denylisting functionality
+
+| setEnableDenylistReads(boolean enabled)
+| Enables or disables read denylisting functionality
+
+| setEnableDenylistRangeReads(boolean enabled)
+| Enables or disables range read denylisting functionality
+
+| denylistKey(String keyspace, String table, String partitionKeyAsString)
+| Adds a specific keyspace, table, and partition key combo to the denylist
+
+| removeDenylistKey(String keyspace, String cf, String partitionKeyAsString)
+| Removes a specific keyspace, table, and partition key combo from the denylist
+
+| setDenylistMaxKeysPerTable(int value)
+| Limits count of allowed keys per table in the denylist
+
+| setDenylistMaxKeysTotal(int value)
+| Limits the total count of allowable denylisted keys in the system
+
+| isKeyDenylisted(String keyspace, String table, String partitionKeyAsString)
+| Indicates whether the keyspace.table has the input partition key denied
+|===
diff --git a/doc/modules/cassandra/pages/operating/fqllogging.adoc b/doc/modules/cassandra/pages/managing/operating/fqllogging.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/fqllogging.adoc
rename to doc/modules/cassandra/pages/managing/operating/fqllogging.adoc
diff --git a/doc/modules/cassandra/pages/operating/hardware.adoc b/doc/modules/cassandra/pages/managing/operating/hardware.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/hardware.adoc
rename to doc/modules/cassandra/pages/managing/operating/hardware.adoc
diff --git a/doc/modules/cassandra/pages/operating/hints.adoc b/doc/modules/cassandra/pages/managing/operating/hints.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/hints.adoc
rename to doc/modules/cassandra/pages/managing/operating/hints.adoc
diff --git a/doc/modules/cassandra/pages/managing/operating/index.adoc b/doc/modules/cassandra/pages/managing/operating/index.adoc
new file mode 100644
index 0000000..41ee1b6
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/index.adoc
@@ -0,0 +1,16 @@
+== Operating Cassandra
+
+* xref:operating/hardware.adoc[Hardware]
+* xref:operating/security.adoc[Security]
+* xref:operating/topo_changes.adoc[Topology changes]
+* xref:operating/hints.adoc[Hints]
+* xref:operating/repair.adoc[Repair]
+* xref:operating/read_repair.adoc[Read repair]
+* xref:operating/backups.adoc[Backups]
+* xref:operating/compression.adoc[Compression]
+* xref:operating/compaction/index.adoc[Compaction]
+* xref:operating/metrics.adoc[Monitoring]
+* xref:operating/bulk_loading.adoc[Bulk loading]
+* xref:operating/cdc.adoc[CDC]
+* xref:operating/bloom_filters.adoc[Bloom filters]
+* xref:operating/denylisting_partitions.adoc[Denylisting partitions]
diff --git a/doc/modules/cassandra/pages/managing/operating/metrics.adoc b/doc/modules/cassandra/pages/managing/operating/metrics.adoc
new file mode 100644
index 0000000..0102285
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/metrics.adoc
@@ -0,0 +1,1097 @@
+= Monitoring
+
+Metrics in Cassandra are managed using the
+http://metrics.dropwizard.io[Dropwizard Metrics] library. These metrics
+can be queried via JMX or pushed to external monitoring systems using a
+number of
+http://metrics.dropwizard.io/3.1.0/getting-started/#other-reporting[built
+in] and http://metrics.dropwizard.io/3.1.0/manual/third-party/[third
+party] reporter plugins.
+
+Metrics are collected for a single node. It's up to the operator to use
+an external monitoring system to aggregate them.
+
+== Metric Types
+
+All metrics reported by cassandra fit into one of the following types.
+
+`Gauge`::
+ An instantaneous measurement of a value.
+`Counter`::
+ A gauge for an `AtomicLong` instance. Typically this is consumed by
+ monitoring the change since the last call to see if there is a large
+ increase compared to the norm.
+`Histogram`::
+ Measures the statistical distribution of values in a stream of data.
+ +
+ In addition to minimum, maximum, mean, etc., it also measures median,
+ 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles.
+`Timer`::
+ Measures both the rate that a particular piece of code is called and
+ the histogram of its duration.
+`Latency`::
+ Special type that tracks latency (in microseconds) with a `Timer` plus
+ a `Counter` that tracks the total latency accrued since starting. The
+ former is useful if you track the change in total latency since the
+ last check. Each metric name of this type will have 'Latency' and
+ 'TotalLatency' appended to it.
+`Meter`::
+ A meter metric which measures mean throughput and one-, five-, and
+ fifteen-minute exponentially-weighted moving average throughputs.
+
+== Table Metrics
+
+Each table in Cassandra has metrics responsible for tracking its state
+and performance.
+
+The metric names are all appended with the specific `Keyspace` and
+`Table` name.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Table.<MetricName>.<Keyspace>.<Table>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Table keyspace=<Keyspace> scope=<Table> name=<MetricName>`
+
+[NOTE]
+====
+There is a special table called '`all`' without a keyspace. This
+represents the aggregation of metrics across *all* tables and keyspaces
+on the node.
+====
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|MemtableOnHeapSize |Gauge<Long> |Total amount of data stored in the
+memtable that resides *on*-heap, including column related overhead and
+partitions overwritten.
+
+|MemtableOffHeapSize |Gauge<Long> |Total amount of data stored in the
+memtable that resides *off*-heap, including column related overhead and
+partitions overwritten.
+
+|MemtableLiveDataSize |Gauge<Long> |Total amount of live data stored in
+the memtable, excluding any data structure overhead.
+
+|AllMemtablesOnHeapSize |Gauge<Long> |Total amount of data stored in the
+memtables (2i and pending flush memtables included) that resides
+*on*-heap.
+
+|AllMemtablesOffHeapSize |Gauge<Long> |Total amount of data stored in
+the memtables (2i and pending flush memtables included) that resides
+*off*-heap.
+
+|AllMemtablesLiveDataSize |Gauge<Long> |Total amount of live data stored
+in the memtables (2i and pending flush memtables included) that resides
+off-heap, excluding any data structure overhead.
+
+|MemtableColumnsCount |Gauge<Long> |Total number of columns present in
+the memtable.
+
+|MemtableSwitchCount |Counter |Number of times flush has resulted in the
+memtable being switched out.
+
+|CompressionRatio |Gauge<Double> |Current compression ratio for all
+SSTables.
+
+|EstimatedPartitionSizeHistogram |Gauge<long[]> |Histogram of estimated
+partition size (in bytes).
+
+|EstimatedPartitionCount |Gauge<Long> |Approximate number of keys in
+table.
+
+|EstimatedColumnCountHistogram |Gauge<long[]> |Histogram of estimated
+number of columns.
+
+|SSTablesPerReadHistogram |Histogram |Histogram of the number of sstable
+data files accessed per single partition read. SSTables skipped due to
+Bloom Filters, min-max key or partition index lookup are not taken into
+acoount.
+
+|ReadLatency |Latency |Local read latency for this table.
+
+|RangeLatency |Latency |Local range scan latency for this table.
+
+|WriteLatency |Latency |Local write latency for this table.
+
+|CoordinatorReadLatency |Timer |Coordinator read latency for this table.
+
+|CoordinatorWriteLatency |Timer |Coordinator write latency for this
+table.
+
+|CoordinatorScanLatency |Timer |Coordinator range scan latency for this
+table.
+
+|PendingFlushes |Counter |Estimated number of flush tasks pending for
+this table.
+
+|BytesFlushed |Counter |Total number of bytes flushed since server
+[re]start.
+
+|CompactionBytesWritten |Counter |Total number of bytes written by
+compaction since server [re]start.
+
+|PendingCompactions |Gauge<Integer> |Estimate of number of pending
+compactions for this table.
+
+|LiveSSTableCount |Gauge<Integer> |Number of SSTables on disk for this
+table.
+
+|LiveDiskSpaceUsed |Counter |Disk space used by SSTables belonging to
+this table (in bytes).
+
+|TotalDiskSpaceUsed |Counter |Total disk space used by SSTables
+belonging to this table, including obsolete ones waiting to be GC'd.
+
+|MaxSSTableSize |Gauge<Long> |Maximum size of SSTable of this table -
+the physical size on disk of all components for such SSTable in bytes. Equals to
+zero if there is not any SSTable on disk.
+
+|MaxSSTableDuration |Gauge<Long> |Maximum duration in milliseconds of an SSTable for this table,
+computed as `maxTimestamp - minTimestamp`. Equals to zero if min or max timestamp is `Long.MAX_VALUE`.
+
+|MinPartitionSize |Gauge<Long> |Size of the smallest compacted partition
+(in bytes).
+
+|MaxPartitionSize |Gauge<Long> |Size of the largest compacted partition
+(in bytes).
+
+|MeanPartitionSize |Gauge<Long> |Size of the average compacted partition
+(in bytes).
+
+|BloomFilterFalsePositives |Gauge<Long> |Number of false positives on
+table's bloom filter.
+
+|BloomFilterFalseRatio |Gauge<Double> |False positive ratio of table's
+bloom filter.
+
+|BloomFilterDiskSpaceUsed |Gauge<Long> |Disk space used by bloom filter
+(in bytes).
+
+|BloomFilterOffHeapMemoryUsed |Gauge<Long> |Off-heap memory used by
+bloom filter.
+
+|IndexSummaryOffHeapMemoryUsed |Gauge<Long> |Off-heap memory used by
+index summary.
+
+|CompressionMetadataOffHeapMemoryUsed |Gauge<Long> |Off-heap memory used
+by compression meta data.
+
+|KeyCacheHitRate |Gauge<Double> |Key cache hit rate for this table.
+
+|TombstoneScannedHistogram |Histogram |Histogram of tombstones scanned
+in queries on this table.
+
+|LiveScannedHistogram |Histogram |Histogram of live cells scanned in
+queries on this table.
+
+|ColUpdateTimeDeltaHistogram |Histogram |Histogram of column update time
+delta on this table.
+
+|ViewLockAcquireTime |Timer |Time taken acquiring a partition lock for
+materialized view updates on this table.
+
+|ViewReadTime |Timer |Time taken during the local read of a materialized
+view update.
+
+|TrueSnapshotsSize |Gauge<Long> |Disk space used by snapshots of this
+table including all SSTable components.
+
+|RowCacheHitOutOfRange |Counter |Number of table row cache hits that do
+not satisfy the query filter, thus went to disk.
+
+|RowCacheHit |Counter |Number of table row cache hits.
+
+|RowCacheMiss |Counter |Number of table row cache misses.
+
+|CasPrepare |Latency |Latency of paxos prepare round.
+
+|CasPropose |Latency |Latency of paxos propose round.
+
+|CasCommit |Latency |Latency of paxos commit round.
+
+|PercentRepaired |Gauge<Double> |Percent of table data that is repaired
+on disk.
+
+|BytesRepaired |Gauge<Long> |Size of table data repaired on disk
+
+|BytesUnrepaired |Gauge<Long> |Size of table data unrepaired on disk
+
+|BytesPendingRepair |Gauge<Long> |Size of table data isolated for an
+ongoing incremental repair
+
+|SpeculativeRetries |Counter |Number of times speculative retries were
+sent for this table.
+
+|SpeculativeFailedRetries |Counter |Number of speculative retries that
+failed to prevent a timeout
+
+|SpeculativeInsufficientReplicas |Counter |Number of speculative retries
+that couldn't be attempted due to lack of replicas
+
+|SpeculativeSampleLatencyNanos |Gauge<Long> |Number of nanoseconds to
+wait before speculation is attempted. Value may be statically configured
+or updated periodically based on coordinator latency.
+
+|WaitingOnFreeMemtableSpace |Histogram |Histogram of time spent waiting
+for free memtable space, either on- or off-heap.
+
+|DroppedMutations |Counter |Number of dropped mutations on this table.
+
+|AnticompactionTime |Timer |Time spent anticompacting before a
+consistent repair.
+
+|ValidationTime |Timer |Time spent doing validation compaction during
+repair.
+
+|SyncTime |Timer |Time spent doing streaming during repair.
+
+|BytesValidated |Histogram |Histogram over the amount of bytes read
+during validation.
+
+|PartitionsValidated |Histogram |Histogram over the number of partitions
+read during validation.
+
+|BytesAnticompacted |Counter |How many bytes we anticompacted.
+
+|BytesMutatedAnticompaction |Counter |How many bytes we avoided
+anticompacting because the sstable was fully contained in the repaired
+range.
+
+|MutatedAnticompactionGauge |Gauge<Double> |Ratio of bytes mutated vs
+total bytes repaired.
+|===
+
+== Keyspace Metrics
+
+Each keyspace in Cassandra has metrics responsible for tracking its
+state and performance.
+
+Most of these metrics are the same as the `Table Metrics` above, only
+they are aggregated at the Keyspace level. The keyspace specific metrics
+are specified in the table below.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.keyspace.<MetricName>.<Keyspace>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Keyspace scope=<Keyspace> name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|WriteFailedIdeaCL |Counter |Number of writes that failed to achieve the
+configured ideal consistency level or 0 if none is configured
+
+|IdealCLWriteLatency |Latency |Coordinator latency of writes at the
+configured ideal consistency level. No values are recorded if ideal
+consistency level is not configured
+
+|RepairTime |Timer |Total time spent as repair coordinator.
+
+|RepairPrepareTime |Timer |Total time spent preparing for repair.
+|===
+
+== ThreadPool Metrics
+
+Cassandra splits work of a particular type into its own thread pool.
+This provides back-pressure and asynchrony for requests on a node. It's
+important to monitor the state of these thread pools since they can tell
+you how saturated a node is.
+
+The metric names are all appended with the specific `ThreadPool` name.
+The thread pools are also categorized under a specific type.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.ThreadPools.<MetricName>.<Path>.<ThreadPoolName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=ThreadPools path=<Path> scope=<ThreadPoolName> name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|ActiveTasks |Gauge<Integer> |Number of tasks being actively worked on
+by this pool.
+
+|PendingTasks |Gauge<Integer> |Number of queued tasks queued up on this
+pool.
+
+|CompletedTasks |Counter |Number of tasks completed.
+
+|TotalBlockedTasks |Counter |Number of tasks that were blocked due to
+queue saturation.
+
+|CurrentlyBlockedTask |Counter |Number of tasks that are currently
+blocked due to queue saturation but on retry will become unblocked.
+
+|MaxPoolSize |Gauge<Integer> |The maximum number of threads in this
+pool.
+
+|MaxTasksQueued |Gauge<Integer> |The maximum number of tasks queued
+before a task get blocked.
+|===
+
+The following thread pools can be monitored.
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Native-Transport-Requests |transport |Handles client CQL requests
+
+|CounterMutationStage |request |Responsible for counter writes
+
+|ViewMutationStage |request |Responsible for materialized view writes
+
+|MutationStage |request |Responsible for all other writes
+
+|ReadRepairStage |request |ReadRepair happens on this thread pool
+
+|ReadStage |request |Local reads run on this thread pool
+
+|RequestResponseStage |request |Coordinator requests to the cluster run
+on this thread pool
+
+|AntiEntropyStage |internal |Builds merkle tree for repairs
+
+|CacheCleanupExecutor |internal |Cache maintenance performed on this
+thread pool
+
+|CompactionExecutor |internal |Compactions are run on these threads
+
+|GossipStage |internal |Handles gossip requests
+
+|HintsDispatcher |internal |Performs hinted handoff
+
+|InternalResponseStage |internal |Responsible for intra-cluster
+callbacks
+
+|MemtableFlushWriter |internal |Writes memtables to disk
+
+|MemtablePostFlush |internal |Cleans up commit log after memtable is
+written to disk
+
+|MemtableReclaimMemory |internal |Memtable recycling
+
+|MigrationStage |internal |Runs schema migrations
+
+|MiscStage |internal |Misceleneous tasks run here
+
+|PendingRangeCalculator |internal |Calculates token range
+
+|PerDiskMemtableFlushWriter_0 |internal |Responsible for writing a spec
+(there is one of these per disk 0-N)
+
+|Sampler |internal |Responsible for re-sampling the index summaries of
+SStables
+
+|SecondaryIndexManagement |internal |Performs updates to secondary
+indexes
+
+|ValidationExecutor |internal |Performs validation compaction or
+scrubbing
+
+|ViewBuildExecutor |internal |Performs materialized views initial build
+|===
+
+== Client Request Metrics
+
+Client requests have their own set of metrics that encapsulate the work
+happening at coordinator level.
+
+Different types of client requests are broken down by `RequestType`.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.ClientRequest.<MetricName>.<RequestType>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=ClientRequest scope=<RequestType> name=<MetricName>`
+
+RequestType::
+ CASRead
+Description::
+ Metrics related to transactional read requests.
+Metrics::
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Timeouts |Counter |Number of timeouts encountered.
+
+|Failures |Counter |Number of transaction failures encountered.
+
+| |Latency |Transaction read latency.
+
+|Unavailables |Counter |Number of unavailable exceptions encountered.
+
+|UnfinishedCommit |Counter |Number of transactions that were committed
+on read.
+
+|ConditionNotMet |Counter |Number of transaction preconditions did not
+match current values.
+
+|ContentionHistogram |Histogram |How many contended reads were
+encountered
+|===
+RequestType::
+ CASWrite
+Description::
+ Metrics related to transactional write requests.
+Metrics::
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Timeouts |Counter |Number of timeouts encountered.
+
+|Failures |Counter |Number of transaction failures encountered.
+
+| |Latency |Transaction write latency.
+
+|Unavailables |Counter |Number of unavailable exceptions encountered.
+
+|UnfinishedCommit |Counter |Number of transactions that were committed
+on write.
+
+|ConditionNotMet |Counter |Number of transaction preconditions did not
+match current values.
+
+|ContentionHistogram |Histogram |How many contended writes were
+encountered
+
+|MutationSizeHistogram |Histogram |Total size in bytes of the requests
+mutations.
+|===
+RequestType::
+ Read
+Description::
+ Metrics related to standard read requests.
+Metrics::
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Timeouts |Counter |Number of timeouts encountered.
+|Failures |Counter |Number of read failures encountered.
+| |Latency |Read latency.
+|Unavailables |Counter |Number of unavailable exceptions encountered.
+|===
+RequestType::
+ RangeSlice
+Description::
+ Metrics related to token range read requests.
+Metrics::
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Timeouts |Counter |Number of timeouts encountered.
+|Failures |Counter |Number of range query failures encountered.
+| |Latency |Range query latency.
+|Unavailables |Counter |Number of unavailable exceptions encountered.
+|===
+RequestType::
+ Write
+Description::
+ Metrics related to regular write requests.
+Metrics::
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Timeouts |Counter |Number of timeouts encountered.
+
+|Failures |Counter |Number of write failures encountered.
+
+| |Latency |Write latency.
+
+|Unavailables |Counter |Number of unavailable exceptions encountered.
+
+|MutationSizeHistogram |Histogram |Total size in bytes of the requests
+mutations.
+|===
+RequestType::
+ ViewWrite
+Description::
+ Metrics related to materialized view write wrtes.
+Metrics::
+[cols=",,",]
+|===
+|Timeouts |Counter |Number of timeouts encountered.
+
+|Failures |Counter |Number of transaction failures encountered.
+
+|Unavailables |Counter |Number of unavailable exceptions encountered.
+
+|ViewReplicasAttempted |Counter |Total number of attempted view
+replica writes.
+
+|ViewReplicasSuccess |Counter |Total number of succeded view replica
+writes.
+
+|ViewPendingMutations |Gauge<Long> |ViewReplicasAttempted -
+ViewReplicasSuccess.
+
+|ViewWriteLatency |Timer |Time between when mutation is applied to
+base table and when CL.ONE is achieved on view.
+|===
+
+== Cache Metrics
+
+Cassandra caches have metrics to track the effectivness of the caches.
+Though the `Table Metrics` might be more useful.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Cache.<MetricName>.<CacheName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Cache scope=<CacheName> name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Capacity |Gauge<Long> |Cache capacity in bytes.
+|Entries |Gauge<Integer> |Total number of cache entries.
+|FifteenMinuteCacheHitRate |Gauge<Double> |15m cache hit rate.
+|FiveMinuteCacheHitRate |Gauge<Double> |5m cache hit rate.
+|OneMinuteCacheHitRate |Gauge<Double> |1m cache hit rate.
+|HitRate |Gauge<Double> |All time cache hit rate.
+|Hits |Meter |Total number of cache hits.
+|Misses |Meter |Total number of cache misses.
+|MissLatency |Timer |Latency of misses.
+|Requests |Gauge<Long> |Total number of cache requests.
+|Size |Gauge<Long> |Total size of occupied cache, in bytes.
+|===
+
+The following caches are covered:
+
+[cols=",",options="header",]
+|===
+|Name |Description
+|CounterCache |Keeps hot counters in memory for performance.
+|ChunkCache |In process uncompressed page cache.
+|KeyCache |Cache for partition to sstable offsets.
+|RowCache |Cache for rows kept in memory.
+|===
+
+[NOTE]
+====
+Misses and MissLatency are only defined for the ChunkCache
+====
+== CQL Metrics
+
+Metrics specific to CQL prepared statement caching.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.CQL.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=CQL name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|PreparedStatementsCount |Gauge<Integer> |Number of cached prepared
+statements.
+
+|PreparedStatementsEvicted |Counter |Number of prepared statements
+evicted from the prepared statement cache
+
+|PreparedStatementsExecuted |Counter |Number of prepared statements
+executed.
+
+|RegularStatementsExecuted |Counter |Number of *non* prepared statements
+executed.
+
+|PreparedStatementsRatio |Gauge<Double> |Percentage of statements that
+are prepared vs unprepared.
+|===
+
+[[dropped-metrics]]
+== DroppedMessage Metrics
+
+Metrics specific to tracking dropped messages for different types of
+requests. Dropped writes are stored and retried by `Hinted Handoff`
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.DroppedMessage.<MetricName>.<Type>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=DroppedMessage scope=<Type> name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|CrossNodeDroppedLatency |Timer |The dropped latency across nodes.
+|InternalDroppedLatency |Timer |The dropped latency within node.
+|Dropped |Meter |Number of dropped messages.
+|===
+
+The different types of messages tracked are:
+
+[cols=",",options="header",]
+|===
+|Name |Description
+|BATCH_STORE |Batchlog write
+|BATCH_REMOVE |Batchlog cleanup (after succesfully applied)
+|COUNTER_MUTATION |Counter writes
+|HINT |Hint replay
+|MUTATION |Regular writes
+|READ |Regular reads
+|READ_REPAIR |Read repair
+|PAGED_SLICE |Paged read
+|RANGE_SLICE |Token range read
+|REQUEST_RESPONSE |RPC Callbacks
+|_TRACE |Tracing writes
+|===
+
+== Streaming Metrics
+
+Metrics reported during `Streaming` operations, such as repair,
+bootstrap, rebuild.
+
+These metrics are specific to a peer endpoint, with the source node
+being the node you are pulling the metrics from.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Streaming.<MetricName>.<PeerIP>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Streaming scope=<PeerIP> name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|IncomingBytes |Counter |Number of bytes streamed to this node from the
+peer.
+
+|OutgoingBytes |Counter |Number of bytes streamed to the peer endpoint
+from this node.
+|===
+
+== Compaction Metrics
+
+Metrics specific to `Compaction` work.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Compaction.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Compaction name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|BytesCompacted |Counter |Total number of bytes compacted since server
+[re]start.
+
+|PendingTasks |Gauge<Integer> |Estimated number of compactions remaining
+to perform.
+
+|CompletedTasks |Gauge<Long> |Number of completed compactions since
+server [re]start.
+
+|TotalCompactionsCompleted |Meter |Throughput of completed compactions
+since server [re]start.
+
+|PendingTasksByTableName |Gauge<Map<String, Map<String, Integer>>>
+|Estimated number of compactions remaining to perform, grouped by
+keyspace and then table name. This info is also kept in `Table Metrics`.
+|===
+
+== CommitLog Metrics
+
+Metrics specific to the `CommitLog`
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.CommitLog.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=CommitLog name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|CompletedTasks |Gauge<Long> |Total number of commit log messages
+written since [re]start.
+
+|PendingTasks |Gauge<Long> |Number of commit log messages written but
+yet to be fsync'd.
+
+|TotalCommitLogSize |Gauge<Long> |Current size, in bytes, used by all
+the commit log segments.
+
+|WaitingOnSegmentAllocation |Timer |Time spent waiting for a
+CommitLogSegment to be allocated - under normal conditions this should
+be zero.
+
+|WaitingOnCommit |Timer |The time spent waiting on CL fsync; for
+Periodic this is only occurs when the sync is lagging its sync interval.
+|===
+
+== Storage Metrics
+
+Metrics specific to the storage engine.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Storage.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Storage name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Exceptions |Counter |Number of internal exceptions caught. Under normal
+exceptions this should be zero.
+
+|Load |Counter |Size, in bytes, of the on disk data size this node
+manages.
+
+|TotalHints |Counter |Number of hint messages written to this node since
+[re]start. Includes one entry for each host to be hinted per hint.
+
+|TotalHintsInProgress |Counter |Number of hints attemping to be sent
+currently.
+|===
+
+[[handoff-metrics]]
+== HintedHandoff Metrics
+
+Metrics specific to Hinted Handoff. There are also some metrics related
+to hints tracked in `Storage Metrics`
+
+These metrics include the peer endpoint *in the metric name*
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.HintedHandOffManager.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=HintedHandOffManager name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Hints_created-<PeerIP> a|
+____
+Counter
+____
+
+a|
+____
+Number of hints on disk for this peer.
+____
+
+|Hints_not_stored-<PeerIP> a|
+____
+Counter
+____
+
+a|
+____
+Number of hints not stored for this peer, due to being down past the
+configured hint window.
+____
+
+|===
+
+== HintsService Metrics
+
+Metrics specific to the Hints delivery service. There are also some
+metrics related to hints tracked in `Storage Metrics`
+
+These metrics include the peer endpoint *in the metric name*
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.HintsService.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=HintsService name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|HintsSucceeded a|
+____
+Meter
+____
+
+a|
+____
+A meter of the hints successfully delivered
+____
+
+|HintsFailed a|
+____
+Meter
+____
+
+a|
+____
+A meter of the hints that failed deliver
+____
+
+|HintsTimedOut a|
+____
+Meter
+____
+
+a|
+____
+A meter of the hints that timed out
+____
+
+|Hint_delays |Histogram |Histogram of hint delivery delays (in
+milliseconds)
+
+|Hint_delays-<PeerIP> |Histogram |Histogram of hint delivery delays (in
+milliseconds) per peer
+|===
+
+== SSTable Index Metrics
+
+Metrics specific to the SSTable index metadata.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Index.<MetricName>.RowIndexEntry`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Index scope=RowIndexEntry name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|IndexedEntrySize |Histogram |Histogram of the on-heap size, in bytes,
+of the index across all SSTables.
+
+|IndexInfoCount |Histogram |Histogram of the number of on-heap index
+entries managed across all SSTables.
+
+|IndexInfoGets |Histogram |Histogram of the number index seeks performed
+per SSTable.
+|===
+
+== BufferPool Metrics
+
+Metrics specific to the internal recycled buffer pool Cassandra manages.
+This pool is meant to keep allocations and GC lower by recycling on and
+off heap buffers.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.BufferPool.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=BufferPool name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Size |Gauge<Long> |Size, in bytes, of the managed buffer pool
+
+|Misses |Meter a|
+____
+The rate of misses in the pool. The higher this is the more allocations
+incurred.
+____
+
+|===
+
+== Client Metrics
+
+Metrics specifc to client managment.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Client.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Client name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|connectedNativeClients |Gauge<Integer> |Number of clients connected to
+this nodes native protocol server
+
+|connections |Gauge<List<Map<String, String>> |List of all connections
+and their state information
+
+|connectedNativeClientsByUser |Gauge<Map<String, Int> |Number of
+connnective native clients by username
+|===
+
+== Batch Metrics
+
+Metrics specifc to batch statements.
+
+Reported name format:
+
+*Metric Name*::
+ `org.apache.cassandra.metrics.Batch.<MetricName>`
+*JMX MBean*::
+ `org.apache.cassandra.metrics:type=Batch name=<MetricName>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|PartitionsPerCounterBatch |Histogram |Distribution of the number of
+partitions processed per counter batch
+
+|PartitionsPerLoggedBatch |Histogram |Distribution of the number of
+partitions processed per logged batch
+
+|PartitionsPerUnloggedBatch |Histogram |Distribution of the number of
+partitions processed per unlogged batch
+|===
+
+== JVM Metrics
+
+JVM metrics such as memory and garbage collection statistics can either
+be accessed by connecting to the JVM using JMX or can be exported using
+<<metric_reporters>>.
+
+=== BufferPool
+
+*Metric Name*::
+ `jvm.buffers.<direct|mapped>.<MetricName>`
+*JMX MBean*::
+ `java.nio:type=BufferPool name=<direct|mapped>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Capacity |Gauge<Long> |Estimated total capacity of the buffers in this
+pool
+
+|Count |Gauge<Long> |Estimated number of buffers in the pool
+
+|Used |Gauge<Long> |Estimated memory that the Java virtual machine is
+using for this buffer pool
+|===
+
+=== FileDescriptorRatio
+
+*Metric Name*::
+ `jvm.fd.<MetricName>`
+*JMX MBean*::
+ `java.lang:type=OperatingSystem name=<OpenFileDescriptorCount|MaxFileDescriptorCount>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Usage |Ratio |Ratio of used to total file descriptors
+|===
+
+=== GarbageCollector
+
+*Metric Name*::
+ `jvm.gc.<gc_type>.<MetricName>`
+*JMX MBean*::
+ `java.lang:type=GarbageCollector name=<gc_type>`
+
+[cols=",,",options="header",]
+|===
+|Name |Type |Description
+|Count |Gauge<Long> |Total number of collections that have occurred
+
+|Time |Gauge<Long> |Approximate accumulated collection elapsed time in
+milliseconds
+|===
+
+=== Memory
+
+*Metric Name*::
+ `jvm.memory.<heap/non-heap/total>.<MetricName>`
+*JMX MBean*::
+ `java.lang:type=Memory`
+
+[cols=",,",]
+|===
+|Committed |Gauge<Long> |Amount of memory in bytes that is committed for
+the JVM to use
+
+|Init |Gauge<Long> |Amount of memory in bytes that the JVM initially
+requests from the OS
+
+|Max |Gauge<Long> |Maximum amount of memory in bytes that can be used
+for memory management
+
+|Usage |Ratio |Ratio of used to maximum memory
+
+|Used |Gauge<Long> |Amount of used memory in bytes
+|===
+
+=== MemoryPool
+
+*Metric Name*::
+ `jvm.memory.pools.<memory_pool>.<MetricName>`
+*JMX MBean*::
+ `java.lang:type=MemoryPool name=<memory_pool>`
+
+[cols=",,",]
+|===
+|Committed |Gauge<Long> |Amount of memory in bytes that is committed for
+the JVM to use
+
+|Init |Gauge<Long> |Amount of memory in bytes that the JVM initially
+requests from the OS
+
+|Max |Gauge<Long> |Maximum amount of memory in bytes that can be used
+for memory management
+
+|Usage |Ratio |Ratio of used to maximum memory
+
+|Used |Gauge<Long> |Amount of used memory in bytes
+|===
+
+== JMX
+
+Any JMX based client can access metrics from cassandra.
+
+If you wish to access JMX metrics over http it's possible to download
+http://mx4j.sourceforge.net/[Mx4jTool] and place `mx4j-tools.jar` into
+the classpath. On startup you will see in the log:
+
+[source,none]
+----
+HttpAdaptor version 3.0.2 started on port 8081
+----
+
+To choose a different port (8081 is the default) or a different listen
+address (0.0.0.0 is not the default) edit `conf/cassandra-env.sh` and
+uncomment:
+
+[source,none]
+----
+#MX4J_ADDRESS="-Dmx4jaddress=0.0.0.0"
+
+#MX4J_PORT="-Dmx4jport=8081"
+----
+
+== Metric Reporters [[metric_reporters]]
+
+As mentioned at the top of this section on monitoring the Cassandra
+metrics can be exported to a number of monitoring system a number of
+http://metrics.dropwizard.io/3.1.0/getting-started/#other-reporting[built
+in] and http://metrics.dropwizard.io/3.1.0/manual/third-party/[third
+party] reporter plugins.
+
+The configuration of these plugins is managed by the
+https://github.com/addthis/metrics-reporter-config[metrics reporter
+config project]. There is a sample configuration file located at
+`conf/metrics-reporter-config-sample.yaml`.
+
+Once configured, you simply start cassandra with the flag
+`-Dcassandra.metricsReporterConfigFile=metrics-reporter-config.yaml`.
+The specified .yaml file plus any 3rd party reporter jars must all be in
+Cassandra's classpath.
diff --git a/doc/modules/cassandra/pages/operating/read_repair.adoc b/doc/modules/cassandra/pages/managing/operating/read_repair.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/read_repair.adoc
rename to doc/modules/cassandra/pages/managing/operating/read_repair.adoc
diff --git a/doc/modules/cassandra/pages/operating/repair.adoc b/doc/modules/cassandra/pages/managing/operating/repair.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/repair.adoc
rename to doc/modules/cassandra/pages/managing/operating/repair.adoc
diff --git a/doc/modules/cassandra/pages/operating/security.adoc b/doc/modules/cassandra/pages/managing/operating/security.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/security.adoc
rename to doc/modules/cassandra/pages/managing/operating/security.adoc
diff --git a/doc/modules/cassandra/pages/operating/topo_changes.adoc b/doc/modules/cassandra/pages/managing/operating/topo_changes.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/topo_changes.adoc
rename to doc/modules/cassandra/pages/managing/operating/topo_changes.adoc
diff --git a/doc/modules/cassandra/pages/operating/transientreplication.adoc b/doc/modules/cassandra/pages/managing/operating/transientreplication.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/operating/transientreplication.adoc
rename to doc/modules/cassandra/pages/managing/operating/transientreplication.adoc
diff --git a/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc b/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc
new file mode 100644
index 0000000..3a0bcb3
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc
@@ -0,0 +1,484 @@
+= Virtual Tables
+
+Apache Cassandra 4.0 implements virtual tables (https://issues.apache.org/jira/browse/CASSANDRA-7622[CASSANDRA-7622]).
+Virtual tables are tables backed by an API instead of data explicitly managed and stored as SSTables.
+Apache Cassandra 4.0 implements a virtual keyspace interface for virtual tables.
+Virtual tables are specific to each node.
+
+Some of the features of virtual tables are the ability to:
+
+* expose metrics through CQL
+* expose YAML configuration information
+
+Virtual keyspaces and tables are quite different from regular tables and keyspaces:
+
+* Virtual tables are created in special keyspaces and not just any keyspace.
+* Virtual tables are managed by Cassandra. Users cannot run DDL to create new virtual tables or DML to modify existing virtual tables.
+* Virtual tables are currently read-only, although that may change in a later version.
+* Virtual tables are local only, non-distributed, and thus not replicated.
+* Virtual tables have no associated SSTables.
+* Consistency level of the queries sent to virtual tables are ignored.
+* All existing virtual tables use `LocalPartitioner`.
+Since a virtual table is not replicated the partitioner sorts in order of partition keys instead of by their hash.
+* Making advanced queries using `ALLOW FILTERING` and aggregation functions can be executed in virtual tables, even though in normal tables we do not recommend it.
+From https://issues.apache.org/jira/browse/CASSANDRA-18238[CASSANDRA-18238], it is not necessary to specify `ALLOW FILTERING` when a query would normally require that, except when querying the table `system_views.system_logs`.
+
+== Virtual Keyspaces
+
+Apache Cassandra 4.0 has added two new keyspaces for virtual tables:
+
+* `system_virtual_schema`
+* `system_views`.
+
+The `system_virtual_schema` keyspace has three tables: `keyspaces`,
+`columns` and `tables` for the virtual keyspace, table, and column definitions, respectively.
+These tables contain schema information for the virtual tables.
+It is used by Cassandra internally and a user should not access it directly.
+
+The `system_views` keyspace contains the actual virtual tables.
+
+== Virtual Table Limitations
+
+Before disccusing virtual keyspaces and tables, note that virtual keyspaces and tables have some limitations.
+These limitations are subject to change.
+Virtual keyspaces cannot be altered or dropped.
+In fact, no operations can be performed against virtual keyspaces.
+
+Virtual tables cannot be created in virtual keyspaces.
+Virtual tables cannot be altered, dropped, or truncated.
+Secondary indexes, types, functions, aggregates, materialized views, and triggers cannot be created for virtual tables.
+Expiring time-to-live (TTL) columns cannot be created.
+Virtual tables do not support conditional updates or deletes.
+Aggregates may be run in SELECT statements.
+
+Conditional batch statements cannot include mutations for virtual tables, nor can a virtual table statement be included in a logged batch.
+In fact, mutations for virtual and regular tables cannot occur in the same batch table.
+
+== Virtual Tables
+
+Each of the virtual tables in the `system_views` virtual keyspace contain different information.
+
+The following table describes the virtual tables:
+
+[width="98%",cols="27%,73%",]
+|===
+|Virtual Table |Description
+
+|caches |Displays the general cache information including cache name, capacity_bytes, entry_count, hit_count, hit_ratio double,
+recent_hit_rate_per_second, recent_request_rate_per_second, request_count, and size_bytes.
+
+|clients |Lists information about all connected clients.
+
+|coordinator_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator reads.
+
+|coordinator_scan |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator scans.
+
+|coordinator_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator writes.
+
+|disk_usage |Records disk usage including disk_space, keyspace_name, and table_name, sorted by system keyspaces.
+
+|internode_inbound |Lists information about the inbound internode messaging.
+
+|internode_outbound |Information about the outbound internode messaging.
+
+|local_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local reads.
+
+|local_scan |Records counts, keyspace_name, table_name, max, median, and per_second for local scans.
+
+|local_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local writes.
+
+|max_partition_size |A table metric for maximum partition size.
+
+|rows_per_read |Records counts, keyspace_name, tablek_name, max, and median for rows read.
+
+|settings |Displays configuration settings in cassandra.yaml.
+
+|sstable_tasks |Lists currently running tasks and progress on SSTables, for operations like compaction and upgrade.
+
+|system_logs |Displays Cassandra logs if logged via CQLLOG appender in logback.xml
+
+|system_properties |Displays environmental system properties set on the node.
+
+|thread_pools |Lists metrics for each thread pool.
+
+|tombstones_per_read |Records counts, keyspace_name, tablek_name, max, and median for tombstones.
+|===
+
+For improved usability, from https://issues.apache.org/jira/browse/CASSANDRA-18238[CASSANDRA-18238],
+all tables except `system_logs` have `ALLOW FILTERING` implicitly added to a query when required by CQL specification.
+
+We shall discuss some of the virtual tables in more detail next.
+
+=== Clients Virtual Table
+
+The `clients` virtual table lists all active connections (connected
+clients) including their ip address, port, client_options, connection stage, driver
+name, driver version, hostname, protocol version, request count, ssl
+enabled, ssl protocol and user name:
+
+....
+cqlsh> EXPAND ON ;
+Now Expanded output is enabled
+cqlsh> SELECT * FROM system_views.clients;
+
+@ Row 1
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50687
+ client_options | {'CQL_VERSION': '3.4.7', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
+ connection_stage | ready
+ driver_name | DataStax Python Driver
+ driver_version | 3.25.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 16
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+@ Row 2
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50688
+ client_options | {'CQL_VERSION': '3.4.7', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
+ connection_stage | ready
+ driver_name | DataStax Python Driver
+ driver_version | 3.25.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 4
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+@ Row 3
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50753
+ client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
+ connection_stage | ready
+ driver_name | DataStax Java driver for Apache Cassandra(R)
+ driver_version | 4.13.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 18
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+@ Row 4
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50755
+ client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
+ connection_stage | ready
+ driver_name | DataStax Java driver for Apache Cassandra(R)
+ driver_version | 4.13.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 7
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+(4 rows)
+....
+
+Some examples of how `clients` can be used are:
+
+* To find applications using old incompatible versions of drivers before
+upgrading and with `nodetool enableoldprotocolversions` and
+`nodetool disableoldprotocolversions` during upgrades.
+* To identify clients sending too many requests.
+* To find if SSL is enabled during the migration to and from ssl.
+
+The virtual tables may be described with `DESCRIBE` statement. The DDL
+listed however cannot be run to create a virtual table. As an example
+describe the `system_views.clients` virtual table:
+
+....
+cqlsh> DESCRIBE TABLE system_views.clients;
+
+/*
+Warning: Table system_views.clients is a virtual table and cannot be recreated with CQL.
+Structure, for reference:
+VIRTUAL TABLE system_views.clients (
+ address inet,
+ port int,
+ client_options frozen<map<text, text>>,
+ connection_stage text,
+ driver_name text,
+ driver_version text,
+ hostname text,
+ protocol_version int,
+ request_count bigint,
+ ssl_cipher_suite text,
+ ssl_enabled boolean,
+ ssl_protocol text,
+ username text,
+ PRIMARY KEY (address, port)
+) WITH CLUSTERING ORDER BY (port ASC)
+ AND comment = 'currently connected clients';
+*/
+....
+
+=== Caches Virtual Table
+
+The `caches` virtual table lists information about the caches. The four
+caches presently created are chunks, counters, keys and rows. A query on
+the `caches` virtual table returns the following details:
+
+....
+cqlsh:system_views> SELECT * FROM system_views.caches;
+name | capacity_bytes | entry_count | hit_count | hit_ratio | recent_hit_rate_per_second | recent_request_rate_per_second | request_count | size_bytes
+---------+----------------+-------------+-----------+-----------+----------------------------+--------------------------------+---------------+------------
+ chunks | 229638144 | 29 | 166 | 0.83 | 5 | 6 | 200 | 475136
+counters | 26214400 | 0 | 0 | NaN | 0 | 0 | 0 | 0
+ keys | 52428800 | 14 | 124 | 0.873239 | 4 | 4 | 142 | 1248
+ rows | 0 | 0 | 0 | NaN | 0 | 0 | 0 | 0
+
+(4 rows)
+....
+
+=== Settings Virtual Table
+
+The `settings` table is rather useful and lists all the current
+configuration settings from the `cassandra.yaml`. The encryption options
+are overridden to hide the sensitive truststore information or
+passwords. The configuration settings however cannot be set using DML on
+the virtual table presently: :
+
+....
+cqlsh:system_views> SELECT * FROM system_views.settings;
+
+name | value
+-------------------------------------+--------------------
+ allocate_tokens_for_keyspace | null
+ audit_logging_options_enabled | false
+ auto_snapshot | true
+ automatic_sstable_upgrade | false
+ cluster_name | Test Cluster
+ enable_transient_replication | false
+ hinted_handoff_enabled | true
+ hints_directory | /home/ec2-user/cassandra/data/hints
+ incremental_backups | false
+ initial_token | null
+ ...
+ ...
+ ...
+ rpc_address | localhost
+ ssl_storage_port | 7001
+ start_native_transport | true
+ storage_port | 7000
+ stream_entire_sstables | true
+ (224 rows)
+....
+
+The `settings` table can be really useful if yaml file has been changed
+since startup and dont know running configuration, or to find if they
+have been modified via jmx/nodetool or virtual tables.
+
+=== Thread Pools Virtual Table
+
+The `thread_pools` table lists information about all thread pools.
+Thread pool information includes active tasks, active tasks limit,
+blocked tasks, blocked tasks all time, completed tasks, and pending
+tasks. A query on the `thread_pools` returns following details:
+
+....
+cqlsh:system_views> select * from system_views.thread_pools;
+
+name | active_tasks | active_tasks_limit | blocked_tasks | blocked_tasks_all_time | completed_tasks | pending_tasks
+------------------------------+--------------+--------------------+---------------+------------------------+-----------------+---------------
+ AntiEntropyStage | 0 | 1 | 0 | 0 | 0 | 0
+ CacheCleanupExecutor | 0 | 1 | 0 | 0 | 0 | 0
+ CompactionExecutor | 0 | 2 | 0 | 0 | 881 | 0
+ CounterMutationStage | 0 | 32 | 0 | 0 | 0 | 0
+ GossipStage | 0 | 1 | 0 | 0 | 0 | 0
+ HintsDispatcher | 0 | 2 | 0 | 0 | 0 | 0
+ InternalResponseStage | 0 | 2 | 0 | 0 | 0 | 0
+ MemtableFlushWriter | 0 | 2 | 0 | 0 | 1 | 0
+ MemtablePostFlush | 0 | 1 | 0 | 0 | 2 | 0
+ MemtableReclaimMemory | 0 | 1 | 0 | 0 | 1 | 0
+ MigrationStage | 0 | 1 | 0 | 0 | 0 | 0
+ MiscStage | 0 | 1 | 0 | 0 | 0 | 0
+ MutationStage | 0 | 32 | 0 | 0 | 0 | 0
+ Native-Transport-Requests | 1 | 128 | 0 | 0 | 130 | 0
+ PendingRangeCalculator | 0 | 1 | 0 | 0 | 1 | 0
+PerDiskMemtableFlushWriter_0 | 0 | 2 | 0 | 0 | 1 | 0
+ ReadStage | 0 | 32 | 0 | 0 | 13 | 0
+ Repair-Task | 0 | 2147483647 | 0 | 0 | 0 | 0
+ RequestResponseStage | 0 | 2 | 0 | 0 | 0 | 0
+ Sampler | 0 | 1 | 0 | 0 | 0 | 0
+ SecondaryIndexManagement | 0 | 1 | 0 | 0 | 0 | 0
+ ValidationExecutor | 0 | 2147483647 | 0 | 0 | 0 | 0
+ ViewBuildExecutor | 0 | 1 | 0 | 0 | 0 | 0
+ ViewMutationStage | 0 | 32 | 0 | 0 | 0 | 0
+....
+
+(24 rows)
+
+=== Internode Inbound Messaging Virtual Table
+
+The `internode_inbound` virtual table is for the internode inbound
+messaging. Initially no internode inbound messaging may get listed. In
+addition to the address, port, datacenter and rack information includes
+corrupt frames recovered, corrupt frames unrecovered, error bytes, error
+count, expired bytes, expired count, processed bytes, processed count,
+received bytes, received count, scheduled bytes, scheduled count,
+throttled count, throttled nanos, using bytes, using reserve bytes. A
+query on the `internode_inbound` returns following details:
+
+....
+cqlsh:system_views> SELECT * FROM system_views.internode_inbound;
+address | port | dc | rack | corrupt_frames_recovered | corrupt_frames_unrecovered |
+error_bytes | error_count | expired_bytes | expired_count | processed_bytes |
+processed_count | received_bytes | received_count | scheduled_bytes | scheduled_count | throttled_count | throttled_nanos | using_bytes | using_reserve_bytes
+---------+------+----+------+--------------------------+----------------------------+-
+----------
+(0 rows)
+....
+
+=== SSTables Tasks Virtual Table
+
+The `sstable_tasks` could be used to get information about running
+tasks. It lists following columns:
+
+....
+cqlsh:system_views> SELECT * FROM sstable_tasks;
+keyspace_name | table_name | task_id | kind | progress | total | unit
+---------------+------------+--------------------------------------+------------+----------+----------+-------
+ basic | wide2 | c3909740-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 60418761 | 70882110 | bytes
+ basic | wide2 | c7556770-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 2995623 | 40314679 | bytes
+....
+
+As another example, to find how much time is remaining for SSTable
+tasks, use the following query:
+
+....
+SELECT total - progress AS remaining
+FROM system_views.sstable_tasks;
+....
+
+=== Other Virtual Tables
+
+Some examples of using other virtual tables are as follows.
+
+Find tables with most disk usage:
+
+....
+cqlsh> SELECT * FROM disk_usage WHERE mebibytes > 1 ALLOW FILTERING;
+
+keyspace_name | table_name | mebibytes
+---------------+------------+-----------
+ keyspace1 | standard1 | 288
+ tlp_stress | keyvalue | 3211
+....
+
+Find queries on table/s with greatest read latency:
+
+....
+cqlsh> SELECT * FROM local_read_latency WHERE per_second > 1 ALLOW FILTERING;
+
+keyspace_name | table_name | p50th_ms | p99th_ms | count | max_ms | per_second
+---------------+------------+----------+----------+----------+---------+------------
+ tlp_stress | keyvalue | 0.043 | 0.152 | 49785158 | 186.563 | 11418.356
+....
+
+
+== Example
+
+[arabic, start=1]
+. To list the keyspaces, enter ``cqlsh`` and run the CQL command ``DESCRIBE KEYSPACES``:
+
+[source, cql]
+----
+cqlsh> DESC KEYSPACES;
+system_schema system system_distributed system_virtual_schema
+system_auth system_traces system_views
+----
+
+[arabic, start=2]
+. To view the virtual table schema, run the CQL commands ``USE system_virtual_schema`` and ``SELECT * FROM tables``:
+
+[source, cql]
+----
+cqlsh> USE system_virtual_schema;
+cqlsh> SELECT * FROM tables;
+----
+
+results in:
+
+[source, cql]
+----
+ keyspace_name | table_name | comment
+-----------------------+---------------------------+--------------------------------------
+ system_views | caches | system caches
+ system_views | clients | currently connected clients
+ system_views | coordinator_read_latency |
+ system_views | coordinator_scan_latency |
+ system_views | coordinator_write_latency |
+ system_views | disk_usage |
+ system_views | internode_inbound |
+ system_views | internode_outbound |
+ system_views | local_read_latency |
+ system_views | local_scan_latency |
+ system_views | local_write_latency |
+ system_views | max_partition_size |
+ system_views | rows_per_read |
+ system_views | settings | current settings
+ system_views | sstable_tasks | current sstable tasks
+ system_views | system_properties | Cassandra relevant system properties
+ system_views | thread_pools |
+ system_views | tombstones_per_read |
+ system_virtual_schema | columns | virtual column definitions
+ system_virtual_schema | keyspaces | virtual keyspace definitions
+ system_virtual_schema | tables | virtual table definitions
+
+(21 rows)
+----
+
+[arabic, start=3]
+. To view the virtual tables, run the CQL commands ``USE system_view`` and ``DESCRIBE tables``:
+
+[source, cql]
+----
+cqlsh> USE system_view;;
+cqlsh> DESCRIBE tables;
+----
+
+results in:
+
+[source, cql]
+----
+sstable_tasks clients coordinator_write_latency
+disk_usage local_write_latency tombstones_per_read
+thread_pools internode_outbound settings
+local_scan_latency coordinator_scan_latency system_properties
+internode_inbound coordinator_read_latency max_partition_size
+local_read_latency rows_per_read caches
+----
+
+[arabic, start=4]
+. To look at any table data, run the CQL command ``SELECT``:
+
+[source, cql]
+----
+cqlsh> USE system_view;;
+cqlsh> SELECT * FROM clients LIMIT 2;
+----
+ results in:
+
+[source, cql]
+----
+ address | port | connection_stage | driver_name | driver_version | hostname | protocol_version | request_count | ssl_cipher_suite | ssl_enabled | ssl_protocol | username
+-----------+-------+------------------+------------------------+----------------+-----------+------------------+---------------+------------------+-------------+--------------+-----------
+ 127.0.0.1 | 37308 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 17 | null | False | null | anonymous
+ 127.0.0.1 | 37310 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 8 | null | False | null | anonymous
+
+(2 rows)
+----
diff --git a/doc/modules/cassandra/pages/managing/tools/cassandra_stress.adoc b/doc/modules/cassandra/pages/managing/tools/cassandra_stress.adoc
new file mode 100644
index 0000000..5d6f0a1
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/tools/cassandra_stress.adoc
@@ -0,0 +1,330 @@
+= Cassandra Stress
+
+The `cassandra-stress` tool is used to benchmark and load-test a Cassandra
+cluster.
+`cassandra-stress` supports testing arbitrary CQL tables and queries, allowing users to benchmark their own data model.
+
+This documentation focuses on user mode to test personal schema.
+
+== Usage
+
+There are several operation types:
+
+* write-only, read-only, and mixed workloads of standard data
+* write-only and read-only workloads for counter columns
+* user configured workloads, running custom queries on custom schemas
+
+The syntax is `cassandra-stress <command> [options]`.
+For more information on a given command or options, run `cassandra-stress help <command|option>`.
+
+Commands:::
+ read:;;
+ Multiple concurrent reads - the cluster must first be populated by a
+ write test
+ write:;;
+ Multiple concurrent writes against the cluster
+ mixed:;;
+ Interleaving of any basic commands, with configurable ratio and
+ distribution - the cluster must first be populated by a write test
+ counter_write:;;
+ Multiple concurrent updates of counters.
+ counter_read:;;
+ Multiple concurrent reads of counters. The cluster must first be
+ populated by a counterwrite test.
+ user:;;
+ Interleaving of user provided queries, with configurable ratio and
+ distribution.
+ help:;;
+ Print help for a command or option
+ print:;;
+ Inspect the output of a distribution definition
+ legacy:;;
+ Legacy support mode
+Primary Options:::
+ -pop:;;
+ Population distribution and intra-partition visit order
+ -insert:;;
+ Insert specific options relating to various methods for batching and
+ splitting partition updates
+ -col:;;
+ Column details such as size and count distribution, data generator,
+ names, comparator and if super columns should be used
+ -rate:;;
+ Thread count, rate limit or automatic mode (default is auto)
+ -mode:;;
+ Thrift or CQL with options
+ -errors:;;
+ How to handle errors when encountered during stress
+ -sample:;;
+ Specify the number of samples to collect for measuring latency
+ -schema:;;
+ Replication settings, compression, compaction, etc.
+ -node:;;
+ Nodes to connect to
+ -log:;;
+ Where to log progress to, and the interval at which to do it
+ -transport:;;
+ Custom transport factories
+ -port:;;
+ The port to connect to cassandra nodes on
+ -graph:;;
+ Graph recorded metrics
+ -tokenrange:;;
+ Token range settings
+ -jmx:;;
+ Username and password for JMX connection
+ -credentials-file <path>:;;
+ Credentials file to specify for CQL, JMX and transport
+ -reporting:;;
+ Frequency of printing statistics and header for stress output
+Suboptions:::
+ Every command and primary option has its own collection of suboptions.
+ These are too numerous to list here. For information on the suboptions
+ for each command or option, please use the help command,
+ `cassandra-stress help <command|option>`.
+
+== User mode
+
+User mode allows you to stress your own schemas, to save you time
+in the long run. Find out if your application can scale using stress test with your schema.
+
+=== Profile
+
+User mode defines a profile using YAML.
+Multiple YAML files may be specified, in which case operations in the ops argument are referenced as
+specname.opname.
+
+An identifier for the profile:
+
+[source,yaml]
+----
+specname: staff_activities
+----
+
+The keyspace for the test:
+
+[source,yaml]
+----
+keyspace: staff
+----
+
+CQL for the keyspace. Optional if the keyspace already exists:
+
+[source,yaml]
+----
+keyspace_definition: |
+ CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};
+----
+
+The table to be stressed:
+
+[source,yaml]
+----
+table: staff_activities
+----
+
+CQL for the table. Optional if the table already exists:
+
+[source,yaml]
+----
+table_definition: |
+ CREATE TABLE staff_activities (
+ name text,
+ when timeuuid,
+ what text,
+ PRIMARY KEY(name, when, what)
+ )
+----
+
+Optional meta-information on the generated columns in the above table.
+The min and max only apply to text and blob types. The distribution
+field represents the total unique population distribution of that column
+across rows:
+
+[source,yaml]
+----
+columnspec:
+ - name: name
+ size: uniform(5..10) # The names of the staff members are between 5-10 characters
+ population: uniform(1..10) # 10 possible staff members to pick from
+ - name: when
+ cluster: uniform(20..500) # Staff members do between 20 and 500 events
+ - name: what
+ size: normal(10..100,50)
+----
+
+Supported types are:
+
+An exponential distribution over the range [min..max]:
+
+[source,yaml]
+----
+EXP(min..max)
+----
+
+An extreme value (Weibull) distribution over the range [min..max]:
+
+[source,yaml]
+----
+EXTREME(min..max,shape)
+----
+
+A gaussian/normal distribution, where mean=(min+max)/2, and stdev is
+(mean-min)/stdvrng:
+
+[source,yaml]
+----
+GAUSSIAN(min..max,stdvrng)
+----
+
+A gaussian/normal distribution, with explicitly defined mean and stdev:
+
+[source,yaml]
+----
+GAUSSIAN(min..max,mean,stdev)
+----
+
+A uniform distribution over the range [min, max]:
+
+[source,yaml]
+----
+UNIFORM(min..max)
+----
+
+A fixed distribution, always returning the same value:
+
+[source,yaml]
+----
+FIXED(val)
+----
+
+If preceded by ~, the distribution is inverted
+
+Defaults for all columns are size: uniform(4..8), population:
+uniform(1..100B), cluster: fixed(1)
+
+Insert distributions:
+
+[source,yaml]
+----
+insert:
+ # How many partition to insert per batch
+ partitions: fixed(1)
+ # How many rows to update per partition
+ select: fixed(1)/500
+ # UNLOGGED or LOGGED batch for insert
+ batchtype: UNLOGGED
+----
+
+Currently all inserts are done inside batches.
+
+Read statements to use during the test:
+
+[source,yaml]
+----
+queries:
+ events:
+ cql: select * from staff_activities where name = ?
+ fields: samerow
+ latest_event:
+ cql: select * from staff_activities where name = ? LIMIT 1
+ fields: samerow
+----
+
+Running a user mode test:
+
+[source,yaml]
+----
+cassandra-stress user profile=./example.yaml duration=1m "ops(insert=1,latest_event=1,events=1)" truncate=once
+----
+
+This will create the schema then run tests for 1 minute with an equal
+number of inserts, latest_event queries and events queries. Additionally
+the table will be truncated once before the test.
+
+The full example can be found here:
+[source, yaml]
+----
+include::example$YAML/stress-example.yaml[]
+----
+
+Running a user mode test with multiple yaml files::::
+ cassandra-stress user profile=./example.yaml,./example2.yaml
+ duration=1m "ops(ex1.insert=1,ex1.latest_event=1,ex2.insert=2)"
+ truncate=once
+This will run operations as specified in both the example.yaml and
+example2.yaml files. example.yaml and example2.yaml can reference the
+same table, although care must be taken that the table definition is identical
+ (data generation specs can be different).
+
+=== Lightweight transaction support
+
+cassandra-stress supports lightweight transactions.
+To use this feature, the command will first read current data from Cassandra, and then uses read values to
+fulfill lightweight transaction conditions.
+
+Lightweight transaction update query:
+
+[source,yaml]
+----
+queries:
+ regularupdate:
+ cql: update blogposts set author = ? where domain = ? and published_date = ?
+ fields: samerow
+ updatewithlwt:
+ cql: update blogposts set author = ? where domain = ? and published_date = ? IF body = ? AND url = ?
+ fields: samerow
+----
+
+The full example can be found here:
+[source, yaml]
+----
+include::example$YAML/stress-lwt-example.yaml[]
+----
+
+== Graphing
+
+Graphs can be generated for each run of stress.
+
+image::example-stress-graph.png[example cassandra-stress graph]
+
+To create a new graph:
+
+[source,yaml]
+----
+cassandra-stress user profile=./stress-example.yaml "ops(insert=1,latest_event=1,events=1)" -graph file=graph.html title="Awesome graph"
+----
+
+To add a new run to an existing graph point to an existing file and add
+a revision name:
+
+[source,yaml]
+----
+cassandra-stress user profile=./stress-example.yaml duration=1m "ops(insert=1,latest_event=1,events=1)" -graph file=graph.html title="Awesome graph" revision="Second run"
+----
+
+== FAQ
+
+*How do you use NetworkTopologyStrategy for the keyspace?*
+
+Use the schema option making sure to either escape the parenthesis or
+enclose in quotes:
+
+[source,yaml]
+----
+cassandra-stress write -schema "replication(strategy=NetworkTopologyStrategy,datacenter1=3)"
+----
+
+*How do you use SSL?*
+
+Use the transport option:
+
+[source,yaml]
+----
+cassandra-stress "write n=100k cl=ONE no-warmup" -transport "truststore=$HOME/jks/truststore.jks truststore-password=cassandra"
+----
+
+*Is Cassandra Stress a secured tool?*
+
+Cassandra stress is not a secured tool. Serialization and other aspects
+of the tool offer no security guarantees.
diff --git a/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc b/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc
new file mode 100644
index 0000000..b5209e9
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc
@@ -0,0 +1,556 @@
+= cqlsh: the CQL shell
+
+`cqlsh` is a command-line interface for interacting with Cassandra using CQL (the Cassandra Query Language).
+It is shipped with every Cassandra package, and can be found in the bin/ directory alongside the cassandra
+executable.
+`cqlsh` is implemented with the Python native protocol driver, and connects to the single specified node.
+
+== Compatibility
+
+`cqlsh` is compatible with Python 2.7.
+
+In general, a given version of `cqlsh` is only guaranteed to work with the
+version of Cassandra that it was released with.
+In some cases, `cqlsh` may work with older or newer versions of Cassandra, but this is not
+officially supported.
+
+== Optional Dependencies
+
+`cqlsh` ships with all essential dependencies. However, there are some
+optional dependencies that can be installed to improve the capabilities
+of `cqlsh`.
+
+=== pytz
+
+By default, `cqlsh` displays all timestamps with a UTC timezone.
+To support display of timestamps with another timezone, install
+the http://pytz.sourceforge.net/[pytz] library.
+See the `timezone` option in xref:cql/tools/cqlsh.adoc#cqlshrc[cqlshrc] for specifying a timezone to
+use.
+
+=== cython
+
+The performance of cqlsh's `COPY` operations can be improved by
+installing http://cython.org/[cython]. This will compile the python
+modules that are central to the performance of `COPY`.
+
+[[cqlshrc]]
+== cqlshrc
+
+The `cqlshrc` file holds configuration options for `cqlsh`.
+By default, the file is located the user's home directory at `~/.cassandra/cqlshrc`, but a
+custom location can be specified with the `--cqlshrc` option.
+
+Example config values and documentation can be found in the
+`conf/cqlshrc.sample` file of a tarball installation.
+You can also view the latest version of the
+https://github.com/apache/cassandra/blob/trunk/conf/cqlshrc.sample[cqlshrc file online].
+
+[[cql_history]]
+== cql history
+
+All CQL commands you execute are written to a history file. By default, CQL history will be written to `~/.cassandra/cql_history`. You can change this default by setting the environment variable `CQL_HISTORY` like `~/some/other/path/to/cqlsh_history` where `cqlsh_history` is a file. All parent directories to history file will be created if they do not exist. If you do not want to persist history, you can do so by setting CQL_HISTORY to /dev/null.
+This feature is supported from Cassandra 4.1.
+
+== Command Line Options
+
+Usage: `cqlsh.py [options] [host [port]]`
+
+CQL Shell for Apache Cassandra
+
+Options:
+
+`--version`::
+ show program's version number and exit
+
+`-h` `--help`::
+ show this help message and exit
+`-C` `--color`::
+ Always use color output
+`--no-color`::
+ Never use color output
+`--browser=BROWSER`::
+ The browser to use to display CQL help, where BROWSER can be:
+ one of the supported browsers in https://docs.python.org/3/library/webbrowser.html.
+ browser path followed by %s, example: /usr/bin/google-chrome-stable %s
+`--ssl`::
+ Use SSL
+
+`-u USERNAME` `--username=USERNAME`::
+ Authenticate as user.
+`-p PASSWORD` `--password=PASSWORD`::
+ Authenticate using password.
+`-k KEYSPACE` `--keyspace=KEYSPACE`::
+ Authenticate to the given keyspace.
+`-f FILE` `--file=FILE`::
+ Execute commands from FILE, then exit
+`--debug`::
+ Show additional debugging information
+`--coverage`::
+ Collect coverage data
+`--encoding=ENCODING`::
+ Specify a non-default encoding for output. (Default: utf-8)
+`--cqlshrc=CQLSHRC`::
+ Specify an alternative cqlshrc file location.
+`--credentials=CREDENTIALS`::
+ Specify an alternative credentials file location.
+`--cqlversion=CQLVERSION`::
+ Specify a particular CQL version, by default the
+ highest version supported by the server will be used.
+ Examples: "3.0.3", "3.1.0"
+`--protocol-version=PROTOCOL_VERSION`::
+ Specify a specific protcol version otherwise the
+ client will default and downgrade as necessary
+`-e EXECUTE` `--execute=EXECUTE`::
+ Execute the statement and quit.
+`--connect-timeout=CONNECT_TIMEOUT`::
+ Specify the connection timeout in seconds (default: 5 seconds).
+`--request-timeout=REQUEST_TIMEOUT`::
+ Specify the default request timeout in seconds
+ (default: 10 seconds).
+`-t, --tty`::
+ Force tty mode (command prompt).
+`-v` `--v`::
+ Print the current version of cqlsh.
+
+== Special Commands
+
+In addition to supporting regular CQL statements, `cqlsh` also supports a
+number of special commands that are not part of CQL. These are detailed
+below.
+
+=== `CONSISTENCY`
+
+`Usage`: `CONSISTENCY <consistency level>`
+
+Sets the consistency level for operations to follow. Valid arguments
+include:
+
+* `ANY`
+* `ONE`
+* `TWO`
+* `THREE`
+* `QUORUM`
+* `ALL`
+* `LOCAL_QUORUM`
+* `LOCAL_ONE`
+* `SERIAL`
+* `LOCAL_SERIAL`
+
+=== `SERIAL CONSISTENCY`
+
+`Usage`: `SERIAL CONSISTENCY <consistency level>`
+
+Sets the serial consistency level for operations to follow. Valid
+arguments include:
+
+* `SERIAL`
+* `LOCAL_SERIAL`
+
+The serial consistency level is only used by conditional updates
+(`INSERT`, `UPDATE` and `DELETE` with an `IF` condition). For those, the
+serial consistency level defines the consistency level of the serial
+phase (or “paxos” phase) while the normal consistency level defines the
+consistency for the “learn” phase, i.e. what type of reads will be
+guaranteed to see the update right away. For example, if a conditional
+write has a consistency level of `QUORUM` (and is successful), then a
+`QUORUM` read is guaranteed to see that write. But if the regular
+consistency level of that write is `ANY`, then only a read with a
+consistency level of `SERIAL` is guaranteed to see it (even a read with
+consistency `ALL` is not guaranteed to be enough).
+
+=== `SHOW VERSION`
+
+Prints the `cqlsh`, Cassandra, CQL, and native protocol versions in use.
+Example:
+
+[source,none]
+----
+cqlsh> SHOW VERSION
+[cqlsh 5.0.1 | Cassandra 3.8 | CQL spec 3.4.2 | Native protocol v4]
+----
+
+=== `SHOW HOST`
+
+Prints the IP address and port of the Cassandra node that `cqlsh` is
+connected to in addition to the cluster name. Example:
+
+[source,none]
+----
+cqlsh> SHOW HOST
+Connected to Prod_Cluster at 192.0.0.1:9042.
+----
+
+=== `SHOW REPLICAS`
+
+Prints the IP addresses of the Cassandra nodes which are replicas for the
+listed given token and keyspace. This command is available from Cassandra 4.2.
+
+`Usage`: `SHOW REPLICAS <token> (<keyspace>)`
+
+Example usage:
+
+[source,none]
+----
+cqlsh> SHOW REPLICAS 95
+['192.0.0.1', '192.0.0.2']
+----
+
+=== `SHOW SESSION`
+
+Pretty prints a specific tracing session.
+
+`Usage`: `SHOW SESSION <session id>`
+
+Example usage:
+
+[source,none]
+----
+cqlsh> SHOW SESSION 95ac6470-327e-11e6-beca-dfb660d92ad8
+
+Tracing session: 95ac6470-327e-11e6-beca-dfb660d92ad8
+
+ activity | timestamp | source | source_elapsed | client
+-----------------------------------------------------------+----------------------------+-----------+----------------+-----------
+ Execute CQL3 query | 2016-06-14 17:23:13.979000 | 127.0.0.1 | 0 | 127.0.0.1
+ Parsing SELECT * FROM system.local; [SharedPool-Worker-1] | 2016-06-14 17:23:13.982000 | 127.0.0.1 | 3843 | 127.0.0.1
+...
+----
+
+=== `SOURCE`
+
+Reads the contents of a file and executes each line as a CQL statement
+or special cqlsh command.
+
+`Usage`: `SOURCE <string filename>`
+
+Example usage:
+
+[source,none]
+----
+cqlsh> SOURCE '/home/calvinhobbs/commands.cql'
+----
+
+=== `CAPTURE`
+
+Begins capturing command output and appending it to a specified file.
+Output will not be shown at the console while it is captured.
+
+`Usage`:
+
+[source,none]
+----
+CAPTURE '<file>';
+CAPTURE OFF;
+CAPTURE;
+----
+
+That is, the path to the file to be appended to must be given inside a
+string literal. The path is interpreted relative to the current working
+directory. The tilde shorthand notation (`'~/mydir'`) is supported for
+referring to `$HOME`.
+
+Only query result output is captured. Errors and output from cqlsh-only
+commands will still be shown in the cqlsh session.
+
+To stop capturing output and show it in the cqlsh session again, use
+`CAPTURE OFF`.
+
+To inspect the current capture configuration, use `CAPTURE` with no
+arguments.
+
+=== `HELP`
+
+Gives information about cqlsh commands. To see available topics, enter
+`HELP` without any arguments. To see help on a topic, use
+`HELP <topic>`. Also see the `--browser` argument for controlling what
+browser is used to display help.
+
+=== `HISTORY`
+
+Prints to the screen the last `n` cqlsh commands executed on the server.
+The number of lines defaults to 50 if not specified. `n` is set for
+the current CQL session so if you set it e.g. to `10`, from that point
+there will be at most 10 last commands returned to you.
+
+`Usage`:
+
+[source,none]
+----
+HISTORY <n>
+----
+
+=== `TRACING`
+
+Enables or disables tracing for queries. When tracing is enabled, once a
+query completes, a trace of the events during the query will be printed.
+
+`Usage`:
+
+[source,none]
+----
+TRACING ON
+TRACING OFF
+----
+
+=== `PAGING`
+
+Enables paging, disables paging, or sets the page size for read queries.
+When paging is enabled, only one page of data will be fetched at a time
+and a prompt will appear to fetch the next page. Generally, it's a good
+idea to leave paging enabled in an interactive session to avoid fetching
+and printing large amounts of data at once.
+
+`Usage`:
+
+[source,none]
+----
+PAGING ON
+PAGING OFF
+PAGING <page size in rows>
+----
+
+=== `EXPAND`
+
+Enables or disables vertical printing of rows. Enabling `EXPAND` is
+useful when many columns are fetched, or the contents of a single column
+are large.
+
+`Usage`:
+
+[source,none]
+----
+EXPAND ON
+EXPAND OFF
+----
+
+=== `LOGIN`
+
+Authenticate as a specified Cassandra user for the current session.
+
+`Usage`:
+
+[source,none]
+----
+LOGIN <username> [<password>]
+----
+
+=== `EXIT`
+
+Ends the current session and terminates the cqlsh process.
+
+`Usage`:
+
+[source,none]
+----
+EXIT
+QUIT
+----
+
+=== `CLEAR`
+
+Clears the console.
+
+`Usage`:
+
+[source,none]
+----
+CLEAR
+CLS
+----
+
+=== `DESCRIBE`
+
+Prints a description (typically a series of DDL statements) of a schema
+element or the cluster. This is useful for dumping all or portions of
+the schema.
+
+`Usage`:
+
+[source,none]
+----
+DESCRIBE CLUSTER
+DESCRIBE SCHEMA
+DESCRIBE KEYSPACES
+DESCRIBE KEYSPACE <keyspace name>
+DESCRIBE TABLES
+DESCRIBE TABLE <table name>
+DESCRIBE INDEX <index name>
+DESCRIBE MATERIALIZED VIEW <view name>
+DESCRIBE TYPES
+DESCRIBE TYPE <type name>
+DESCRIBE FUNCTIONS
+DESCRIBE FUNCTION <function name>
+DESCRIBE AGGREGATES
+DESCRIBE AGGREGATE <aggregate function name>
+----
+
+In any of the commands, `DESC` may be used in place of `DESCRIBE`.
+
+The `DESCRIBE CLUSTER` command prints the cluster name and partitioner:
+
+[source,none]
+----
+cqlsh> DESCRIBE CLUSTER
+
+Cluster: Test Cluster
+Partitioner: Murmur3Partitioner
+----
+
+The `DESCRIBE SCHEMA` command prints the DDL statements needed to
+recreate the entire schema. This is especially useful for dumping the
+schema in order to clone a cluster or restore from a backup.
+
+=== `COPY TO`
+
+Copies data from a table to a CSV file.
+
+`Usage`:
+
+[source,none]
+----
+COPY <table name> [(<column>, ...)] TO <file name> WITH <copy option> [AND <copy option> ...]
+----
+
+If no columns are specified, all columns from the table will be copied
+to the CSV file. A subset of columns to copy may be specified by adding
+a comma-separated list of column names surrounded by parenthesis after
+the table name.
+
+The `<file name>` should be a string literal (with single quotes)
+representing a path to the destination file. This can also the special
+value `STDOUT` (without single quotes) to print the CSV to stdout.
+
+See `shared-copy-options` for options that apply to both `COPY TO` and
+`COPY FROM`.
+
+==== Options for `COPY TO`
+
+`MAXREQUESTS`::
+ The maximum number token ranges to fetch simultaneously. Defaults to
+ 6.
+`PAGESIZE`::
+ The number of rows to fetch in a single page. Defaults to 1000.
+`PAGETIMEOUT`::
+ By default the page timeout is 10 seconds per 1000 entries in the page
+ size or 10 seconds if pagesize is smaller.
+`BEGINTOKEN`, `ENDTOKEN`::
+ Token range to export. Defaults to exporting the full ring.
+`MAXOUTPUTSIZE`::
+ The maximum size of the output file measured in number of lines;
+ beyond this maximum the output file will be split into segments. -1
+ means unlimited, and is the default.
+`ENCODING`::
+ The encoding used for characters. Defaults to `utf8`.
+
+=== `COPY FROM`
+
+Copies data from a CSV file to table.
+
+`Usage`:
+
+[source,none]
+----
+COPY <table name> [(<column>, ...)] FROM <file name> WITH <copy option> [AND <copy option> ...]
+----
+
+If no columns are specified, all columns from the CSV file will be
+copied to the table. A subset of columns to copy may be specified by
+adding a comma-separated list of column names surrounded by parenthesis
+after the table name.
+
+The `<file name>` should be a string literal (with single quotes)
+representing a path to the source file. This can also the special value
+`STDIN` (without single quotes) to read the CSV data from stdin.
+
+See `shared-copy-options` for options that apply to both `COPY TO` and
+`COPY FROM`.
+
+==== Options for `COPY FROM`
+
+`INGESTRATE`::
+ The maximum number of rows to process per second. Defaults to 100000.
+`MAXROWS`::
+ The maximum number of rows to import. -1 means unlimited, and is the
+ default.
+`SKIPROWS`::
+ A number of initial rows to skip. Defaults to 0.
+`SKIPCOLS`::
+ A comma-separated list of column names to ignore. By default, no
+ columns are skipped.
+`MAXPARSEERRORS`::
+ The maximum global number of parsing errors to ignore. -1 means
+ unlimited, and is the default.
+`MAXINSERTERRORS`::
+ The maximum global number of insert errors to ignore. -1 means
+ unlimited. The default is 1000.
+`ERRFILE` =::
+ A file to store all rows that could not be imported, by default this
+ is `import_<ks>_<table>.err` where `<ks>` is your keyspace and
+ `<table>` is your table name.
+`MAXBATCHSIZE`::
+ The max number of rows inserted in a single batch. Defaults to 20.
+`MINBATCHSIZE`::
+ The min number of rows inserted in a single batch. Defaults to 10.
+`CHUNKSIZE`::
+ The number of rows that are passed to child worker processes from the
+ main process at a time. Defaults to 5000.
+
+==== Shared COPY Options
+
+Options that are common to both `COPY TO` and `COPY FROM`.
+
+`NULLVAL`::
+ The string placeholder for null values. Defaults to `null`.
+`HEADER`::
+ For `COPY TO`, controls whether the first line in the CSV output file
+ will contain the column names. For COPY FROM, specifies whether the
+ first line in the CSV input file contains column names. Defaults to
+ `false`.
+`DECIMALSEP`::
+ The character that is used as the decimal point separator. Defaults to
+ `.`.
+`THOUSANDSSEP`::
+ The character that is used to separate thousands. Defaults to the
+ empty string.
+`BOOLSTYlE`::
+ The string literal format for boolean values. Defaults to
+ `True,False`.
+`NUMPROCESSES`::
+ The number of child worker processes to create for `COPY` tasks.
+ Defaults to 16 for `COPY` tasks. However, at most (num_cores - 1)
+ processes will be created.
+`MAXATTEMPTS`::
+ The maximum number of failed attempts to fetch a range of data (when
+ using `COPY TO`) or insert a chunk of data (when using `COPY FROM`)
+ before giving up. Defaults to 5.
+`REPORTFREQUENCY`::
+ How often status updates are refreshed, in seconds. Defaults to 0.25.
+`RATEFILE`::
+ An optional file to output rate statistics to. By default, statistics
+ are not output to a file.
+
+== Escaping Quotes
+
+Dates, IP addresses, and strings need to be enclosed in single quotation marks. To use a single quotation mark itself in a string literal, escape it using a single quotation mark.
+
+When fetching simple text data, `cqlsh` will return an unquoted string. However, when fetching text data from complex types (collections, user-defined types, etc.) `cqlsh` will return a quoted string containing the escaped characters. For example:
+
+Simple data
+[source,none]
+----
+cqlsh> CREATE TABLE test.simple_data (id int, data text, PRIMARY KEY (id));
+cqlsh> INSERT INTO test.simple_data (id, data) values(1, 'I''m fine');
+cqlsh> SELECT data from test.simple_data; data
+----------
+ I'm fine
+----
+Complex data
+[source,none]
+----
+cqlsh> CREATE TABLE test.complex_data (id int, data map<int, text>, PRIMARY KEY (id));
+cqlsh> INSERT INTO test.complex_data (id, data) values(1, {1:'I''m fine'});
+cqlsh> SELECT data from test.complex_data; data
+------------------
+ {1: 'I''m fine'}
+----
diff --git a/doc/modules/cassandra/pages/tools/hash_password.adoc b/doc/modules/cassandra/pages/managing/tools/hash_password.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/hash_password.adoc
rename to doc/modules/cassandra/pages/managing/tools/hash_password.adoc
diff --git a/doc/modules/cassandra/pages/tools/index.adoc b/doc/modules/cassandra/pages/managing/tools/index.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/index.adoc
rename to doc/modules/cassandra/pages/managing/tools/index.adoc
diff --git a/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc
new file mode 100644
index 0000000..cc7637c
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc
@@ -0,0 +1,21 @@
+= SSTable Tools
+
+This section describes the functionality of the various sstable tools.
+
+Cassandra must be stopped before these tools are executed, or unexpected
+results will occur. Note: the scripts do not verify that Cassandra is
+stopped.
+
+* xref:tools/sstable/sstabledump.adoc[sstabledump]
+* xref:tools/sstable/sstableexpiredblockers.adoc[sstableexpiredblockers]
+* xref:tools/sstable/sstablelevelreset.adoc[sstablelevelreset]
+* xref:tools/sstable/sstableloader.adoc[sstableloader]
+* xref:tools/sstable/sstablemetadata.adoc[sstablemetadata]
+* xref:tools/sstable/sstableofflinerelevel.adoc[sstableofflinerelevel]
+* xref:tools/sstable/sstablepartitions.adoc[sstablepartitions]
+* xref:tools/sstable/sstablerepairedset.adoc[sstablerepairdset]
+* xref:tools/sstable/sstablescrub.adoc[sstablescrub]
+* xref:tools/sstable/sstablesplit.adoc[sstablesplit]
+* xref:tools/sstable/sstableupgrade.adoc[sstableupgrade]
+* xref:tools/sstable/sstableutil.adoc[sstableutil]
+* xref:tools/sstable/sstableverify.adoc[sstableverify]
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstabledump.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstabledump.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstabledump.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstabledump.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstableexpiredblockers.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstableexpiredblockers.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstableexpiredblockers.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstableexpiredblockers.adoc
diff --git a/doc/modules/cassandra/pages/managing/tools/sstable/sstablelevelreset.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablelevelreset.adoc
new file mode 100644
index 0000000..69bbf42
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/tools/sstable/sstablelevelreset.adoc
@@ -0,0 +1,69 @@
+= sstablelevelreset
+
+If LeveledCompactionStrategy is set, this script can be used to reset
+level to 0 on a given set of sstables. This is useful if you want to,
+for example, change the minimum sstable size, and therefore restart the
+compaction process using this new configuration.
+
+See
+https://cassandra.apache.org/doc/latest/operating/compaction/lcs.html#lcs
+for information on how levels are used in this compaction strategy.
+
+Cassandra must be stopped before this tool is executed, or unexpected
+results will occur. Note: the script does not verify that Cassandra is
+stopped.
+
+ref: https://issues.apache.org/jira/browse/CASSANDRA-5271
+
+== Usage
+
+sstablelevelreset --really-reset <keyspace> <table>
+
+The really-reset flag is required, to ensure this intrusive command is
+not run accidentally.
+
+== Table not found
+
+If the keyspace and/or table is not in the schema (e.g., if you
+misspelled the table name), the script will return an error.
+
+Example:
+
+....
+ColumnFamily not found: keyspace/evenlog.
+....
+
+== Table has no sstables
+
+Example:
+
+....
+Found no sstables, did you give the correct keyspace/table?
+....
+
+== Table already at level 0
+
+The script will not set the level if it is already set to 0.
+
+Example:
+
+....
+Skipped /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db since it is already on level 0
+....
+
+== Table levels reduced to 0
+
+If the level is not already 0, then this will reset it to 0.
+
+Example:
+
+....
+sstablemetadata /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db | grep -i level
+SSTable Level: 1
+
+sstablelevelreset --really-reset keyspace eventlog
+Changing level from 1 to 0 on /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db
+
+sstablemetadata /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db | grep -i level
+SSTable Level: 0
+....
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstableloader.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstableloader.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstableloader.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstableloader.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstablemetadata.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstablemetadata.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstableofflinerelevel.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstableofflinerelevel.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstableofflinerelevel.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstableofflinerelevel.adoc
diff --git a/doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc
new file mode 100644
index 0000000..6684946
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc
@@ -0,0 +1,171 @@
+= sstablepartitions
+
+Identifies large partitions of SSTables and outputs the partition size in bytes, row count, cell count, and tombstone count.
+
+You can supply any number of sstables file paths, or directories containing sstables. Each sstable will be analyzed separately.
+
+If a metrics threshold such as `--min-size`, `--min-rows`, `--min-cells` or `--min-tombstones` is provided,
+then the partition keys exceeding of the threshold will be printed in the output.
+It also prints a summary of metrics for the table. The percentiles in the metrics are estimates,
+while the min/max/count metrics are accurate.
+
+The default output of this tool is meant to be read by human eyes.
+Future versions might include small formatting changes or present new data that can fool scripts reading it.
+Scripts or other automatic tools should use the `--csv` flag to produce machine-readable output.
+Future versions will not change the format of the CSV output except for maybe adding new columns,
+so a proper CSV parser consuming the output should keep working.
+
+Cassandra doesn't need to be running before this tool is executed.
+
+== Usage
+
+sstablepartitions <options> <sstable files or directories>
+
+[cols=",",]
+|===
+|-t, --min-size <arg> |Partition size threshold, expressed as either the number of bytes or a size with unit of the form 10KiB, 20MiB, 30GiB, etc.
+|-w, --min-rows <arg> |Partition row count threshold.
+|-c, --min-cells <arg> |Partition cell count threshold
+|-o, --min-tombstones <arg> |Partition tombstone count threshold.
+|-k, --key <arg> |Partition keys to include, instead of scanning all partitions.
+|-x, --exclude-key <arg> |Partition keys to exclude.
+|-r, --recursive |Scan for sstables recursively
+|-b, --backups |Include backups present in data directories when scanning directories
+|-s, --snaphsots |Include snapshots present in data directories when scanning directories
+|-u, --current-timestamp <arg> |Timestamp (seconds since epoch, unit time) for TTL expired calculation.
+|-y, --partitions-only |Only brief partition information. Exclude per-partition detailed row/cell/tombstone information from process and output.
+|-m, --csv |Produced CSV output (machine readable)
+|===
+
+== Examples
+
+=== Analyze partition statistics for a single SSTable
+
+Use the path to the SSTable file as the only argument.
+
+Example:
+
+....
+sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db
+
+Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk)
+ Partition size Row count Cell count Tombstone count
+ ~p50 767.519 KiB 770 1916 0
+ ~p75 2.238 MiB 2299 5722 0
+ ~p90 3.867 MiB 3311 9887 50
+ ~p95 16.629 MiB 14237 42510 446
+ ~p99 148.267 MiB 126934 379022 1331
+ ~p999 368.936 MiB 315852 943127 2759
+ min 56.854 KiB 100 150 0
+ max 356.067 MiB 310706 932118 2450
+ count 210
+....
+
+=== Analyze partition statistics for all SSTables in a directory
+
+Use the path to the SSTables directory as the only argument.
+
+Example:
+
+....
+sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb
+
+Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk)
+ Partition size Row count Cell count Tombstone count
+ ~p50 767.519 KiB 770 1916 0
+ ~p75 2.238 MiB 2299 5722 0
+ ~p90 3.867 MiB 3311 9887 50
+ ~p95 16.629 MiB 14237 42510 446
+ ~p99 148.267 MiB 126934 379022 1331
+ ~p999 368.936 MiB 315852 943127 2759
+ min 56.854 KiB 100 150 0
+ max 356.067 MiB 310706 932118 2450
+ count 210
+
+Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #9 (big-nc) (457.540 MiB uncompressed, 174.880 MiB on disk)
+ Partition size Row count Cell count Tombstone count
+ ~p50 1.865 MiB 1597 4768 0
+ ~p75 13.858 MiB 14237 42510 0
+ ~p90 28.735 MiB 29521 73457 50
+ ~p95 34.482 MiB 29521 88148 8239
+ ~p99 49.654 MiB 42510 126934 14237
+ ~p999 49.654 MiB 42510 126934 14237
+ min 47.272 KiB 100 150 0
+ max 45.133 MiB 39429 118287 13030
+ count 57
+....
+
+=== Output only partitions over 100MiB in size
+
+Use the `--min-size` option to specify the minimum size a partition must have to be included in the output.
+
+Example:
+
+....
+sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db --min-size 100MiB
+
+Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk)
+ Partition: '13' (0000000d) live, size: 105.056 MiB, rows: 91490, cells: 274470, tombstones: 50 (row:50, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+ Partition: '1' (00000001) live, size: 127.241 MiB, rows: 111065, cells: 333195, tombstones: 50 (row:50, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+ Partition: '8' (00000008) live, size: 356.067 MiB, rows: 310706, cells: 932118, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+ Partition: '2' (00000002) live, size: 213.341 MiB, rows: 186582, cells: 559125, tombstones: 978 (row:978, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+Summary of k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc):
+ File: /Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db
+ 4 partitions match
+ Keys: 13 1 8 2
+ Partition size Row count Cell count Tombstone count
+ ~p50 767.519 KiB 770 1916 0
+ ~p75 2.238 MiB 2299 5722 0
+ ~p90 3.867 MiB 3311 9887 50
+ ~p95 16.629 MiB 14237 42510 446
+ ~p99 148.267 MiB 126934 379022 1331
+ ~p999 368.936 MiB 315852 943127 2759
+ min 56.854 KiB 100 150 0
+ max 356.067 MiB 310706 932118 2450
+ count 210
+....
+
+=== Output only partitions with more than 1000 tombstones
+
+Use the `--min-tombstones` option to specify the minimum number of tombstones a partition must have to be included in the output.
+
+Example:
+
+....
+sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db --min-tombstones 1000
+
+Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk)
+ Partition: '55' (00000037) live, size: 1.290 MiB, rows: 2317, cells: 3474, tombstones: 1159 (row:1159, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+ Partition: '28' (0000001c) live, size: 1.198 MiB, rows: 2099, cells: 3147, tombstones: 1050 (row:1050, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+ Partition: '89' (00000059) live, size: 1.346 MiB, rows: 2226, cells: 3339, tombstones: 1113 (row:1113, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+ Partition: '21' (00000015) live, size: 3.853 MiB, rows: 4900, cells: 9927, tombstones: 2450 (row:2450, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)
+Summary of k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc):
+ File: /Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db
+ 4 partitions match
+ Keys: 55 28 89 21
+ Partition size Row count Cell count Tombstone count
+ ~p50 767.519 KiB 770 1916 0
+ ~p75 2.238 MiB 2299 5722 0
+ ~p90 3.867 MiB 3311 9887 50
+ ~p95 16.629 MiB 14237 42510 446
+ ~p99 148.267 MiB 126934 379022 1331
+ ~p999 368.936 MiB 315852 943127 2759
+ min 56.854 KiB 100 150 0
+ max 356.067 MiB 310706 932118 2450
+ count 210
+....
+
+=== Output CSV machine-readable output
+
+Use the `--csv` option to output a CSV machine-readable output, combined with any threshold value.
+
+Example:
+
+....
+sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db --min-size 100MiB --csv
+key,keyBinary,live,offset,size,rowCount,cellCount,tombstoneCount,rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount,cellTombstoneCount,rowTtlExpired,cellTtlExpired,directory,keyspace,table,index,snapshot,backup,generation,format,version
+"13",0000000d,true,186403543,110158965,91490,274470,50,50,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc
+"1",00000001,true,325141542,133422183,111065,333195,50,50,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc
+"8",00000008,true,477133752,373362819,310706,932118,0,0,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc
+"2",00000002,true,851841363,223704192,186582,559125,978,978,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc
+....
\ No newline at end of file
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstablerepairedset.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablerepairedset.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstablerepairedset.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstablerepairedset.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstablescrub.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablescrub.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstablescrub.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstablescrub.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstablesplit.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablesplit.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstablesplit.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstablesplit.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstableupgrade.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstableupgrade.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstableupgrade.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstableupgrade.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstableutil.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstableutil.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstableutil.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstableutil.adoc
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstableverify.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstableverify.adoc
similarity index 100%
rename from doc/modules/cassandra/pages/tools/sstable/sstableverify.adoc
rename to doc/modules/cassandra/pages/managing/tools/sstable/sstableverify.adoc
diff --git a/doc/modules/cassandra/pages/managing/virtualtables.adoc b/doc/modules/cassandra/pages/managing/virtualtables.adoc
new file mode 100644
index 0000000..22262c1
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/virtualtables.adoc
@@ -0,0 +1,517 @@
+= Virtual Tables
+
+Apache Cassandra 4.0 implements virtual tables (https://issues.apache.org/jira/browse/CASSANDRA-7622[CASSANDRA-7622]).
+Virtual tables are tables backed by an API instead of data explicitly managed and stored as SSTables.
+Apache Cassandra 4.0 implements a virtual keyspace interface for virtual tables.
+Virtual tables are specific to each node.
+
+Some of the features of virtual tables are the ability to:
+
+* expose metrics through CQL
+* expose YAML configuration information
+
+Virtual keyspaces and tables are quite different from regular tables and keyspaces:
+
+* Virtual tables are created in special keyspaces and not just any keyspace.
+* Virtual tables are managed by Cassandra. Users cannot run DDL to create new virtual tables or DML to modify existing virtual tables.
+* Virtual tables are currently read-only, although that may change in a later version.
+* Virtual tables are local only, non-distributed, and thus not replicated.
+* Virtual tables have no associated SSTables.
+* Consistency level of the queries sent to virtual tables are ignored.
+* All existing virtual tables use `LocalPartitioner`.
+Since a virtual table is not replicated the partitioner sorts in order of partition keys instead of by their hash.
+* Making advanced queries using `ALLOW FILTERING` and aggregation functions can be executed in virtual tables, even though in normal tables we dont recommend it.
+
+== Virtual Keyspaces
+
+Apache Cassandra 4.0 has added two new keyspaces for virtual tables:
+
+* `system_virtual_schema`
+* `system_views`.
+
+The `system_virtual_schema` keyspace has three tables: `keyspaces`,
+`columns` and `tables` for the virtual keyspace, table, and column definitions, respectively.
+These tables contain schema information for the virtual tables.
+It is used by Cassandra internally and a user should not access it directly.
+
+The `system_views` keyspace contains the actual virtual tables.
+
+== Virtual Table Limitations
+
+Before disccusing virtual keyspaces and tables, note that virtual keyspaces and tables have some limitations.
+These limitations are subject to change.
+Virtual keyspaces cannot be altered or dropped.
+In fact, no operations can be performed against virtual keyspaces.
+
+Virtual tables cannot be created in virtual keyspaces.
+Virtual tables cannot be altered, dropped, or truncated.
+Secondary indexes, types, functions, aggregates, materialized views, and triggers cannot be created for virtual tables.
+Expiring time-to-live (TTL) columns cannot be created.
+Virtual tables do not support conditional updates or deletes.
+Aggregates may be run in SELECT statements.
+
+Conditional batch statements cannot include mutations for virtual tables, nor can a virtual table statement be included in a logged batch.
+In fact, mutations for virtual and regular tables cannot occur in the same batch table.
+
+== Virtual Tables
+
+Each of the virtual tables in the `system_views` virtual keyspace contain different information.
+
+The following table describes the virtual tables:
+
+[width="98%",cols="27%,73%",]
+|===
+|Virtual Table |Description
+
+|caches |Displays the general cache information including cache name, capacity_bytes, entry_count, hit_count, hit_ratio double,
+recent_hit_rate_per_second, recent_request_rate_per_second, request_count, and size_bytes.
+
+|clients |Lists information about all connected clients.
+
+|coordinator_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator reads.
+
+|coordinator_scan |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator scans.
+
+|coordinator_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator writes.
+
+|cql_metrics |Metrics specific to CQL prepared statement caching.
+
+|disk_usage |Records disk usage including disk_space, keyspace_name, and table_name, sorted by system keyspaces.
+
+|gossip_info |Lists the gossip information for the cluster.
+
+|internode_inbound |Lists information about the inbound internode messaging.
+
+|internode_outbound |Information about the outbound internode messaging.
+
+|local_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local reads.
+
+|local_scan |Records counts, keyspace_name, table_name, max, median, and per_second for local scans.
+
+|local_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local writes.
+
+|max_partition_size |A table metric for maximum partition size.
+
+|rows_per_read |Records counts, keyspace_name, tablek_name, max, and median for rows read.
+
+|settings |Displays configuration settings in cassandra.yaml.
+
+|sstable_tasks |Lists currently running tasks and progress on SSTables, for operations like compaction and upgrade.
+
+|system_properties |Displays environmental system properties set on the node.
+
+|thread_pools |Lists metrics for each thread pool.
+
+|tombstones_per_read |Records counts, keyspace_name, tablek_name, max, and median for tombstones.
+
+|===
+
+We shall discuss some of the virtual tables in more detail next.
+
+=== Clients Virtual Table
+
+The `clients` virtual table lists all active connections (connected
+clients) including their ip address, port, client_options, connection stage, driver
+name, driver version, hostname, protocol version, request count, ssl
+enabled, ssl protocol and user name:
+
+....
+cqlsh> EXPAND ON ;
+Now Expanded output is enabled
+cqlsh> SELECT * FROM system_views.clients;
+
+@ Row 1
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50687
+ client_options | {'CQL_VERSION': '3.4.7', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
+ connection_stage | ready
+ driver_name | DataStax Python Driver
+ driver_version | 3.25.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 16
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+@ Row 2
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50688
+ client_options | {'CQL_VERSION': '3.4.7', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
+ connection_stage | ready
+ driver_name | DataStax Python Driver
+ driver_version | 3.25.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 4
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+@ Row 3
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50753
+ client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
+ connection_stage | ready
+ driver_name | DataStax Java driver for Apache Cassandra(R)
+ driver_version | 4.13.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 18
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+@ Row 4
+------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ address | 127.0.0.1
+ port | 50755
+ client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
+ connection_stage | ready
+ driver_name | DataStax Java driver for Apache Cassandra(R)
+ driver_version | 4.13.0
+ hostname | localhost
+ protocol_version | 5
+ request_count | 7
+ ssl_cipher_suite | null
+ ssl_enabled | False
+ ssl_protocol | null
+ username | anonymous
+
+(4 rows)
+....
+
+Some examples of how `clients` can be used are:
+
+* To find applications using old incompatible versions of drivers before
+upgrading and with `nodetool enableoldprotocolversions` and
+`nodetool disableoldprotocolversions` during upgrades.
+* To identify clients sending too many requests.
+* To find if SSL is enabled during the migration to and from ssl.
+* To identify all options the client is sending, e.g. APPLICATION_NAME and APPLICATION_VERSION
+
+The virtual tables may be described with `DESCRIBE` statement. The DDL
+listed however cannot be run to create a virtual table. As an example
+describe the `system_views.clients` virtual table:
+
+....
+cqlsh> DESCRIBE TABLE system_views.clients;
+
+/*
+Warning: Table system_views.clients is a virtual table and cannot be recreated with CQL.
+Structure, for reference:
+VIRTUAL TABLE system_views.clients (
+ address inet,
+ port int,
+ client_options frozen<map<text, text>>,
+ connection_stage text,
+ driver_name text,
+ driver_version text,
+ hostname text,
+ protocol_version int,
+ request_count bigint,
+ ssl_cipher_suite text,
+ ssl_enabled boolean,
+ ssl_protocol text,
+ username text,
+ PRIMARY KEY (address, port)
+) WITH CLUSTERING ORDER BY (port ASC)
+ AND comment = 'currently connected clients';
+*/
+....
+
+=== Caches Virtual Table
+
+The `caches` virtual table lists information about the caches. The four
+caches presently created are chunks, counters, keys and rows. A query on
+the `caches` virtual table returns the following details:
+
+....
+cqlsh:system_views> SELECT * FROM system_views.caches;
+name | capacity_bytes | entry_count | hit_count | hit_ratio | recent_hit_rate_per_second | recent_request_rate_per_second | request_count | size_bytes
+---------+----------------+-------------+-----------+-----------+----------------------------+--------------------------------+---------------+------------
+ chunks | 229638144 | 29 | 166 | 0.83 | 5 | 6 | 200 | 475136
+counters | 26214400 | 0 | 0 | NaN | 0 | 0 | 0 | 0
+ keys | 52428800 | 14 | 124 | 0.873239 | 4 | 4 | 142 | 1248
+ rows | 0 | 0 | 0 | NaN | 0 | 0 | 0 | 0
+
+(4 rows)
+....
+
+=== CQL metrics Virtual Table
+The `cql_metrics` virtual table lists metrics specific to CQL prepared statement caching. A query on `cql_metrics` virtual table lists below metrics.
+
+....
+cqlsh> select * from system_views.cql_metrics ;
+
+ name | value
+------------------------------+-------
+ prepared_statements_count | 0
+ prepared_statements_evicted | 0
+ prepared_statements_executed | 0
+ prepared_statements_ratio | 0
+ regular_statements_executed | 17
+....
+
+=== Settings Virtual Table
+
+The `settings` table is rather useful and lists all the current
+configuration settings from the `cassandra.yaml`. The encryption options
+are overridden to hide the sensitive truststore information or
+passwords. The configuration settings however cannot be set using DML on
+the virtual table presently: :
+
+....
+cqlsh:system_views> SELECT * FROM system_views.settings;
+
+name | value
+-------------------------------------+--------------------
+ allocate_tokens_for_keyspace | null
+ audit_logging_options_enabled | false
+ auto_snapshot | true
+ automatic_sstable_upgrade | false
+ cluster_name | Test Cluster
+ transient_replication_enabled | false
+ hinted_handoff_enabled | true
+ hints_directory | /home/ec2-user/cassandra/data/hints
+ incremental_backups | false
+ initial_token | null
+ ...
+ ...
+ ...
+ rpc_address | localhost
+ ssl_storage_port | 7001
+ start_native_transport | true
+ storage_port | 7000
+ stream_entire_sstables | true
+ (224 rows)
+....
+
+The `settings` table can be really useful if yaml file has been changed
+since startup and dont know running configuration, or to find if they
+have been modified via jmx/nodetool or virtual tables.
+
+=== Thread Pools Virtual Table
+
+The `thread_pools` table lists information about all thread pools.
+Thread pool information includes active tasks, active tasks limit,
+blocked tasks, blocked tasks all time, completed tasks, and pending
+tasks. A query on the `thread_pools` returns following details:
+
+....
+cqlsh:system_views> select * from system_views.thread_pools;
+
+name | active_tasks | active_tasks_limit | blocked_tasks | blocked_tasks_all_time | completed_tasks | pending_tasks
+------------------------------+--------------+--------------------+---------------+------------------------+-----------------+---------------
+ AntiEntropyStage | 0 | 1 | 0 | 0 | 0 | 0
+ CacheCleanupExecutor | 0 | 1 | 0 | 0 | 0 | 0
+ CompactionExecutor | 0 | 2 | 0 | 0 | 881 | 0
+ CounterMutationStage | 0 | 32 | 0 | 0 | 0 | 0
+ GossipStage | 0 | 1 | 0 | 0 | 0 | 0
+ HintsDispatcher | 0 | 2 | 0 | 0 | 0 | 0
+ InternalResponseStage | 0 | 2 | 0 | 0 | 0 | 0
+ MemtableFlushWriter | 0 | 2 | 0 | 0 | 1 | 0
+ MemtablePostFlush | 0 | 1 | 0 | 0 | 2 | 0
+ MemtableReclaimMemory | 0 | 1 | 0 | 0 | 1 | 0
+ MigrationStage | 0 | 1 | 0 | 0 | 0 | 0
+ MiscStage | 0 | 1 | 0 | 0 | 0 | 0
+ MutationStage | 0 | 32 | 0 | 0 | 0 | 0
+ Native-Transport-Requests | 1 | 128 | 0 | 0 | 130 | 0
+ PendingRangeCalculator | 0 | 1 | 0 | 0 | 1 | 0
+PerDiskMemtableFlushWriter_0 | 0 | 2 | 0 | 0 | 1 | 0
+ ReadStage | 0 | 32 | 0 | 0 | 13 | 0
+ Repair-Task | 0 | 2147483647 | 0 | 0 | 0 | 0
+ RequestResponseStage | 0 | 2 | 0 | 0 | 0 | 0
+ Sampler | 0 | 1 | 0 | 0 | 0 | 0
+ SecondaryIndexManagement | 0 | 1 | 0 | 0 | 0 | 0
+ ValidationExecutor | 0 | 2147483647 | 0 | 0 | 0 | 0
+ ViewBuildExecutor | 0 | 1 | 0 | 0 | 0 | 0
+ ViewMutationStage | 0 | 32 | 0 | 0 | 0 | 0
+....
+
+(24 rows)
+
+=== Internode Inbound Messaging Virtual Table
+
+The `internode_inbound` virtual table is for the internode inbound
+messaging. Initially no internode inbound messaging may get listed. In
+addition to the address, port, datacenter and rack information includes
+corrupt frames recovered, corrupt frames unrecovered, error bytes, error
+count, expired bytes, expired count, processed bytes, processed count,
+received bytes, received count, scheduled bytes, scheduled count,
+throttled count, throttled nanos, using bytes, using reserve bytes. A
+query on the `internode_inbound` returns following details:
+
+....
+cqlsh:system_views> SELECT * FROM system_views.internode_inbound;
+address | port | dc | rack | corrupt_frames_recovered | corrupt_frames_unrecovered |
+error_bytes | error_count | expired_bytes | expired_count | processed_bytes |
+processed_count | received_bytes | received_count | scheduled_bytes | scheduled_count | throttled_count | throttled_nanos | using_bytes | using_reserve_bytes
+---------+------+----+------+--------------------------+----------------------------+-
+----------
+(0 rows)
+....
+
+=== SSTables Tasks Virtual Table
+
+The `sstable_tasks` could be used to get information about running
+tasks. It lists following columns:
+
+....
+cqlsh:system_views> SELECT * FROM sstable_tasks;
+keyspace_name | table_name | task_id | kind | progress | total | unit
+---------------+------------+--------------------------------------+------------+----------+----------+-------
+ basic | wide2 | c3909740-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 60418761 | 70882110 | bytes
+ basic | wide2 | c7556770-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 2995623 | 40314679 | bytes
+....
+
+As another example, to find how much time is remaining for SSTable
+tasks, use the following query:
+
+....
+SELECT total - progress AS remaining
+FROM system_views.sstable_tasks;
+....
+
+=== Gossip Information Virtual Table
+
+The `gossip_info` virtual table lists the Gossip information for the cluster. An example query is as follows:
+
+....
+cqlsh> select address, port, generation, heartbeat, load, dc, rack from system_views.gossip_info;
+
+ address | port | generation | heartbeat | load | dc | rack
+-----------+------+------------+-----------+---------+-------------+-------
+ 127.0.0.1 | 7000 | 1645575140 | 312 | 70542.0 | datacenter1 | rack1
+ 127.0.0.2 | 7000 | 1645575135 | 318 | 70499.0 | datacenter1 | rack1
+ 127.0.0.3 | 7000 | 1645575140 | 312 | 70504.0 | datacenter1 | rack1
+ 127.0.0.4 | 7000 | 1645575141 | 311 | 70502.0 | datacenter1 | rack1
+ 127.0.0.5 | 7000 | 1645575136 | 315 | 70500.0 | datacenter1 | rack1
+
+(5 rows)
+....
+
+=== Other Virtual Tables
+
+Some examples of using other virtual tables are as follows.
+
+Find tables with most disk usage:
+
+....
+cqlsh> SELECT * FROM disk_usage WHERE mebibytes > 1 ALLOW FILTERING;
+
+keyspace_name | table_name | mebibytes
+---------------+------------+-----------
+ keyspace1 | standard1 | 288
+ tlp_stress | keyvalue | 3211
+....
+
+Find queries on table/s with greatest read latency:
+
+....
+cqlsh> SELECT * FROM local_read_latency WHERE per_second > 1 ALLOW FILTERING;
+
+keyspace_name | table_name | p50th_ms | p99th_ms | count | max_ms | per_second
+---------------+------------+----------+----------+----------+---------+------------
+ tlp_stress | keyvalue | 0.043 | 0.152 | 49785158 | 186.563 | 11418.356
+....
+
+
+== Example
+
+[arabic, start=1]
+. To list the keyspaces, enter ``cqlsh`` and run the CQL command ``DESCRIBE KEYSPACES``:
+
+[source, cql]
+----
+cqlsh> DESC KEYSPACES;
+system_schema system system_distributed system_virtual_schema
+system_auth system_traces system_views
+----
+
+[arabic, start=2]
+. To view the virtual table schema, run the CQL commands ``USE system_virtual_schema`` and ``SELECT * FROM tables``:
+
+[source, cql]
+----
+cqlsh> USE system_virtual_schema;
+cqlsh> SELECT * FROM tables;
+----
+
+results in:
+
+[source, cql]
+----
+ keyspace_name | table_name | comment
+-----------------------+---------------------------+--------------------------------------
+ system_views | caches | system caches
+ system_views | clients | currently connected clients
+ system_views | coordinator_read_latency |
+ system_views | coordinator_scan_latency |
+ system_views | coordinator_write_latency |
+ system_views | disk_usage |
+ system_views | internode_inbound |
+ system_views | internode_outbound |
+ system_views | local_read_latency |
+ system_views | local_scan_latency |
+ system_views | local_write_latency |
+ system_views | max_partition_size |
+ system_views | rows_per_read |
+ system_views | settings | current settings
+ system_views | sstable_tasks | current sstable tasks
+ system_views | system_properties | Cassandra relevant system properties
+ system_views | thread_pools |
+ system_views | tombstones_per_read |
+ system_virtual_schema | columns | virtual column definitions
+ system_virtual_schema | keyspaces | virtual keyspace definitions
+ system_virtual_schema | tables | virtual table definitions
+
+(21 rows)
+----
+
+[arabic, start=3]
+. To view the virtual tables, run the CQL commands ``USE system_view`` and ``DESCRIBE tables``:
+
+[source, cql]
+----
+cqlsh> USE system_view;;
+cqlsh> DESCRIBE tables;
+----
+
+results in:
+
+[source, cql]
+----
+sstable_tasks clients coordinator_write_latency
+disk_usage local_write_latency tombstones_per_read
+thread_pools internode_outbound settings
+local_scan_latency coordinator_scan_latency system_properties
+internode_inbound coordinator_read_latency max_partition_size
+local_read_latency rows_per_read caches
+----
+
+[arabic, start=4]
+. To look at any table data, run the CQL command ``SELECT``:
+
+[source, cql]
+----
+cqlsh> USE system_view;;
+cqlsh> SELECT * FROM clients LIMIT 2;
+----
+ results in:
+
+[source, cql]
+----
+ address | port | connection_stage | driver_name | driver_version | hostname | protocol_version | request_count | ssl_cipher_suite | ssl_enabled | ssl_protocol | username
+-----------+-------+------------------+------------------------+----------------+-----------+------------------+---------------+------------------+-------------+--------------+-----------
+ 127.0.0.1 | 37308 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 17 | null | False | null | anonymous
+ 127.0.0.1 | 37310 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 8 | null | False | null | anonymous
+
+(2 rows)
+----
diff --git a/doc/modules/cassandra/pages/new/index.adoc b/doc/modules/cassandra/pages/new/index.adoc
index a6545a7..1a6028c 100644
--- a/doc/modules/cassandra/pages/new/index.adoc
+++ b/doc/modules/cassandra/pages/new/index.adoc
@@ -1,20 +1,16 @@
-= New Features in Apache Cassandra 4.1
+= New Features
-This section covers the new features in Apache Cassandra 4.1.
+== New Features in Apache Cassandra 5.0
+This section covers the new features in Apache Cassandra 5.0.
-* https://issues.apache.org/jira/browse/CASSANDRA-17164[Paxos v2]
-* link:/_/blog/Apache-Cassandra-4.1-Features-Guardrails-Framework.html[Guardrails]
-* link:/_/blog/Apache-Cassandra-4.1-Configuration-Standardization.html[New and Improved Configuration Format]
-* link:/_/blog/Apache-Cassandra-4.1-Features-Client-side-Password-Hashing.html[Client-side Password Hashing]
-* link:/_/blog/Apache-Cassandra-4.1-Denylisting-Partitions.html[Partition Denylist]
-* Lots of CQL improvements
-* link:/_/blog/Apache-Cassandra-4.1-New-SSTable-Identifiers.html[New SSTable Identifiers]
-* https://issues.apache.org/jira/browse/CASSANDRA-17423[Native Transport rate limiting]
-* https://issues.apache.org/jira/browse/CASSANDRA-16310[Top partition tracking per table]
-* https://issues.apache.org/jira/browse/CASSANDRA-14309[Hint Window consistency]
-* https://issues.apache.org/jira/browse/CASSANDRA-17044[Pluggability]
-** link:/_/blog/Apache-Cassandra-4.1-Features-Pluggable-Memtable-Implementations.html[Memtable]
-** Encryption
-** link:/_/blog/Apache-Cassandra-4.1-Features-Authentication-Plugin-Support-for-CQLSH.html[CQLSH Authentication]
-* and much link:https://github.com/apache/cassandra/blob/cassandra-4.1/NEWS.txt[more]
+* https://cwiki.apache.org/confluence/x/FQRACw[ACID Transactions (Accord)]
+* https://issues.apache.org/jira/browse/CASSANDRA-16052[Storage Attached Indexes]
+* https://issues.apache.org/jira/browse/CASSANDRA-17240[Trie Memtables]
+* https://github.com/apache/cassandra/blob/trunk/NEWS.txt[More Guardrails]
+* https://issues.apache.org/jira/browse/CASSANDRA-8877[TTL and writetime on collections and UDTs]
+* https://cwiki.apache.org/confluence/x/YyD1D[Transactional Cluster Metadata]
+* https://issues.apache.org/jira/browse/CASSANDRA-16895[JDK 17]
+* https://issues.apache.org/jira/browse/CASSANDRA-17221[Add new Mathematical CQL functions: abs, exp, log, log10 and round]
+* https://issues.apache.org/jira/browse/CASSANDRA-18060[Added new CQL native scalar functions for collections]
+** The new functions are mostly analogous to the existing aggregation functions, but they operate on the elements of collection columns. The new functions are `map_keys`, `map_values`, `collection_count`, `collection_min`, `collection_max`, `collection_sum` and `collection_avg`.
diff --git a/doc/modules/cassandra/pages/new/virtualtables.adoc b/doc/modules/cassandra/pages/new/virtualtables.adoc
deleted file mode 100644
index 7a7a4be..0000000
--- a/doc/modules/cassandra/pages/new/virtualtables.adoc
+++ /dev/null
@@ -1,517 +0,0 @@
-= Virtual Tables
-
-Apache Cassandra 4.0 implements virtual tables (https://issues.apache.org/jira/browse/CASSANDRA-7622[CASSANDRA-7622]).
-Virtual tables are tables backed by an API instead of data explicitly managed and stored as SSTables.
-Apache Cassandra 4.0 implements a virtual keyspace interface for virtual tables.
-Virtual tables are specific to each node.
-
-Some of the features of virtual tables are the ability to:
-
-* expose metrics through CQL
-* expose YAML configuration information
-
-Virtual keyspaces and tables are quite different from regular tables and keyspaces:
-
-* Virtual tables are created in special keyspaces and not just any keyspace.
-* Virtual tables are managed by Cassandra. Users cannot run DDL to create new virtual tables or DML to modify existing virtual tables.
-* Virtual tables are currently read-only, although that may change in a later version.
-* Virtual tables are local only, non-distributed, and thus not replicated.
-* Virtual tables have no associated SSTables.
-* Consistency level of the queries sent to virtual tables are ignored.
-* All existing virtual tables use `LocalPartitioner`.
-Since a virtual table is not replicated the partitioner sorts in order of partition keys instead of by their hash.
-* Making advanced queries using `ALLOW FILTERING` and aggregation functions can be executed in virtual tables, even though in normal tables we dont recommend it.
-
-== Virtual Keyspaces
-
-Apache Cassandra 4.0 has added two new keyspaces for virtual tables:
-
-* `system_virtual_schema`
-* `system_views`.
-
-The `system_virtual_schema` keyspace has three tables: `keyspaces`,
-`columns` and `tables` for the virtual keyspace, table, and column definitions, respectively.
-These tables contain schema information for the virtual tables.
-It is used by Cassandra internally and a user should not access it directly.
-
-The `system_views` keyspace contains the actual virtual tables.
-
-== Virtual Table Limitations
-
-Before disccusing virtual keyspaces and tables, note that virtual keyspaces and tables have some limitations.
-These limitations are subject to change.
-Virtual keyspaces cannot be altered or dropped.
-In fact, no operations can be performed against virtual keyspaces.
-
-Virtual tables cannot be created in virtual keyspaces.
-Virtual tables cannot be altered, dropped, or truncated.
-Secondary indexes, types, functions, aggregates, materialized views, and triggers cannot be created for virtual tables.
-Expiring time-to-live (TTL) columns cannot be created.
-Virtual tables do not support conditional updates or deletes.
-Aggregates may be run in SELECT statements.
-
-Conditional batch statements cannot include mutations for virtual tables, nor can a virtual table statement be included in a logged batch.
-In fact, mutations for virtual and regular tables cannot occur in the same batch table.
-
-== Virtual Tables
-
-Each of the virtual tables in the `system_views` virtual keyspace contain different information.
-
-The following table describes the virtual tables:
-
-[width="98%",cols="27%,73%",]
-|===
-|Virtual Table |Description
-
-|caches |Displays the general cache information including cache name, capacity_bytes, entry_count, hit_count, hit_ratio double,
-recent_hit_rate_per_second, recent_request_rate_per_second, request_count, and size_bytes.
-
-|clients |Lists information about all connected clients.
-
-|coordinator_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator reads.
-
-|coordinator_scan |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator scans.
-
-|coordinator_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator writes.
-
-|cql_metrics |Metrics specific to CQL prepared statement caching.
-
-|disk_usage |Records disk usage including disk_space, keyspace_name, and table_name, sorted by system keyspaces.
-
-|gossip_info |Lists the gossip information for the cluster.
-
-|internode_inbound |Lists information about the inbound internode messaging.
-
-|internode_outbound |Information about the outbound internode messaging.
-
-|local_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local reads.
-
-|local_scan |Records counts, keyspace_name, table_name, max, median, and per_second for local scans.
-
-|local_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local writes.
-
-|max_partition_size |A table metric for maximum partition size.
-
-|rows_per_read |Records counts, keyspace_name, tablek_name, max, and median for rows read.
-
-|settings |Displays configuration settings in cassandra.yaml.
-
-|sstable_tasks |Lists currently running tasks and progress on SSTables, for operations like compaction and upgrade.
-
-|system_properties |Displays environmental system properties set on the node.
-
-|thread_pools |Lists metrics for each thread pool.
-
-|tombstones_per_read |Records counts, keyspace_name, tablek_name, max, and median for tombstones.
-
-|===
-
-We shall discuss some of the virtual tables in more detail next.
-
-=== Clients Virtual Table
-
-The `clients` virtual table lists all active connections (connected
-clients) including their ip address, port, client_options, connection stage, driver
-name, driver version, hostname, protocol version, request count, ssl
-enabled, ssl protocol and user name:
-
-....
-cqlsh> EXPAND ON ;
-Now Expanded output is enabled
-cqlsh> SELECT * FROM system_views.clients;
-
-@ Row 1
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50687
- client_options | {'CQL_VERSION': '3.4.6', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
- connection_stage | ready
- driver_name | DataStax Python Driver
- driver_version | 3.25.0
- hostname | localhost
- protocol_version | 5
- request_count | 16
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-@ Row 2
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50688
- client_options | {'CQL_VERSION': '3.4.6', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
- connection_stage | ready
- driver_name | DataStax Python Driver
- driver_version | 3.25.0
- hostname | localhost
- protocol_version | 5
- request_count | 4
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-@ Row 3
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50753
- client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
- connection_stage | ready
- driver_name | DataStax Java driver for Apache Cassandra(R)
- driver_version | 4.13.0
- hostname | localhost
- protocol_version | 5
- request_count | 18
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-@ Row 4
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50755
- client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
- connection_stage | ready
- driver_name | DataStax Java driver for Apache Cassandra(R)
- driver_version | 4.13.0
- hostname | localhost
- protocol_version | 5
- request_count | 7
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-(4 rows)
-....
-
-Some examples of how `clients` can be used are:
-
-* To find applications using old incompatible versions of drivers before
-upgrading and with `nodetool enableoldprotocolversions` and
-`nodetool disableoldprotocolversions` during upgrades.
-* To identify clients sending too many requests.
-* To find if SSL is enabled during the migration to and from ssl.
-* To identify all options the client is sending, e.g. APPLICATION_NAME and APPLICATION_VERSION
-
-The virtual tables may be described with `DESCRIBE` statement. The DDL
-listed however cannot be run to create a virtual table. As an example
-describe the `system_views.clients` virtual table:
-
-....
-cqlsh> DESCRIBE TABLE system_views.clients;
-
-/*
-Warning: Table system_views.clients is a virtual table and cannot be recreated with CQL.
-Structure, for reference:
-VIRTUAL TABLE system_views.clients (
- address inet,
- port int,
- client_options frozen<map<text, text>>,
- connection_stage text,
- driver_name text,
- driver_version text,
- hostname text,
- protocol_version int,
- request_count bigint,
- ssl_cipher_suite text,
- ssl_enabled boolean,
- ssl_protocol text,
- username text,
- PRIMARY KEY (address, port)
-) WITH CLUSTERING ORDER BY (port ASC)
- AND comment = 'currently connected clients';
-*/
-....
-
-=== Caches Virtual Table
-
-The `caches` virtual table lists information about the caches. The four
-caches presently created are chunks, counters, keys and rows. A query on
-the `caches` virtual table returns the following details:
-
-....
-cqlsh:system_views> SELECT * FROM system_views.caches;
-name | capacity_bytes | entry_count | hit_count | hit_ratio | recent_hit_rate_per_second | recent_request_rate_per_second | request_count | size_bytes
----------+----------------+-------------+-----------+-----------+----------------------------+--------------------------------+---------------+------------
- chunks | 229638144 | 29 | 166 | 0.83 | 5 | 6 | 200 | 475136
-counters | 26214400 | 0 | 0 | NaN | 0 | 0 | 0 | 0
- keys | 52428800 | 14 | 124 | 0.873239 | 4 | 4 | 142 | 1248
- rows | 0 | 0 | 0 | NaN | 0 | 0 | 0 | 0
-
-(4 rows)
-....
-
-=== CQL metrics Virtual Table
-The `cql_metrics` virtual table lists metrics specific to CQL prepared statement caching. A query on `cql_metrics` virtual table lists below metrics.
-
-....
-cqlsh> select * from system_views.cql_metrics ;
-
- name | value
-------------------------------+-------
- prepared_statements_count | 0
- prepared_statements_evicted | 0
- prepared_statements_executed | 0
- prepared_statements_ratio | 0
- regular_statements_executed | 17
-....
-
-=== Settings Virtual Table
-
-The `settings` table is rather useful and lists all the current
-configuration settings from the `cassandra.yaml`. The encryption options
-are overridden to hide the sensitive truststore information or
-passwords. The configuration settings however cannot be set using DML on
-the virtual table presently: :
-
-....
-cqlsh:system_views> SELECT * FROM system_views.settings;
-
-name | value
--------------------------------------+--------------------
- allocate_tokens_for_keyspace | null
- audit_logging_options_enabled | false
- auto_snapshot | true
- automatic_sstable_upgrade | false
- cluster_name | Test Cluster
- transient_replication_enabled | false
- hinted_handoff_enabled | true
- hints_directory | /home/ec2-user/cassandra/data/hints
- incremental_backups | false
- initial_token | null
- ...
- ...
- ...
- rpc_address | localhost
- ssl_storage_port | 7001
- start_native_transport | true
- storage_port | 7000
- stream_entire_sstables | true
- (224 rows)
-....
-
-The `settings` table can be really useful if yaml file has been changed
-since startup and dont know running configuration, or to find if they
-have been modified via jmx/nodetool or virtual tables.
-
-=== Thread Pools Virtual Table
-
-The `thread_pools` table lists information about all thread pools.
-Thread pool information includes active tasks, active tasks limit,
-blocked tasks, blocked tasks all time, completed tasks, and pending
-tasks. A query on the `thread_pools` returns following details:
-
-....
-cqlsh:system_views> select * from system_views.thread_pools;
-
-name | active_tasks | active_tasks_limit | blocked_tasks | blocked_tasks_all_time | completed_tasks | pending_tasks
-------------------------------+--------------+--------------------+---------------+------------------------+-----------------+---------------
- AntiEntropyStage | 0 | 1 | 0 | 0 | 0 | 0
- CacheCleanupExecutor | 0 | 1 | 0 | 0 | 0 | 0
- CompactionExecutor | 0 | 2 | 0 | 0 | 881 | 0
- CounterMutationStage | 0 | 32 | 0 | 0 | 0 | 0
- GossipStage | 0 | 1 | 0 | 0 | 0 | 0
- HintsDispatcher | 0 | 2 | 0 | 0 | 0 | 0
- InternalResponseStage | 0 | 2 | 0 | 0 | 0 | 0
- MemtableFlushWriter | 0 | 2 | 0 | 0 | 1 | 0
- MemtablePostFlush | 0 | 1 | 0 | 0 | 2 | 0
- MemtableReclaimMemory | 0 | 1 | 0 | 0 | 1 | 0
- MigrationStage | 0 | 1 | 0 | 0 | 0 | 0
- MiscStage | 0 | 1 | 0 | 0 | 0 | 0
- MutationStage | 0 | 32 | 0 | 0 | 0 | 0
- Native-Transport-Requests | 1 | 128 | 0 | 0 | 130 | 0
- PendingRangeCalculator | 0 | 1 | 0 | 0 | 1 | 0
-PerDiskMemtableFlushWriter_0 | 0 | 2 | 0 | 0 | 1 | 0
- ReadStage | 0 | 32 | 0 | 0 | 13 | 0
- Repair-Task | 0 | 2147483647 | 0 | 0 | 0 | 0
- RequestResponseStage | 0 | 2 | 0 | 0 | 0 | 0
- Sampler | 0 | 1 | 0 | 0 | 0 | 0
- SecondaryIndexManagement | 0 | 1 | 0 | 0 | 0 | 0
- ValidationExecutor | 0 | 2147483647 | 0 | 0 | 0 | 0
- ViewBuildExecutor | 0 | 1 | 0 | 0 | 0 | 0
- ViewMutationStage | 0 | 32 | 0 | 0 | 0 | 0
-....
-
-(24 rows)
-
-=== Internode Inbound Messaging Virtual Table
-
-The `internode_inbound` virtual table is for the internode inbound
-messaging. Initially no internode inbound messaging may get listed. In
-addition to the address, port, datacenter and rack information includes
-corrupt frames recovered, corrupt frames unrecovered, error bytes, error
-count, expired bytes, expired count, processed bytes, processed count,
-received bytes, received count, scheduled bytes, scheduled count,
-throttled count, throttled nanos, using bytes, using reserve bytes. A
-query on the `internode_inbound` returns following details:
-
-....
-cqlsh:system_views> SELECT * FROM system_views.internode_inbound;
-address | port | dc | rack | corrupt_frames_recovered | corrupt_frames_unrecovered |
-error_bytes | error_count | expired_bytes | expired_count | processed_bytes |
-processed_count | received_bytes | received_count | scheduled_bytes | scheduled_count | throttled_count | throttled_nanos | using_bytes | using_reserve_bytes
----------+------+----+------+--------------------------+----------------------------+-
-----------
-(0 rows)
-....
-
-=== SSTables Tasks Virtual Table
-
-The `sstable_tasks` could be used to get information about running
-tasks. It lists following columns:
-
-....
-cqlsh:system_views> SELECT * FROM sstable_tasks;
-keyspace_name | table_name | task_id | kind | progress | total | unit
----------------+------------+--------------------------------------+------------+----------+----------+-------
- basic | wide2 | c3909740-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 60418761 | 70882110 | bytes
- basic | wide2 | c7556770-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 2995623 | 40314679 | bytes
-....
-
-As another example, to find how much time is remaining for SSTable
-tasks, use the following query:
-
-....
-SELECT total - progress AS remaining
-FROM system_views.sstable_tasks;
-....
-
-=== Gossip Information Virtual Table
-
-The `gossip_info` virtual table lists the Gossip information for the cluster. An example query is as follows:
-
-....
-cqlsh> select address, port, generation, heartbeat, load, dc, rack from system_views.gossip_info;
-
- address | port | generation | heartbeat | load | dc | rack
------------+------+------------+-----------+---------+-------------+-------
- 127.0.0.1 | 7000 | 1645575140 | 312 | 70542.0 | datacenter1 | rack1
- 127.0.0.2 | 7000 | 1645575135 | 318 | 70499.0 | datacenter1 | rack1
- 127.0.0.3 | 7000 | 1645575140 | 312 | 70504.0 | datacenter1 | rack1
- 127.0.0.4 | 7000 | 1645575141 | 311 | 70502.0 | datacenter1 | rack1
- 127.0.0.5 | 7000 | 1645575136 | 315 | 70500.0 | datacenter1 | rack1
-
-(5 rows)
-....
-
-=== Other Virtual Tables
-
-Some examples of using other virtual tables are as follows.
-
-Find tables with most disk usage:
-
-....
-cqlsh> SELECT * FROM disk_usage WHERE mebibytes > 1 ALLOW FILTERING;
-
-keyspace_name | table_name | mebibytes
----------------+------------+-----------
- keyspace1 | standard1 | 288
- tlp_stress | keyvalue | 3211
-....
-
-Find queries on table/s with greatest read latency:
-
-....
-cqlsh> SELECT * FROM local_read_latency WHERE per_second > 1 ALLOW FILTERING;
-
-keyspace_name | table_name | p50th_ms | p99th_ms | count | max_ms | per_second
----------------+------------+----------+----------+----------+---------+------------
- tlp_stress | keyvalue | 0.043 | 0.152 | 49785158 | 186.563 | 11418.356
-....
-
-
-== Example
-
-[arabic, start=1]
-. To list the keyspaces, enter ``cqlsh`` and run the CQL command ``DESCRIBE KEYSPACES``:
-
-[source, cql]
-----
-cqlsh> DESC KEYSPACES;
-system_schema system system_distributed system_virtual_schema
-system_auth system_traces system_views
-----
-
-[arabic, start=2]
-. To view the virtual table schema, run the CQL commands ``USE system_virtual_schema`` and ``SELECT * FROM tables``:
-
-[source, cql]
-----
-cqlsh> USE system_virtual_schema;
-cqlsh> SELECT * FROM tables;
-----
-
-results in:
-
-[source, cql]
-----
- keyspace_name | table_name | comment
------------------------+---------------------------+--------------------------------------
- system_views | caches | system caches
- system_views | clients | currently connected clients
- system_views | coordinator_read_latency |
- system_views | coordinator_scan_latency |
- system_views | coordinator_write_latency |
- system_views | disk_usage |
- system_views | internode_inbound |
- system_views | internode_outbound |
- system_views | local_read_latency |
- system_views | local_scan_latency |
- system_views | local_write_latency |
- system_views | max_partition_size |
- system_views | rows_per_read |
- system_views | settings | current settings
- system_views | sstable_tasks | current sstable tasks
- system_views | system_properties | Cassandra relevant system properties
- system_views | thread_pools |
- system_views | tombstones_per_read |
- system_virtual_schema | columns | virtual column definitions
- system_virtual_schema | keyspaces | virtual keyspace definitions
- system_virtual_schema | tables | virtual table definitions
-
-(21 rows)
-----
-
-[arabic, start=3]
-. To view the virtual tables, run the CQL commands ``USE system_view`` and ``DESCRIBE tables``:
-
-[source, cql]
-----
-cqlsh> USE system_view;;
-cqlsh> DESCRIBE tables;
-----
-
-results in:
-
-[source, cql]
-----
-sstable_tasks clients coordinator_write_latency
-disk_usage local_write_latency tombstones_per_read
-thread_pools internode_outbound settings
-local_scan_latency coordinator_scan_latency system_properties
-internode_inbound coordinator_read_latency max_partition_size
-local_read_latency rows_per_read caches
-----
-
-[arabic, start=4]
-. To look at any table data, run the CQL command ``SELECT``:
-
-[source, cql]
-----
-cqlsh> USE system_view;;
-cqlsh> SELECT * FROM clients LIMIT 2;
-----
- results in:
-
-[source, cql]
-----
- address | port | connection_stage | driver_name | driver_version | hostname | protocol_version | request_count | ssl_cipher_suite | ssl_enabled | ssl_protocol | username
------------+-------+------------------+------------------------+----------------+-----------+------------------+---------------+------------------+-------------+--------------+-----------
- 127.0.0.1 | 37308 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 17 | null | False | null | anonymous
- 127.0.0.1 | 37310 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 8 | null | False | null | anonymous
-
-(2 rows)
-----
diff --git a/doc/modules/cassandra/pages/operating/auditlogging.adoc b/doc/modules/cassandra/pages/operating/auditlogging.adoc
deleted file mode 100644
index a479921..0000000
--- a/doc/modules/cassandra/pages/operating/auditlogging.adoc
+++ /dev/null
@@ -1,475 +0,0 @@
-= Audit Logging
-
-Audit Logging is a new feature in Apache Cassandra 4.0 (https://issues.apache.org/jira/browse/CASSANDRA-12151[CASSANDRA-12151]).
-This new feature is safe for production use, with configurable limits to heap memory and disk space to prevent out-of-memory errors.
-All database activity is logged per-node as file-based records to a specified local filesystem directory.
-The audit log files are rolled periodically based on a configurable value.
-
-Some of the features of audit logging are:
-
-* No additional database capacity is needed to store audit logs.
-* No query tool is required to store the audit logs.
-* Latency of database operations is not affected, so there is no performance impact.
-* Heap memory usage is bounded by a weighted queue, with configurable maximum weight sitting in front of logging thread.
-* Disk utilization is bounded by a configurable size, deleting old log segments once the limit is reached.
-* Can be enabled, disabled, or reset (to delete on-disk data) using the JMX tool, ``nodetool``.
-* Can configure the settings in either the `cassandra.yaml` file or by using ``nodetool``.
-
-Audit logging includes all CQL requests, both successful and failed.
-It also captures all successful and failed authentication and authorization events, such as login attempts.
-The difference between Full Query Logging (FQL) and audit logging is that FQL captures only successful CQL requests, which allow replay or comparison of logs.
-Audit logs are useful for compliance and debugging, while FQL is useful for debugging, performance benchmarking, testing and auditing CQL queries.
-
-== Audit information logged
-
-The audit log contains:
-
-* all events in the configured keyspaces to include
-* all events in the configured categories to include
-* all events executed by the configured users to include
-
-The audit log does not contain:
-
-* configuration changes made in `cassandra.yaml` file
-* `nodetool` commands
-* Passwords mentioned as part of DCL statements: Passwords will be obfuscated as \*\*\*\*\*\*\*.
- ** Statements that fail to parse will have everything after the appearance of the word password obfuscated as \*\*\*\*\*\*\*.
- ** Statements with a mistyped word 'password' will be logged without obfuscation. Please make sure to use a different password on retries.
-
-The audit log is a series of log entries.
-An audit log entry contains:
-
-* keyspace (String) - Keyspace on which request is made
-* operation (String) - Database operation such as CQL command
-* user (String) - User name
-* scope (String) - Scope of request such as Table/Function/Aggregate name
-* type (AuditLogEntryType) - Type of request
-** CQL Audit Log Entry Type
-** Common Audit Log Entry Type
-* source (InetAddressAndPort) - Source IP Address from which request originated
-* timestamp (long ) - Timestamp of the request
-* batch (UUID) - Batch of request
-* options (QueryOptions) - CQL Query options
-* state (QueryState) - State related to a given query
-
-Each entry contains all applicable attributes for the given event, concatenated with a pipe (|).
-
-CQL audit log entry types are the following CQL commands. Each command is assigned to a particular specified category to log:
-
-[width="100%",cols="20%,80%",options="header",]
-|===
-| Category | CQL commands
-
-| DDL | ALTER_KEYSPACE, CREATE_KEYSPACE, DROP_KEYSPACE,
-ALTER_TABLE, CREATE_TABLE, DROP_TABLE,
-CREATE_FUNCTION, DROP_FUNCTION,
-CREATE_AGGREGATE, DROP_AGGREGATE,
-CREATE_INDEX, DROP_INDEX,
-ALTER_TYPE, CREATE_TYPE, DROP_TYPE,
-CREATE_TRIGGER, DROP_TRIGGER,
-ALTER_VIEW, CREATE_VIEW, DROP_VIEW,
-TRUNCATE
-| DML | BATCH, DELETE, UPDATE
-| DCL | GRANT, REVOKE,
-ALTER_ROLE, CREATE_ROLE, DROP_ROLE,
-LIST_ROLES, LIST_PERMISSIONS, LIST_USERS
-| OTHER | USE_KEYSPACE
-| QUERY | SELECT
-| PREPARE | PREPARE_STATEMENT
-|===
-
-Common audit log entry types are one of the following:
-
-[width="100%",cols="50%,50%",options="header",]
-|===
-| Category | CQL commands
-
-| AUTH | LOGIN_SUCCESS, LOGIN_ERROR, UNAUTHORIZED_ATTEMPT
-| ERROR | REQUEST_FAILURE
-|===
-
-== Configuring audit logging in cassandra.yaml
-
-The `cassandra.yaml` file can be used to configure and enable audit logging.
-Configuration and enablement may be the same or different on each node, depending on the `cassandra.yaml` file settings.
-Audit logs are generated on each enabled node, so logs on each node will have that node's queries.
-All options for audit logging can be set in the `cassandra.yaml` file under the ``audit_logging_options:``.
-
-The file includes the following options that can be uncommented for use:
-
-[source, yaml]
-----
-# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
-# on audit_logging for full details about the various configuration options.
-audit_logging_options:
- enabled: false
- logger:
- - class_name: BinAuditLogger
- # audit_logs_dir:
- # included_keyspaces:
- # excluded_keyspaces: system, system_schema, system_virtual_schema
- # included_categories:
- # excluded_categories:
- # included_users:
- # excluded_users:
- # roll_cycle: HOURLY
- # block: true
- # max_queue_weight: 268435456 # 256 MiB
- # max_log_size: 17179869184 # 16 GiB
- ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
- # archive_command:
- # max_archive_retries: 10
-----
-
-=== enabled
-
-Audit logging is enabled by setting the `enabled` option to `true` in
-the `audit_logging_options` setting.
-If this option is enabled, audit logging will start when Cassandra is started.
-For example, ``enabled: true``.
-
-=== logger
-
-The type of audit logger is set with the `logger` option.
-Supported values are: `BinAuditLogger` (default), `FileAuditLogger` and `NoOpAuditLogger`.
-`BinAuditLogger` logs events to a file in binary format.
-`FileAuditLogger` uses the standard logging mechanism, `slf4j` to log events to the `audit/audit.log` file. It is a synchronous, file-based audit logger. The roll_cycle will be set in the `logback.xml` file.
-`NoOpAuditLogger` is a no-op implementation of the audit logger that shoudl be specified when audit logging is disabled.
-
-For example:
-
-[source, yaml]
-----
-logger:
- - class_name: FileAuditLogger
-----
-
-=== audit_logs_dir
-
-To write audit logs, an existing directory must be set in ``audit_logs_dir``.
-
-The directory must have appropriate permissions set to allow reading, writing, and executing.
-Logging will recursively delete the directory contents as needed.
-Do not place links in this directory to other sections of the filesystem.
-For example, ``audit_logs_dir: /cassandra/audit/logs/hourly``.
-
-The audit log directory can also be configured using the system property `cassandra.logdir.audit`, which by default is set to `cassandra.logdir + /audit/`.
-
-=== included_keyspaces and excluded_keyspaces
-
-Set the keyspaces to include with the `included_keyspaces` option and
-the keyspaces to exclude with the `excluded_keyspaces` option.
-By default, `system`, `system_schema` and `system_virtual_schema` are excluded, and all other keyspaces are included.
-
-For example:
-[source, yaml]
-----
-included_keyspaces: test, demo
-excluded_keyspaces: system, system_schema, system_virtual_schema
-----
-
-=== included_categories and excluded_categories
-
-The categories of database operations to include are specified with the `included_categories` option as a comma-separated list.
-The categories of database operations to exclude are specified with `excluded_categories` option as a comma-separated list.
-The supported categories for audit log are: `AUTH`, `DCL`, `DDL`, `DML`, `ERROR`, `OTHER`, `PREPARE`, and `QUERY`.
-By default all supported categories are included, and no category is excluded.
-
-[source, yaml]
-----
-included_categories: AUTH, ERROR, DCL
-excluded_categories: DDL, DML, QUERY, PREPARE
-----
-
-=== included_users and excluded_users
-
-Users to audit log are set with the `included_users` and `excluded_users` options.
-The `included_users` option specifies a comma-separated list of users to include explicitly.
-The `excluded_users` option specifies a comma-separated list of users to exclude explicitly.
-By default all users are included, and no users are excluded.
-
-[source, yaml]
-----
-included_users:
-excluded_users: john, mary
-----
-
-=== roll_cycle
-
-The ``roll_cycle`` defines the frequency with which the audit log segments are rolled.
-Supported values are ``HOURLY`` (default), ``MINUTELY``, and ``DAILY``.
-For example: ``roll_cycle: DAILY``
-
-=== block
-
-The ``block`` option specifies whether audit logging should block writing or drop log records if the audit logging falls behind. Supported boolean values are ``true`` (default) or ``false``.
-For example: ``block: false`` to drop records
-
-=== max_queue_weight
-
-The ``max_queue_weight`` option sets the maximum weight of in-memory queue for records waiting to be written to the file before blocking or dropping. The option must be set to a positive value. The default value is 268435456, or 256 MiB.
-For example, to change the default: ``max_queue_weight: 134217728 # 128 MiB``
-
-=== max_log_size
-
-The ``max_log_size`` option sets the maximum size of the rolled files to retain on disk before deleting the oldest file. The option must be set to a positive value. The default is 17179869184, or 16 GiB.
-For example, to change the default: ``max_log_size: 34359738368 # 32 GiB``
-
-=== archive_command
-
-The ``archive_command`` option sets the user-defined archive script to execute on rolled log files.
-For example: ``archive_command: /usr/local/bin/archiveit.sh %path # %path is the file being rolled``
-
-=== max_archive_retries
-
-The ``max_archive_retries`` option sets the max number of retries of failed archive commands. The default is 10.
-For example: ``max_archive_retries: 10``
-
-
-An audit log file could get rolled for other reasons as well such as a
-log file reaches the configured size threshold.
-
-Audit logging can also be configured using ``nodetool` when enabling the feature, and will override any values set in the `cassandra.yaml` file, as discussed in the next section.
-
-
-== Enabling Audit Logging with ``nodetool``
-
-Audit logging is enabled on a per-node basis using the ``nodetool enableauditlog`` command. The logging directory must be defined with ``audit_logs_dir`` in the `cassandra.yaml` file or uses the default value ``cassandra.logdir.audit``.
-
-The syntax of the ``nodetool enableauditlog`` command has all the same options that can be set in the ``cassandra.yaml`` file except ``audit_logs_dir``.
-In addition, ``nodetool`` has options to set which host and port to run the command on, and username and password if the command requires authentication.
-
-[source, plaintext]
-----
- nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]
- [(-pp | --print-port)] [(-pw <password> | --password <password>)]
- [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]
- [(-u <username> | --username <username>)] enableauditlog
- [--excluded-categories <excluded_categories>]
- [--excluded-keyspaces <excluded_keyspaces>]
- [--excluded-users <excluded_users>]
- [--included-categories <included_categories>]
- [--included-keyspaces <included_keyspaces>]
- [--included-users <included_users>] [--logger <logger>]
-
-OPTIONS
- --excluded-categories <excluded_categories>
- Comma separated list of Audit Log Categories to be excluded for
- audit log. If not set the value from cassandra.yaml will be used
-
- --excluded-keyspaces <excluded_keyspaces>
- Comma separated list of keyspaces to be excluded for audit log. If
- not set the value from cassandra.yaml will be used
-
- --excluded-users <excluded_users>
- Comma separated list of users to be excluded for audit log. If not
- set the value from cassandra.yaml will be used
-
- -h <host>, --host <host>
- Node hostname or ip address
-
- --included-categories <included_categories>
- Comma separated list of Audit Log Categories to be included for
- audit log. If not set the value from cassandra.yaml will be used
-
- --included-keyspaces <included_keyspaces>
- Comma separated list of keyspaces to be included for audit log. If
- not set the value from cassandra.yaml will be used
-
- --included-users <included_users>
- Comma separated list of users to be included for audit log. If not
- set the value from cassandra.yaml will be used
-
- --logger <logger>
- Logger name to be used for AuditLogging. Default BinAuditLogger. If
- not set the value from cassandra.yaml will be used
-
- -p <port>, --port <port>
- Remote jmx agent port number
-
- -pp, --print-port
- Operate in 4.0 mode with hosts disambiguated by port number
-
- -pw <password>, --password <password>
- Remote jmx agent password
-
- -pwf <passwordFilePath>, --password-file <passwordFilePath>
- Path to the JMX password file
-
- -u <username>, --username <username>
- Remote jmx agent username
-----
-
-To enable audit logging, run following command on each node in the cluster on which you want to enable logging:
-
-[source, bash]
-----
-$ nodetool enableauditlog
-----
-
-== Disabling audit logging
-
-Use the `nodetool disableauditlog` command to disable audit logging.
-
-== Viewing audit logs
-
-The `auditlogviewer` tool is used to view (dump) audit logs if the logger was ``BinAuditLogger``..
-``auditlogviewer`` converts the binary log files into human-readable format; only the audit log directory must be supplied as a command-line option.
-If the logger ``FileAuditLogger`` was set, the log file are already in human-readable format and ``auditlogviewer`` is not needed to read files.
-
-
-The syntax of `auditlogviewer` is:
-
-[source, plaintext]
-----
-auditlogviewer
-
-Audit log files directory path is a required argument.
-usage: auditlogviewer <path1> [<path2>...<pathN>] [options]
---
-View the audit log contents in human readable format
---
-Options are:
--f,--follow Upon reaching the end of the log continue indefinitely
- waiting for more records
--h,--help display this help message
--r,--roll_cycle How often to roll the log file was rolled. May be
- necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY,
- DAILY). Default HOURLY.
-----
-
-== Example
-
-[arabic, start=1]
-. To demonstrate audit logging, first configure the ``cassandra.yaml`` file with the following settings:
-
-[source, yaml]
-----
-audit_logging_options:
- enabled: true
- logger: BinAuditLogger
- audit_logs_dir: "/cassandra/audit/logs/hourly"
- # included_keyspaces:
- # excluded_keyspaces: system, system_schema, system_virtual_schema
- # included_categories:
- # excluded_categories:
- # included_users:
- # excluded_users:
- roll_cycle: HOURLY
- # block: true
- # max_queue_weight: 268435456 # 256 MiB
- # max_log_size: 17179869184 # 16 GiB
- ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
- # archive_command:
- # max_archive_retries: 10
-----
-
-[arabic, start=2]
-. Create the audit log directory `/cassandra/audit/logs/hourly` and set the directory permissions to read, write, and execute for all.
-
-[arabic, start=3]
-. Now create a demo keyspace and table and insert some data using ``cqlsh``:
-
-[source, cql]
-----
- cqlsh> CREATE KEYSPACE auditlogkeyspace
- ... WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
- cqlsh> USE auditlogkeyspace;
- cqlsh:auditlogkeyspace> CREATE TABLE t (
- ...id int,
- ...k int,
- ...v text,
- ...PRIMARY KEY (id)
- ... );
- cqlsh:auditlogkeyspace> INSERT INTO t (id, k, v) VALUES (0, 0, 'val0');
- cqlsh:auditlogkeyspace> INSERT INTO t (id, k, v) VALUES (0, 1, 'val1');
-----
-
-All the supported CQL commands will be logged to the audit log directory.
-
-[arabic, start=4]
-. Change directory to the audit logs directory.
-
-[source, bash]
-----
-$ cd /cassandra/audit/logs/hourly
-----
-
-[arabic, start=5]
-. List the audit log files and directories.
-
-[source, bash]
-----
-$ ls -l
-----
-
-You should see results similar to:
-
-[source, plaintext]
-----
-total 28
--rw-rw-r--. 1 ec2-user ec2-user 65536 Aug 2 03:01 directory-listing.cq4t
--rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 03:01 20190802-02.cq4
--rw-rw-r--. 1 ec2-user ec2-user 83886080 Aug 2 03:01 20190802-03.cq4
-----
-
-The audit log files will all be listed with a `.cq4` file type. The audit directory is of `.cq4t` type.
-
-[arabic, start=6]
-. Run `auditlogviewer` tool to view the audit logs.
-
-[source, bash]
-----
-$ auditlogviewer /cassandra/audit/logs/hourly
-----
-
-This command will return a readable version of the log. Here is a partial sample of the log for the commands in this demo:
-
-[source, plaintext]
-----
-WARN 03:12:11,124 Using Pauser.sleepy() as not enough processors, have 2, needs 8+
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711427328|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE AuditLogKeyspace;
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711427329|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE "auditlogkeyspace"
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564711446279|type :SELECT|category:QUERY|ks:auditlogkeyspace|scope:t|operation:SELECT * FROM t;
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564713878834|type :DROP_TABLE|category:DDL|ks:auditlogkeyspace|scope:t|operation:DROP TABLE IF EXISTS
-AuditLogKeyspace.t;
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/3.91.56.164|port:42382|timestamp:1564714618360|ty
-pe:REQUEST_FAILURE|category:ERROR|operation:CREATE KEYSPACE AuditLogKeyspace
-WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};; Cannot add
-existing keyspace "auditlogkeyspace"
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564714690968|type :DROP_KEYSPACE|category:DDL|ks:auditlogkeyspace|operation:DROP KEYSPACE AuditLogKeyspace;
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/3.91.56.164|port:42406|timestamp:1564714708329|ty pe:CREATE_KEYSPACE|category:DDL|ks:auditlogkeyspace|operation:CREATE KEYSPACE
-AuditLogKeyspace
-WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
-Type: AuditLog
-LogMessage:
-user:anonymous|host:10.0.2.238:7000|source:/127.0.0.1|port:46264|timestamp:1564714870678|type :USE_KEYSPACE|category:OTHER|ks:auditlogkeyspace|operation:USE auditlogkeyspace;
-
-Password obfuscation examples:
-LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630496708|type:CREATE_ROLE|category:DCL|operation:CREATE ROLE role1 WITH PASSWORD = '*******';
-Type: audit
-LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630634552|type:ALTER_ROLE|category:DCL|operation:ATLER ROLE role1 WITH PASSWORD = '*******';
-Type: audit
-LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630698686|type:CREATE_ROLE|category:DCL|operation:CREATE USER user1 WITH PASSWORD '*******';
-Type: audit
-LogMessage: user:cassandra|host:localhost/127.0.0.1:7000|source:/127.0.0.1|port:65282|timestamp:1622630747344|type:ALTER_ROLE|category:DCL|operation:ALTER USER user1 WITH PASSWORD '*******';
-----
-
-== Diagnostic events for user audit logging
-
-Any native transport-enabled client can subscribe to audit log events for diagnosing cluster issues.
-These events can be consumed by external tools to implement a Cassandra user auditing solution.
diff --git a/doc/modules/cassandra/pages/operating/backups.adoc b/doc/modules/cassandra/pages/operating/backups.adoc
deleted file mode 100644
index a083d5b..0000000
--- a/doc/modules/cassandra/pages/operating/backups.adoc
+++ /dev/null
@@ -1,517 +0,0 @@
-= Backups
-
-Apache Cassandra stores data in immutable SSTable files. Backups in
-Apache Cassandra database are backup copies of the database data that is
-stored as SSTable files. Backups are used for several purposes including
-the following:
-
-* To store a data copy for durability
-* To be able to restore a table if table data is lost due to
-node/partition/network failure
-* To be able to transfer the SSTable files to a different machine; for
-portability
-
-== Types of Backups
-
-Apache Cassandra supports two kinds of backup strategies.
-
-* Snapshots
-* Incremental Backups
-
-A _snapshot_ is a copy of a table’s SSTable files at a given time,
-created via hard links.
-The DDL to create the table is stored as well.
-Snapshots may be created by a user or created automatically.
-The setting `snapshot_before_compaction` in the `cassandra.yaml` file determines if
-snapshots are created before each compaction.
-By default, `snapshot_before_compaction` is set to false.
-Snapshots may be created automatically before keyspace truncation or dropping of a table by
-setting `auto_snapshot` to true (default) in `cassandra.yaml`.
-Truncates could be delayed due to the auto snapshots and another setting in
-`cassandra.yaml` determines how long the coordinator should wait for
-truncates to complete.
-By default Cassandra waits 60 seconds for auto snapshots to complete.
-
-An _incremental backup_ is a copy of a table’s SSTable files created by
-a hard link when memtables are flushed to disk as SSTables.
-Typically incremental backups are paired with snapshots to reduce the backup time
-as well as reduce disk space.
-Incremental backups are not enabled by default and must be enabled explicitly in `cassandra.yaml` (with
-`incremental_backups` setting) or with `nodetool`.
-Once enabled, Cassandra creates a hard link to each SSTable flushed or streamed
-locally in a `backups/` subdirectory of the keyspace data.
-Incremental backups of system tables are also created.
-
-== Data Directory Structure
-
-The directory structure of Cassandra data consists of different
-directories for keyspaces, and tables with the data files within the
-table directories.
-Directories backups and snapshots to store backups
-and snapshots respectively for a particular table are also stored within
-the table directory.
-The directory structure for Cassandra is illustrated in Figure 1.
-
-image::Figure_1_backups.jpg[Data directory structure for backups]
-
-Figure 1. Directory Structure for Cassandra Data
-
-=== Setting Up Example Tables for Backups and Snapshots
-
-In this section we shall create some example data that could be used to
-demonstrate incremental backups and snapshots.
-We have used a three node Cassandra cluster.
-First, the keyspaces are created.
-Then tables are created within a keyspace and table data is added.
-We have used two keyspaces `cqlkeyspace` and `catalogkeyspace` with two tables within
-each.
-
-Create the keyspace `cqlkeyspace`:
-
-[source,cql]
-----
-include::example$CQL/create_ks_backup.cql[]
-----
-
-Create two tables `t` and `t2` in the `cqlkeyspace` keyspace.
-
-[source,cql]
-----
-include::example$CQL/create_table_backup.cql[]
-----
-
-Add data to the tables:
-
-[source,cql]
-----
-include::example$CQL/insert_data_backup.cql[]
-----
-
-Query the table to list the data:
-
-[source,cql]
-----
-include::example$CQL/select_data_backup.cql[]
-----
-
-results in
-
-[source,cql]
-----
-include::example$RESULTS/select_data_backup.result[]
-----
-
-Create a second keyspace `catalogkeyspace`:
-
-[source,cql]
-----
-include::example$CQL/create_ks2_backup.cql[]
-----
-
-Create two tables `journal` and `magazine` in `catalogkeyspace`:
-
-[source,cql]
-----
-include::example$CQL/create_table2_backup.cql[]
-----
-
-Add data to the tables:
-
-[source,cql]
-----
-include::example$CQL/insert_data2_backup.cql[]
-----
-
-Query the tables to list the data:
-
-[source,cql]
-----
-include::example$CQL/select_data2_backup.cql[]
-----
-
-results in
-
-[source,cql]
-----
-include::example$RESULTS/select_data2_backup.result[]
-----
-
-== Snapshots
-
-In this section, we demonstrate creating snapshots.
-The command used to create a snapshot is `nodetool snapshot` with the usage:
-
-[source,bash]
-----
-include::example$BASH/nodetool_snapshot.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/nodetool_snapshot_help.result[]
-----
-
-=== Configuring for Snapshots
-
-To demonstrate creating snapshots with Nodetool on the commandline we
-have set `auto_snapshots` setting to `false` in the `cassandra.yaml` file:
-
-[source,yaml]
-----
-include::example$YAML/auto_snapshot.yaml[]
-----
-
-Also set `snapshot_before_compaction` to `false` to disable creating
-snapshots automatically before compaction:
-
-[source,yaml]
-----
-include::example$YAML/snapshot_before_compaction.yaml[]
-----
-
-=== Creating Snapshots
-
-Before creating any snapshots, search for snapshots and none will be listed:
-
-[source,bash]
-----
-include::example$BASH/find_snapshots.sh[]
-----
-
-We shall be using the example keyspaces and tables to create snapshots.
-
-==== Taking Snapshots of all Tables in a Keyspace
-
-Using the syntax above, create a snapshot called `catalog-ks` for all the tables
-in the `catalogkeyspace` keyspace:
-
-[source,bash]
-----
-include::example$BASH/snapshot_backup2.sh[]
-----
-
-results in
-
-[source,none]
-----
-include::example$RESULTS/snapshot_backup2.result[]
-----
-
-Using the `find` command above, the snapshots and `snapshots` directories
-are now found with listed files similar to:
-
-[source, plaintext]
-----
-include::example$RESULTS/snapshot_backup2_find.result[]
-----
-
-Snapshots of all tables in multiple keyspaces may be created similarly:
-
-[source,bash]
-----
-include::example$BASH/snapshot_both_backups.sh[]
-----
-
-==== Taking Snapshots of Single Table in a Keyspace
-
-To take a snapshot of a single table the `nodetool snapshot` command
-syntax becomes as follows:
-
-[source,bash]
-----
-include::example$BASH/snapshot_one_table.sh[]
-----
-
-Using the syntax above, create a snapshot for table `magazine` in keyspace `catalogkeyspace`:
-
-[source,bash]
-----
-include::example$BASH/snapshot_one_table2.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/snapshot_one_table2.result[]
-----
-
-==== Taking Snapshot of Multiple Tables from same Keyspace
-
-To take snapshots of multiple tables in a keyspace the list of
-_Keyspace.table_ must be specified with option `--kt-list`.
-For example, create snapshots for tables `t` and `t2` in the `cqlkeyspace` keyspace:
-
-[source,bash]
-----
-include::example$BASH/snapshot_mult_tables.sh[]
-----
-
-results in
-
-[source,plaintext]
-----
-include::example$RESULTS/snapshot_mult_tables.result[]
-----
-
-Multiple snapshots of the same set of tables may be created and tagged with a different name.
-As an example, create another snapshot for the same set of tables `t` and `t2` in the `cqlkeyspace`
-keyspace and tag the snapshots differently:
-
-[source,bash]
-----
-include::example$BASH/snapshot_mult_tables_again.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/snapshot_mult_tables_again.result[]
-----
-
-==== Taking Snapshot of Multiple Tables from Different Keyspaces
-
-To take snapshots of multiple tables that are in different keyspaces the
-command syntax is the same as when multiple tables are in the same
-keyspace.
-Each <keyspace>.<table> must be specified separately in the
-`--kt-list` option.
-
-For example, create a snapshot for table `t` in
-the `cqlkeyspace` and table `journal` in the catalogkeyspace and tag the
-snapshot `multi-ks`.
-
-[source,bash]
-----
-include::example$BASH/snapshot_mult_ks.sh[]
-----
-
-results in
-[source, plaintext]
-----
-include::example$RESULTS/snapshot_mult_ks.result[]
-----
-
-=== Listing Snapshots
-
-To list snapshots use the `nodetool listsnapshots` command. All the
-snapshots that we created in the preceding examples get listed:
-
-[source,bash]
-----
-include::example$BASH/nodetool_list_snapshots.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/nodetool_list_snapshots.result[]
-----
-
-=== Finding Snapshots Directories
-
-The `snapshots` directories may be listed with `find –name snapshots`
-command:
-
-[source,bash]
-----
-include::example$BASH/find_snapshots.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/snapshot_all.result[]
-----
-
-To list the snapshots for a particular table first change to the snapshots directory for that table.
-For example, list the snapshots for the `catalogkeyspace/journal` table:
-
-[source,bash]
-----
-include::example$BASH/find_two_snapshots.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/find_two_snapshots.result[]
-----
-
-A `snapshots` directory lists the SSTable files in the snapshot.
-A `schema.cql` file is also created in each snapshot that defines schema
-that can recreate the table with CQL when restoring from a snapshot:
-
-[source,bash]
-----
-include::example$BASH/snapshot_files.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/snapshot_files.result[]
-----
-
-=== Clearing Snapshots
-
-Snapshots may be cleared or deleted with the `nodetool clearsnapshot`
-command. Either a specific snapshot name must be specified or the `–all`
-option must be specified.
-
-For example, delete a snapshot called `magazine` from keyspace `cqlkeyspace`:
-
-[source,bash]
-----
-include::example$BASH/nodetool_clearsnapshot.sh[]
-----
-
-or delete all snapshots from `cqlkeyspace` with the –all option:
-
-[source,bash]
-----
-include::example$BASH/nodetool_clearsnapshot_all.sh[]
-----
-
-== Incremental Backups
-
-In the following sections, we shall discuss configuring and creating
-incremental backups.
-
-=== Configuring for Incremental Backups
-
-To create incremental backups set `incremental_backups` to `true` in
-`cassandra.yaml`.
-
-[source,yaml]
-----
-include::example$YAML/incremental_bups.yaml[]
-----
-
-This is the only setting needed to create incremental backups.
-By default `incremental_backups` setting is set to `false` because a new
-set of SSTable files is created for each data flush and if several CQL
-statements are to be run the `backups` directory could fill up quickly
-and use up storage that is needed to store table data.
-Incremental backups may also be enabled on the command line with the nodetool
-command `nodetool enablebackup`.
-Incremental backups may be disabled with `nodetool disablebackup` command.
-Status of incremental backups, whether they are enabled may be checked with `nodetool statusbackup`.
-
-=== Creating Incremental Backups
-
-After each table is created flush the table data with `nodetool flush`
-command. Incremental backups get created.
-
-[source,bash]
-----
-include::example$BASH/nodetool_flush.sh[]
-----
-
-=== Finding Incremental Backups
-
-Incremental backups are created within the Cassandra’s `data` directory
-within a table directory. Backups may be found with following command.
-
-[source,bash]
-----
-include::example$BASH/find_backups.sh[]
-----
-results in
-[source,none]
-----
-include::example$RESULTS/find_backups.result[]
-----
-
-=== Creating an Incremental Backup
-
-This section discusses how incremental backups are created in more
-detail using the keyspace and table previously created.
-
-Flush the keyspace and table:
-
-[source,bash]
-----
-include::example$BASH/nodetool_flush_table.sh[]
-----
-
-A search for backups and a `backups` directory will list a backup directory,
-even if we have added no table data yet.
-
-[source,bash]
-----
-include::example$BASH/find_backups.sh[]
-----
-
-results in
-
-[source,plaintext]
-----
-include::example$RESULTS/find_backups_table.result[]
-----
-
-Checking the `backups` directory will show that there are also no backup files:
-
-[source,bash]
-----
-include::example$BASH/check_backups.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/no_bups.result[]
-----
-
-If a row of data is added to the data, running the `nodetool flush` command will
-flush the table data and an incremental backup will be created:
-
-[source,bash]
-----
-include::example$BASH/flush_and_check.sh[]
-----
-
-results in
-
-[source, plaintext]
-----
-include::example$RESULTS/flush_and_check.result[]
-----
-
-[NOTE]
-.note
-====
-The `backups` directory for any table, such as `cqlkeyspace/t` is created in the
-`data` directory for that table.
-====
-
-Adding another row of data and flushing will result in another set of incremental backup files.
-The SSTable files are timestamped, which distinguishes the first incremental backup from the
-second:
-
-[source,none]
-----
-include::example$RESULTS/flush_and_check2.result[]
-----
-
-== Restoring from Incremental Backups and Snapshots
-
-The two main tools/commands for restoring a table after it has been
-dropped are:
-
-* sstableloader
-* nodetool import
-
-A snapshot contains essentially the same set of SSTable files as an
-incremental backup does with a few additional files. A snapshot includes
-a `schema.cql` file for the schema DDL to create a table in CQL. A table
-backup does not include DDL which must be obtained from a snapshot when
-restoring from an incremental backup.
diff --git a/doc/modules/cassandra/pages/operating/compaction/index.adoc b/doc/modules/cassandra/pages/operating/compaction/index.adoc
deleted file mode 100644
index be20656..0000000
--- a/doc/modules/cassandra/pages/operating/compaction/index.adoc
+++ /dev/null
@@ -1,338 +0,0 @@
-= Compaction
-
-== Strategies
-
-Picking the right compaction strategy for your workload will ensure the
-best performance for both querying and for compaction itself.
-
-xref:operating/compaction/stcs.adoc[`Size Tiered Compaction Strategy (STCS)`]::
- The default compaction strategy. Useful as a fallback when other
- strategies don't fit the workload. Most useful for non pure time
- series workloads with spinning disks, or when the I/O from `LCS`
- is too high.
-xref:operating/compaction/lcs.adoc[`Leveled Compaction Strategy (LCS)`]::
- Leveled Compaction Strategy (LCS) is optimized for read heavy
- workloads, or workloads with lots of updates and deletes. It is not a
- good choice for immutable time series data.
-xref:operating/compaction/twcs.adoc[`Time Window Compaction Strategy (TWCS)`]::
- Time Window Compaction Strategy is designed for TTL'ed, mostly
- immutable time series data.
-
-== Types of compaction
-
-The concept of compaction is used for different kinds of operations in
-Cassandra, the common thing about these operations is that it takes one
-or more SSTables and output new SSTables. The types of compactions are:
-
-Minor compaction::
- triggered automatically in Cassandra.
-Major compaction::
- a user executes a compaction over all SSTables on the node.
-User defined compaction::
- a user triggers a compaction on a given set of SSTables.
-Scrub::
- try to fix any broken SSTables. This can actually remove valid data if
- that data is corrupted, if that happens you will need to run a full
- repair on the node.
-UpgradeSSTables::
- upgrade SSTables to the latest version. Run this after upgrading to a
- new major version.
-Cleanup::
- remove any ranges this node does not own anymore, typically triggered
- on neighbouring nodes after a node has been bootstrapped since that
- node will take ownership of some ranges from those nodes.
-Secondary index rebuild::
- rebuild the secondary indexes on the node.
-Anticompaction::
- after repair the ranges that were actually repaired are split out of
- the SSTables that existed when repair started.
-Sub range compaction::
- It is possible to only compact a given sub range - this could be
- useful if you know a token that has been misbehaving - either
- gathering many updates or many deletes.
- (`nodetool compact -st x -et y`) will pick all SSTables containing the
- range between x and y and issue a compaction for those SSTables. For
- STCS this will most likely include all SSTables but with LCS it can
- issue the compaction for a subset of the SSTables. With LCS the
- resulting sstable will end up in L0.
-
-== When is a minor compaction triggered?
-
-* When an sstable is added to the node through flushing/streaming
-* When autocompaction is enabled after being disabled (`nodetool enableautocompaction`)
-* When compaction adds new SSTables
-* A check for new minor compactions every 5 minutes
-
-== Merging SSTables
-
-Compaction is about merging SSTables, since partitions in SSTables are
-sorted based on the hash of the partition key it is possible to
-efficiently merge separate SSTables. Content of each partition is also
-sorted so each partition can be merged efficiently.
-
-== Tombstones and Garbage Collection (GC) Grace
-
-=== Why Tombstones
-
-When a delete request is received by Cassandra it does not actually
-remove the data from the underlying store. Instead it writes a special
-piece of data known as a tombstone. The Tombstone represents the delete
-and causes all values which occurred before the tombstone to not appear
-in queries to the database. This approach is used instead of removing
-values because of the distributed nature of Cassandra.
-
-=== Deletes without tombstones
-
-Imagine a three node cluster which has the value [A] replicated to every
-node.:
-
-[source,none]
-----
-[A], [A], [A]
-----
-
-If one of the nodes fails and and our delete operation only removes
-existing values we can end up with a cluster that looks like:
-
-[source,none]
-----
-[], [], [A]
-----
-
-Then a repair operation would replace the value of [A] back onto the two
-nodes which are missing the value.:
-
-[source,none]
-----
-[A], [A], [A]
-----
-
-This would cause our data to be resurrected even though it had been
-deleted.
-
-=== Deletes with Tombstones
-
-Starting again with a three node cluster which has the value [A]
-replicated to every node.:
-
-[source,none]
-----
-[A], [A], [A]
-----
-
-If instead of removing data we add a tombstone record, our single node
-failure situation will look like this.:
-
-[source,none]
-----
-[A, Tombstone[A]], [A, Tombstone[A]], [A]
-----
-
-Now when we issue a repair the Tombstone will be copied to the replica,
-rather than the deleted data being resurrected.:
-
-[source,none]
-----
-[A, Tombstone[A]], [A, Tombstone[A]], [A, Tombstone[A]]
-----
-
-Our repair operation will correctly put the state of the system to what
-we expect with the record [A] marked as deleted on all nodes. This does
-mean we will end up accruing Tombstones which will permanently
-accumulate disk space. To avoid keeping tombstones forever we have a
-parameter known as `gc_grace_seconds` for every table in Cassandra.
-
-=== The gc_grace_seconds parameter and Tombstone Removal
-
-The table level `gc_grace_seconds` parameter controls how long Cassandra
-will retain tombstones through compaction events before finally removing
-them. This duration should directly reflect the amount of time a user
-expects to allow before recovering a failed node. After
-`gc_grace_seconds` has expired the tombstone may be removed (meaning
-there will no longer be any record that a certain piece of data was
-deleted), but as a tombstone can live in one sstable and the data it
-covers in another, a compaction must also include both sstable for a
-tombstone to be removed. More precisely, to be able to drop an actual
-tombstone the following needs to be true;
-
-* The tombstone must be older than `gc_grace_seconds`
-* If partition X contains the tombstone, the sstable containing the
-partition plus all SSTables containing data older than the tombstone
-containing X must be included in the same compaction. We don't need to
-care if the partition is in an sstable if we can guarantee that all data
-in that sstable is newer than the tombstone. If the tombstone is older
-than the data it cannot shadow that data.
-* If the option `only_purge_repaired_tombstones` is enabled, tombstones
-are only removed if the data has also been repaired.
-
-If a node remains down or disconnected for longer than
-`gc_grace_seconds` it's deleted data will be repaired back to the other
-nodes and re-appear in the cluster. This is basically the same as in the
-"Deletes without Tombstones" section. Note that tombstones will not be
-removed until a compaction event even if `gc_grace_seconds` has elapsed.
-
-The default value for `gc_grace_seconds` is 864000 which is equivalent
-to 10 days. This can be set when creating or altering a table using
-`WITH gc_grace_seconds`.
-
-== TTL
-
-Data in Cassandra can have an additional property called time to live -
-this is used to automatically drop data that has expired once the time
-is reached. Once the TTL has expired the data is converted to a
-tombstone which stays around for at least `gc_grace_seconds`. Note that
-if you mix data with TTL and data without TTL (or just different length
-of the TTL) Cassandra will have a hard time dropping the tombstones
-created since the partition might span many SSTables and not all are
-compacted at once.
-
-== Fully expired SSTables
-
-If an sstable contains only tombstones and it is guaranteed that that
-sstable is not shadowing data in any other sstable compaction can drop
-that sstable. If you see SSTables with only tombstones (note that TTL:ed
-data is considered tombstones once the time to live has expired) but it
-is not being dropped by compaction, it is likely that other SSTables
-contain older data. There is a tool called `sstableexpiredblockers` that
-will list which SSTables are droppable and which are blocking them from
-being dropped. This is especially useful for time series compaction with
-`TimeWindowCompactionStrategy` (and the deprecated
-`DateTieredCompactionStrategy`). With `TimeWindowCompactionStrategy` it
-is possible to remove the guarantee (not check for shadowing data) by
-enabling `unsafe_aggressive_sstable_expiration`.
-
-== Repaired/unrepaired data
-
-With incremental repairs Cassandra must keep track of what data is
-repaired and what data is unrepaired. With anticompaction repaired data
-is split out into repaired and unrepaired SSTables. To avoid mixing up
-the data again separate compaction strategy instances are run on the two
-sets of data, each instance only knowing about either the repaired or
-the unrepaired SSTables. This means that if you only run incremental
-repair once and then never again, you might have very old data in the
-repaired SSTables that block compaction from dropping tombstones in the
-unrepaired (probably newer) SSTables.
-
-== Data directories
-
-Since tombstones and data can live in different SSTables it is important
-to realize that losing an sstable might lead to data becoming live again
-- the most common way of losing SSTables is to have a hard drive break
-down. To avoid making data live tombstones and actual data are always in
-the same data directory. This way, if a disk is lost, all versions of a
-partition are lost and no data can get undeleted. To achieve this a
-compaction strategy instance per data directory is run in addition to
-the compaction strategy instances containing repaired/unrepaired data,
-this means that if you have 4 data directories there will be 8
-compaction strategy instances running. This has a few more benefits than
-just avoiding data getting undeleted:
-
-* It is possible to run more compactions in parallel - leveled
-compaction will have several totally separate levelings and each one can
-run compactions independently from the others.
-* Users can backup and restore a single data directory.
-* Note though that currently all data directories are considered equal,
-so if you have a tiny disk and a big disk backing two data directories,
-the big one will be limited the by the small one. One work around to
-this is to create more data directories backed by the big disk.
-
-== Single sstable tombstone compaction
-
-When an sstable is written a histogram with the tombstone expiry times
-is created and this is used to try to find SSTables with very many
-tombstones and run single sstable compaction on that sstable in hope of
-being able to drop tombstones in that sstable. Before starting this it
-is also checked how likely it is that any tombstones will actually will
-be able to be dropped how much this sstable overlaps with other
-SSTables. To avoid most of these checks the compaction option
-`unchecked_tombstone_compaction` can be enabled.
-
-[[compaction-options]]
-== Common options
-
-There is a number of common options for all the compaction strategies;
-
-`enabled` (default: true)::
- Whether minor compactions should run. Note that you can have
- 'enabled': true as a compaction option and then do 'nodetool
- enableautocompaction' to start running compactions.
-`tombstone_threshold` (default: 0.2)::
- How much of the sstable should be tombstones for us to consider doing
- a single sstable compaction of that sstable.
-`tombstone_compaction_interval` (default: 86400s (1 day))::
- Since it might not be possible to drop any tombstones when doing a
- single sstable compaction we need to make sure that one sstable is not
- constantly getting recompacted - this option states how often we
- should try for a given sstable.
-`log_all` (default: false)::
- New detailed compaction logging, see
- `below <detailed-compaction-logging>`.
-`unchecked_tombstone_compaction` (default: false)::
- The single sstable compaction has quite strict checks for whether it
- should be started, this option disables those checks and for some
- usecases this might be needed. Note that this does not change anything
- for the actual compaction, tombstones are only dropped if it is safe
- to do so - it might just rewrite an sstable without being able to drop
- any tombstones.
-`only_purge_repaired_tombstone` (default: false)::
- Option to enable the extra safety of making sure that tombstones are
- only dropped if the data has been repaired.
-`min_threshold` (default: 4)::
- Lower limit of number of SSTables before a compaction is triggered.
- Not used for `LeveledCompactionStrategy`.
-`max_threshold` (default: 32)::
- Upper limit of number of SSTables before a compaction is triggered.
- Not used for `LeveledCompactionStrategy`.
-
-Further, see the section on each strategy for specific additional
-options.
-
-== Compaction nodetool commands
-
-The `nodetool <nodetool>` utility provides a number of commands related
-to compaction:
-
-`enableautocompaction`::
- Enable compaction.
-`disableautocompaction`::
- Disable compaction.
-`setcompactionthroughput`::
- How fast compaction should run at most - defaults to 64MiB/s.
-`compactionstats`::
- Statistics about current and pending compactions.
-`compactionhistory`::
- List details about the last compactions.
-`setcompactionthreshold`::
- Set the min/max sstable count for when to trigger compaction, defaults
- to 4/32.
-
-== Switching the compaction strategy and options using JMX
-
-It is possible to switch compaction strategies and its options on just a
-single node using JMX, this is a great way to experiment with settings
-without affecting the whole cluster. The mbean is:
-
-[source,none]
-----
-org.apache.cassandra.db:type=ColumnFamilies,keyspace=<keyspace_name>,columnfamily=<table_name>
-----
-
-and the attribute to change is `CompactionParameters` or
-`CompactionParametersJson` if you use jconsole or jmc. The syntax for
-the json version is the same as you would use in an
-`ALTER TABLE <alter-table-statement>` statement -for example:
-
-[source,none]
-----
-{ 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 123, 'fanout_size': 10}
-----
-
-The setting is kept until someone executes an
-`ALTER TABLE <alter-table-statement>` that touches the compaction
-settings or restarts the node.
-
-[[detailed-compaction-logging]]
-== More detailed compaction logging
-
-Enable with the compaction option `log_all` and a more detailed
-compaction log file will be produced in your log directory.
diff --git a/doc/modules/cassandra/pages/operating/index.adoc b/doc/modules/cassandra/pages/operating/index.adoc
deleted file mode 100644
index 367430a..0000000
--- a/doc/modules/cassandra/pages/operating/index.adoc
+++ /dev/null
@@ -1,15 +0,0 @@
-== Operating Cassandra
-
-* xref:operating/hardware.adoc[Hardware]
-* xref:operating/security.adoc[Security]
-* xref:operating/topo_changes.adoc[Topology changes]
-* xref:operating/hints.adoc[Hints]
-* xref:operating/repair.adoc[Repair]
-* xref:operating/read_repair.adoc[Read repair]
-* xref:operating/backups.adoc[Backups]
-* xref:operating/compression.adoc[Compression]
-* xref:operating/compaction/index.adoc[Compaction]
-* xref:operating/metrics.adoc[Monitoring]
-* xref:operating/bulk_loading.adoc[Bulk loading]
-* xref:operating/cdc.adoc[CDC]
-* xref:operating/bloom_filters.adoc[Bloom filters]
diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc
deleted file mode 100644
index 1eb8156..0000000
--- a/doc/modules/cassandra/pages/operating/metrics.adoc
+++ /dev/null
@@ -1,1088 +0,0 @@
-= Monitoring
-
-Metrics in Cassandra are managed using the
-http://metrics.dropwizard.io[Dropwizard Metrics] library. These metrics
-can be queried via JMX or pushed to external monitoring systems using a
-number of
-http://metrics.dropwizard.io/3.1.0/getting-started/#other-reporting[built
-in] and http://metrics.dropwizard.io/3.1.0/manual/third-party/[third
-party] reporter plugins.
-
-Metrics are collected for a single node. It's up to the operator to use
-an external monitoring system to aggregate them.
-
-== Metric Types
-
-All metrics reported by cassandra fit into one of the following types.
-
-`Gauge`::
- An instantaneous measurement of a value.
-`Counter`::
- A gauge for an `AtomicLong` instance. Typically this is consumed by
- monitoring the change since the last call to see if there is a large
- increase compared to the norm.
-`Histogram`::
- Measures the statistical distribution of values in a stream of data.
- +
- In addition to minimum, maximum, mean, etc., it also measures median,
- 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles.
-`Timer`::
- Measures both the rate that a particular piece of code is called and
- the histogram of its duration.
-`Latency`::
- Special type that tracks latency (in microseconds) with a `Timer` plus
- a `Counter` that tracks the total latency accrued since starting. The
- former is useful if you track the change in total latency since the
- last check. Each metric name of this type will have 'Latency' and
- 'TotalLatency' appended to it.
-`Meter`::
- A meter metric which measures mean throughput and one-, five-, and
- fifteen-minute exponentially-weighted moving average throughputs.
-
-== Table Metrics
-
-Each table in Cassandra has metrics responsible for tracking its state
-and performance.
-
-The metric names are all appended with the specific `Keyspace` and
-`Table` name.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Table.<MetricName>.<Keyspace>.<Table>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Table keyspace=<Keyspace> scope=<Table> name=<MetricName>`
-
-[NOTE]
-.Note
-====
-There is a special table called '`all`' without a keyspace. This
-represents the aggregation of metrics across *all* tables and keyspaces
-on the node.
-====[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|MemtableOnHeapSize |Gauge<Long> |Total amount of data stored in the
-memtable that resides *on*-heap, including column related overhead and
-partitions overwritten.
-
-|MemtableOffHeapSize |Gauge<Long> |Total amount of data stored in the
-memtable that resides *off*-heap, including column related overhead and
-partitions overwritten.
-
-|MemtableLiveDataSize |Gauge<Long> |Total amount of live data stored in
-the memtable, excluding any data structure overhead.
-
-|AllMemtablesOnHeapSize |Gauge<Long> |Total amount of data stored in the
-memtables (2i and pending flush memtables included) that resides
-*on*-heap.
-
-|AllMemtablesOffHeapSize |Gauge<Long> |Total amount of data stored in
-the memtables (2i and pending flush memtables included) that resides
-*off*-heap.
-
-|AllMemtablesLiveDataSize |Gauge<Long> |Total amount of live data stored
-in the memtables (2i and pending flush memtables included) that resides
-off-heap, excluding any data structure overhead.
-
-|MemtableColumnsCount |Gauge<Long> |Total number of columns present in
-the memtable.
-
-|MemtableSwitchCount |Counter |Number of times flush has resulted in the
-memtable being switched out.
-
-|CompressionRatio |Gauge<Double> |Current compression ratio for all
-SSTables.
-
-|EstimatedPartitionSizeHistogram |Gauge<long[]> |Histogram of estimated
-partition size (in bytes).
-
-|EstimatedPartitionCount |Gauge<Long> |Approximate number of keys in
-table.
-
-|EstimatedColumnCountHistogram |Gauge<long[]> |Histogram of estimated
-number of columns.
-
-|SSTablesPerReadHistogram |Histogram |Histogram of the number of sstable
-data files accessed per single partition read. SSTables skipped due to
-Bloom Filters, min-max key or partition index lookup are not taken into
-acoount.
-
-|ReadLatency |Latency |Local read latency for this table.
-
-|RangeLatency |Latency |Local range scan latency for this table.
-
-|WriteLatency |Latency |Local write latency for this table.
-
-|CoordinatorReadLatency |Timer |Coordinator read latency for this table.
-
-|CoordinatorWriteLatency |Timer |Coordinator write latency for this
-table.
-
-|CoordinatorScanLatency |Timer |Coordinator range scan latency for this
-table.
-
-|PendingFlushes |Counter |Estimated number of flush tasks pending for
-this table.
-
-|BytesFlushed |Counter |Total number of bytes flushed since server
-[re]start.
-
-|CompactionBytesWritten |Counter |Total number of bytes written by
-compaction since server [re]start.
-
-|PendingCompactions |Gauge<Integer> |Estimate of number of pending
-compactions for this table.
-
-|LiveSSTableCount |Gauge<Integer> |Number of SSTables on disk for this
-table.
-
-|LiveDiskSpaceUsed |Counter |Disk space used by SSTables belonging to
-this table (in bytes).
-
-|TotalDiskSpaceUsed |Counter |Total disk space used by SSTables
-belonging to this table, including obsolete ones waiting to be GC'd.
-
-|MinPartitionSize |Gauge<Long> |Size of the smallest compacted partition
-(in bytes).
-
-|MaxPartitionSize |Gauge<Long> |Size of the largest compacted partition
-(in bytes).
-
-|MeanPartitionSize |Gauge<Long> |Size of the average compacted partition
-(in bytes).
-
-|BloomFilterFalsePositives |Gauge<Long> |Number of false positives on
-table's bloom filter.
-
-|BloomFilterFalseRatio |Gauge<Double> |False positive ratio of table's
-bloom filter.
-
-|BloomFilterDiskSpaceUsed |Gauge<Long> |Disk space used by bloom filter
-(in bytes).
-
-|BloomFilterOffHeapMemoryUsed |Gauge<Long> |Off-heap memory used by
-bloom filter.
-
-|IndexSummaryOffHeapMemoryUsed |Gauge<Long> |Off-heap memory used by
-index summary.
-
-|CompressionMetadataOffHeapMemoryUsed |Gauge<Long> |Off-heap memory used
-by compression meta data.
-
-|KeyCacheHitRate |Gauge<Double> |Key cache hit rate for this table.
-
-|TombstoneScannedHistogram |Histogram |Histogram of tombstones scanned
-in queries on this table.
-
-|LiveScannedHistogram |Histogram |Histogram of live cells scanned in
-queries on this table.
-
-|ColUpdateTimeDeltaHistogram |Histogram |Histogram of column update time
-delta on this table.
-
-|ViewLockAcquireTime |Timer |Time taken acquiring a partition lock for
-materialized view updates on this table.
-
-|ViewReadTime |Timer |Time taken during the local read of a materialized
-view update.
-
-|TrueSnapshotsSize |Gauge<Long> |Disk space used by snapshots of this
-table including all SSTable components.
-
-|RowCacheHitOutOfRange |Counter |Number of table row cache hits that do
-not satisfy the query filter, thus went to disk.
-
-|RowCacheHit |Counter |Number of table row cache hits.
-
-|RowCacheMiss |Counter |Number of table row cache misses.
-
-|CasPrepare |Latency |Latency of paxos prepare round.
-
-|CasPropose |Latency |Latency of paxos propose round.
-
-|CasCommit |Latency |Latency of paxos commit round.
-
-|PercentRepaired |Gauge<Double> |Percent of table data that is repaired
-on disk.
-
-|BytesRepaired |Gauge<Long> |Size of table data repaired on disk
-
-|BytesUnrepaired |Gauge<Long> |Size of table data unrepaired on disk
-
-|BytesPendingRepair |Gauge<Long> |Size of table data isolated for an
-ongoing incremental repair
-
-|SpeculativeRetries |Counter |Number of times speculative retries were
-sent for this table.
-
-|SpeculativeFailedRetries |Counter |Number of speculative retries that
-failed to prevent a timeout
-
-|SpeculativeInsufficientReplicas |Counter |Number of speculative retries
-that couldn't be attempted due to lack of replicas
-
-|SpeculativeSampleLatencyNanos |Gauge<Long> |Number of nanoseconds to
-wait before speculation is attempted. Value may be statically configured
-or updated periodically based on coordinator latency.
-
-|WaitingOnFreeMemtableSpace |Histogram |Histogram of time spent waiting
-for free memtable space, either on- or off-heap.
-
-|DroppedMutations |Counter |Number of dropped mutations on this table.
-
-|AnticompactionTime |Timer |Time spent anticompacting before a
-consistent repair.
-
-|ValidationTime |Timer |Time spent doing validation compaction during
-repair.
-
-|SyncTime |Timer |Time spent doing streaming during repair.
-
-|BytesValidated |Histogram |Histogram over the amount of bytes read
-during validation.
-
-|PartitionsValidated |Histogram |Histogram over the number of partitions
-read during validation.
-
-|BytesAnticompacted |Counter |How many bytes we anticompacted.
-
-|BytesMutatedAnticompaction |Counter |How many bytes we avoided
-anticompacting because the sstable was fully contained in the repaired
-range.
-
-|MutatedAnticompactionGauge |Gauge<Double> |Ratio of bytes mutated vs
-total bytes repaired.
-|===
-
-== Keyspace Metrics
-
-Each keyspace in Cassandra has metrics responsible for tracking its
-state and performance.
-
-Most of these metrics are the same as the `Table Metrics` above, only
-they are aggregated at the Keyspace level. The keyspace specific metrics
-are specified in the table below.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.keyspace.<MetricName>.<Keyspace>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Keyspace scope=<Keyspace> name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|WriteFailedIdeaCL |Counter |Number of writes that failed to achieve the
-configured ideal consistency level or 0 if none is configured
-
-|IdealCLWriteLatency |Latency |Coordinator latency of writes at the
-configured ideal consistency level. No values are recorded if ideal
-consistency level is not configured
-
-|RepairTime |Timer |Total time spent as repair coordinator.
-
-|RepairPrepareTime |Timer |Total time spent preparing for repair.
-|===
-
-== ThreadPool Metrics
-
-Cassandra splits work of a particular type into its own thread pool.
-This provides back-pressure and asynchrony for requests on a node. It's
-important to monitor the state of these thread pools since they can tell
-you how saturated a node is.
-
-The metric names are all appended with the specific `ThreadPool` name.
-The thread pools are also categorized under a specific type.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.ThreadPools.<MetricName>.<Path>.<ThreadPoolName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=ThreadPools path=<Path> scope=<ThreadPoolName> name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|ActiveTasks |Gauge<Integer> |Number of tasks being actively worked on
-by this pool.
-
-|PendingTasks |Gauge<Integer> |Number of queued tasks queued up on this
-pool.
-
-|CompletedTasks |Counter |Number of tasks completed.
-
-|TotalBlockedTasks |Counter |Number of tasks that were blocked due to
-queue saturation.
-
-|CurrentlyBlockedTask |Counter |Number of tasks that are currently
-blocked due to queue saturation but on retry will become unblocked.
-
-|MaxPoolSize |Gauge<Integer> |The maximum number of threads in this
-pool.
-
-|MaxTasksQueued |Gauge<Integer> |The maximum number of tasks queued
-before a task get blocked.
-|===
-
-The following thread pools can be monitored.
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Native-Transport-Requests |transport |Handles client CQL requests
-
-|CounterMutationStage |request |Responsible for counter writes
-
-|ViewMutationStage |request |Responsible for materialized view writes
-
-|MutationStage |request |Responsible for all other writes
-
-|ReadRepairStage |request |ReadRepair happens on this thread pool
-
-|ReadStage |request |Local reads run on this thread pool
-
-|RequestResponseStage |request |Coordinator requests to the cluster run
-on this thread pool
-
-|AntiEntropyStage |internal |Builds merkle tree for repairs
-
-|CacheCleanupExecutor |internal |Cache maintenance performed on this
-thread pool
-
-|CompactionExecutor |internal |Compactions are run on these threads
-
-|GossipStage |internal |Handles gossip requests
-
-|HintsDispatcher |internal |Performs hinted handoff
-
-|InternalResponseStage |internal |Responsible for intra-cluster
-callbacks
-
-|MemtableFlushWriter |internal |Writes memtables to disk
-
-|MemtablePostFlush |internal |Cleans up commit log after memtable is
-written to disk
-
-|MemtableReclaimMemory |internal |Memtable recycling
-
-|MigrationStage |internal |Runs schema migrations
-
-|MiscStage |internal |Misceleneous tasks run here
-
-|PendingRangeCalculator |internal |Calculates token range
-
-|PerDiskMemtableFlushWriter_0 |internal |Responsible for writing a spec
-(there is one of these per disk 0-N)
-
-|Sampler |internal |Responsible for re-sampling the index summaries of
-SStables
-
-|SecondaryIndexManagement |internal |Performs updates to secondary
-indexes
-
-|ValidationExecutor |internal |Performs validation compaction or
-scrubbing
-
-|ViewBuildExecutor |internal |Performs materialized views initial build
-|===
-
-== Client Request Metrics
-
-Client requests have their own set of metrics that encapsulate the work
-happening at coordinator level.
-
-Different types of client requests are broken down by `RequestType`.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.ClientRequest.<MetricName>.<RequestType>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=ClientRequest scope=<RequestType> name=<MetricName>`
-
-RequestType::
- CASRead
-Description::
- Metrics related to transactional read requests.
-Metrics::
- [cols=",,",options="header",]
- |===
- |Name |Type |Description
- |Timeouts |Counter |Number of timeouts encountered.
-
- |Failures |Counter |Number of transaction failures encountered.
-
- | |Latency |Transaction read latency.
-
- |Unavailables |Counter |Number of unavailable exceptions encountered.
-
- |UnfinishedCommit |Counter |Number of transactions that were committed
- on read.
-
- |ConditionNotMet |Counter |Number of transaction preconditions did not
- match current values.
-
- |ContentionHistogram |Histogram |How many contended reads were
- encountered
- |===
-RequestType::
- CASWrite
-Description::
- Metrics related to transactional write requests.
-Metrics::
- [cols=",,",options="header",]
- |===
- |Name |Type |Description
- |Timeouts |Counter |Number of timeouts encountered.
-
- |Failures |Counter |Number of transaction failures encountered.
-
- | |Latency |Transaction write latency.
-
- |UnfinishedCommit |Counter |Number of transactions that were committed
- on write.
-
- |ConditionNotMet |Counter |Number of transaction preconditions did not
- match current values.
-
- |ContentionHistogram |Histogram |How many contended writes were
- encountered
-
- |MutationSizeHistogram |Histogram |Total size in bytes of the requests
- mutations.
- |===
-RequestType::
- Read
-Description::
- Metrics related to standard read requests.
-Metrics::
- [cols=",,",options="header",]
- |===
- |Name |Type |Description
- |Timeouts |Counter |Number of timeouts encountered.
- |Failures |Counter |Number of read failures encountered.
- | |Latency |Read latency.
- |Unavailables |Counter |Number of unavailable exceptions encountered.
- |===
-RequestType::
- RangeSlice
-Description::
- Metrics related to token range read requests.
-Metrics::
- [cols=",,",options="header",]
- |===
- |Name |Type |Description
- |Timeouts |Counter |Number of timeouts encountered.
- |Failures |Counter |Number of range query failures encountered.
- | |Latency |Range query latency.
- |Unavailables |Counter |Number of unavailable exceptions encountered.
- |===
-RequestType::
- Write
-Description::
- Metrics related to regular write requests.
-Metrics::
- [cols=",,",options="header",]
- |===
- |Name |Type |Description
- |Timeouts |Counter |Number of timeouts encountered.
-
- |Failures |Counter |Number of write failures encountered.
-
- | |Latency |Write latency.
-
- |Unavailables |Counter |Number of unavailable exceptions encountered.
-
- |MutationSizeHistogram |Histogram |Total size in bytes of the requests
- mutations.
- |===
-RequestType::
- ViewWrite
-Description::
- Metrics related to materialized view write wrtes.
-Metrics::
- [cols=",,",]
- |===
- |Timeouts |Counter |Number of timeouts encountered.
-
- |Failures |Counter |Number of transaction failures encountered.
-
- |Unavailables |Counter |Number of unavailable exceptions encountered.
-
- |ViewReplicasAttempted |Counter |Total number of attempted view
- replica writes.
-
- |ViewReplicasSuccess |Counter |Total number of succeded view replica
- writes.
-
- |ViewPendingMutations |Gauge<Long> |ViewReplicasAttempted -
- ViewReplicasSuccess.
-
- |ViewWriteLatency |Timer |Time between when mutation is applied to
- base table and when CL.ONE is achieved on view.
- |===
-
-== Cache Metrics
-
-Cassandra caches have metrics to track the effectivness of the caches.
-Though the `Table Metrics` might be more useful.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Cache.<MetricName>.<CacheName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Cache scope=<CacheName> name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Capacity |Gauge<Long> |Cache capacity in bytes.
-|Entries |Gauge<Integer> |Total number of cache entries.
-|FifteenMinuteCacheHitRate |Gauge<Double> |15m cache hit rate.
-|FiveMinuteCacheHitRate |Gauge<Double> |5m cache hit rate.
-|OneMinuteCacheHitRate |Gauge<Double> |1m cache hit rate.
-|HitRate |Gauge<Double> |All time cache hit rate.
-|Hits |Meter |Total number of cache hits.
-|Misses |Meter |Total number of cache misses.
-|MissLatency |Timer |Latency of misses.
-|Requests |Gauge<Long> |Total number of cache requests.
-|Size |Gauge<Long> |Total size of occupied cache, in bytes.
-|===
-
-The following caches are covered:
-
-[cols=",",options="header",]
-|===
-|Name |Description
-|CounterCache |Keeps hot counters in memory for performance.
-|ChunkCache |In process uncompressed page cache.
-|KeyCache |Cache for partition to sstable offsets.
-|RowCache |Cache for rows kept in memory.
-|===
-
-[NOTE]
-.Note
-====
-Misses and MissLatency are only defined for the ChunkCache
-====== CQL Metrics
-
-Metrics specific to CQL prepared statement caching.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.CQL.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=CQL name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|PreparedStatementsCount |Gauge<Integer> |Number of cached prepared
-statements.
-
-|PreparedStatementsEvicted |Counter |Number of prepared statements
-evicted from the prepared statement cache
-
-|PreparedStatementsExecuted |Counter |Number of prepared statements
-executed.
-
-|RegularStatementsExecuted |Counter |Number of *non* prepared statements
-executed.
-
-|PreparedStatementsRatio |Gauge<Double> |Percentage of statements that
-are prepared vs unprepared.
-|===
-
-[[dropped-metrics]]
-== DroppedMessage Metrics
-
-Metrics specific to tracking dropped messages for different types of
-requests. Dropped writes are stored and retried by `Hinted Handoff`
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.DroppedMessage.<MetricName>.<Type>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=DroppedMessage scope=<Type> name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|CrossNodeDroppedLatency |Timer |The dropped latency across nodes.
-|InternalDroppedLatency |Timer |The dropped latency within node.
-|Dropped |Meter |Number of dropped messages.
-|===
-
-The different types of messages tracked are:
-
-[cols=",",options="header",]
-|===
-|Name |Description
-|BATCH_STORE |Batchlog write
-|BATCH_REMOVE |Batchlog cleanup (after succesfully applied)
-|COUNTER_MUTATION |Counter writes
-|HINT |Hint replay
-|MUTATION |Regular writes
-|READ |Regular reads
-|READ_REPAIR |Read repair
-|PAGED_SLICE |Paged read
-|RANGE_SLICE |Token range read
-|REQUEST_RESPONSE |RPC Callbacks
-|_TRACE |Tracing writes
-|===
-
-== Streaming Metrics
-
-Metrics reported during `Streaming` operations, such as repair,
-bootstrap, rebuild.
-
-These metrics are specific to a peer endpoint, with the source node
-being the node you are pulling the metrics from.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Streaming.<MetricName>.<PeerIP>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Streaming scope=<PeerIP> name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|IncomingBytes |Counter |Number of bytes streamed to this node from the
-peer.
-
-|OutgoingBytes |Counter |Number of bytes streamed to the peer endpoint
-from this node.
-|===
-
-== Compaction Metrics
-
-Metrics specific to `Compaction` work.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Compaction.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Compaction name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|BytesCompacted |Counter |Total number of bytes compacted since server
-[re]start.
-
-|PendingTasks |Gauge<Integer> |Estimated number of compactions remaining
-to perform.
-
-|CompletedTasks |Gauge<Long> |Number of completed compactions since
-server [re]start.
-
-|TotalCompactionsCompleted |Meter |Throughput of completed compactions
-since server [re]start.
-
-|PendingTasksByTableName |Gauge<Map<String, Map<String, Integer>>>
-|Estimated number of compactions remaining to perform, grouped by
-keyspace and then table name. This info is also kept in `Table Metrics`.
-|===
-
-== CommitLog Metrics
-
-Metrics specific to the `CommitLog`
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.CommitLog.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=CommitLog name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|CompletedTasks |Gauge<Long> |Total number of commit log messages
-written since [re]start.
-
-|PendingTasks |Gauge<Long> |Number of commit log messages written but
-yet to be fsync'd.
-
-|TotalCommitLogSize |Gauge<Long> |Current size, in bytes, used by all
-the commit log segments.
-
-|WaitingOnSegmentAllocation |Timer |Time spent waiting for a
-CommitLogSegment to be allocated - under normal conditions this should
-be zero.
-
-|WaitingOnCommit |Timer |The time spent waiting on CL fsync; for
-Periodic this is only occurs when the sync is lagging its sync interval.
-|===
-
-== Storage Metrics
-
-Metrics specific to the storage engine.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Storage.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Storage name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Exceptions |Counter |Number of internal exceptions caught. Under normal
-exceptions this should be zero.
-
-|Load |Counter |Size, in bytes, of the on disk data size this node
-manages.
-
-|TotalHints |Counter |Number of hint messages written to this node since
-[re]start. Includes one entry for each host to be hinted per hint.
-
-|TotalHintsInProgress |Counter |Number of hints attemping to be sent
-currently.
-|===
-
-[[handoff-metrics]]
-== HintedHandoff Metrics
-
-Metrics specific to Hinted Handoff. There are also some metrics related
-to hints tracked in `Storage Metrics`
-
-These metrics include the peer endpoint *in the metric name*
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.HintedHandOffManager.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=HintedHandOffManager name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Hints_created-<PeerIP> a|
-____
-Counter
-____
-
-a|
-____
-Number of hints on disk for this peer.
-____
-
-|Hints_not_stored-<PeerIP> a|
-____
-Counter
-____
-
-a|
-____
-Number of hints not stored for this peer, due to being down past the
-configured hint window.
-____
-
-|===
-
-== HintsService Metrics
-
-Metrics specific to the Hints delivery service. There are also some
-metrics related to hints tracked in `Storage Metrics`
-
-These metrics include the peer endpoint *in the metric name*
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.HintsService.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=HintsService name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|HintsSucceeded a|
-____
-Meter
-____
-
-a|
-____
-A meter of the hints successfully delivered
-____
-
-|HintsFailed a|
-____
-Meter
-____
-
-a|
-____
-A meter of the hints that failed deliver
-____
-
-|HintsTimedOut a|
-____
-Meter
-____
-
-a|
-____
-A meter of the hints that timed out
-____
-
-|Hint_delays |Histogram |Histogram of hint delivery delays (in
-milliseconds)
-
-|Hint_delays-<PeerIP> |Histogram |Histogram of hint delivery delays (in
-milliseconds) per peer
-|===
-
-== SSTable Index Metrics
-
-Metrics specific to the SSTable index metadata.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Index.<MetricName>.RowIndexEntry`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Index scope=RowIndexEntry name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|IndexedEntrySize |Histogram |Histogram of the on-heap size, in bytes,
-of the index across all SSTables.
-
-|IndexInfoCount |Histogram |Histogram of the number of on-heap index
-entries managed across all SSTables.
-
-|IndexInfoGets |Histogram |Histogram of the number index seeks performed
-per SSTable.
-|===
-
-== BufferPool Metrics
-
-Metrics specific to the internal recycled buffer pool Cassandra manages.
-This pool is meant to keep allocations and GC lower by recycling on and
-off heap buffers.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.BufferPool.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=BufferPool name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Size |Gauge<Long> |Size, in bytes, of the managed buffer pool
-
-|Misses |Meter a|
-____
-The rate of misses in the pool. The higher this is the more allocations
-incurred.
-____
-
-|===
-
-== Client Metrics
-
-Metrics specifc to client managment.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Client.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Client name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|connectedNativeClients |Gauge<Integer> |Number of clients connected to
-this nodes native protocol server
-
-|connections |Gauge<List<Map<String, String>> |List of all connections
-and their state information
-
-|connectedNativeClientsByUser |Gauge<Map<String, Int> |Number of
-connnective native clients by username
-|===
-
-== Batch Metrics
-
-Metrics specifc to batch statements.
-
-Reported name format:
-
-*Metric Name*::
- `org.apache.cassandra.metrics.Batch.<MetricName>`
-*JMX MBean*::
- `org.apache.cassandra.metrics:type=Batch name=<MetricName>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|PartitionsPerCounterBatch |Histogram |Distribution of the number of
-partitions processed per counter batch
-
-|PartitionsPerLoggedBatch |Histogram |Distribution of the number of
-partitions processed per logged batch
-
-|PartitionsPerUnloggedBatch |Histogram |Distribution of the number of
-partitions processed per unlogged batch
-|===
-
-== JVM Metrics
-
-JVM metrics such as memory and garbage collection statistics can either
-be accessed by connecting to the JVM using JMX or can be exported using
-link:#metric-reporters[Metric Reporters].
-
-=== BufferPool
-
-*Metric Name*::
- `jvm.buffers.<direct|mapped>.<MetricName>`
-*JMX MBean*::
- `java.nio:type=BufferPool name=<direct|mapped>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Capacity |Gauge<Long> |Estimated total capacity of the buffers in this
-pool
-
-|Count |Gauge<Long> |Estimated number of buffers in the pool
-
-|Used |Gauge<Long> |Estimated memory that the Java virtual machine is
-using for this buffer pool
-|===
-
-=== FileDescriptorRatio
-
-*Metric Name*::
- `jvm.fd.<MetricName>`
-*JMX MBean*::
- `java.lang:type=OperatingSystem name=<OpenFileDescriptorCount|MaxFileDescriptorCount>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Usage |Ratio |Ratio of used to total file descriptors
-|===
-
-=== GarbageCollector
-
-*Metric Name*::
- `jvm.gc.<gc_type>.<MetricName>`
-*JMX MBean*::
- `java.lang:type=GarbageCollector name=<gc_type>`
-
-[cols=",,",options="header",]
-|===
-|Name |Type |Description
-|Count |Gauge<Long> |Total number of collections that have occurred
-
-|Time |Gauge<Long> |Approximate accumulated collection elapsed time in
-milliseconds
-|===
-
-=== Memory
-
-*Metric Name*::
- `jvm.memory.<heap/non-heap/total>.<MetricName>`
-*JMX MBean*::
- `java.lang:type=Memory`
-
-[cols=",,",]
-|===
-|Committed |Gauge<Long> |Amount of memory in bytes that is committed for
-the JVM to use
-
-|Init |Gauge<Long> |Amount of memory in bytes that the JVM initially
-requests from the OS
-
-|Max |Gauge<Long> |Maximum amount of memory in bytes that can be used
-for memory management
-
-|Usage |Ratio |Ratio of used to maximum memory
-
-|Used |Gauge<Long> |Amount of used memory in bytes
-|===
-
-=== MemoryPool
-
-*Metric Name*::
- `jvm.memory.pools.<memory_pool>.<MetricName>`
-*JMX MBean*::
- `java.lang:type=MemoryPool name=<memory_pool>`
-
-[cols=",,",]
-|===
-|Committed |Gauge<Long> |Amount of memory in bytes that is committed for
-the JVM to use
-
-|Init |Gauge<Long> |Amount of memory in bytes that the JVM initially
-requests from the OS
-
-|Max |Gauge<Long> |Maximum amount of memory in bytes that can be used
-for memory management
-
-|Usage |Ratio |Ratio of used to maximum memory
-
-|Used |Gauge<Long> |Amount of used memory in bytes
-|===
-
-== JMX
-
-Any JMX based client can access metrics from cassandra.
-
-If you wish to access JMX metrics over http it's possible to download
-http://mx4j.sourceforge.net/[Mx4jTool] and place `mx4j-tools.jar` into
-the classpath. On startup you will see in the log:
-
-[source,none]
-----
-HttpAdaptor version 3.0.2 started on port 8081
-----
-
-To choose a different port (8081 is the default) or a different listen
-address (0.0.0.0 is not the default) edit `conf/cassandra-env.sh` and
-uncomment:
-
-[source,none]
-----
-#MX4J_ADDRESS="-Dmx4jaddress=0.0.0.0"
-
-#MX4J_PORT="-Dmx4jport=8081"
-----
-
-== Metric Reporters
-
-As mentioned at the top of this section on monitoring the Cassandra
-metrics can be exported to a number of monitoring system a number of
-http://metrics.dropwizard.io/3.1.0/getting-started/#other-reporting[built
-in] and http://metrics.dropwizard.io/3.1.0/manual/third-party/[third
-party] reporter plugins.
-
-The configuration of these plugins is managed by the
-https://github.com/addthis/metrics-reporter-config[metrics reporter
-config project]. There is a sample configuration file located at
-`conf/metrics-reporter-config-sample.yaml`.
-
-Once configured, you simply start cassandra with the flag
-`-Dcassandra.metricsReporterConfigFile=metrics-reporter-config.yaml`.
-The specified .yaml file plus any 3rd party reporter jars must all be in
-Cassandra's classpath.
diff --git a/doc/modules/cassandra/pages/operating/virtualtables.adoc b/doc/modules/cassandra/pages/operating/virtualtables.adoc
deleted file mode 100644
index 2963ecb..0000000
--- a/doc/modules/cassandra/pages/operating/virtualtables.adoc
+++ /dev/null
@@ -1,478 +0,0 @@
-= Virtual Tables
-
-Apache Cassandra 4.0 implements virtual tables (https://issues.apache.org/jira/browse/CASSANDRA-7622[CASSANDRA-7622]).
-Virtual tables are tables backed by an API instead of data explicitly managed and stored as SSTables.
-Apache Cassandra 4.0 implements a virtual keyspace interface for virtual tables.
-Virtual tables are specific to each node.
-
-Some of the features of virtual tables are the ability to:
-
-* expose metrics through CQL
-* expose YAML configuration information
-
-Virtual keyspaces and tables are quite different from regular tables and keyspaces:
-
-* Virtual tables are created in special keyspaces and not just any keyspace.
-* Virtual tables are managed by Cassandra. Users cannot run DDL to create new virtual tables or DML to modify existing virtual tables.
-* Virtual tables are currently read-only, although that may change in a later version.
-* Virtual tables are local only, non-distributed, and thus not replicated.
-* Virtual tables have no associated SSTables.
-* Consistency level of the queries sent to virtual tables are ignored.
-* All existing virtual tables use `LocalPartitioner`.
-Since a virtual table is not replicated the partitioner sorts in order of partition keys instead of by their hash.
-* Making advanced queries using `ALLOW FILTERING` and aggregation functions can be executed in virtual tables, even though in normal tables we dont recommend it.
-
-== Virtual Keyspaces
-
-Apache Cassandra 4.0 has added two new keyspaces for virtual tables:
-
-* `system_virtual_schema`
-* `system_views`.
-
-The `system_virtual_schema` keyspace has three tables: `keyspaces`,
-`columns` and `tables` for the virtual keyspace, table, and column definitions, respectively.
-These tables contain schema information for the virtual tables.
-It is used by Cassandra internally and a user should not access it directly.
-
-The `system_views` keyspace contains the actual virtual tables.
-
-== Virtual Table Limitations
-
-Before disccusing virtual keyspaces and tables, note that virtual keyspaces and tables have some limitations.
-These limitations are subject to change.
-Virtual keyspaces cannot be altered or dropped.
-In fact, no operations can be performed against virtual keyspaces.
-
-Virtual tables cannot be created in virtual keyspaces.
-Virtual tables cannot be altered, dropped, or truncated.
-Secondary indexes, types, functions, aggregates, materialized views, and triggers cannot be created for virtual tables.
-Expiring time-to-live (TTL) columns cannot be created.
-Virtual tables do not support conditional updates or deletes.
-Aggregates may be run in SELECT statements.
-
-Conditional batch statements cannot include mutations for virtual tables, nor can a virtual table statement be included in a logged batch.
-In fact, mutations for virtual and regular tables cannot occur in the same batch table.
-
-== Virtual Tables
-
-Each of the virtual tables in the `system_views` virtual keyspace contain different information.
-
-The following table describes the virtual tables:
-
-[width="98%",cols="27%,73%",]
-|===
-|Virtual Table |Description
-
-|caches |Displays the general cache information including cache name, capacity_bytes, entry_count, hit_count, hit_ratio double,
-recent_hit_rate_per_second, recent_request_rate_per_second, request_count, and size_bytes.
-
-|clients |Lists information about all connected clients.
-
-|coordinator_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator reads.
-
-|coordinator_scan |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator scans.
-
-|coordinator_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for coordinator writes.
-
-|disk_usage |Records disk usage including disk_space, keyspace_name, and table_name, sorted by system keyspaces.
-
-|internode_inbound |Lists information about the inbound internode messaging.
-
-|internode_outbound |Information about the outbound internode messaging.
-
-|local_read_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local reads.
-
-|local_scan |Records counts, keyspace_name, table_name, max, median, and per_second for local scans.
-
-|local_write_latency |Records counts, keyspace_name, table_name, max, median, and per_second for local writes.
-
-|max_partition_size |A table metric for maximum partition size.
-
-|rows_per_read |Records counts, keyspace_name, tablek_name, max, and median for rows read.
-
-|settings |Displays configuration settings in cassandra.yaml.
-
-|sstable_tasks |Lists currently running tasks and progress on SSTables, for operations like compaction and upgrade.
-
-|system_properties |Displays environmental system properties set on the node.
-
-|thread_pools |Lists metrics for each thread pool.
-
-|tombstones_per_read |Records counts, keyspace_name, tablek_name, max, and median for tombstones.
-|===
-
-We shall discuss some of the virtual tables in more detail next.
-
-=== Clients Virtual Table
-
-The `clients` virtual table lists all active connections (connected
-clients) including their ip address, port, client_options, connection stage, driver
-name, driver version, hostname, protocol version, request count, ssl
-enabled, ssl protocol and user name:
-
-....
-cqlsh> EXPAND ON ;
-Now Expanded output is enabled
-cqlsh> SELECT * FROM system_views.clients;
-
-@ Row 1
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50687
- client_options | {'CQL_VERSION': '3.4.6', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
- connection_stage | ready
- driver_name | DataStax Python Driver
- driver_version | 3.25.0
- hostname | localhost
- protocol_version | 5
- request_count | 16
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-@ Row 2
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50688
- client_options | {'CQL_VERSION': '3.4.6', 'DRIVER_NAME': 'DataStax Python Driver', 'DRIVER_VERSION': '3.25.0'}
- connection_stage | ready
- driver_name | DataStax Python Driver
- driver_version | 3.25.0
- hostname | localhost
- protocol_version | 5
- request_count | 4
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-@ Row 3
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50753
- client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
- connection_stage | ready
- driver_name | DataStax Java driver for Apache Cassandra(R)
- driver_version | 4.13.0
- hostname | localhost
- protocol_version | 5
- request_count | 18
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-@ Row 4
-------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- address | 127.0.0.1
- port | 50755
- client_options | {'APPLICATION_NAME': 'TestApp', 'APPLICATION_VERSION': '1.0.0', 'CLIENT_ID': '55b3efbd-c56b-469d-8cca-016b860b2f03', 'CQL_VERSION': '3.0.0', 'DRIVER_NAME': 'DataStax Java driver for Apache Cassandra(R)', 'DRIVER_VERSION': '4.13.0'}
- connection_stage | ready
- driver_name | DataStax Java driver for Apache Cassandra(R)
- driver_version | 4.13.0
- hostname | localhost
- protocol_version | 5
- request_count | 7
- ssl_cipher_suite | null
- ssl_enabled | False
- ssl_protocol | null
- username | anonymous
-
-(4 rows)
-....
-
-Some examples of how `clients` can be used are:
-
-* To find applications using old incompatible versions of drivers before
-upgrading and with `nodetool enableoldprotocolversions` and
-`nodetool disableoldprotocolversions` during upgrades.
-* To identify clients sending too many requests.
-* To find if SSL is enabled during the migration to and from ssl.
-
-The virtual tables may be described with `DESCRIBE` statement. The DDL
-listed however cannot be run to create a virtual table. As an example
-describe the `system_views.clients` virtual table:
-
-....
-cqlsh> DESCRIBE TABLE system_views.clients;
-
-/*
-Warning: Table system_views.clients is a virtual table and cannot be recreated with CQL.
-Structure, for reference:
-VIRTUAL TABLE system_views.clients (
- address inet,
- port int,
- client_options frozen<map<text, text>>,
- connection_stage text,
- driver_name text,
- driver_version text,
- hostname text,
- protocol_version int,
- request_count bigint,
- ssl_cipher_suite text,
- ssl_enabled boolean,
- ssl_protocol text,
- username text,
- PRIMARY KEY (address, port)
-) WITH CLUSTERING ORDER BY (port ASC)
- AND comment = 'currently connected clients';
-*/
-....
-
-=== Caches Virtual Table
-
-The `caches` virtual table lists information about the caches. The four
-caches presently created are chunks, counters, keys and rows. A query on
-the `caches` virtual table returns the following details:
-
-....
-cqlsh:system_views> SELECT * FROM system_views.caches;
-name | capacity_bytes | entry_count | hit_count | hit_ratio | recent_hit_rate_per_second | recent_request_rate_per_second | request_count | size_bytes
----------+----------------+-------------+-----------+-----------+----------------------------+--------------------------------+---------------+------------
- chunks | 229638144 | 29 | 166 | 0.83 | 5 | 6 | 200 | 475136
-counters | 26214400 | 0 | 0 | NaN | 0 | 0 | 0 | 0
- keys | 52428800 | 14 | 124 | 0.873239 | 4 | 4 | 142 | 1248
- rows | 0 | 0 | 0 | NaN | 0 | 0 | 0 | 0
-
-(4 rows)
-....
-
-=== Settings Virtual Table
-
-The `settings` table is rather useful and lists all the current
-configuration settings from the `cassandra.yaml`. The encryption options
-are overridden to hide the sensitive truststore information or
-passwords. The configuration settings however cannot be set using DML on
-the virtual table presently: :
-
-....
-cqlsh:system_views> SELECT * FROM system_views.settings;
-
-name | value
--------------------------------------+--------------------
- allocate_tokens_for_keyspace | null
- audit_logging_options_enabled | false
- auto_snapshot | true
- automatic_sstable_upgrade | false
- cluster_name | Test Cluster
- enable_transient_replication | false
- hinted_handoff_enabled | true
- hints_directory | /home/ec2-user/cassandra/data/hints
- incremental_backups | false
- initial_token | null
- ...
- ...
- ...
- rpc_address | localhost
- ssl_storage_port | 7001
- start_native_transport | true
- storage_port | 7000
- stream_entire_sstables | true
- (224 rows)
-....
-
-The `settings` table can be really useful if yaml file has been changed
-since startup and dont know running configuration, or to find if they
-have been modified via jmx/nodetool or virtual tables.
-
-=== Thread Pools Virtual Table
-
-The `thread_pools` table lists information about all thread pools.
-Thread pool information includes active tasks, active tasks limit,
-blocked tasks, blocked tasks all time, completed tasks, and pending
-tasks. A query on the `thread_pools` returns following details:
-
-....
-cqlsh:system_views> select * from system_views.thread_pools;
-
-name | active_tasks | active_tasks_limit | blocked_tasks | blocked_tasks_all_time | completed_tasks | pending_tasks
-------------------------------+--------------+--------------------+---------------+------------------------+-----------------+---------------
- AntiEntropyStage | 0 | 1 | 0 | 0 | 0 | 0
- CacheCleanupExecutor | 0 | 1 | 0 | 0 | 0 | 0
- CompactionExecutor | 0 | 2 | 0 | 0 | 881 | 0
- CounterMutationStage | 0 | 32 | 0 | 0 | 0 | 0
- GossipStage | 0 | 1 | 0 | 0 | 0 | 0
- HintsDispatcher | 0 | 2 | 0 | 0 | 0 | 0
- InternalResponseStage | 0 | 2 | 0 | 0 | 0 | 0
- MemtableFlushWriter | 0 | 2 | 0 | 0 | 1 | 0
- MemtablePostFlush | 0 | 1 | 0 | 0 | 2 | 0
- MemtableReclaimMemory | 0 | 1 | 0 | 0 | 1 | 0
- MigrationStage | 0 | 1 | 0 | 0 | 0 | 0
- MiscStage | 0 | 1 | 0 | 0 | 0 | 0
- MutationStage | 0 | 32 | 0 | 0 | 0 | 0
- Native-Transport-Requests | 1 | 128 | 0 | 0 | 130 | 0
- PendingRangeCalculator | 0 | 1 | 0 | 0 | 1 | 0
-PerDiskMemtableFlushWriter_0 | 0 | 2 | 0 | 0 | 1 | 0
- ReadStage | 0 | 32 | 0 | 0 | 13 | 0
- Repair-Task | 0 | 2147483647 | 0 | 0 | 0 | 0
- RequestResponseStage | 0 | 2 | 0 | 0 | 0 | 0
- Sampler | 0 | 1 | 0 | 0 | 0 | 0
- SecondaryIndexManagement | 0 | 1 | 0 | 0 | 0 | 0
- ValidationExecutor | 0 | 2147483647 | 0 | 0 | 0 | 0
- ViewBuildExecutor | 0 | 1 | 0 | 0 | 0 | 0
- ViewMutationStage | 0 | 32 | 0 | 0 | 0 | 0
-....
-
-(24 rows)
-
-=== Internode Inbound Messaging Virtual Table
-
-The `internode_inbound` virtual table is for the internode inbound
-messaging. Initially no internode inbound messaging may get listed. In
-addition to the address, port, datacenter and rack information includes
-corrupt frames recovered, corrupt frames unrecovered, error bytes, error
-count, expired bytes, expired count, processed bytes, processed count,
-received bytes, received count, scheduled bytes, scheduled count,
-throttled count, throttled nanos, using bytes, using reserve bytes. A
-query on the `internode_inbound` returns following details:
-
-....
-cqlsh:system_views> SELECT * FROM system_views.internode_inbound;
-address | port | dc | rack | corrupt_frames_recovered | corrupt_frames_unrecovered |
-error_bytes | error_count | expired_bytes | expired_count | processed_bytes |
-processed_count | received_bytes | received_count | scheduled_bytes | scheduled_count | throttled_count | throttled_nanos | using_bytes | using_reserve_bytes
----------+------+----+------+--------------------------+----------------------------+-
-----------
-(0 rows)
-....
-
-=== SSTables Tasks Virtual Table
-
-The `sstable_tasks` could be used to get information about running
-tasks. It lists following columns:
-
-....
-cqlsh:system_views> SELECT * FROM sstable_tasks;
-keyspace_name | table_name | task_id | kind | progress | total | unit
----------------+------------+--------------------------------------+------------+----------+----------+-------
- basic | wide2 | c3909740-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 60418761 | 70882110 | bytes
- basic | wide2 | c7556770-cdf7-11e9-a8ed-0f03de2d9ae1 | compaction | 2995623 | 40314679 | bytes
-....
-
-As another example, to find how much time is remaining for SSTable
-tasks, use the following query:
-
-....
-SELECT total - progress AS remaining
-FROM system_views.sstable_tasks;
-....
-
-=== Other Virtual Tables
-
-Some examples of using other virtual tables are as follows.
-
-Find tables with most disk usage:
-
-....
-cqlsh> SELECT * FROM disk_usage WHERE mebibytes > 1 ALLOW FILTERING;
-
-keyspace_name | table_name | mebibytes
----------------+------------+-----------
- keyspace1 | standard1 | 288
- tlp_stress | keyvalue | 3211
-....
-
-Find queries on table/s with greatest read latency:
-
-....
-cqlsh> SELECT * FROM local_read_latency WHERE per_second > 1 ALLOW FILTERING;
-
-keyspace_name | table_name | p50th_ms | p99th_ms | count | max_ms | per_second
----------------+------------+----------+----------+----------+---------+------------
- tlp_stress | keyvalue | 0.043 | 0.152 | 49785158 | 186.563 | 11418.356
-....
-
-
-== Example
-
-[arabic, start=1]
-. To list the keyspaces, enter ``cqlsh`` and run the CQL command ``DESCRIBE KEYSPACES``:
-
-[source, cql]
-----
-cqlsh> DESC KEYSPACES;
-system_schema system system_distributed system_virtual_schema
-system_auth system_traces system_views
-----
-
-[arabic, start=2]
-. To view the virtual table schema, run the CQL commands ``USE system_virtual_schema`` and ``SELECT * FROM tables``:
-
-[source, cql]
-----
-cqlsh> USE system_virtual_schema;
-cqlsh> SELECT * FROM tables;
-----
-
-results in:
-
-[source, cql]
-----
- keyspace_name | table_name | comment
------------------------+---------------------------+--------------------------------------
- system_views | caches | system caches
- system_views | clients | currently connected clients
- system_views | coordinator_read_latency |
- system_views | coordinator_scan_latency |
- system_views | coordinator_write_latency |
- system_views | disk_usage |
- system_views | internode_inbound |
- system_views | internode_outbound |
- system_views | local_read_latency |
- system_views | local_scan_latency |
- system_views | local_write_latency |
- system_views | max_partition_size |
- system_views | rows_per_read |
- system_views | settings | current settings
- system_views | sstable_tasks | current sstable tasks
- system_views | system_properties | Cassandra relevant system properties
- system_views | thread_pools |
- system_views | tombstones_per_read |
- system_virtual_schema | columns | virtual column definitions
- system_virtual_schema | keyspaces | virtual keyspace definitions
- system_virtual_schema | tables | virtual table definitions
-
-(21 rows)
-----
-
-[arabic, start=3]
-. To view the virtual tables, run the CQL commands ``USE system_view`` and ``DESCRIBE tables``:
-
-[source, cql]
-----
-cqlsh> USE system_view;;
-cqlsh> DESCRIBE tables;
-----
-
-results in:
-
-[source, cql]
-----
-sstable_tasks clients coordinator_write_latency
-disk_usage local_write_latency tombstones_per_read
-thread_pools internode_outbound settings
-local_scan_latency coordinator_scan_latency system_properties
-internode_inbound coordinator_read_latency max_partition_size
-local_read_latency rows_per_read caches
-----
-
-[arabic, start=4]
-. To look at any table data, run the CQL command ``SELECT``:
-
-[source, cql]
-----
-cqlsh> USE system_view;;
-cqlsh> SELECT * FROM clients LIMIT 2;
-----
- results in:
-
-[source, cql]
-----
- address | port | connection_stage | driver_name | driver_version | hostname | protocol_version | request_count | ssl_cipher_suite | ssl_enabled | ssl_protocol | username
------------+-------+------------------+------------------------+----------------+-----------+------------------+---------------+------------------+-------------+--------------+-----------
- 127.0.0.1 | 37308 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 17 | null | False | null | anonymous
- 127.0.0.1 | 37310 | ready | DataStax Python Driver | 3.21.0.post0 | localhost | 4 | 8 | null | False | null | anonymous
-
-(2 rows)
-----
diff --git a/doc/modules/cassandra/pages/overview/faq/index.adoc b/doc/modules/cassandra/pages/overview/faq/index.adoc
new file mode 100644
index 0000000..a745e138
--- /dev/null
+++ b/doc/modules/cassandra/pages/overview/faq/index.adoc
@@ -0,0 +1,275 @@
+= Frequently Asked Questions
+
+[[why-cant-list-all]]
+== Why can't I set `listen_address` to listen on 0.0.0.0 (all my addresses)?
+
+Cassandra is a gossip-based distributed system and `listen_address` is
+the address a node tells other nodes to reach it at. Telling other nodes
+"contact me on any of my addresses" is a bad idea; if different nodes in
+the cluster pick different addresses for you, Bad Things happen.
+
+If you don't want to manually specify an IP to `listen_address` for each
+node in your cluster (understandable!), leave it blank and Cassandra
+will use `InetAddress.getLocalHost()` to pick an address. Then it's up
+to you or your ops team to make things resolve correctly (`/etc/hosts/`,
+dns, etc).
+
+One exception to this process is JMX, which by default binds to 0.0.0.0
+(Java bug 6425769).
+
+See `256` and `43` for more gory details.
+
+[[what-ports]]
+== What ports does Cassandra use?
+
+By default, Cassandra uses 7000 for cluster communication (7001 if SSL
+is enabled), 9042 for native protocol clients, and 7199 for JMX. The
+internode communication and native protocol ports are configurable in
+the `cassandra-yaml`. The JMX port is configurable in `cassandra-env.sh`
+(through JVM options). All ports are TCP.
+
+[[what-happens-on-joins]]
+== What happens to existing data in my cluster when I add new nodes?
+
+When a new nodes joins a cluster, it will automatically contact the
+other nodes in the cluster and copy the right data to itself. See
+`topology-changes`.
+
+[[asynch-deletes]]
+== I delete data from Cassandra, but disk usage stays the same. What gives?
+
+Data you write to Cassandra gets persisted to SSTables. Since SSTables
+are immutable, the data can't actually be removed when you perform a
+delete, instead, a marker (also called a "tombstone") is written to
+indicate the value's new status. Never fear though, on the first
+compaction that occurs between the data and the tombstone, the data will
+be expunged completely and the corresponding disk space recovered. See
+`compaction` for more detail.
+
+[[one-entry-ring]]
+== Why does nodetool ring only show one entry, even though my nodes logged that they see each other joining the ring?
+
+This happens when you have the same token assigned to each node. Don't
+do that.
+
+Most often this bites people who deploy by installing Cassandra on a VM
+(especially when using the Debian package, which auto-starts Cassandra
+after installation, thus generating and saving a token), then cloning
+that VM to other nodes.
+
+The easiest fix is to wipe the data and commitlog directories, thus
+making sure that each node will generate a random token on the next
+restart.
+
+[[change-replication-factor]]
+== Can I change the replication factor (a a keyspace) on a live cluster?
+
+Yes, but it will require running a full repair (or cleanup) to change
+the replica count of existing data:
+
+* `Alter <alter-keyspace-statement>` the replication factor for desired
+keyspace (using cqlsh for instance).
+* If you're reducing the replication factor, run `nodetool cleanup` on
+the cluster to remove surplus replicated data. Cleanup runs on a
+per-node basis.
+* If you're increasing the replication factor, run
+`nodetool repair -full` to ensure data is replicated according to the
+new configuration. Repair runs on a per-replica set basis. This is an
+intensive process that may result in adverse cluster performance. It's
+highly recommended to do rolling repairs, as an attempt to repair the
+entire cluster at once will most likely swamp it. Note that you will
+need to run a full repair (`-full`) to make sure that already repaired
+sstables are not skipped. You should use `ConsistencyLevel.QUORUM` or
+`ALL` (depending on your existing replication factor) to make sure that
+a replica that actually has the data is consulted. Otherwise some
+clients potentially being told no data exists until repair is done.
+
+[[can-large-blob]]
+== Can I Store (large) BLOBs in Cassandra?
+
+Cassandra isn't optimized for large file or BLOB storage and a single
+`blob` value is always read and send to the client entirely. As such,
+storing small blobs (less than single digit MB) should not be a problem,
+but it is advised to manually split large blobs into smaller chunks.
+
+Please note in particular that by default, any value greater than 16MiB
+will be rejected by Cassandra due the `max_mutation_size`
+configuration of the `cassandra-yaml` file (which default to half of
+`commitlog_segment_size`, which itself default to 32MiB).
+
+[[nodetool-connection-refused]]
+== Nodetool says "Connection refused to host: 127.0.1.1" for any remote host. What gives?
+
+Nodetool relies on JMX, which in turn relies on RMI, which in turn sets
+up its own listeners and connectors as needed on each end of the
+exchange. Normally all of this happens behind the scenes transparently,
+but incorrect name resolution for either the host connecting, or the one
+being connected to, can result in crossed wires and confusing
+exceptions.
+
+If you are not using DNS, then make sure that your `/etc/hosts` files
+are accurate on both ends. If that fails, try setting the
+`-Djava.rmi.server.hostname=<public name>` JVM option near the bottom of
+`cassandra-env.sh` to an interface that you can reach from the remote
+machine.
+
+[[to-batch-or-not-to-batch]]
+== Will batching my operations speed up my bulk load?
+
+No. Using batches to load data will generally just add "spikes" of
+latency. Use asynchronous INSERTs instead, or use true `bulk-loading`.
+
+An exception is batching updates to a single partition, which can be a
+Good Thing (as long as the size of a single batch stay reasonable). But
+never ever blindly batch everything!
+
+[[selinux]]
+== On RHEL nodes are unable to join the ring
+
+Check if https://en.wikipedia.org/wiki/Security-Enhanced_Linux[SELinux]
+is on; if it is, turn it off.
+
+[[how-to-unsubscribe]]
+== How do I unsubscribe from the email list?
+
+Send an email to `user-unsubscribe@cassandra.apache.org`.
+
+[[cassandra-eats-all-my-memory]]
+== Why does top report that Cassandra is using a lot more memory than the Java heap max?
+
+Cassandra uses https://en.wikipedia.org/wiki/Memory-mapped_file[Memory
+Mapped Files] (mmap) internally. That is, we use the operating system's
+virtual memory system to map a number of on-disk files into the
+Cassandra process' address space. This will "use" virtual memory; i.e.
+address space, and will be reported by tools like top accordingly, but
+on 64 bit systems virtual address space is effectively unlimited so you
+should not worry about that.
+
+What matters from the perspective of "memory use" in the sense as it is
+normally meant, is the amount of data allocated on brk() or mmap'd
+/dev/zero, which represent real memory used. The key issue is that for a
+mmap'd file, there is never a need to retain the data resident in
+physical memory. Thus, whatever you do keep resident in physical memory
+is essentially just there as a cache, in the same way as normal I/O will
+cause the kernel page cache to retain data that you read/write.
+
+The difference between normal I/O and mmap() is that in the mmap() case
+the memory is actually mapped to the process, thus affecting the virtual
+size as reported by top. The main argument for using mmap() instead of
+standard I/O is the fact that reading entails just touching memory - in
+the case of the memory being resident, you just read it - you don't even
+take a page fault (so no overhead in entering the kernel and doing a
+semi-context switch). This is covered in more detail
+http://www.varnish-cache.org/trac/wiki/ArchitectNotes[here].
+
+== What are seeds?
+
+Seeds are used during startup to discover the cluster.
+
+If you configure your nodes to refer some node as seed, nodes in your
+ring tend to send Gossip message to seeds more often (also see the
+`section on gossip <gossip>`) than to non-seeds. In other words, seeds
+are worked as hubs of Gossip network. With seeds, each node can detect
+status changes of other nodes quickly.
+
+Seeds are also referred by new nodes on bootstrap to learn other nodes
+in ring. When you add a new node to ring, you need to specify at least
+one live seed to contact. Once a node join the ring, it learns about the
+other nodes, so it doesn't need seed on subsequent boot.
+
+You can make a seed a node at any time. There is nothing special about
+seed nodes. If you list the node in seed list it is a seed
+
+Seeds do not auto bootstrap (i.e. if a node has itself in its seed list
+it will not automatically transfer data to itself) If you want a node to
+do that, bootstrap it first and then add it to seeds later. If you have
+no data (new install) you do not have to worry about bootstrap at all.
+
+Recommended usage of seeds:
+
+* pick two (or more) nodes per data center as seed nodes.
+* sync the seed list to all your nodes
+
+[[are-seeds-SPOF]]
+== Does single seed mean single point of failure?
+
+The ring can operate or boot without a seed; however, you will not be
+able to add new nodes to the cluster. It is recommended to configure
+multiple seeds in production system.
+
+[[cant-call-jmx-method]]
+== Why can't I call jmx method X on jconsole?
+
+Some of JMX operations use array argument and as jconsole doesn't
+support array argument, those operations can't be called with jconsole
+(the buttons are inactive for them). You need to write a JMX client to
+call such operations or need array-capable JMX monitoring tool.
+
+[[why-message-dropped]]
+== Why do I see "... messages dropped ..." in the logs?
+
+This is a symptom of load shedding -- Cassandra defending itself against
+more requests than it can handle.
+
+Internode messages which are received by a node, but do not get not to
+be processed within their proper timeout (see `read_request_timeout`,
+`write_request_timeout`, ... in the `cassandra-yaml`), are dropped
+rather than processed (since the as the coordinator node will no longer
+be waiting for a response).
+
+For writes, this means that the mutation was not applied to all replicas
+it was sent to. The inconsistency will be repaired by read repair, hints
+or a manual repair. The write operation may also have timeouted as a
+result.
+
+For reads, this means a read request may not have completed.
+
+Load shedding is part of the Cassandra architecture, if this is a
+persistent issue it is generally a sign of an overloaded node or
+cluster.
+
+[[oom-map-failed]]
+== Cassandra dies with `java.lang.OutOfMemoryError: Map failed`
+
+If Cassandra is dying *specifically* with the "Map failed" message, it
+means the OS is denying java the ability to lock more memory. In linux,
+this typically means memlock is limited. Check
+`/proc/<pid of cassandra>/limits` to verify this and raise it (eg, via
+ulimit in bash). You may also need to increase `vm.max_map_count.` Note
+that the debian package handles this for you automatically.
+
+[[what-on-same-timestamp-update]]
+== What happens if two updates are made with the same timestamp?
+
+Updates must be commutative, since they may arrive in different orders
+on different replicas. As long as Cassandra has a deterministic way to
+pick the winner (in a timestamp tie), the one selected is as valid as
+any other, and the specifics should be treated as an implementation
+detail. That said, in the case of a timestamp tie, Cassandra follows two
+rules: first, deletes take precedence over inserts/updates. Second, if
+there are two updates, the one with the lexically larger value is
+selected.
+
+[[why-bootstrapping-stream-error]]
+== Why bootstrapping a new node fails with a "Stream failed" error?
+
+Two main possibilities:
+
+. the GC may be creating long pauses disrupting the streaming process
+. compactions happening in the background hold streaming long enough
+that the TCP connection fails
+
+In the first case, regular GC tuning advices apply. In the second case,
+you need to set TCP keepalive to a lower value (default is very high on
+Linux). Try to just run the following:
+
+....
+$ sudo /sbin/sysctl -w net.ipv4.tcp_keepalive_time=60 net.ipv4.tcp_keepalive_intvl=60 net.ipv4.tcp_keepalive_probes=5
+....
+
+To make those settings permanent, add them to your `/etc/sysctl.conf`
+file.
+
+Note: https://cloud.google.com/compute/[GCE]'s firewall will always
+interrupt TCP connections that are inactive for more than 10 min.
+Running the above command is highly recommended in that environment.
diff --git a/doc/modules/cassandra/pages/overview/terminology.adoc b/doc/modules/cassandra/pages/overview/terminology.adoc
new file mode 100644
index 0000000..8200056
--- /dev/null
+++ b/doc/modules/cassandra/pages/overview/terminology.adoc
@@ -0,0 +1,23 @@
+= Terminology
+
+a | b | xref:#c[c] | d | e | f
+
+[[c]]
+cluster::
+A ring of nodes that holds a database.
+
+node::
+A machine that holds Cassandra replicas.
+Each node holds a portion of the whole database.
+
+replica::
+A copy of a portion of the whole database. Each node holds some replicas.
+
+replication::
+The process of creating replicas across nodes in a cluster.
+
+replication factor (RF)::
+A scalar value that sets the number of replicas of each partition in a cluster.
+For example, and RF=3 means that three nodes hold a replica of each partition.
+
+
diff --git a/doc/modules/cassandra/pages/references/java11.adoc b/doc/modules/cassandra/pages/references/java11.adoc
new file mode 100644
index 0000000..ded5c2e
--- /dev/null
+++ b/doc/modules/cassandra/pages/references/java11.adoc
@@ -0,0 +1,206 @@
+= Support for Java 11
+
+In the new Java release cadence a new Java version is made available
+every six months. The more frequent release cycle is favored as it
+brings new Java features to the developers as and when they are
+developed without the wait that the earlier 3 year release model
+incurred. Not every Java version is a Long Term Support (LTS) version.
+After Java 8 the next LTS version is Java 11. Java 9, 10, 12 and 13 are
+all non-LTS versions.
+
+One of the objectives of the Apache Cassandra 4.0 version is to support
+the recent LTS Java versions 8 and 11
+(https://issues.apache.org/jira/browse/CASSANDRA-9608[CASSANDRA-9608]).
+Java 8 and Java 11 may be used to build and run Apache Cassandra 4.0. Effective Cassandra
+4.0.2 there is full Java 11 support, it is not experimental anymore.
+
+== Support Matrix
+
+The support matrix for the Java versions for compiling and running
+Apache Cassandra 4.0 is detailed in Table 1. The build version is along
+the vertical axis and the run version is along the horizontal axis.
+
+Table 1 : Support Matrix for Java
+
+[width="68%",cols="34%,30%,36%",]
+|===
+| |Java 8 (Run) |Java 11 (Run)
+|Java 8 (Build) |Supported |Supported
+|Java 11(Build) |Not Supported |Experimental
+|===
+
+Apache 4.0 source code built with Java 11 cannot be run with
+Java 8.
+
+All binary releases are built with Java 8.
+
+Next, we shall discuss using each of Java 8 and 11 to build and
+run Apache Cassandra 4.0.
+
+== Using Java 8 to Build
+
+To start with, install Java 8. As an example, for installing Java 8 on
+RedHat Linux the command is as follows:
+
+....
+$ sudo yum install java-1.8.0-openjdk-devel
+....
+
+Set the environment variables `JAVA_HOME` and `PATH`.
+
+....
+$ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+$ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
+....
+
+Download and install Apache Cassandra 4.0 source code from the Git along
+with the dependencies.
+
+....
+$ git clone https://github.com/apache/cassandra.git
+....
+
+If Cassandra is already running stop Cassandra with the following
+command.
+
+....
+$ ./nodetool stopdaemon
+....
+
+Build the source code from the `cassandra` directory, which has the
+`build.xml` build script. The Apache Ant uses the Java version set in
+the `JAVA_HOME` environment variable.
+
+....
+$ cd ~/cassandra
+$ ant
+....
+
+Apache Cassandra 4.0 gets built with Java 8. Set the environment
+variable for `CASSANDRA_HOME` in the bash script. Also add the
+`CASSANDRA_HOME/bin` to the `PATH` variable.
+
+....
+$ export CASSANDRA_HOME=~/cassandra
+$ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$CASSANDRA_HOME/bin
+....
+
+To run Apache Cassandra 4.0 with either of Java 8 or Java 11 run the
+Cassandra application in the `CASSANDRA_HOME/bin` directory, which is in
+the `PATH` env variable.
+
+....
+$ cassandra
+....
+
+The Java version used to run Cassandra gets output as Cassandra is
+getting started. As an example if Java 11 is used, the run output should
+include similar to the following output snippet:
+
+....
+INFO [main] 2019-07-31 21:18:16,862 CassandraDaemon.java:480 - Hostname: ip-172-30-3-
+146.ec2.internal:7000:7001
+INFO [main] 2019-07-31 21:18:16,862 CassandraDaemon.java:487 - JVM vendor/version: OpenJDK
+64-Bit Server VM/11.0.3
+INFO [main] 2019-07-31 21:18:16,863 CassandraDaemon.java:488 - Heap size:
+1004.000MiB/1004.000MiB
+....
+
+The following output indicates a single node Cassandra 4.0 cluster has
+started.
+
+....
+INFO [main] 2019-07-31 21:18:19,687 InboundConnectionInitiator.java:130 - Listening on
+address: (127.0.0.1:7000), nic: lo, encryption: enabled (openssl)
+...
+...
+INFO [main] 2019-07-31 21:18:19,850 StorageService.java:512 - Unable to gossip with any
+peers but continuing anyway since node is in its own seed list
+INFO [main] 2019-07-31 21:18:19,864 StorageService.java:695 - Loading persisted ring state
+INFO [main] 2019-07-31 21:18:19,865 StorageService.java:814 - Starting up server gossip
+INFO [main] 2019-07-31 21:18:20,088 BufferPool.java:216 - Global buffer pool is enabled,
+when pool is exhausted (max is 251.000MiB) it will allocate on heap
+INFO [main] 2019-07-31 21:18:20,110 StorageService.java:875 - This node will not auto
+bootstrap because it is configured to be a seed node.
+...
+...
+INFO [main] 2019-07-31 21:18:20,809 StorageService.java:1507 - JOINING: Finish joining ring
+INFO [main] 2019-07-31 21:18:20,921 StorageService.java:2508 - Node 127.0.0.1:7000 state
+jump to NORMAL
+....
+
+== Using Java 11 to Build
+
+If Java 11 is used to build Apache Cassandra 4.0, first Java 11 must be
+installed and the environment variables set. As an example, to download
+and install Java 11 on RedHat Linux run the following command.
+
+....
+$ yum install java-11-openjdk-devel
+....
+
+Set the environment variables `JAVA_HOME` and `PATH`.
+
+....
+$ export JAVA_HOME=/usr/lib/jvm/java-11-openjdk
+$ export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
+....
+
+The build output should now include the following.
+
+....
+[echo] Non default JDK version used: 11
+...
+...
+_build_java:
+ [echo] Compiling for Java 11
+...
+...
+build:
+
+_main-jar:
+ [copy] Copying 1 file to /home/ec2-user/cassandra/build/classes/main/META-INF
+ [jar] Building jar: /home/ec2-user/cassandra/build/apache-cassandra-4.0-SNAPSHOT.jar
+...
+...
+_build-test:
+ [javac] Compiling 739 source files to /home/ec2-user/cassandra/build/test/classes
+ [copy] Copying 25 files to /home/ec2-user/cassandra/build/test/classes
+...
+...
+jar:
+ [mkdir] Created dir: /home/ec2-user/cassandra/build/classes/stress/META-INF
+ [mkdir] Created dir: /home/ec2-user/cassandra/build/tools/lib
+ [jar] Building jar: /home/ec2-user/cassandra/build/tools/lib/stress.jar
+ [mkdir] Created dir: /home/ec2-user/cassandra/build/classes/fqltool/META-INF
+ [jar] Building jar: /home/ec2-user/cassandra/build/tools/lib/fqltool.jar
+
+BUILD SUCCESSFUL
+Total time: 1 minute 3 seconds
+....
+
+== Common Issues
+
+The Java 11 built Apache Cassandra 4.0 source code may be run with Java
+11 only. If a Java 11 built code is run with Java 8 the following error
+message gets output.
+
+....
+# ssh -i cassandra.pem ec2-user@ec2-3-85-85-75.compute-1.amazonaws.com
+Last login: Wed Jul 31 20:47:26 2019 from 75.155.255.51
+$ echo $JAVA_HOME
+/usr/lib/jvm/java-1.8.0-openjdk
+$ cassandra
+...
+...
+Error: A JNI error has occurred, please check your installation and try again
+Exception in thread "main" java.lang.UnsupportedClassVersionError:
+org/apache/cassandra/service/CassandraDaemon has been compiled by a more recent version of
+the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes
+class file versions up to 52.0
+ at java.lang.ClassLoader.defineClass1(Native Method)
+ at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
+ at ...
+...
+....
+
diff --git a/test/data/serialization/2.0/db.WriteResponse.bin b/doc/modules/cassandra/pages/tooling/cass-stress-deprecated
similarity index 100%
copy from test/data/serialization/2.0/db.WriteResponse.bin
copy to doc/modules/cassandra/pages/tooling/cass-stress-deprecated
diff --git a/test/data/serialization/2.0/db.WriteResponse.bin b/doc/modules/cassandra/pages/tooling/generate-tokens
similarity index 100%
copy from test/data/serialization/2.0/db.WriteResponse.bin
copy to doc/modules/cassandra/pages/tooling/generate-tokens
diff --git a/test/data/serialization/2.0/db.WriteResponse.bin b/doc/modules/cassandra/pages/tooling/hash-password-tool
similarity index 100%
copy from test/data/serialization/2.0/db.WriteResponse.bin
copy to doc/modules/cassandra/pages/tooling/hash-password-tool
diff --git a/doc/modules/cassandra/pages/tools/cassandra_stress.adoc b/doc/modules/cassandra/pages/tools/cassandra_stress.adoc
deleted file mode 100644
index 7cf3548..0000000
--- a/doc/modules/cassandra/pages/tools/cassandra_stress.adoc
+++ /dev/null
@@ -1,324 +0,0 @@
-= Cassandra Stress
-
-The `cassandra-stress` tool is used to benchmark and load-test a Cassandra
-cluster.
-`cassandra-stress` supports testing arbitrary CQL tables and queries, allowing users to benchmark their own data model.
-
-This documentation focuses on user mode to test personal schema.
-
-== Usage
-
-There are several operation types:
-
-* write-only, read-only, and mixed workloads of standard data
-* write-only and read-only workloads for counter columns
-* user configured workloads, running custom queries on custom schemas
-
-The syntax is `cassandra-stress <command> [options]`.
-For more information on a given command or options, run `cassandra-stress help <command|option>`.
-
-Commands:::
- read:;;
- Multiple concurrent reads - the cluster must first be populated by a
- write test
- write:;;
- Multiple concurrent writes against the cluster
- mixed:;;
- Interleaving of any basic commands, with configurable ratio and
- distribution - the cluster must first be populated by a write test
- counter_write:;;
- Multiple concurrent updates of counters.
- counter_read:;;
- Multiple concurrent reads of counters. The cluster must first be
- populated by a counterwrite test.
- user:;;
- Interleaving of user provided queries, with configurable ratio and
- distribution.
- help:;;
- Print help for a command or option
- print:;;
- Inspect the output of a distribution definition
- legacy:;;
- Legacy support mode
-Primary Options:::
- -pop:;;
- Population distribution and intra-partition visit order
- -insert:;;
- Insert specific options relating to various methods for batching and
- splitting partition updates
- -col:;;
- Column details such as size and count distribution, data generator,
- names, comparator and if super columns should be used
- -rate:;;
- Thread count, rate limit or automatic mode (default is auto)
- -mode:;;
- Thrift or CQL with options
- -errors:;;
- How to handle errors when encountered during stress
- -sample:;;
- Specify the number of samples to collect for measuring latency
- -schema:;;
- Replication settings, compression, compaction, etc.
- -node:;;
- Nodes to connect to
- -log:;;
- Where to log progress to, and the interval at which to do it
- -transport:;;
- Custom transport factories
- -port:;;
- The port to connect to cassandra nodes on
- -graph:;;
- Graph recorded metrics
- -tokenrange:;;
- Token range settings
-Suboptions:::
- Every command and primary option has its own collection of suboptions.
- These are too numerous to list here. For information on the suboptions
- for each command or option, please use the help command,
- `cassandra-stress help <command|option>`.
-
-== User mode
-
-User mode allows you to stress your own schemas, to save you time
-in the long run. Find out if your application can scale using stress test with your schema.
-
-=== Profile
-
-User mode defines a profile using YAML.
-Multiple YAML files may be specified, in which case operations in the ops argument are referenced as
-specname.opname.
-
-An identifier for the profile:
-
-[source,yaml]
-----
-specname: staff_activities
-----
-
-The keyspace for the test:
-
-[source,yaml]
-----
-keyspace: staff
-----
-
-CQL for the keyspace. Optional if the keyspace already exists:
-
-[source,yaml]
-----
-keyspace_definition: |
- CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};
-----
-
-The table to be stressed:
-
-[source,yaml]
-----
-table: staff_activities
-----
-
-CQL for the table. Optional if the table already exists:
-
-[source,yaml]
-----
-table_definition: |
- CREATE TABLE staff_activities (
- name text,
- when timeuuid,
- what text,
- PRIMARY KEY(name, when, what)
- )
-----
-
-Optional meta-information on the generated columns in the above table.
-The min and max only apply to text and blob types. The distribution
-field represents the total unique population distribution of that column
-across rows:
-
-[source,yaml]
-----
-columnspec:
- - name: name
- size: uniform(5..10) # The names of the staff members are between 5-10 characters
- population: uniform(1..10) # 10 possible staff members to pick from
- - name: when
- cluster: uniform(20..500) # Staff members do between 20 and 500 events
- - name: what
- size: normal(10..100,50)
-----
-
-Supported types are:
-
-An exponential distribution over the range [min..max]:
-
-[source,yaml]
-----
-EXP(min..max)
-----
-
-An extreme value (Weibull) distribution over the range [min..max]:
-
-[source,yaml]
-----
-EXTREME(min..max,shape)
-----
-
-A gaussian/normal distribution, where mean=(min+max)/2, and stdev is
-(mean-min)/stdvrng:
-
-[source,yaml]
-----
-GAUSSIAN(min..max,stdvrng)
-----
-
-A gaussian/normal distribution, with explicitly defined mean and stdev:
-
-[source,yaml]
-----
-GAUSSIAN(min..max,mean,stdev)
-----
-
-A uniform distribution over the range [min, max]:
-
-[source,yaml]
-----
-UNIFORM(min..max)
-----
-
-A fixed distribution, always returning the same value:
-
-[source,yaml]
-----
-FIXED(val)
-----
-
-If preceded by ~, the distribution is inverted
-
-Defaults for all columns are size: uniform(4..8), population:
-uniform(1..100B), cluster: fixed(1)
-
-Insert distributions:
-
-[source,yaml]
-----
-insert:
- # How many partition to insert per batch
- partitions: fixed(1)
- # How many rows to update per partition
- select: fixed(1)/500
- # UNLOGGED or LOGGED batch for insert
- batchtype: UNLOGGED
-----
-
-Currently all inserts are done inside batches.
-
-Read statements to use during the test:
-
-[source,yaml]
-----
-queries:
- events:
- cql: select * from staff_activities where name = ?
- fields: samerow
- latest_event:
- cql: select * from staff_activities where name = ? LIMIT 1
- fields: samerow
-----
-
-Running a user mode test:
-
-[source,yaml]
-----
-cassandra-stress user profile=./example.yaml duration=1m "ops(insert=1,latest_event=1,events=1)" truncate=once
-----
-
-This will create the schema then run tests for 1 minute with an equal
-number of inserts, latest_event queries and events queries. Additionally
-the table will be truncated once before the test.
-
-The full example can be found here:
-[source, yaml]
-----
-include::example$YAML/stress-example.yaml[]
-----
-
-Running a user mode test with multiple yaml files::::
- cassandra-stress user profile=./example.yaml,./example2.yaml
- duration=1m "ops(ex1.insert=1,ex1.latest_event=1,ex2.insert=2)"
- truncate=once
-This will run operations as specified in both the example.yaml and
-example2.yaml files. example.yaml and example2.yaml can reference the
-same table, although care must be taken that the table definition is identical
- (data generation specs can be different).
-
-=== Lightweight transaction support
-
-cassandra-stress supports lightweight transactions.
-To use this feature, the command will first read current data from Cassandra, and then uses read values to
-fulfill lightweight transaction conditions.
-
-Lightweight transaction update query:
-
-[source,yaml]
-----
-queries:
- regularupdate:
- cql: update blogposts set author = ? where domain = ? and published_date = ?
- fields: samerow
- updatewithlwt:
- cql: update blogposts set author = ? where domain = ? and published_date = ? IF body = ? AND url = ?
- fields: samerow
-----
-
-The full example can be found here:
-[source, yaml]
-----
-include::example$YAML/stress-lwt-example.yaml[]
-----
-
-== Graphing
-
-Graphs can be generated for each run of stress.
-
-image::example-stress-graph.png[example cassandra-stress graph]
-
-To create a new graph:
-
-[source,yaml]
-----
-cassandra-stress user profile=./stress-example.yaml "ops(insert=1,latest_event=1,events=1)" -graph file=graph.html title="Awesome graph"
-----
-
-To add a new run to an existing graph point to an existing file and add
-a revision name:
-
-[source,yaml]
-----
-cassandra-stress user profile=./stress-example.yaml duration=1m "ops(insert=1,latest_event=1,events=1)" -graph file=graph.html title="Awesome graph" revision="Second run"
-----
-
-== FAQ
-
-*How do you use NetworkTopologyStrategy for the keyspace?*
-
-Use the schema option making sure to either escape the parenthesis or
-enclose in quotes:
-
-[source,yaml]
-----
-cassandra-stress write -schema "replication(strategy=NetworkTopologyStrategy,datacenter1=3)"
-----
-
-*How do you use SSL?*
-
-Use the transport option:
-
-[source,yaml]
-----
-cassandra-stress "write n=100k cl=ONE no-warmup" -transport "truststore=$HOME/jks/truststore.jks truststore-password=cassandra"
-----
-
-*Is Cassandra Stress a secured tool?*
-
-Cassandra stress is not a secured tool. Serialization and other aspects
-of the tool offer no security guarantees.
diff --git a/doc/modules/cassandra/pages/tools/cqlsh.adoc b/doc/modules/cassandra/pages/tools/cqlsh.adoc
deleted file mode 100644
index 8050ee5..0000000
--- a/doc/modules/cassandra/pages/tools/cqlsh.adoc
+++ /dev/null
@@ -1,502 +0,0 @@
-= cqlsh: the CQL shell
-
-`cqlsh` is a command-line interface for interacting with Cassandra using CQL (the Cassandra Query Language).
-It is shipped with every Cassandra package, and can be found in the bin/ directory alongside the cassandra
-executable.
-`cqlsh` is implemented with the Python native protocol driver, and connects to the single specified node.
-
-== Compatibility
-
-`cqlsh` is compatible with Python 2.7.
-
-In general, a given version of `cqlsh` is only guaranteed to work with the
-version of Cassandra that it was released with.
-In some cases, `cqlsh` may work with older or newer versions of Cassandra, but this is not
-officially supported.
-
-== Optional Dependencies
-
-`cqlsh` ships with all essential dependencies. However, there are some
-optional dependencies that can be installed to improve the capabilities
-of `cqlsh`.
-
-=== pytz
-
-By default, `cqlsh` displays all timestamps with a UTC timezone.
-To support display of timestamps with another timezone, install
-the http://pytz.sourceforge.net/[pytz] library.
-See the `timezone` option in xref:cql/tools/cqlsh.adoc#cqlshrc[cqlshrc] for specifying a timezone to
-use.
-
-=== cython
-
-The performance of cqlsh's `COPY` operations can be improved by
-installing http://cython.org/[cython]. This will compile the python
-modules that are central to the performance of `COPY`.
-
-[[cqlshrc]]
-== cqlshrc
-
-The `cqlshrc` file holds configuration options for `cqlsh`.
-By default, the file is locagted the user's home directory at `~/.cassandra/cqlsh`, but a
-custom location can be specified with the `--cqlshrc` option.
-
-Example config values and documentation can be found in the
-`conf/cqlshrc.sample` file of a tarball installation.
-You can also view the latest version of the
-https://github.com/apache/cassandra/blob/trunk/conf/cqlshrc.sample[cqlshrc file online].
-
-[[cql_history]]
-== cql history
-
-All CQL commands you execute are written to a history file. By default, CQL history will be written to `~/.cassandra/cql_history`. You can change this default by setting the environment variable `CQL_HISTORY` like `~/some/other/path/to/cqlsh_history` where `cqlsh_history` is a file. All parent directories to history file will be created if they do not exist. If you do not want to persist history, you can do so by setting CQL_HISTORY to /dev/null.
-This feature is supported from Cassandra 4.1.
-
-== Command Line Options
-
-Usage: `cqlsh.py [options] [host [port]]`
-
-CQL Shell for Apache Cassandra
-
-Options:
-
-`--version`::
- show program's version number and exit
-
-`-h` `--help`::
- show this help message and exit
-`-C` `--color`::
- Always use color output
-`--no-color`::
- Never use color output
-`--browser=BROWSER`::
- The browser to use to display CQL help, where BROWSER can be:
- one of the supported browsers in https://docs.python.org/3/library/webbrowser.html.
- browser path followed by %s, example: /usr/bin/google-chrome-stable %s
-`--ssl`::
- Use SSL
-
-`-u USERNAME` `--username=USERNAME`::
- Authenticate as user.
-`-p PASSWORD` `--password=PASSWORD`::
- Authenticate using password.
-`-k KEYSPACE` `--keyspace=KEYSPACE`::
- Authenticate to the given keyspace.
-`-f FILE` `--file=FILE`::
- Execute commands from FILE, then exit
-`--debug`::
- Show additional debugging information
-`--coverage`::
- Collect coverage data
-`--encoding=ENCODING`::
- Specify a non-default encoding for output. (Default: utf-8)
-`--cqlshrc=CQLSHRC`::
- Specify an alternative cqlshrc file location.
-`--credentials=CREDENTIALS`::
- Specify an alternative credentials file location.
-`--cqlversion=CQLVERSION`::
- Specify a particular CQL version, by default the
- highest version supported by the server will be used.
- Examples: "3.0.3", "3.1.0"
-`--protocol-version=PROTOCOL_VERSION`::
- Specify a specific protcol version otherwise the
- client will default and downgrade as necessary
-`-e EXECUTE` `--execute=EXECUTE`::
- Execute the statement and quit.
-`--connect-timeout=CONNECT_TIMEOUT`::
- Specify the connection timeout in seconds (default: 5 seconds).
-`--request-timeout=REQUEST_TIMEOUT`::
- Specify the default request timeout in seconds
- (default: 10 seconds).
-`-t, --tty`::
- Force tty mode (command prompt).
-`-v` `--v`::
- Print the current version of cqlsh.
-
-== Special Commands
-
-In addition to supporting regular CQL statements, `cqlsh` also supports a
-number of special commands that are not part of CQL. These are detailed
-below.
-
-=== `CONSISTENCY`
-
-`Usage`: `CONSISTENCY <consistency level>`
-
-Sets the consistency level for operations to follow. Valid arguments
-include:
-
-* `ANY`
-* `ONE`
-* `TWO`
-* `THREE`
-* `QUORUM`
-* `ALL`
-* `LOCAL_QUORUM`
-* `LOCAL_ONE`
-* `SERIAL`
-* `LOCAL_SERIAL`
-
-=== `SERIAL CONSISTENCY`
-
-`Usage`: `SERIAL CONSISTENCY <consistency level>`
-
-Sets the serial consistency level for operations to follow. Valid
-arguments include:
-
-* `SERIAL`
-* `LOCAL_SERIAL`
-
-The serial consistency level is only used by conditional updates
-(`INSERT`, `UPDATE` and `DELETE` with an `IF` condition). For those, the
-serial consistency level defines the consistency level of the serial
-phase (or “paxos” phase) while the normal consistency level defines the
-consistency for the “learn” phase, i.e. what type of reads will be
-guaranteed to see the update right away. For example, if a conditional
-write has a consistency level of `QUORUM` (and is successful), then a
-`QUORUM` read is guaranteed to see that write. But if the regular
-consistency level of that write is `ANY`, then only a read with a
-consistency level of `SERIAL` is guaranteed to see it (even a read with
-consistency `ALL` is not guaranteed to be enough).
-
-=== `SHOW VERSION`
-
-Prints the `cqlsh`, Cassandra, CQL, and native protocol versions in use.
-Example:
-
-[source,none]
-----
-cqlsh> SHOW VERSION
-[cqlsh 5.0.1 | Cassandra 3.8 | CQL spec 3.4.2 | Native protocol v4]
-----
-
-=== `SHOW HOST`
-
-Prints the IP address and port of the Cassandra node that `cqlsh` is
-connected to in addition to the cluster name. Example:
-
-[source,none]
-----
-cqlsh> SHOW HOST
-Connected to Prod_Cluster at 192.0.0.1:9042.
-----
-
-=== `SHOW SESSION`
-
-Pretty prints a specific tracing session.
-
-`Usage`: `SHOW SESSION <session id>`
-
-Example usage:
-
-[source,none]
-----
-cqlsh> SHOW SESSION 95ac6470-327e-11e6-beca-dfb660d92ad8
-
-Tracing session: 95ac6470-327e-11e6-beca-dfb660d92ad8
-
- activity | timestamp | source | source_elapsed | client
------------------------------------------------------------+----------------------------+-----------+----------------+-----------
- Execute CQL3 query | 2016-06-14 17:23:13.979000 | 127.0.0.1 | 0 | 127.0.0.1
- Parsing SELECT * FROM system.local; [SharedPool-Worker-1] | 2016-06-14 17:23:13.982000 | 127.0.0.1 | 3843 | 127.0.0.1
-...
-----
-
-=== `SOURCE`
-
-Reads the contents of a file and executes each line as a CQL statement
-or special cqlsh command.
-
-`Usage`: `SOURCE <string filename>`
-
-Example usage:
-
-[source,none]
-----
-cqlsh> SOURCE '/home/calvinhobbs/commands.cql'
-----
-
-=== `CAPTURE`
-
-Begins capturing command output and appending it to a specified file.
-Output will not be shown at the console while it is captured.
-
-`Usage`:
-
-[source,none]
-----
-CAPTURE '<file>';
-CAPTURE OFF;
-CAPTURE;
-----
-
-That is, the path to the file to be appended to must be given inside a
-string literal. The path is interpreted relative to the current working
-directory. The tilde shorthand notation (`'~/mydir'`) is supported for
-referring to `$HOME`.
-
-Only query result output is captured. Errors and output from cqlsh-only
-commands will still be shown in the cqlsh session.
-
-To stop capturing output and show it in the cqlsh session again, use
-`CAPTURE OFF`.
-
-To inspect the current capture configuration, use `CAPTURE` with no
-arguments.
-
-=== `HELP`
-
-Gives information about cqlsh commands. To see available topics, enter
-`HELP` without any arguments. To see help on a topic, use
-`HELP <topic>`. Also see the `--browser` argument for controlling what
-browser is used to display help.
-
-=== `TRACING`
-
-Enables or disables tracing for queries. When tracing is enabled, once a
-query completes, a trace of the events during the query will be printed.
-
-`Usage`:
-
-[source,none]
-----
-TRACING ON
-TRACING OFF
-----
-
-=== `PAGING`
-
-Enables paging, disables paging, or sets the page size for read queries.
-When paging is enabled, only one page of data will be fetched at a time
-and a prompt will appear to fetch the next page. Generally, it's a good
-idea to leave paging enabled in an interactive session to avoid fetching
-and printing large amounts of data at once.
-
-`Usage`:
-
-[source,none]
-----
-PAGING ON
-PAGING OFF
-PAGING <page size in rows>
-----
-
-=== `EXPAND`
-
-Enables or disables vertical printing of rows. Enabling `EXPAND` is
-useful when many columns are fetched, or the contents of a single column
-are large.
-
-`Usage`:
-
-[source,none]
-----
-EXPAND ON
-EXPAND OFF
-----
-
-=== `LOGIN`
-
-Authenticate as a specified Cassandra user for the current session.
-
-`Usage`:
-
-[source,none]
-----
-LOGIN <username> [<password>]
-----
-
-=== `EXIT`
-
-Ends the current session and terminates the cqlsh process.
-
-`Usage`:
-
-[source,none]
-----
-EXIT
-QUIT
-----
-
-=== `CLEAR`
-
-Clears the console.
-
-`Usage`:
-
-[source,none]
-----
-CLEAR
-CLS
-----
-
-=== `DESCRIBE`
-
-Prints a description (typically a series of DDL statements) of a schema
-element or the cluster. This is useful for dumping all or portions of
-the schema.
-
-`Usage`:
-
-[source,none]
-----
-DESCRIBE CLUSTER
-DESCRIBE SCHEMA
-DESCRIBE KEYSPACES
-DESCRIBE KEYSPACE <keyspace name>
-DESCRIBE TABLES
-DESCRIBE TABLE <table name>
-DESCRIBE INDEX <index name>
-DESCRIBE MATERIALIZED VIEW <view name>
-DESCRIBE TYPES
-DESCRIBE TYPE <type name>
-DESCRIBE FUNCTIONS
-DESCRIBE FUNCTION <function name>
-DESCRIBE AGGREGATES
-DESCRIBE AGGREGATE <aggregate function name>
-----
-
-In any of the commands, `DESC` may be used in place of `DESCRIBE`.
-
-The `DESCRIBE CLUSTER` command prints the cluster name and partitioner:
-
-[source,none]
-----
-cqlsh> DESCRIBE CLUSTER
-
-Cluster: Test Cluster
-Partitioner: Murmur3Partitioner
-----
-
-The `DESCRIBE SCHEMA` command prints the DDL statements needed to
-recreate the entire schema. This is especially useful for dumping the
-schema in order to clone a cluster or restore from a backup.
-
-=== `COPY TO`
-
-Copies data from a table to a CSV file.
-
-`Usage`:
-
-[source,none]
-----
-COPY <table name> [(<column>, ...)] TO <file name> WITH <copy option> [AND <copy option> ...]
-----
-
-If no columns are specified, all columns from the table will be copied
-to the CSV file. A subset of columns to copy may be specified by adding
-a comma-separated list of column names surrounded by parenthesis after
-the table name.
-
-The `<file name>` should be a string literal (with single quotes)
-representing a path to the destination file. This can also the special
-value `STDOUT` (without single quotes) to print the CSV to stdout.
-
-See `shared-copy-options` for options that apply to both `COPY TO` and
-`COPY FROM`.
-
-==== Options for `COPY TO`
-
-`MAXREQUESTS`::
- The maximum number token ranges to fetch simultaneously. Defaults to
- 6.
-`PAGESIZE`::
- The number of rows to fetch in a single page. Defaults to 1000.
-`PAGETIMEOUT`::
- By default the page timeout is 10 seconds per 1000 entries in the page
- size or 10 seconds if pagesize is smaller.
-`BEGINTOKEN`, `ENDTOKEN`::
- Token range to export. Defaults to exporting the full ring.
-`MAXOUTPUTSIZE`::
- The maximum size of the output file measured in number of lines;
- beyond this maximum the output file will be split into segments. -1
- means unlimited, and is the default.
-`ENCODING`::
- The encoding used for characters. Defaults to `utf8`.
-
-=== `COPY FROM`
-
-Copies data from a CSV file to table.
-
-`Usage`:
-
-[source,none]
-----
-COPY <table name> [(<column>, ...)] FROM <file name> WITH <copy option> [AND <copy option> ...]
-----
-
-If no columns are specified, all columns from the CSV file will be
-copied to the table. A subset of columns to copy may be specified by
-adding a comma-separated list of column names surrounded by parenthesis
-after the table name.
-
-The `<file name>` should be a string literal (with single quotes)
-representing a path to the source file. This can also the special value
-`STDIN` (without single quotes) to read the CSV data from stdin.
-
-See `shared-copy-options` for options that apply to both `COPY TO` and
-`COPY FROM`.
-
-==== Options for `COPY TO`
-
-`INGESTRATE`::
- The maximum number of rows to process per second. Defaults to 100000.
-`MAXROWS`::
- The maximum number of rows to import. -1 means unlimited, and is the
- default.
-`SKIPROWS`::
- A number of initial rows to skip. Defaults to 0.
-`SKIPCOLS`::
- A comma-separated list of column names to ignore. By default, no
- columns are skipped.
-`MAXPARSEERRORS`::
- The maximum global number of parsing errors to ignore. -1 means
- unlimited, and is the default.
-`MAXINSERTERRORS`::
- The maximum global number of insert errors to ignore. -1 means
- unlimited. The default is 1000.
-`ERRFILE` =::
- A file to store all rows that could not be imported, by default this
- is `import_<ks>_<table>.err` where `<ks>` is your keyspace and
- `<table>` is your table name.
-`MAXBATCHSIZE`::
- The max number of rows inserted in a single batch. Defaults to 20.
-`MINBATCHSIZE`::
- The min number of rows inserted in a single batch. Defaults to 2.
-`CHUNKSIZE`::
- The number of rows that are passed to child worker processes from the
- main process at a time. Defaults to 1000.
-
-==== Shared COPY Options
-
-Options that are common to both `COPY TO` and `COPY FROM`.
-
-`NULLVAL`::
- The string placeholder for null values. Defaults to `null`.
-`HEADER`::
- For `COPY TO`, controls whether the first line in the CSV output file
- will contain the column names. For COPY FROM, specifies whether the
- first line in the CSV input file contains column names. Defaults to
- `false`.
-`DECIMALSEP`::
- The character that is used as the decimal point separator. Defaults to
- `.`.
-`THOUSANDSSEP`::
- The character that is used to separate thousands. Defaults to the
- empty string.
-`BOOLSTYlE`::
- The string literal format for boolean values. Defaults to
- `True,False`.
-`NUMPROCESSES`::
- The number of child worker processes to create for `COPY` tasks.
- Defaults to a max of 4 for `COPY FROM` and 16 for `COPY TO`. However,
- at most (num_cores - 1) processes will be created.
-`MAXATTEMPTS`::
- The maximum number of failed attempts to fetch a range of data (when
- using `COPY TO`) or insert a chunk of data (when using `COPY FROM`)
- before giving up. Defaults to 5.
-`REPORTFREQUENCY`::
- How often status updates are refreshed, in seconds. Defaults to 0.25.
-`RATEFILE`::
- An optional file to output rate statistics to. By default, statistics
- are not output to a file.
diff --git a/doc/modules/cassandra/pages/tools/sstable/index.adoc b/doc/modules/cassandra/pages/tools/sstable/index.adoc
deleted file mode 100644
index cb787ec..0000000
--- a/doc/modules/cassandra/pages/tools/sstable/index.adoc
+++ /dev/null
@@ -1,20 +0,0 @@
-= SSTable Tools
-
-This section describes the functionality of the various sstable tools.
-
-Cassandra must be stopped before these tools are executed, or unexpected
-results will occur. Note: the scripts do not verify that Cassandra is
-stopped.
-
-* xref:tools/sstable/sstabledump.adoc[sstabledump]
-* xref:tools/sstable/sstableexpiredblockers.adoc[sstableexpiredblockers]
-* xref:tools/sstable/sstablelevelreset.adoc[sstablelevelreset]
-* xref:tools/sstable/sstableloader.adoc[sstableloader]
-* xref:tools/sstable/sstablemetadata.adoc[sstablemetadata]
-* xref:tools/sstable/sstableofflinerelevel.adoc[sstableofflinerelevel]
-* xref:tools/sstable/sstablerepairedset.adoc[sstablerepairdset]
-* xref:tools/sstable/sstablescrub.adoc[sstablescrub]
-* xref:tools/sstable/sstablesplit.adoc[sstablesplit]
-* xref:tools/sstable/sstableupgrade.adoc[sstableupgrade]
-* xref:tools/sstable/sstableutil.adoc[sstableutil]
-* xref:tools/sstable/sstableverify.adoc[sstableverify]
diff --git a/doc/modules/cassandra/pages/tools/sstable/sstablelevelreset.adoc b/doc/modules/cassandra/pages/tools/sstable/sstablelevelreset.adoc
deleted file mode 100644
index 65dc02e..0000000
--- a/doc/modules/cassandra/pages/tools/sstable/sstablelevelreset.adoc
+++ /dev/null
@@ -1,69 +0,0 @@
-= sstablelevelreset
-
-If LeveledCompactionStrategy is set, this script can be used to reset
-level to 0 on a given set of sstables. This is useful if you want to,
-for example, change the minimum sstable size, and therefore restart the
-compaction process using this new configuration.
-
-See
-http://cassandra.apache.org/doc/latest/operating/compaction.html#leveled-compaction-strategy
-for information on how levels are used in this compaction strategy.
-
-Cassandra must be stopped before this tool is executed, or unexpected
-results will occur. Note: the script does not verify that Cassandra is
-stopped.
-
-ref: https://issues.apache.org/jira/browse/CASSANDRA-5271
-
-== Usage
-
-sstablelevelreset --really-reset <keyspace> <table>
-
-The really-reset flag is required, to ensure this intrusive command is
-not run accidentally.
-
-== Table not found
-
-If the keyspace and/or table is not in the schema (e.g., if you
-misspelled the table name), the script will return an error.
-
-Example:
-
-....
-ColumnFamily not found: keyspace/evenlog.
-....
-
-== Table has no sstables
-
-Example:
-
-....
-Found no sstables, did you give the correct keyspace/table?
-....
-
-== Table already at level 0
-
-The script will not set the level if it is already set to 0.
-
-Example:
-
-....
-Skipped /var/lib/cassandra/data/keyspace/eventlog-65c429e08c5a11e8939edf4f403979ef/mc-1-big-Data.db since it is already on level 0
-....
-
-== Table levels reduced to 0
-
-If the level is not already 0, then this will reset it to 0.
-
-Example:
-
-....
-sstablemetadata /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db | grep -i level
-SSTable Level: 1
-
-sstablelevelreset --really-reset keyspace eventlog
-Changing level from 1 to 0 on /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db
-
-sstablemetadata /var/lib/cassandra/data/keyspace/eventlog-6365332094dd11e88f324f9c503e4753/mc-8-big-Data.db | grep -i level
-SSTable Level: 0
-....
diff --git a/doc/modules/cassandra/partials/masking_functions.adoc b/doc/modules/cassandra/partials/masking_functions.adoc
new file mode 100644
index 0000000..afadcd0
--- /dev/null
+++ b/doc/modules/cassandra/partials/masking_functions.adoc
@@ -0,0 +1,61 @@
+[cols=",",options="header",]
+|===
+|Function | Description
+
+| `mask_null(value)` | Replaces the first argument by a `null` column. The returned value is always an absent column, as it didn't exist, and not a not-null column representing a `null` value.
+
+Examples:
+
+`mask_null('Alice')` -> `null`
+
+`mask_null(123)` -> `null`
+
+| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default value of the same type. This will be `\***\***` for text values, zero for numeric values, `false` for booleans, etc.
+
+Examples:
+
+`mask_default('Alice')` -> `'\****'`
+
+`mask_default(123)` -> `0`
+
+| `mask_replace(value, replacement])` | Replaces the first argument by the replacement value on the second argument. The replacement value needs to have the same type as the replaced value.
+
+Examples:
+
+`mask_replace('Alice', 'REDACTED')` -> `'REDACTED'`
+
+`mask_replace(123, -1)` -> `-1`
+
+| `mask_inner(value, begin, end, [padding])` | Returns a copy of the first `text`, `varchar` or `ascii` argument, replacing each character except the first and last ones by a padding character. The 2nd and 3rd arguments are the size of the exposed prefix and suffix. The optional 4th argument is the padding character, `\*` by default.
+
+Examples:
+
+`mask_inner('Alice', 1, 2)` -> `'A**ce'`
+
+`mask_inner('Alice', 1, null)` -> `'A****'`
+
+`mask_inner('Alice', null, 2)` -> `'***ce'`
+
+`mask_inner('Alice', 2, 1, '\#')` -> `'Al##e'`
+
+| `mask_outer(value, begin, end, [padding])` | Returns a copy of the first `text`, `varchar` or `ascii` argument, replacing the first and last character by a padding character. The 2nd and 3rd arguments are the size of the exposed prefix and suffix. The optional 4th argument is the padding character, `\*` by default.
+
+Examples:
+
+`mask_outer('Alice', 1, 2)` -> `'*li**'`
+
+`mask_outer('Alice', 1, null)` -> `'*lice'`
+
+`mask_outer('Alice', null, 2)` -> `'Ali**'`
+
+`mask_outer('Alice', 2, 1, '\#')` -> `'##ic#'`
+
+| `mask_hash(value, [algorithm])` | Returns a `blob` containing the hash of the first argument. The optional 2nd argument is the hashing algorithm to be used, according the available Java security provider. The default hashing algorithm is `SHA-256`.
+
+Examples:
+
+`mask_hash('Alice')`
+
+`mask_hash('Alice', 'SHA-512')`
+
+|===
\ No newline at end of file
diff --git a/doc/modules/cassandra/partials/nodetool_and_cqlsh.adoc b/doc/modules/cassandra/partials/nodetool_and_cqlsh.adoc
index d1c4e73..80da915 100644
--- a/doc/modules/cassandra/partials/nodetool_and_cqlsh.adoc
+++ b/doc/modules/cassandra/partials/nodetool_and_cqlsh.adoc
@@ -1,5 +1,5 @@
NOTE: For information on how to configure your installation, see
-{cass_url}doc/latest/getting_started/configuring.html[Configuring
+{cass_url}doc/latest/getting-started/configuring.html[Configuring
Cassandra].
[arabic, start=7]
diff --git a/doc/modules/cassandra/partials/nodetool_and_cqlsh_nobin.adoc b/doc/modules/cassandra/partials/nodetool_and_cqlsh_nobin.adoc
index c17949c..bf74ad2 100644
--- a/doc/modules/cassandra/partials/nodetool_and_cqlsh_nobin.adoc
+++ b/doc/modules/cassandra/partials/nodetool_and_cqlsh_nobin.adoc
@@ -1,5 +1,5 @@
NOTE: For information on how to configure your installation, see
-{cass_url}doc/latest/getting_started/configuring.html[Configuring
+{cass_url}doc/latest/getting-started/configuring.html[Configuring
Cassandra].
[arabic, start=7]
diff --git a/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactory.java b/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactory.java
index fb11c91..18e211b 100644
--- a/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactory.java
+++ b/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactory.java
@@ -125,6 +125,8 @@
private String pemEncodedCertificates;
private PEMBasedSslContextFactory pemBasedSslContextFactory;
+ public boolean checkedExpiry = false;
+
public KubernetesSecretsPEMSslContextFactory()
{
pemBasedSslContextFactory = new PEMBasedSslContextFactory();
@@ -165,7 +167,7 @@
protected KeyManagerFactory buildKeyManagerFactory() throws SSLException
{
KeyManagerFactory kmf = pemBasedSslContextFactory.buildKeyManagerFactory();
- checkedExpiry = pemBasedSslContextFactory.checkedExpiry;
+ checkedExpiry = pemBasedSslContextFactory.keystoreContext.checkedExpiry;
return kmf;
}
diff --git a/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsSslContextFactory.java b/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsSslContextFactory.java
index c83fb03..ebeac6c 100644
--- a/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsSslContextFactory.java
+++ b/examples/ssl-factory/src/org/apache/cassandra/security/KubernetesSecretsSslContextFactory.java
@@ -149,12 +149,12 @@
public KubernetesSecretsSslContextFactory()
{
- keystore = getString(EncryptionOptions.ConfigKey.KEYSTORE.toString(), KEYSTORE_PATH_VALUE);
- keystore_password = getValueFromEnv(KEYSTORE_PASSWORD_ENV_VAR_NAME,
- DEFAULT_KEYSTORE_PASSWORD);
- truststore = getString(EncryptionOptions.ConfigKey.TRUSTSTORE.toString(), TRUSTSTORE_PATH_VALUE);
- truststore_password = getValueFromEnv(TRUSTSTORE_PASSWORD_ENV_VAR_NAME,
- DEFAULT_TRUSTSTORE_PASSWORD);
+ keystoreContext = new FileBasedStoreContext(getString(EncryptionOptions.ConfigKey.KEYSTORE.toString(), KEYSTORE_PATH_VALUE),
+ getValueFromEnv(KEYSTORE_PASSWORD_ENV_VAR_NAME, DEFAULT_KEYSTORE_PASSWORD));
+
+ trustStoreContext = new FileBasedStoreContext(getString(EncryptionOptions.ConfigKey.TRUSTSTORE.toString(), TRUSTSTORE_PATH_VALUE),
+ getValueFromEnv(TRUSTSTORE_PASSWORD_ENV_VAR_NAME, DEFAULT_TRUSTSTORE_PASSWORD));
+
keystoreLastUpdatedTime = System.nanoTime();
keystoreUpdatedTimeSecretKeyPath = getString(ConfigKeys.KEYSTORE_UPDATED_TIMESTAMP_PATH,
KEYSTORE_UPDATED_TIMESTAMP_PATH_VALUE);
@@ -166,12 +166,13 @@
public KubernetesSecretsSslContextFactory(Map<String, Object> parameters)
{
super(parameters);
- keystore = getString(EncryptionOptions.ConfigKey.KEYSTORE.toString(), KEYSTORE_PATH_VALUE);
- keystore_password = getValueFromEnv(getString(ConfigKeys.KEYSTORE_PASSWORD_ENV_VAR,
- KEYSTORE_PASSWORD_ENV_VAR_NAME), DEFAULT_KEYSTORE_PASSWORD);
- truststore = getString(EncryptionOptions.ConfigKey.TRUSTSTORE.toString(), TRUSTSTORE_PATH_VALUE);
- truststore_password = getValueFromEnv(getString(ConfigKeys.TRUSTSTORE_PASSWORD_ENV_VAR,
- TRUSTSTORE_PASSWORD_ENV_VAR_NAME), DEFAULT_TRUSTSTORE_PASSWORD);
+ keystoreContext = new FileBasedStoreContext(getString(EncryptionOptions.ConfigKey.KEYSTORE.toString(), KEYSTORE_PATH_VALUE),
+ getValueFromEnv(getString(ConfigKeys.KEYSTORE_PASSWORD_ENV_VAR,
+ KEYSTORE_PASSWORD_ENV_VAR_NAME), DEFAULT_KEYSTORE_PASSWORD));
+
+ trustStoreContext = new FileBasedStoreContext(getString(EncryptionOptions.ConfigKey.TRUSTSTORE.toString(), TRUSTSTORE_PATH_VALUE),
+ getValueFromEnv(getString(ConfigKeys.TRUSTSTORE_PASSWORD_ENV_VAR,
+ TRUSTSTORE_PASSWORD_ENV_VAR_NAME), DEFAULT_TRUSTSTORE_PASSWORD));
keystoreLastUpdatedTime = System.nanoTime();
keystoreUpdatedTimeSecretKeyPath = getString(ConfigKeys.KEYSTORE_UPDATED_TIMESTAMP_PATH,
KEYSTORE_UPDATED_TIMESTAMP_PATH_VALUE);
diff --git a/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactoryTest.java b/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactoryTest.java
index 2d127f3..f101e89 100644
--- a/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactoryTest.java
+++ b/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsPEMSslContextFactoryTest.java
@@ -147,7 +147,7 @@
KubernetesSecretsPEMSslContextFactory kubernetesSecretsSslContextFactory =
new KubernetesSecretsPEMSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
}
@@ -158,7 +158,7 @@
config.putAll(commonConfig);
KubernetesSecretsPEMSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsPEMSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
}
@@ -172,7 +172,7 @@
KubernetesSecretsPEMSslContextFactory kubernetesSecretsSslContextFactory =
new KubernetesSecretsPEMSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.keystoreContext.checkedExpiry = false;
kubernetesSecretsSslContextFactory.buildKeyManagerFactory();
}
@@ -262,7 +262,7 @@
addKeystoreOptions(config);
KubernetesSecretsPEMSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsPEMSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
Assert.assertFalse(kubernetesSecretsSslContextFactory.shouldReload());
@@ -282,7 +282,7 @@
addKeystoreOptions(config);
KubernetesSecretsPEMSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsPEMSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.keystoreContext.checkedExpiry = false;
KeyManagerFactory keyManagerFactory = kubernetesSecretsSslContextFactory.buildKeyManagerFactory();
Assert.assertNotNull(keyManagerFactory);
Assert.assertFalse(kubernetesSecretsSslContextFactory.shouldReload());
diff --git a/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsSslContextFactoryTest.java b/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsSslContextFactoryTest.java
index d37992a..0c36419 100644
--- a/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsSslContextFactoryTest.java
+++ b/examples/ssl-factory/test/unit/org/apache/cassandra/security/KubernetesSecretsSslContextFactoryTest.java
@@ -20,8 +20,8 @@
import java.io.IOException;
import java.io.OutputStream;
+import java.io.File;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
@@ -38,7 +38,6 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.EncryptionOptions;
-import org.apache.cassandra.io.util.File;
import static org.apache.cassandra.security.KubernetesSecretsSslContextFactory.ConfigKeys.KEYSTORE_PASSWORD_ENV_VAR;
import static org.apache.cassandra.security.KubernetesSecretsSslContextFactory.ConfigKeys.KEYSTORE_UPDATED_TIMESTAMP_PATH;
@@ -63,11 +62,10 @@
private static void deleteFileIfExists(String file)
{
- Path filePath = Paths.get(file);
- boolean deleted = new File(filePath).toJavaIOFile().delete();
+ boolean deleted = new File(file).delete();
if (!deleted)
{
- logger.warn("File {} could not be deleted.", filePath);
+ logger.warn("File {} could not be deleted.", file);
}
}
@@ -106,7 +104,7 @@
config.put(TRUSTSTORE_PATH, "/this/is/probably/not/a/file/on/your/test/machine");
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
}
@@ -119,7 +117,7 @@
config.put(KubernetesSecretsSslContextFactory.DEFAULT_TRUSTSTORE_PASSWORD_ENV_VAR_NAME, "HomeOfBadPasswords");
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
}
@@ -133,7 +131,7 @@
config.put(KubernetesSecretsSslContextFactory.DEFAULT_TRUSTSTORE_PASSWORD_ENV_VAR_NAME, "");
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
}
@@ -144,7 +142,7 @@
config.putAll(commonConfig);
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
}
@@ -159,7 +157,7 @@
config.put("MY_KEYSTORE_PASSWORD","ThisWontMatter");
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.keystoreContext.checkedExpiry = false;
kubernetesSecretsSslContextFactory.buildKeyManagerFactory();
}
@@ -183,20 +181,20 @@
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory1 = new KubernetesSecretsSslContextFactoryForTestOnly(config);
// Make sure the exiry check didn't happen so far for the private key
- Assert.assertFalse(kubernetesSecretsSslContextFactory1.checkedExpiry);
+ Assert.assertFalse(kubernetesSecretsSslContextFactory1.keystoreContext.checkedExpiry);
addKeystoreOptions(config);
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory2 = new KubernetesSecretsSslContextFactoryForTestOnly(config);
// Trigger the private key loading. That will also check for expired private key
kubernetesSecretsSslContextFactory2.buildKeyManagerFactory();
// Now we should have checked the private key's expiry
- Assert.assertTrue(kubernetesSecretsSslContextFactory2.checkedExpiry);
+ Assert.assertTrue(kubernetesSecretsSslContextFactory2.keystoreContext.checkedExpiry);
// Make sure that new factory object preforms the fresh private key expiry check
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory3 = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- Assert.assertFalse(kubernetesSecretsSslContextFactory3.checkedExpiry);
+ Assert.assertFalse(kubernetesSecretsSslContextFactory3.keystoreContext.checkedExpiry);
kubernetesSecretsSslContextFactory3.buildKeyManagerFactory();
- Assert.assertTrue(kubernetesSecretsSslContextFactory3.checkedExpiry);
+ Assert.assertTrue(kubernetesSecretsSslContextFactory3.keystoreContext.checkedExpiry);
}
@Test
@@ -207,7 +205,7 @@
addKeystoreOptions(config);
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.trustStoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = kubernetesSecretsSslContextFactory.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
Assert.assertFalse(kubernetesSecretsSslContextFactory.shouldReload());
@@ -227,7 +225,7 @@
addKeystoreOptions(config);
KubernetesSecretsSslContextFactory kubernetesSecretsSslContextFactory = new KubernetesSecretsSslContextFactoryForTestOnly(config);
- kubernetesSecretsSslContextFactory.checkedExpiry = false;
+ kubernetesSecretsSslContextFactory.keystoreContext.checkedExpiry = false;
KeyManagerFactory keyManagerFactory = kubernetesSecretsSslContextFactory.buildKeyManagerFactory();
Assert.assertNotNull(keyManagerFactory);
Assert.assertFalse(kubernetesSecretsSslContextFactory.shouldReload());
diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml
index 8aea315..ffd7530 100644
--- a/ide/idea/workspace.xml
+++ b/ide/idea/workspace.xml
@@ -151,6 +151,7 @@
-Dcassandra.storagedir=$PROJECT_DIR$/data
-Djava.library.path=$PROJECT_DIR$/lib/sigar-bin
-Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml
+ -XX:HeapDumpPath=build/test
-ea" />
<option name="PROGRAM_PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="" />
@@ -198,7 +199,8 @@
-Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml
-Dmigration-sstable-root=$PROJECT_DIR$/test/data/migration-sstables
-XX:ActiveProcessorCount=2
- -XX:MaxMetaspaceSize=384M
+ -XX:HeapDumpPath=build/test
+ -XX:MaxMetaspaceSize=1G
-XX:SoftRefLRUPolicyMSPerMB=0
-ea" />
<option name="PARAMETERS" value="" />
@@ -230,6 +232,7 @@
-Dcassandra.triggers_dir=$PROJECT_DIR$/conf/triggers
-Djava.library.path=$PROJECT_DIR$/lib/sigar-bin
-Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml
+ -XX:HeapDumpPath=build/test
-Xmx1G
-ea" />
<option name="PROGRAM_PARAMETERS" value="" />
diff --git a/ide/nbproject/project.xml b/ide/nbproject/project.xml
index 3900f1f..5ea2225 100644
--- a/ide/nbproject/project.xml
+++ b/ide/nbproject/project.xml
@@ -7,7 +7,7 @@
<properties>
<property name="project.dir">..</property>
<!-- the compile classpaths should be distinct per compilation unit… but it is kept simple and the build will catch errors -->
- <property name="cassandra.classpath.jars">${project.dir}/build/lib/jars/HdrHistogram-2.1.9.jar:${project.dir}/build/lib/jars/ST4-4.0.8.jar:${project.dir}/build/lib/jars/airline-0.8.jar:${project.dir}/build/lib/jars/antlr-3.5.2.jar:${project.dir}/build/lib/jars/antlr-runtime-3.5.2.jar:${project.dir}/build/lib/jars/asm-9.1.jar:${project.dir}/build/lib/jars/assertj-core-3.15.0.jar:${project.dir}/build/lib/jars/byteman-4.0.6.jar:${project.dir}/build/lib/jars/byteman-bmunit-4.0.6.jar:${project.dir}/build/lib/jars/byteman-install-4.0.6.jar:${project.dir}/build/lib/jars/byteman-submit-4.0.6.jar:${project.dir}/build/lib/jars/caffeine-2.9.2.jar:${project.dir}/build/lib/jars/cassandra-driver-core-3.11.0-shaded.jar:${project.dir}/build/lib/jars/checker-qual-3.10.0.jar:${project.dir}/build/lib/jars/chronicle-bytes-2.20.111.jar:${project.dir}/build/lib/jars/chronicle-core-2.20.126.jar:${project.dir}/build/lib/jars/chronicle-queue-5.20.123.jar:${project.dir}/build/lib/jars/chronicle-threads-2.20.111.jar:${project.dir}/build/lib/jars/chronicle-wire-2.20.117.jar:${project.dir}/build/lib/jars/commons-beanutils-1.7.0.jar:${project.dir}/build/lib/jars/commons-beanutils-core-1.8.0.jar:${project.dir}/build/lib/jars/commons-cli-1.1.jar:${project.dir}/build/lib/jars/commons-codec-1.9.jar:${project.dir}/build/lib/jars/commons-collections-3.2.1.jar:${project.dir}/build/lib/jars/commons-configuration-1.6.jar:${project.dir}/build/lib/jars/commons-digester-1.8.jar:${project.dir}/build/lib/jars/commons-el-1.0.jar:${project.dir}/build/lib/jars/commons-httpclient-3.0.1.jar:${project.dir}/build/lib/jars/commons-lang3-3.11.jar:${project.dir}/build/lib/jars/commons-math-2.1.jar:${project.dir}/build/lib/jars/commons-math3-3.2.jar:${project.dir}/build/lib/jars/commons-net-1.4.1.jar:${project.dir}/build/lib/jars/compile-command-annotations-1.2.0.jar:${project.dir}/build/lib/jars/compress-lzf-0.8.4.jar:${project.dir}/build/lib/jars/concurrent-trees-2.4.0.jar:${project.dir}/build/lib/jars/ecj-4.6.1.jar:${project.dir}/build/lib/jars/error_prone_annotations-2.5.1.jar:${project.dir}/build/lib/jars/ftplet-api-1.0.0.jar:${project.dir}/build/lib/jars/ftpserver-core-1.0.0.jar:${project.dir}/build/lib/jars/ftpserver-deprecated-1.0.0-M2.jar:${project.dir}/build/lib/jars/guava-27.0-jre.jar:${project.dir}/build/lib/jars/hadoop-core-1.0.3.jar:${project.dir}/build/lib/jars/hadoop-minicluster-1.0.3.jar:${project.dir}/build/lib/jars/hadoop-test-1.0.3.jar:${project.dir}/build/lib/jars/high-scale-lib-1.0.6.jar:${project.dir}/build/lib/jars/hppc-0.8.1.jar:${project.dir}/build/lib/jars/hsqldb-1.8.0.10.jar:${project.dir}/build/lib/jars/ipaddress-5.3.3.jar:${project.dir}/build/lib/jars/j2objc-annotations-1.3.jar:${project.dir}/build/lib/jars/jackson-annotations-2.13.2.jar:${project.dir}/build/lib/jars/jackson-core-2.13.2.jar:${project.dir}/build/lib/jars/jackson-databind-2.13.2.2.jar:${project.dir}/build/lib/jars/jackson-datatype-jsr310-2.13.2.jar:${project.dir}/build/lib/jars/jacocoagent.jar:${project.dir}/build/lib/jars/jamm-0.3.2.jar:${project.dir}/build/lib/jars/jasper-compiler-5.5.12.jar:${project.dir}/build/lib/jars/jasper-runtime-5.5.12.jar:${project.dir}/build/lib/jars/java-cup-runtime-11b-20160615.jar:${project.dir}/build/lib/jars/javax.inject-1.jar:${project.dir}/build/lib/jars/jbcrypt-0.4.jar:${project.dir}/build/lib/jars/jcl-over-slf4j-1.7.25.jar:${project.dir}/build/lib/jars/jcommander-1.30.jar:${project.dir}/build/lib/jars/jctools-core-3.1.0.jar:${project.dir}/build/lib/jars/jersey-core-1.0.jar:${project.dir}/build/lib/jars/jersey-server-1.0.jar:${project.dir}/build/lib/jars/jets3t-0.7.1.jar:${project.dir}/build/lib/jars/jetty-6.1.26.jar:${project.dir}/build/lib/jars/jetty-util-6.1.26.jar:${project.dir}/build/lib/jars/jflex-1.8.2.jar:${project.dir}/build/lib/jars/jna-5.9.0.jar:${project.dir}/build/lib/jars/json-simple-1.1.jar:${project.dir}/build/lib/jars/jsp-2.1-6.1.14.jar:${project.dir}/build/lib/jars/jsp-api-2.1-6.1.14.jar:${project.dir}/build/lib/jars/jsr305-2.0.2.jar:${project.dir}/build/lib/jars/jsr311-api-1.0.jar:${project.dir}/build/lib/jars/jvm-attach-api-1.5.jar:${project.dir}/build/lib/jars/kfs-0.3.jar:${project.dir}/build/lib/jars/log4j-over-slf4j-1.7.25.jar:${project.dir}/build/lib/jars/logback-classic-1.2.9.jar:${project.dir}/build/lib/jars/logback-core-1.2.9.jar:${project.dir}/build/lib/jars/lz4-java-1.8.0.jar:${project.dir}/build/lib/jars/metrics-core-3.1.5.jar:${project.dir}/build/lib/jars/metrics-jvm-3.1.5.jar:${project.dir}/build/lib/jars/metrics-logback-3.1.5.jar:${project.dir}/build/lib/jars/mina-core-2.0.0-M5.jar:${project.dir}/build/lib/jars/mxdump-0.14.jar:${project.dir}/build/lib/jars/netty-all-4.1.58.Final.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.36.Final.jar:${project.dir}/build/lib/jars/ohc-core-0.5.1.jar:${project.dir}/build/lib/jars/ohc-core-j8-0.5.1.jar:${project.dir}/build/lib/jars/oro-2.0.8.jar:${project.dir}/build/lib/jars/psjava-0.1.19.jar:${project.dir}/build/lib/jars/reporter-config-base-3.0.3.jar:${project.dir}/build/lib/jars/reporter-config3-3.0.3.jar:${project.dir}/build/lib/jars/servlet-api-2.5-6.1.14.jar:${project.dir}/build/lib/jars/sigar-1.6.4.jar:${project.dir}/build/lib/jars/sjk-cli-0.14.jar:${project.dir}/build/lib/jars/sjk-core-0.14.jar:${project.dir}/build/lib/jars/sjk-json-0.14.jar:${project.dir}/build/lib/jars/sjk-stacktrace-0.14.jar:${project.dir}/build/lib/jars/slf4j-api-1.7.25.jar:${project.dir}/build/lib/jars/snakeyaml-1.26.jar:${project.dir}/build/lib/jars/snappy-java-1.1.8.4.jar:${project.dir}/build/lib/jars/snowball-stemmer-1.3.0.581.1.jar:${project.dir}/build/lib/jars/stream-2.5.2.jar:${project.dir}/build/lib/jars/xmlenc-0.52.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.0-4.jar:${project.dir}/build/test/lib/jars/Saxon-HE-10.3.jar:${project.dir}/build/test/lib/jars/ant-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-junit-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-launcher-1.10.12.jar:${project.dir}/build/test/lib/jars/antlr-2.7.7.jar:${project.dir}/build/test/lib/jars/antlr4-runtime-4.9.1.jar:${project.dir}/build/test/lib/jars/asm-analysis-9.1.jar:${project.dir}/build/test/lib/jars/asm-commons-9.1.jar:${project.dir}/build/test/lib/jars/asm-tree-9.1.jar:${project.dir}/build/test/lib/jars/asm-util-9.1.jar:${project.dir}/build/test/lib/jars/asm-xml-6.0.jar:${project.dir}/build/test/lib/jars/awaitility-4.0.3.jar:${project.dir}/build/test/lib/jars/byte-buddy-1.10.5.jar:${project.dir}/build/test/lib/jars/byte-buddy-agent-1.10.5.jar:${project.dir}/build/test/lib/jars/checkstyle-8.40.jar:${project.dir}/build/test/lib/jars/commons-beanutils-1.9.4.jar:${project.dir}/build/test/lib/jars/commons-collections-3.2.2.jar:${project.dir}/build/test/lib/jars/commons-io-2.6.jar:${project.dir}/build/test/lib/jars/commons-logging-1.2.jar:${project.dir}/build/test/lib/jars/commons-math3-3.2.jar:${project.dir}/build/test/lib/jars/compile-command-annotations-1.2.0.jar:${project.dir}/build/test/lib/jars/dtest-api-0.0.13.jar:${project.dir}/build/test/lib/jars/guava-18.0.jar:${project.dir}/build/test/lib/jars/hamcrest-2.2.jar:${project.dir}/build/test/lib/jars/harry-core-0.0.1.jar:${project.dir}/build/test/lib/jars/jackson-annotations-2.11.3.jar:${project.dir}/build/test/lib/jars/jackson-core-2.13.2.jar:${project.dir}/build/test/lib/jars/jackson-databind-2.11.3.jar:${project.dir}/build/test/lib/jars/jackson-dataformat-yaml-2.13.2.jar:${project.dir}/build/test/lib/jars/java-allocation-instrumenter-3.1.0.jar:${project.dir}/build/test/lib/jars/javassist-3.28.0-GA.jar:${project.dir}/build/test/lib/jars/jimfs-1.1.jar:${project.dir}/build/test/lib/jars/jmh-core-1.21.jar:${project.dir}/build/test/lib/jars/jmh-generator-annprocess-1.21.jar:${project.dir}/build/test/lib/jars/jopt-simple-4.6.jar:${project.dir}/build/test/lib/jars/jsr305-3.0.2.jar:${project.dir}/build/test/lib/jars/junit-4.12.jar:${project.dir}/build/test/lib/jars/mockito-core-3.2.4.jar:${project.dir}/build/test/lib/jars/objenesis-2.6.jar:${project.dir}/build/test/lib/jars/org.jacoco.agent-0.8.6.jar:${project.dir}/build/test/lib/jars/org.jacoco.ant-0.8.6.jar:${project.dir}/build/test/lib/jars/org.jacoco.core-0.8.6.jar:${project.dir}/build/test/lib/jars/org.jacoco.report-0.8.6.jar:${project.dir}/build/test/lib/jars/picocli-4.6.1.jar:${project.dir}/build/test/lib/jars/quicktheories-0.26.jar:${project.dir}/build/test/lib/jars/reflections-0.10.2.jar:${project.dir}/build/test/lib/jars/semver4j-3.1.0.jar:${project.dir}/build/test/lib/jars/simulator-asm.jar:${project.dir}/build/test/lib/jars/simulator-bootstrap.jar:${project.dir}/build/test/lib/jars/slf4j-api-1.7.32.jar:</property>
+ <property name="cassandra.classpath.jars">${project.dir}/build/lib/jars/HdrHistogram-2.1.9.jar:${project.dir}/build/lib/jars/ST4-4.0.8.jar:${project.dir}/build/lib/jars/agrona-1.17.1.jar:${project.dir}/build/lib/jars/airline-0.8.jar:${project.dir}/build/lib/jars/antlr-3.5.2.jar:${project.dir}/build/lib/jars/antlr-runtime-3.5.2.jar:${project.dir}/build/lib/jars/asm-9.3.jar:${project.dir}/build/lib/jars/assertj-core-3.15.0.jar:${project.dir}/build/lib/jars/big-math-2.3.0.jar:${project.dir}/build/lib/jars/byteman-4.0.20.jar:${project.dir}/build/lib/jars/byteman-bmunit-4.0.20.jar:${project.dir}/build/lib/jars/byteman-install-4.0.20.jar:${project.dir}/build/lib/jars/byteman-submit-4.0.20.jar:${project.dir}/build/lib/jars/caffeine-2.9.2.jar:${project.dir}/build/lib/jars/cassandra-driver-core-3.11.0-shaded.jar:${project.dir}/build/lib/jars/checker-qual-3.10.0.jar:${project.dir}/build/lib/jars/chronicle-bytes-2.20.111.jar:${project.dir}/build/lib/jars/chronicle-core-2.20.126.jar:${project.dir}/build/lib/jars/chronicle-queue-5.20.123.jar:${project.dir}/build/lib/jars/chronicle-threads-2.20.111.jar:${project.dir}/build/lib/jars/chronicle-wire-2.20.117.jar:${project.dir}/build/lib/jars/commons-beanutils-1.7.0.jar:${project.dir}/build/lib/jars/commons-beanutils-core-1.8.0.jar:${project.dir}/build/lib/jars/commons-cli-1.1.jar:${project.dir}/build/lib/jars/commons-codec-1.9.jar:${project.dir}/build/lib/jars/commons-collections-3.2.1.jar:${project.dir}/build/lib/jars/commons-configuration-1.6.jar:${project.dir}/build/lib/jars/commons-digester-1.8.jar:${project.dir}/build/lib/jars/commons-el-1.0.jar:${project.dir}/build/lib/jars/commons-httpclient-3.0.1.jar:${project.dir}/build/lib/jars/commons-lang3-3.11.jar:${project.dir}/build/lib/jars/commons-math-2.1.jar:${project.dir}/build/lib/jars/commons-math3-3.2.jar:${project.dir}/build/lib/jars/commons-net-1.4.1.jar:${project.dir}/build/lib/jars/compile-command-annotations-1.2.0.jar:${project.dir}/build/lib/jars/compress-lzf-0.8.4.jar:${project.dir}/build/lib/jars/concurrent-trees-2.4.0.jar:${project.dir}/build/lib/jars/ecj-4.6.1.jar:${project.dir}/build/lib/jars/error_prone_annotations-2.5.1.jar:${project.dir}/build/lib/jars/ftplet-api-1.0.0.jar:${project.dir}/build/lib/jars/ftpserver-core-1.0.0.jar:${project.dir}/build/lib/jars/ftpserver-deprecated-1.0.0-M2.jar:${project.dir}/build/lib/jars/guava-27.0-jre.jar:${project.dir}/build/lib/jars/hadoop-core-1.0.3.jar:${project.dir}/build/lib/jars/hadoop-minicluster-1.0.3.jar:${project.dir}/build/lib/jars/hadoop-test-1.0.3.jar:${project.dir}/build/lib/jars/high-scale-lib-1.0.6.jar:${project.dir}/build/lib/jars/hppc-0.8.1.jar:${project.dir}/build/lib/jars/hsqldb-1.8.0.10.jar:${project.dir}/build/lib/jars/ipaddress-5.3.3.jar:${project.dir}/build/lib/jars/j2objc-annotations-1.3.jar:${project.dir}/build/lib/jars/jackson-annotations-2.13.2.jar:${project.dir}/build/lib/jars/jackson-core-2.13.2.jar:${project.dir}/build/lib/jars/jackson-databind-2.13.2.2.jar:${project.dir}/build/lib/jars/jackson-datatype-jsr310-2.13.2.jar:${project.dir}/build/lib/jars/jacocoagent.jar:${project.dir}/build/lib/jars/jamm-0.3.2.jar:${project.dir}/build/lib/jars/jasper-compiler-5.5.12.jar:${project.dir}/build/lib/jars/jasper-runtime-5.5.12.jar:${project.dir}/build/lib/jars/java-cup-runtime-11b-20160615.jar:${project.dir}/build/lib/jars/javax.inject-1.jar:${project.dir}/build/lib/jars/jbcrypt-0.4.jar:${project.dir}/build/lib/jars/jcl-over-slf4j-1.7.25.jar:${project.dir}/build/lib/jars/jcommander-1.30.jar:${project.dir}/build/lib/jars/jctools-core-3.1.0.jar:${project.dir}/build/lib/jars/jersey-core-1.0.jar:${project.dir}/build/lib/jars/jersey-server-1.0.jar:${project.dir}/build/lib/jars/jets3t-0.7.1.jar:${project.dir}/build/lib/jars/jetty-6.1.26.jar:${project.dir}/build/lib/jars/jetty-util-6.1.26.jar:${project.dir}/build/lib/jars/jflex-1.8.2.jar:${project.dir}/build/lib/jars/jna-5.13.0.jar:${project.dir}/build/lib/jars/json-simple-1.1.jar:${project.dir}/build/lib/jars/jsp-2.1-6.1.14.jar:${project.dir}/build/lib/jars/jsp-api-2.1-6.1.14.jar:${project.dir}/build/lib/jars/jsr305-2.0.2.jar:${project.dir}/build/lib/jars/jsr311-api-1.0.jar:${project.dir}/build/lib/jars/jvm-attach-api-1.5.jar:${project.dir}/build/lib/jars/kfs-0.3.jar:${project.dir}/build/lib/jars/log4j-over-slf4j-1.7.25.jar:${project.dir}/build/lib/jars/logback-classic-1.2.9.jar:${project.dir}/build/lib/jars/logback-core-1.2.9.jar:${project.dir}/build/lib/jars/lz4-java-1.8.0.jar:${project.dir}/build/lib/jars/metrics-core-3.1.5.jar:${project.dir}/build/lib/jars/metrics-jvm-3.1.5.jar:${project.dir}/build/lib/jars/metrics-logback-3.1.5.jar:${project.dir}/build/lib/jars/mina-core-2.0.0-M5.jar:${project.dir}/build/lib/jars/mxdump-0.14.jar:${project.dir}/build/lib/jars/netty-all-4.1.58.Final.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.36.Final.jar:${project.dir}/build/lib/jars/ohc-core-0.5.1.jar:${project.dir}/build/lib/jars/ohc-core-j8-0.5.1.jar:${project.dir}/build/lib/jars/oro-2.0.8.jar:${project.dir}/build/lib/jars/psjava-0.1.19.jar:${project.dir}/build/lib/jars/reporter-config-base-3.0.3.jar:${project.dir}/build/lib/jars/reporter-config3-3.0.3.jar:${project.dir}/build/lib/jars/servlet-api-2.5-6.1.14.jar:${project.dir}/build/lib/jars/sigar-1.6.4.jar:${project.dir}/build/lib/jars/sjk-cli-0.14.jar:${project.dir}/build/lib/jars/sjk-core-0.14.jar:${project.dir}/build/lib/jars/sjk-json-0.14.jar:${project.dir}/build/lib/jars/sjk-stacktrace-0.14.jar:${project.dir}/build/lib/jars/slf4j-api-1.7.25.jar:${project.dir}/build/lib/jars/snakeyaml-1.26.jar:${project.dir}/build/lib/jars/snappy-java-1.1.8.4.jar:${project.dir}/build/lib/jars/snowball-stemmer-1.3.0.581.1.jar:${project.dir}/build/lib/jars/stream-2.5.2.jar:${project.dir}/build/lib/jars/xmlenc-0.52.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.4-1.jar:${project.dir}/build/test/lib/jars/Saxon-HE-10.3.jar:${project.dir}/build/test/lib/jars/ant-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-junit-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-launcher-1.10.12.jar:${project.dir}/build/test/lib/jars/antlr-2.7.7.jar:${project.dir}/build/test/lib/jars/antlr4-runtime-4.9.1.jar:${project.dir}/build/test/lib/jars/asm-9.3.jar:${project.dir}/build/test/lib/jars/asm-analysis-9.3.jar:${project.dir}/build/test/lib/jars/asm-commons-9.3.jar:${project.dir}/build/test/lib/jars/asm-tree-9.3.jar:${project.dir}/build/test/lib/jars/asm-util-9.3.jar:${project.dir}/build/test/lib/jars/asm-xml-6.0.jar:${project.dir}/build/test/lib/jars/awaitility-4.0.3.jar:${project.dir}/build/test/lib/jars/byte-buddy-1.12.13.jar:${project.dir}/build/test/lib/jars/byte-buddy-agent-1.12.13.jar:${project.dir}/build/test/lib/jars/checkstyle-8.40.jar:${project.dir}/build/test/lib/jars/commons-beanutils-1.9.4.jar:${project.dir}/build/test/lib/jars/commons-collections-3.2.2.jar:${project.dir}/build/test/lib/jars/commons-io-2.6.jar:${project.dir}/build/test/lib/jars/commons-logging-1.2.jar:${project.dir}/build/test/lib/jars/commons-math3-3.2.jar:${project.dir}/build/test/lib/jars/dtest-api-0.0.13.jar:${project.dir}/build/test/lib/jars/guava-18.0.jar:${project.dir}/build/test/lib/jars/hamcrest-2.2.jar:${project.dir}/build/test/lib/jars/harry-core-0.0.1.jar:${project.dir}/build/test/lib/jars/jackson-annotations-2.11.3.jar:${project.dir}/build/test/lib/jars/jackson-core-2.13.2.jar:${project.dir}/build/test/lib/jars/jackson-databind-2.11.3.jar:${project.dir}/build/test/lib/jars/jackson-dataformat-yaml-2.13.2.jar:${project.dir}/build/test/lib/jars/java-allocation-instrumenter-3.1.0.jar:${project.dir}/build/test/lib/jars/javassist-3.28.0-GA.jar:${project.dir}/build/test/lib/jars/jimfs-1.1.jar:${project.dir}/build/test/lib/jars/jmh-core-1.21.jar:${project.dir}/build/test/lib/jars/jmh-generator-annprocess-1.21.jar:${project.dir}/build/test/lib/jars/jopt-simple-4.6.jar:${project.dir}/build/test/lib/jars/jsr305-3.0.2.jar:${project.dir}/build/test/lib/jars/junit-4.12.jar:${project.dir}/build/test/lib/jars/mockito-core-4.7.0.jar:${project.dir}/build/test/lib/jars/mockito-inline-4.7.0.jar:${project.dir}/build/test/lib/jars/objenesis-3.2.jar:${project.dir}/build/test/lib/jars/org.jacoco.agent-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.ant-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.core-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.report-0.8.8.jar:${project.dir}/build/test/lib/jars/picocli-4.6.1.jar:${project.dir}/build/test/lib/jars/quicktheories-0.26.jar:${project.dir}/build/test/lib/jars/reflections-0.10.2.jar:${project.dir}/build/test/lib/jars/semver4j-3.1.0.jar:${project.dir}/build/test/lib/jars/slf4j-api-1.7.32.jar:</property>
</properties>
<folders>
<source-folder>
diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh
index ad6ae75..f6b1f21 100755
--- a/pylib/cassandra-cqlsh-tests.sh
+++ b/pylib/cassandra-cqlsh-tests.sh
@@ -41,18 +41,9 @@
export CCM_CONFIG_DIR=${WORKSPACE}/.ccm
export NUM_TOKENS="16"
export CASSANDRA_DIR=${WORKSPACE}
-export TESTSUITE_NAME="cqlshlib.${PYTHON_VERSION}"
-if [ -z "$CASSANDRA_USE_JDK11" ]; then
- export CASSANDRA_USE_JDK11=false
-fi
-
-if [ "$CASSANDRA_USE_JDK11" = true ] ; then
- TESTSUITE_NAME="${TESTSUITE_NAME}.jdk11"
-else
- TESTSUITE_NAME="${TESTSUITE_NAME}.jdk8"
- unset JAVA11_HOME
-fi
+java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
+export TESTSUITE_NAME="cqlshlib.${PYTHON_VERSION}.jdk${java_version}"
ant -buildfile ${CASSANDRA_DIR}/build.xml realclean
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
@@ -97,7 +88,6 @@
ccm remove test || true # in case an old ccm cluster is left behind
ccm create test -n 1 --install-dir=${CASSANDRA_DIR}
ccm updateconf "user_defined_functions_enabled: true"
-ccm updateconf "scripted_user_defined_functions_enabled: true"
version_from_build=$(ccm node1 versionfrombuild)
export pre_or_post_cdc=$(python -c """from distutils.version import LooseVersion
diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py
index 03e06d8..7ff252b 100644
--- a/pylib/cqlshlib/cql3handling.py
+++ b/pylib/cqlshlib/cql3handling.py
@@ -43,14 +43,17 @@
class Cql3ParsingRuleSet(CqlParsingRuleSet):
columnfamily_layout_options = (
+ ('allow_auto_snapshot', None),
('bloom_filter_fp_chance', None),
('comment', None),
('gc_grace_seconds', None),
+ ('incremental_backups', None),
('min_index_interval', None),
('max_index_interval', None),
('default_time_to_live', None),
('speculative_retry', None),
('additional_write_policy', None),
+ ('memtable', None),
('memtable_flush_period_in_ms', None),
('cdc', None),
('read_repair', None),
@@ -94,14 +97,6 @@
'fanout_size'
)
- date_tiered_compaction_strategy_options = (
- 'base_time_seconds',
- 'max_sstable_age_days',
- 'min_threshold',
- 'max_window_size_seconds',
- 'timestamp_resolution'
- )
-
time_window_compaction_strategy_options = (
'compaction_window_unit',
'compaction_window_size',
@@ -310,7 +305,9 @@
<userType> ::= utname=<cfOrKsName> ;
-<storageType> ::= <simpleStorageType> | <collectionType> | <frozenCollectionType> | <userType> ;
+<storageType> ::= ( <simpleStorageType> | <collectionType> | <frozenCollectionType> | <userType> ) ( <column_mask> )? ;
+
+<column_mask> ::= "MASKED" "WITH" ( "DEFAULT" | <functionName> <selectionFunctionArguments> );
# Note: autocomplete for frozen collection types does not handle nesting past depth 1 properly,
# but that's a lot of work to fix for little benefit.
@@ -519,6 +516,10 @@
return [Hint('<true|false>')]
if this_opt in ('read_repair'):
return [Hint('<\'none\'|\'blocking\'>')]
+ if this_opt == 'allow_auto_snapshot':
+ return [Hint('<boolean>')]
+ if this_opt == 'incremental_backups':
+ return [Hint('<boolean>')]
return [Hint('<option_value>')]
@@ -547,8 +548,6 @@
opts = opts.union(set(CqlRuleSet.size_tiered_compaction_strategy_options))
elif csc == 'LeveledCompactionStrategy':
opts = opts.union(set(CqlRuleSet.leveled_compaction_strategy_options))
- elif csc == 'DateTieredCompactionStrategy':
- opts = opts.union(set(CqlRuleSet.date_tiered_compaction_strategy_options))
elif csc == 'TimeWindowCompactionStrategy':
opts = opts.union(set(CqlRuleSet.time_window_compaction_strategy_options))
@@ -736,6 +735,7 @@
<selector> ::= [colname]=<cident> ( "[" ( <term> ( ".." <term> "]" )? | <term> ".." ) )?
| <udtSubfieldSelection>
| "WRITETIME" "(" [colname]=<cident> ")"
+ | "MAXWRITETIME" "(" [colname]=<cident> ")"
| "TTL" "(" [colname]=<cident> ")"
| "COUNT" "(" star=( "*" | "1" ) ")"
| "CAST" "(" <selector> "AS" <storageType> ")"
@@ -1308,7 +1308,7 @@
( "WITH" <cfamProperty> ( "AND" <cfamProperty> )* )?
;
-<createUserTypeStatement> ::= "CREATE" "TYPE" ( ks=<nonSystemKeyspaceName> dot="." )? typename=<cfOrKsName> "(" newcol=<cident> <storageType>
+<createUserTypeStatement> ::= "CREATE" "TYPE" ("IF" "NOT" "EXISTS")? ( ks=<nonSystemKeyspaceName> dot="." )? typename=<cfOrKsName> "(" newcol=<cident> <storageType>
( "," [newcolname]=<cident> <storageType> )*
")"
;
@@ -1372,7 +1372,7 @@
<dropMaterializedViewStatement> ::= "DROP" "MATERIALIZED" "VIEW" ("IF" "EXISTS")? mv=<materializedViewName>
;
-<dropUserTypeStatement> ::= "DROP" "TYPE" ut=<userTypeName>
+<dropUserTypeStatement> ::= "DROP" "TYPE" ( "IF" "EXISTS" )? ut=<userTypeName>
;
<dropFunctionStatement> ::= "DROP" "FUNCTION" ( "IF" "EXISTS" )? <userFunctionName>
@@ -1420,6 +1420,7 @@
| "WITH" <cfamProperty> ( "AND" <cfamProperty> )*
| "RENAME" ("IF" "EXISTS")? existcol=<cident> "TO" newcol=<cident>
( "AND" existcol=<cident> "TO" newcol=<cident> )*
+ | "ALTER" ("IF" "EXISTS")? existcol=<cident> ( <column_mask> | "DROP" "MASKED" )
;
<alterUserTypeStatement> ::= "ALTER" "TYPE" ("IF" "EXISTS")? ut=<userTypeName>
@@ -1483,12 +1484,12 @@
| <unreservedKeyword>
;
-<createRoleStatement> ::= "CREATE" "ROLE" <rolename>
+<createRoleStatement> ::= "CREATE" "ROLE" ("IF" "NOT" "EXISTS")? <rolename>
( "WITH" <roleProperty> ("AND" <roleProperty>)*)?
;
<alterRoleStatement> ::= "ALTER" "ROLE" ("IF" "EXISTS")? <rolename>
- ( "WITH" <roleProperty> ("AND" <roleProperty>)*)?
+ ( "WITH" <roleProperty> ("AND" <roleProperty>)*)
;
<roleProperty> ::= (("HASHED")? "PASSWORD") "=" <stringLiteral>
@@ -1499,7 +1500,7 @@
| "ACCESS" "TO" "ALL" "DATACENTERS"
;
-<dropRoleStatement> ::= "DROP" "ROLE" <rolename>
+<dropRoleStatement> ::= "DROP" "ROLE" ("IF" "EXISTS")? <rolename>
;
<grantRoleStatement> ::= "GRANT" <rolename> "TO" <rolename>
@@ -1532,6 +1533,8 @@
| "MODIFY"
| "DESCRIBE"
| "EXECUTE"
+ | "UNMASK"
+ | "SELECT_MASKED"
;
<permissionExpr> ::= ( [newpermission]=<permission> "PERMISSION"? ( "," [newpermission]=<permission> "PERMISSION"? )* )
diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py
index ca12a25..5ea0391 100644
--- a/pylib/cqlshlib/cqlhandling.py
+++ b/pylib/cqlshlib/cqlhandling.py
@@ -49,7 +49,6 @@
available_compaction_classes = (
'LeveledCompactionStrategy',
'SizeTieredCompactionStrategy',
- 'DateTieredCompactionStrategy',
'TimeWindowCompactionStrategy'
)
diff --git a/pylib/cqlshlib/cqlshhandling.py b/pylib/cqlshlib/cqlshhandling.py
index aa1fbc0..e6a121f 100644
--- a/pylib/cqlshlib/cqlshhandling.py
+++ b/pylib/cqlshlib/cqlshhandling.py
@@ -37,7 +37,8 @@
'exit',
'quit',
'clear',
- 'cls'
+ 'cls',
+ 'history'
)
cqlsh_syntax_completers = []
@@ -73,6 +74,7 @@
| <exitCommand>
| <pagingCommand>
| <clearCommand>
+ | <historyCommand>
;
'''
@@ -131,7 +133,7 @@
'''
cqlsh_show_cmd_syntax_rules = r'''
-<showCommand> ::= "SHOW" what=( "VERSION" | "HOST" | "SESSION" sessionid=<uuid> )
+<showCommand> ::= "SHOW" what=( "VERSION" | "HOST" | "SESSION" sessionid=<uuid> | "REPLICAS" token=<integer> (keyspace=<keyspaceName>)? )
;
'''
@@ -141,7 +143,7 @@
'''
cqlsh_capture_cmd_syntax_rules = r'''
-<captureCommand> ::= "CAPTURE" ( fname=( <stringLiteral> | "OFF" ) )?
+<captureCommand> ::= "CAPTURE" ( fname=( <stringLiteral>) | "OFF" )?
;
'''
@@ -188,7 +190,7 @@
'''
cqlsh_paging_cmd_syntax_rules = r'''
-<pagingCommand> ::= "PAGING" ( switch=( "ON" | "OFF" | /[0-9]+/) )?
+<pagingCommand> ::= "PAGING" ( switch=( "ON" | "OFF" | <wholenumber>) )?
;
'''
@@ -207,6 +209,11 @@
;
'''
+cqlsh_history_cmd_syntax_rules = r'''
+<historyCommand> ::= "history" (n=<wholenumber>)?
+ ;
+'''
+
cqlsh_question_mark = r'''
<qmark> ::= "?" ;
'''
@@ -232,6 +239,7 @@
cqlsh_login_cmd_syntax_rules + \
cqlsh_exit_cmd_syntax_rules + \
cqlsh_clear_cmd_syntax_rules + \
+ cqlsh_history_cmd_syntax_rules + \
cqlsh_question_mark
diff --git a/pylib/cqlshlib/cqlshmain.py b/pylib/cqlshlib/cqlshmain.py
new file mode 100755
index 0000000..7cffd68
--- /dev/null
+++ b/pylib/cqlshlib/cqlshmain.py
@@ -0,0 +1,2374 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cmd
+import codecs
+import configparser
+import csv
+import getpass
+import optparse
+import os
+import re
+import subprocess
+import sys
+import traceback
+import warnings
+import webbrowser
+from contextlib import contextmanager
+from io import StringIO
+from uuid import UUID
+
+UTF8 = 'utf-8'
+
+description = "CQL Shell for Apache Cassandra"
+version = "6.2.0"
+
+readline = None
+try:
+ # check if tty first, cause readline doesn't check, and only cares
+ # about $TERM. we don't want the funky escape code stuff to be
+ # output if not a tty.
+ if sys.stdin.isatty():
+ import readline
+except ImportError:
+ pass
+
+# On Linux, the Python webbrowser module uses the 'xdg-open' executable
+# to open a file/URL. But that only works, if the current session has been
+# opened from _within_ a desktop environment. I.e. 'xdg-open' will fail,
+# if the session's been opened via ssh to a remote box.
+#
+try:
+ webbrowser.register_standard_browsers() # registration is otherwise lazy in Python3
+except AttributeError:
+ pass
+if webbrowser._tryorder and webbrowser._tryorder[0] == 'xdg-open' and os.environ.get('XDG_DATA_DIRS', '') == '':
+ # only on Linux (some OS with xdg-open)
+ webbrowser._tryorder.remove('xdg-open')
+ webbrowser._tryorder.append('xdg-open')
+
+warnings.filterwarnings("ignore", r".*blist.*")
+
+import cassandra
+from cassandra.auth import PlainTextAuthProvider
+from cassandra.cluster import Cluster
+from cassandra.cqltypes import cql_typename
+from cassandra.marshal import int64_unpack
+from cassandra.metadata import (ColumnMetadata, KeyspaceMetadata, TableMetadata)
+from cassandra.policies import WhiteListRoundRobinPolicy
+from cassandra.query import SimpleStatement, ordered_dict_factory, TraceUnavailable
+from cassandra.util import datetime_from_timestamp
+
+from cqlshlib import cql3handling, pylexotron, sslhandling, cqlshhandling, authproviderhandling
+from cqlshlib.copyutil import ExportTask, ImportTask
+from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
+ RED, WHITE, FormattedValue, colorme)
+from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
+ DEFAULT_TIMESTAMP_FORMAT, CqlType, DateTimeFormat,
+ format_by_type)
+from cqlshlib.tracing import print_trace, print_trace_session
+from cqlshlib.util import get_file_encoding_bomsize
+from cqlshlib.util import is_file_secure
+
+
+DEFAULT_HOST = '127.0.0.1'
+DEFAULT_PORT = 9042
+DEFAULT_SSL = False
+DEFAULT_CONNECT_TIMEOUT_SECONDS = 5
+DEFAULT_REQUEST_TIMEOUT_SECONDS = 10
+
+DEFAULT_FLOAT_PRECISION = 5
+DEFAULT_DOUBLE_PRECISION = 5
+DEFAULT_MAX_TRACE_WAIT = 10
+
+if readline is not None and readline.__doc__ is not None and 'libedit' in readline.__doc__:
+ DEFAULT_COMPLETEKEY = '\t'
+else:
+ DEFAULT_COMPLETEKEY = 'tab'
+
+cqldocs = None
+cqlruleset = None
+CASSANDRA_CQL_HTML = None
+
+epilog = """Connects to %(DEFAULT_HOST)s:%(DEFAULT_PORT)d by default. These
+defaults can be changed by setting $CQLSH_HOST and/or $CQLSH_PORT. When a
+host (and optional port number) are given on the command line, they take
+precedence over any defaults.""" % globals()
+
+parser = optparse.OptionParser(description=description, epilog=epilog,
+ usage="Usage: %prog [options] [host [port]]",
+ version='cqlsh ' + version)
+parser.add_option("-C", "--color", action='store_true', dest='color',
+ help='Always use color output')
+parser.add_option("--no-color", action='store_false', dest='color',
+ help='Never use color output')
+parser.add_option("--browser", dest='browser', help="""The browser to use to display CQL help, where BROWSER can be:
+ - one of the supported browsers in https://docs.python.org/3/library/webbrowser.html.
+ - browser path followed by %s, example: /usr/bin/google-chrome-stable %s""")
+parser.add_option('--ssl', action='store_true', help='Use SSL', default=False)
+parser.add_option("-u", "--username", help="Authenticate as user.")
+parser.add_option("-p", "--password", help="Authenticate using password.")
+parser.add_option('-k', '--keyspace', help='Authenticate to the given keyspace.')
+parser.add_option("-f", "--file", help="Execute commands from FILE, then exit")
+parser.add_option('--debug', action='store_true',
+ help='Show additional debugging information')
+parser.add_option('--coverage', action='store_true',
+ help='Collect coverage data')
+parser.add_option("--encoding", help="Specify a non-default encoding for output."
+ + " (Default: %s)" % (UTF8,))
+parser.add_option("--cqlshrc", help="Specify an alternative cqlshrc file location.")
+parser.add_option("--credentials", help="Specify an alternative credentials file location.")
+parser.add_option('--cqlversion', default=None,
+ help='Specify a particular CQL version, '
+ 'by default the highest version supported by the server will be used.'
+ ' Examples: "3.0.3", "3.1.0"')
+parser.add_option("--protocol-version", type="int", default=None,
+ help='Specify a specific protcol version otherwise the client will default and downgrade as necessary')
+
+parser.add_option("-e", "--execute", help='Execute the statement and quit.')
+parser.add_option("--connect-timeout", default=DEFAULT_CONNECT_TIMEOUT_SECONDS, dest='connect_timeout',
+ help='Specify the connection timeout in seconds (default: %default seconds).')
+parser.add_option("--request-timeout", default=DEFAULT_REQUEST_TIMEOUT_SECONDS, dest='request_timeout',
+ help='Specify the default request timeout in seconds (default: %default seconds).')
+parser.add_option("-t", "--tty", action='store_true', dest='tty',
+ help='Force tty mode (command prompt).')
+parser.add_option('-v', action="version", help='Print the current version of cqlsh.')
+
+# This is a hidden option to suppress the warning when the -p/--password command line option is used.
+# Power users may use this option if they know no other people has access to the system where cqlsh is run or don't care about security.
+# Use of this option in scripting is discouraged. Please use a (temporary) credentials file where possible.
+# The Cassandra distributed tests (dtests) also use this option in some tests when a well-known password is supplied via the command line.
+parser.add_option("--insecure-password-without-warning", action='store_true', dest='insecure_password_without_warning',
+ help=optparse.SUPPRESS_HELP)
+
+# use cfoptions for config file
+
+opt_values = optparse.Values()
+(cfoptions, arguments) = parser.parse_args(sys.argv[1:], values=opt_values)
+
+# BEGIN history config
+
+
+def mkdirp(path):
+ """Creates all parent directories up to path parameter or fails when path exists, but it is not a directory."""
+
+ try:
+ os.makedirs(path)
+ except OSError:
+ if not os.path.isdir(path):
+ raise
+
+
+def resolve_cql_history_file():
+ default_cql_history = os.path.expanduser(os.path.join('~', '.cassandra', 'cqlsh_history'))
+ if 'CQL_HISTORY' in os.environ:
+ return os.environ['CQL_HISTORY']
+ else:
+ return default_cql_history
+
+
+HISTORY = resolve_cql_history_file()
+HISTORY_DIR = os.path.dirname(HISTORY)
+
+try:
+ mkdirp(HISTORY_DIR)
+except OSError:
+ print('\nWarning: Cannot create directory at `%s`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.\n' % HISTORY_DIR)
+
+
+# END history config
+
+
+DEFAULT_CQLSHRC = os.path.expanduser(os.path.join('~', '.cassandra', 'cqlshrc'))
+
+if hasattr(cfoptions, 'cqlshrc'):
+ CONFIG_FILE = os.path.expanduser(cfoptions.cqlshrc)
+ if not os.path.exists(CONFIG_FILE):
+ print('\nWarning: Specified cqlshrc location `%s` does not exist. Using `%s` instead.\n' % (CONFIG_FILE, DEFAULT_CQLSHRC))
+ CONFIG_FILE = DEFAULT_CQLSHRC
+else:
+ CONFIG_FILE = DEFAULT_CQLSHRC
+
+CQL_DIR = os.path.dirname(CONFIG_FILE)
+
+CQL_ERRORS = (
+ cassandra.AlreadyExists, cassandra.AuthenticationFailed, cassandra.CoordinationFailure,
+ cassandra.InvalidRequest, cassandra.Timeout, cassandra.Unauthorized, cassandra.OperationTimedOut,
+ cassandra.cluster.NoHostAvailable,
+ cassandra.connection.ConnectionBusy, cassandra.connection.ProtocolError, cassandra.connection.ConnectionException,
+ cassandra.protocol.ErrorMessage, cassandra.protocol.InternalError, cassandra.query.TraceUnavailable
+)
+
+debug_completion = bool(os.environ.get('CQLSH_DEBUG_COMPLETION', '') == 'YES')
+
+
+class NoKeyspaceError(Exception):
+ pass
+
+
+class KeyspaceNotFound(Exception):
+ pass
+
+
+class ColumnFamilyNotFound(Exception):
+ pass
+
+
+class IndexNotFound(Exception):
+ pass
+
+
+class MaterializedViewNotFound(Exception):
+ pass
+
+
+class ObjectNotFound(Exception):
+ pass
+
+
+class VersionNotSupported(Exception):
+ pass
+
+
+class UserTypeNotFound(Exception):
+ pass
+
+
+class FunctionNotFound(Exception):
+ pass
+
+
+class AggregateNotFound(Exception):
+ pass
+
+
+class DecodeError(Exception):
+ verb = 'decode'
+
+ def __init__(self, thebytes, err, colname=None):
+ self.thebytes = thebytes
+ self.err = err
+ self.colname = colname
+
+ def __str__(self):
+ return str(self.thebytes)
+
+ def message(self):
+ what = 'value %r' % (self.thebytes,)
+ if self.colname is not None:
+ what = 'value %r (for column %r)' % (self.thebytes, self.colname)
+ return 'Failed to %s %s : %s' \
+ % (self.verb, what, self.err)
+
+ def __repr__(self):
+ return '<%s %s>' % (self.__class__.__name__, self.message())
+
+
+def maybe_ensure_text(val):
+ return str(val) if val else val
+
+
+class FormatError(DecodeError):
+ verb = 'format'
+
+
+def full_cql_version(ver):
+ while ver.count('.') < 2:
+ ver += '.0'
+ ver_parts = ver.split('-', 1) + ['']
+ vertuple = tuple(list(map(int, ver_parts[0].split('.'))) + [ver_parts[1]])
+ return ver, vertuple
+
+
+def format_value(val, cqltype, encoding, addcolor=False, date_time_format=None,
+ float_precision=None, colormap=None, nullval=None):
+ if isinstance(val, DecodeError):
+ if addcolor:
+ return colorme(repr(val.thebytes), colormap, 'error')
+ else:
+ return FormattedValue(repr(val.thebytes))
+ return format_by_type(val, cqltype=cqltype, encoding=encoding, colormap=colormap,
+ addcolor=addcolor, nullval=nullval, date_time_format=date_time_format,
+ float_precision=float_precision)
+
+
+def show_warning_without_quoting_line(message, category, filename, lineno, file=None, line=None):
+ if file is None:
+ file = sys.stderr
+ try:
+ file.write(warnings.formatwarning(message, category, filename, lineno, line=''))
+ except IOError:
+ pass
+
+
+warnings.showwarning = show_warning_without_quoting_line
+warnings.filterwarnings('always', category=cql3handling.UnexpectedTableStructure)
+
+
+class Shell(cmd.Cmd):
+ custom_prompt = os.getenv('CQLSH_PROMPT', '')
+ if custom_prompt != '':
+ custom_prompt += "\n"
+ default_prompt = custom_prompt + "cqlsh> "
+ continue_prompt = " ... "
+ keyspace_prompt = custom_prompt + "cqlsh:{}> "
+ keyspace_continue_prompt = "{} ... "
+ show_line_nums = False
+ debug = False
+ coverage = False
+ coveragerc_path = None
+ stop = False
+ last_hist = None
+ shunted_query_out = None
+ use_paging = True
+
+ default_page_size = 100
+
+ def __init__(self, hostname, port, color=False,
+ username=None, encoding=None, stdin=None, tty=True,
+ completekey=DEFAULT_COMPLETEKEY, browser=None, use_conn=None,
+ cqlver=None, keyspace=None,
+ tracing_enabled=False, expand_enabled=False,
+ display_nanotime_format=DEFAULT_NANOTIME_FORMAT,
+ display_timestamp_format=DEFAULT_TIMESTAMP_FORMAT,
+ display_date_format=DEFAULT_DATE_FORMAT,
+ display_float_precision=DEFAULT_FLOAT_PRECISION,
+ display_double_precision=DEFAULT_DOUBLE_PRECISION,
+ display_timezone=None,
+ max_trace_wait=DEFAULT_MAX_TRACE_WAIT,
+ ssl=False,
+ single_statement=None,
+ request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,
+ protocol_version=None,
+ connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
+ is_subshell=False,
+ auth_provider=None):
+ cmd.Cmd.__init__(self, completekey=completekey)
+ self.hostname = hostname
+ self.port = port
+ self.auth_provider = auth_provider
+ self.username = username
+
+ if isinstance(auth_provider, PlainTextAuthProvider):
+ self.username = auth_provider.username
+ if not auth_provider.password:
+ # if no password is provided, we need to query the user to get one.
+ password = getpass.getpass()
+ self.auth_provider = PlainTextAuthProvider(username=auth_provider.username, password=password)
+
+ self.keyspace = keyspace
+ self.ssl = ssl
+ self.tracing_enabled = tracing_enabled
+ self.page_size = self.default_page_size
+ self.expand_enabled = expand_enabled
+ if use_conn:
+ self.conn = use_conn
+ else:
+ kwargs = {}
+ if protocol_version is not None:
+ kwargs['protocol_version'] = protocol_version
+ self.conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=cqlver,
+ auth_provider=self.auth_provider,
+ ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None,
+ load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
+ control_connection_timeout=connect_timeout,
+ connect_timeout=connect_timeout,
+ **kwargs)
+ self.owns_connection = not use_conn
+
+ if keyspace:
+ self.session = self.conn.connect(keyspace)
+ else:
+ self.session = self.conn.connect()
+
+ if browser == "":
+ browser = None
+ self.browser = browser
+ self.color = color
+
+ self.display_nanotime_format = display_nanotime_format
+ self.display_timestamp_format = display_timestamp_format
+ self.display_date_format = display_date_format
+
+ self.display_float_precision = display_float_precision
+ self.display_double_precision = display_double_precision
+
+ self.display_timezone = display_timezone
+
+ self.session.default_timeout = request_timeout
+ self.session.row_factory = ordered_dict_factory
+ self.session.default_consistency_level = cassandra.ConsistencyLevel.ONE
+ self.get_connection_versions()
+ self.set_expanded_cql_version(self.connection_versions['cql'])
+
+ self.current_keyspace = keyspace
+
+ self.max_trace_wait = max_trace_wait
+ self.session.max_trace_wait = max_trace_wait
+
+ self.tty = tty
+ self.encoding = encoding
+
+ self.output_codec = codecs.lookup(encoding)
+
+ self.statement = StringIO()
+ self.lineno = 1
+ self.in_comment = False
+
+ self.prompt = ''
+ if stdin is None:
+ stdin = sys.stdin
+
+ if tty:
+ self.reset_prompt()
+ self.report_connection()
+ print('Use HELP for help.')
+ else:
+ self.show_line_nums = True
+ self.stdin = stdin
+ self.query_out = sys.stdout
+ self.consistency_level = cassandra.ConsistencyLevel.ONE
+ self.serial_consistency_level = cassandra.ConsistencyLevel.SERIAL
+
+ self.empty_lines = 0
+ self.statement_error = False
+ self.single_statement = single_statement
+ self.is_subshell = is_subshell
+
+ @property
+ def batch_mode(self):
+ return not self.tty
+
+ def set_expanded_cql_version(self, ver):
+ ver, vertuple = full_cql_version(ver)
+ self.cql_version = ver
+ self.cql_ver_tuple = vertuple
+
+ def cqlver_atleast(self, major, minor=0, patch=0):
+ return self.cql_ver_tuple[:3] >= (major, minor, patch)
+
+ def myformat_value(self, val, cqltype=None, **kwargs):
+ if isinstance(val, DecodeError):
+ self.decoding_errors.append(val)
+ try:
+ dtformats = DateTimeFormat(timestamp_format=self.display_timestamp_format,
+ date_format=self.display_date_format, nanotime_format=self.display_nanotime_format,
+ timezone=self.display_timezone)
+ precision = self.display_double_precision if cqltype is not None and cqltype.type_name == 'double' \
+ else self.display_float_precision
+ return format_value(val, cqltype=cqltype, encoding=self.output_codec.name,
+ addcolor=self.color, date_time_format=dtformats,
+ float_precision=precision, **kwargs)
+ except Exception as e:
+ err = FormatError(val, e)
+ self.decoding_errors.append(err)
+ return format_value(err, cqltype=cqltype, encoding=self.output_codec.name, addcolor=self.color)
+
+ def myformat_colname(self, name, table_meta=None):
+ column_colors = COLUMN_NAME_COLORS.copy()
+ # check column role and color appropriately
+ if table_meta:
+ if name in [col.name for col in table_meta.partition_key]:
+ column_colors.default_factory = lambda: RED
+ elif name in [col.name for col in table_meta.clustering_key]:
+ column_colors.default_factory = lambda: CYAN
+ elif name in table_meta.columns and table_meta.columns[name].is_static:
+ column_colors.default_factory = lambda: WHITE
+ return self.myformat_value(name, colormap=column_colors)
+
+ def report_connection(self):
+ self.show_host()
+ self.show_version()
+
+ def show_host(self):
+ print("Connected to {0} at {1}:{2}"
+ .format(self.applycolor(self.get_cluster_name(), BLUE),
+ self.hostname,
+ self.port))
+
+ def show_version(self):
+ vers = self.connection_versions.copy()
+ vers['shver'] = version
+ # system.Versions['cql'] apparently does not reflect changes with
+ # set_cql_version.
+ vers['cql'] = self.cql_version
+ print("[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers)
+
+ def show_session(self, sessionid, partial_session=False):
+ print_trace_session(self, self.session, sessionid, partial_session)
+
+ def show_replicas(self, token_value, keyspace=None):
+ ks = self.current_keyspace if keyspace is None else keyspace
+ token_map = self.conn.metadata.token_map
+ nodes = token_map.get_replicas(ks, token_map.token_class(token_value))
+ addresses = [x.address for x in nodes]
+ print(f"{addresses}")
+
+ def get_connection_versions(self):
+ result, = self.session.execute("select * from system.local where key = 'local'")
+ vers = {
+ 'build': result['release_version'],
+ 'protocol': self.conn.protocol_version,
+ 'cql': result['cql_version'],
+ }
+ self.connection_versions = vers
+
+ def get_keyspace_names(self):
+ return list(self.conn.metadata.keyspaces)
+
+ def get_columnfamily_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return list(self.get_keyspace_meta(ksname).tables)
+
+ def get_materialized_view_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return list(self.get_keyspace_meta(ksname).views)
+
+ def get_index_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return list(self.get_keyspace_meta(ksname).indexes)
+
+ def get_column_names(self, ksname, cfname):
+ if ksname is None:
+ ksname = self.current_keyspace
+ layout = self.get_table_meta(ksname, cfname)
+ return list(layout.columns)
+
+ def get_usertype_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return list(self.get_keyspace_meta(ksname).user_types)
+
+ def get_usertype_layout(self, ksname, typename):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ ks_meta = self.get_keyspace_meta(ksname)
+
+ try:
+ user_type = ks_meta.user_types[typename]
+ except KeyError:
+ raise UserTypeNotFound("User type {!r} not found".format(typename))
+
+ return list(zip(user_type.field_names, user_type.field_types))
+
+ def get_userfunction_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return [f.name for f in list(self.get_keyspace_meta(ksname).functions.values())]
+
+ def get_useraggregate_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return [f.name for f in list(self.get_keyspace_meta(ksname).aggregates.values())]
+
+ def get_cluster_name(self):
+ return self.conn.metadata.cluster_name
+
+ def get_partitioner(self):
+ return self.conn.metadata.partitioner
+
+ def get_keyspace_meta(self, ksname):
+ if ksname in self.conn.metadata.keyspaces:
+ return self.conn.metadata.keyspaces[ksname]
+
+ raise KeyspaceNotFound('Keyspace %r not found.' % ksname)
+
+ def get_keyspaces(self):
+ return list(self.conn.metadata.keyspaces.values())
+
+ def get_ring(self, ks):
+ self.conn.metadata.token_map.rebuild_keyspace(ks, build_if_absent=True)
+ return self.conn.metadata.token_map.tokens_to_hosts_by_ks[ks]
+
+ def get_table_meta(self, ksname, tablename):
+ if ksname is None:
+ ksname = self.current_keyspace
+ ksmeta = self.get_keyspace_meta(ksname)
+ if tablename not in ksmeta.tables:
+ if ksname == 'system_auth' and tablename in ['roles', 'role_permissions']:
+ self.get_fake_auth_table_meta(ksname, tablename)
+ else:
+ raise ColumnFamilyNotFound("Column family {} not found".format(tablename))
+ else:
+ return ksmeta.tables[tablename]
+
+ def get_fake_auth_table_meta(self, ksname, tablename):
+ # may be using external auth implementation so internal tables
+ # aren't actually defined in schema. In this case, we'll fake
+ # them up
+ if tablename == 'roles':
+ ks_meta = KeyspaceMetadata(ksname, True, None, None)
+ table_meta = TableMetadata(ks_meta, 'roles')
+ table_meta.columns['role'] = ColumnMetadata(table_meta, 'role', cassandra.cqltypes.UTF8Type)
+ table_meta.columns['is_superuser'] = ColumnMetadata(table_meta, 'is_superuser', cassandra.cqltypes.BooleanType)
+ table_meta.columns['can_login'] = ColumnMetadata(table_meta, 'can_login', cassandra.cqltypes.BooleanType)
+ elif tablename == 'role_permissions':
+ ks_meta = KeyspaceMetadata(ksname, True, None, None)
+ table_meta = TableMetadata(ks_meta, 'role_permissions')
+ table_meta.columns['role'] = ColumnMetadata(table_meta, 'role', cassandra.cqltypes.UTF8Type)
+ table_meta.columns['resource'] = ColumnMetadata(table_meta, 'resource', cassandra.cqltypes.UTF8Type)
+ table_meta.columns['permission'] = ColumnMetadata(table_meta, 'permission', cassandra.cqltypes.UTF8Type)
+ else:
+ raise ColumnFamilyNotFound("Column family {} not found".format(tablename))
+
+ def get_index_meta(self, ksname, idxname):
+ if ksname is None:
+ ksname = self.current_keyspace
+ ksmeta = self.get_keyspace_meta(ksname)
+
+ if idxname not in ksmeta.indexes:
+ raise IndexNotFound("Index {} not found".format(idxname))
+
+ return ksmeta.indexes[idxname]
+
+ def get_view_meta(self, ksname, viewname):
+ if ksname is None:
+ ksname = self.current_keyspace
+ ksmeta = self.get_keyspace_meta(ksname)
+
+ if viewname not in ksmeta.views:
+ raise MaterializedViewNotFound("Materialized view '{}' not found".format(viewname))
+ return ksmeta.views[viewname]
+
+ def get_object_meta(self, ks, name):
+ if name is None:
+ if ks and ks in self.conn.metadata.keyspaces:
+ return self.conn.metadata.keyspaces[ks]
+ elif self.current_keyspace is None:
+ raise ObjectNotFound("'{}' not found in keyspaces".format(ks))
+ else:
+ name = ks
+ ks = self.current_keyspace
+
+ if ks is None:
+ ks = self.current_keyspace
+
+ ksmeta = self.get_keyspace_meta(ks)
+
+ if name in ksmeta.tables:
+ return ksmeta.tables[name]
+ elif name in ksmeta.indexes:
+ return ksmeta.indexes[name]
+ elif name in ksmeta.views:
+ return ksmeta.views[name]
+
+ raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks))
+
+ def get_trigger_names(self, ksname=None):
+ if ksname is None:
+ ksname = self.current_keyspace
+
+ return [trigger.name
+ for table in list(self.get_keyspace_meta(ksname).tables.values())
+ for trigger in list(table.triggers.values())]
+
+ def reset_statement(self):
+ self.reset_prompt()
+ self.statement.truncate(0)
+ self.statement.seek(0)
+ self.empty_lines = 0
+
+ def reset_prompt(self):
+ if self.current_keyspace is None:
+ self.set_prompt(self.default_prompt, True)
+ else:
+ self.set_prompt(self.keyspace_prompt.format(self.current_keyspace), True)
+
+ def set_continue_prompt(self):
+ if self.empty_lines >= 3:
+ self.set_prompt("Statements are terminated with a ';'. You can press CTRL-C to cancel an incomplete statement.")
+ self.empty_lines = 0
+ return
+ if self.current_keyspace is None:
+ self.set_prompt(self.continue_prompt)
+ else:
+ spaces = ' ' * len(str(self.current_keyspace))
+ self.set_prompt(self.keyspace_continue_prompt.format(spaces))
+ self.empty_lines = self.empty_lines + 1 if not self.lastcmd else 0
+
+ @contextmanager
+ def prepare_loop(self):
+ readline = None
+ if self.tty and self.completekey:
+ try:
+ import readline
+ except ImportError:
+ pass
+ else:
+ old_completer = readline.get_completer()
+ readline.set_completer(self.complete)
+ if readline.__doc__ is not None and 'libedit' in readline.__doc__:
+ readline.parse_and_bind("bind -e")
+ readline.parse_and_bind("bind '" + self.completekey + "' rl_complete")
+ readline.parse_and_bind("bind ^R em-inc-search-prev")
+ else:
+ readline.parse_and_bind(self.completekey + ": complete")
+ # start coverage collection if requested, unless in subshell
+ if self.coverage and not self.is_subshell:
+ # check for coveragerc file, write it if missing
+ if os.path.exists(CQL_DIR):
+ self.coveragerc_path = os.path.join(CQL_DIR, '.coveragerc')
+ covdata_path = os.path.join(CQL_DIR, '.coverage')
+ if not os.path.isfile(self.coveragerc_path):
+ with open(self.coveragerc_path, 'w') as f:
+ f.writelines(["[run]\n",
+ "concurrency = multiprocessing\n",
+ "data_file = {}\n".format(covdata_path),
+ "parallel = true\n"]
+ )
+ # start coverage
+ import coverage
+ self.cov = coverage.Coverage(config_file=self.coveragerc_path)
+ self.cov.start()
+ try:
+ yield
+ finally:
+ if readline is not None:
+ readline.set_completer(old_completer)
+ if self.coverage and not self.is_subshell:
+ self.stop_coverage()
+
+ def get_input_line(self, prompt=''):
+ if self.tty:
+ self.lastcmd = input(str(prompt))
+ line = self.lastcmd + '\n'
+ else:
+ self.lastcmd = self.stdin.readline()
+ line = self.lastcmd
+ if not len(line):
+ raise EOFError
+ self.lineno += 1
+ return line
+
+ def use_stdin_reader(self, until='', prompt=''):
+ until += '\n'
+ while True:
+ try:
+ newline = self.get_input_line(prompt=prompt)
+ except EOFError:
+ return
+ if newline == until:
+ return
+ yield newline
+
+ def cmdloop(self, intro=None):
+ """
+ Adapted from cmd.Cmd's version, because there is literally no way with
+ cmd.Cmd.cmdloop() to tell the difference between "EOF" showing up in
+ input and an actual EOF.
+ """
+ with self.prepare_loop():
+ while not self.stop:
+ try:
+ if self.single_statement:
+ line = self.single_statement
+ self.stop = True
+ else:
+ line = self.get_input_line(self.prompt)
+ self.statement.write(line)
+ if self.onecmd(self.statement.getvalue()):
+ self.reset_statement()
+ except EOFError:
+ self.handle_eof()
+ except CQL_ERRORS as cqlerr:
+ self.printerr(cqlerr.message)
+ except KeyboardInterrupt:
+ self.reset_statement()
+ print('')
+
+ def strip_comment_blocks(self, statementtext):
+ comment_block_in_literal_string = re.search('["].*[/][*].*[*][/].*["]', statementtext)
+ if not comment_block_in_literal_string:
+ result = re.sub('[/][*].*[*][/]', "", statementtext)
+ if '*/' in result and '/*' not in result and not self.in_comment:
+ raise SyntaxError("Encountered comment block terminator without being in comment block")
+ if '/*' in result:
+ result = re.sub('[/][*].*', "", result)
+ self.in_comment = True
+ if '*/' in result:
+ result = re.sub('.*[*][/]', "", result)
+ self.in_comment = False
+ if self.in_comment and not re.findall('[/][*]|[*][/]', statementtext):
+ result = ''
+ return result
+ return statementtext
+
+ def onecmd(self, statementtext):
+ """
+ Returns true if the statement is complete and was handled (meaning it
+ can be reset).
+ """
+ statementtext = self.strip_comment_blocks(statementtext)
+ try:
+ statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext)
+ except pylexotron.LexingError as e:
+ if self.show_line_nums:
+ self.printerr('Invalid syntax at line {0}, char {1}'
+ .format(e.linenum, e.charnum))
+ else:
+ self.printerr('Invalid syntax at char {0}'.format(e.charnum))
+ statementline = statementtext.split('\n')[e.linenum - 1]
+ self.printerr(' {0}'.format(statementline))
+ self.printerr(' {0}^'.format(' ' * e.charnum))
+ return True
+
+ while statements and not statements[-1]:
+ statements = statements[:-1]
+ if not statements:
+ return True
+ if endtoken_escaped or statements[-1][-1][0] != 'endtoken':
+ self.set_continue_prompt()
+ return
+ for st in statements:
+ try:
+ self.handle_statement(st, statementtext)
+ except Exception as e:
+ if self.debug:
+ traceback.print_exc()
+ else:
+ self.printerr(e)
+ return True
+
+ def handle_eof(self):
+ if self.tty:
+ print('')
+ statement = self.statement.getvalue()
+ if statement.strip():
+ if not self.onecmd(statement):
+ self.printerr('Incomplete statement at end of file')
+ self.do_exit()
+
+ def handle_statement(self, tokens, srcstr):
+ # Concat multi-line statements and insert into history
+ if readline is not None:
+ nl_count = srcstr.count("\n")
+
+ new_hist = srcstr.replace("\n", " ").rstrip()
+
+ if nl_count > 1 and self.last_hist != new_hist:
+ readline.add_history(new_hist)
+
+ self.last_hist = new_hist
+ cmdword = tokens[0][1]
+ if cmdword == '?':
+ cmdword = 'help'
+ custom_handler = getattr(self, 'do_' + cmdword.lower(), None)
+ if custom_handler:
+ parsed = cqlruleset.cql_whole_parse_tokens(tokens, srcstr=srcstr,
+ startsymbol='cqlshCommand')
+ if parsed and not parsed.remainder:
+ # successful complete parse
+ return custom_handler(parsed)
+ else:
+ return self.handle_parse_error(cmdword, tokens, parsed, srcstr)
+ return self.perform_statement(cqlruleset.cql_extract_orig(tokens, srcstr))
+
+ def handle_parse_error(self, cmdword, tokens, parsed, srcstr):
+ if cmdword.lower() in ('select', 'insert', 'update', 'delete', 'truncate',
+ 'create', 'drop', 'alter', 'grant', 'revoke',
+ 'batch', 'list'):
+ # hey, maybe they know about some new syntax we don't. type
+ # assumptions won't work, but maybe the query will.
+ return self.perform_statement(cqlruleset.cql_extract_orig(tokens, srcstr))
+ if parsed:
+ self.printerr('Improper %s command (problem at %r).' % (cmdword, parsed.remainder[0]))
+ else:
+ self.printerr(f'Improper {cmdword} command.')
+
+ def do_use(self, parsed):
+ ksname = parsed.get_binding('ksname')
+ success, _ = self.perform_simple_statement(SimpleStatement(parsed.extract_orig()))
+ if success:
+ if ksname[0] == '"' and ksname[-1] == '"':
+ self.current_keyspace = self.cql_unprotect_name(ksname)
+ else:
+ self.current_keyspace = ksname.lower()
+
+ def do_select(self, parsed):
+ tracing_was_enabled = self.tracing_enabled
+ ksname = parsed.get_binding('ksname')
+ stop_tracing = ksname == 'system_traces' or (ksname is None and self.current_keyspace == 'system_traces')
+ self.tracing_enabled = self.tracing_enabled and not stop_tracing
+ statement = parsed.extract_orig()
+ self.perform_statement(statement)
+ self.tracing_enabled = tracing_was_enabled
+
+ def perform_statement(self, statement):
+
+ stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None)
+ success, future = self.perform_simple_statement(stmt)
+
+ if future:
+ if future.warnings:
+ self.print_warnings(future.warnings)
+
+ if self.tracing_enabled:
+ try:
+ for trace in future.get_all_query_traces(max_wait_per=self.max_trace_wait, query_cl=self.consistency_level):
+ print_trace(self, trace)
+ except TraceUnavailable:
+ msg = "Statement trace did not complete within %d seconds; trace data may be incomplete." % (self.session.max_trace_wait,)
+ self.writeresult(msg, color=RED)
+ for trace_id in future.get_query_trace_ids():
+ self.show_session(trace_id, partial_session=True)
+ except Exception as err:
+ self.printerr("Unable to fetch query trace: %s" % (str(err),))
+
+ return success
+
+ def parse_for_select_meta(self, query_string):
+ try:
+ parsed = cqlruleset.cql_parse(query_string)[1]
+ except IndexError:
+ return None
+ ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
+ name = self.cql_unprotect_name(parsed.get_binding('cfname', None))
+ try:
+ return self.get_table_meta(ks, name)
+ except ColumnFamilyNotFound:
+ try:
+ return self.get_view_meta(ks, name)
+ except MaterializedViewNotFound:
+ raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks))
+
+ def parse_for_update_meta(self, query_string):
+ try:
+ parsed = cqlruleset.cql_parse(query_string)[1]
+ except IndexError:
+ return None
+ ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
+ cf = self.cql_unprotect_name(parsed.get_binding('cfname'))
+ return self.get_table_meta(ks, cf)
+
+ def perform_simple_statement(self, statement):
+ if not statement:
+ return False, None
+
+ future = self.session.execute_async(statement, trace=self.tracing_enabled)
+ result = None
+ try:
+ result = future.result()
+ except CQL_ERRORS as err:
+ err_msg = err.message if hasattr(err, 'message') else str(err)
+ self.printerr(str(err.__class__.__name__) + ": " + err_msg)
+ except Exception:
+ import traceback
+ self.printerr(traceback.format_exc())
+
+ # Even if statement failed we try to refresh schema if not agreed (see CASSANDRA-9689)
+ if not future.is_schema_agreed:
+ try:
+ self.conn.refresh_schema_metadata(5) # will throw exception if there is a schema mismatch
+ except Exception:
+ self.printerr("Warning: schema version mismatch detected; check the schema versions of your "
+ "nodes in system.local and system.peers.")
+ self.conn.refresh_schema_metadata(-1)
+
+ if result is None:
+ return False, None
+
+ if statement.query_string[:6].lower() == 'select':
+ self.print_result(result, self.parse_for_select_meta(statement.query_string))
+ elif statement.query_string.lower().startswith("list users") or statement.query_string.lower().startswith("list roles"):
+ self.print_result(result, self.get_table_meta('system_auth', 'roles'))
+ elif statement.query_string.lower().startswith("list"):
+ self.print_result(result, self.get_table_meta('system_auth', 'role_permissions'))
+ elif result:
+ # CAS INSERT/UPDATE
+ self.writeresult("")
+ self.print_static_result(result, self.parse_for_update_meta(statement.query_string), with_header=True, tty=self.tty)
+ self.flush_output()
+ return True, future
+
+ def print_result(self, result, table_meta):
+ self.decoding_errors = []
+
+ self.writeresult("")
+
+ def print_all(result, table_meta, tty):
+ # Return the number of rows in total
+ num_rows = 0
+ is_first = True
+ while True:
+ # Always print for the first page even it is empty
+ if result.current_rows or is_first:
+ with_header = is_first or tty
+ self.print_static_result(result, table_meta, with_header, tty, num_rows)
+ num_rows += len(result.current_rows)
+ if result.has_more_pages:
+ if self.shunted_query_out is None and tty:
+ # Only pause when not capturing.
+ input("---MORE---")
+ result.fetch_next_page()
+ else:
+ if not tty:
+ self.writeresult("")
+ break
+ is_first = False
+ return num_rows
+
+ num_rows = print_all(result, table_meta, self.tty)
+ self.writeresult("(%d rows)" % num_rows)
+
+ if self.decoding_errors:
+ for err in self.decoding_errors[:2]:
+ self.writeresult(err.message(), color=RED)
+ if len(self.decoding_errors) > 2:
+ self.writeresult('%d more decoding errors suppressed.'
+ % (len(self.decoding_errors) - 2), color=RED)
+
+ def print_static_result(self, result, table_meta, with_header, tty, row_count_offset=0):
+ if not result.column_names and not table_meta:
+ return
+
+ column_names = result.column_names or list(table_meta.columns.keys())
+ formatted_names = [self.myformat_colname(name, table_meta) for name in column_names]
+ if not result.current_rows:
+ # print header only
+ self.print_formatted_result(formatted_names, None, with_header=True, tty=tty)
+ return
+
+ cql_types = []
+ if result.column_types:
+ ks_name = table_meta.keyspace_name if table_meta else self.current_keyspace
+ ks_meta = self.conn.metadata.keyspaces.get(ks_name, None)
+ cql_types = [CqlType(cql_typename(t), ks_meta) for t in result.column_types]
+
+ formatted_values = [list(map(self.myformat_value, [row[c] for c in column_names], cql_types)) for row in result.current_rows]
+
+ if self.expand_enabled:
+ self.print_formatted_result_vertically(formatted_names, formatted_values, row_count_offset)
+ else:
+ self.print_formatted_result(formatted_names, formatted_values, with_header, tty)
+
+ def print_formatted_result(self, formatted_names, formatted_values, with_header, tty):
+ # determine column widths
+ widths = [n.displaywidth for n in formatted_names]
+ if formatted_values is not None:
+ for fmtrow in formatted_values:
+ for num, col in enumerate(fmtrow):
+ widths[num] = max(widths[num], col.displaywidth)
+
+ # print header
+ if with_header:
+ header = ' | '.join(hdr.ljust(w, color=self.color) for (hdr, w) in zip(formatted_names, widths))
+ self.writeresult(' ' + header.rstrip())
+ self.writeresult('-%s-' % '-+-'.join('-' * w for w in widths))
+
+ # stop if there are no rows
+ if formatted_values is None:
+ self.writeresult("")
+ return
+
+ # print row data
+ for row in formatted_values:
+ line = ' | '.join(col.rjust(w, color=self.color) for (col, w) in zip(row, widths))
+ self.writeresult(' ' + line)
+
+ if tty:
+ self.writeresult("")
+
+ def print_formatted_result_vertically(self, formatted_names, formatted_values, row_count_offset):
+ max_col_width = max([n.displaywidth for n in formatted_names])
+ max_val_width = max([n.displaywidth for row in formatted_values for n in row])
+
+ # for each row returned, list all the column-value pairs
+ for i, row in enumerate(formatted_values):
+ self.writeresult("@ Row %d" % (row_count_offset + i + 1))
+ self.writeresult('-%s-' % '-+-'.join(['-' * max_col_width, '-' * max_val_width]))
+ for field_id, field in enumerate(row):
+ column = formatted_names[field_id].ljust(max_col_width, color=self.color)
+ value = field.ljust(field.displaywidth, color=self.color)
+ self.writeresult(' ' + " | ".join([column, value]))
+ self.writeresult('')
+
+ def print_warnings(self, warnings):
+ if warnings is None or len(warnings) == 0:
+ return
+
+ self.writeresult('')
+ self.writeresult('Warnings :')
+ for warning in warnings:
+ self.writeresult(warning)
+ self.writeresult('')
+
+ def emptyline(self):
+ pass
+
+ def parseline(self, line):
+ # this shouldn't be needed
+ raise NotImplementedError
+
+ def complete(self, text, state):
+ if readline is None:
+ return
+ if state == 0:
+ try:
+ self.completion_matches = self.find_completions(text)
+ except Exception:
+ if debug_completion:
+ import traceback
+ traceback.print_exc()
+ else:
+ raise
+ try:
+ return self.completion_matches[state]
+ except IndexError:
+ return None
+
+ def find_completions(self, text):
+ curline = readline.get_line_buffer()
+ prevlines = self.statement.getvalue()
+ wholestmt = prevlines + curline
+ begidx = readline.get_begidx() + len(prevlines)
+ stuff_to_complete = wholestmt[:begidx]
+ return cqlruleset.cql_complete(stuff_to_complete, text, cassandra_conn=self,
+ debug=debug_completion, startsymbol='cqlshCommand')
+
+ def set_prompt(self, prompt, prepend_user=False):
+ if prepend_user and self.username:
+ self.prompt = "{0}@{1}".format(self.username, prompt)
+ return
+ self.prompt = prompt
+
+ def cql_unprotect_name(self, namestr):
+ if namestr is None:
+ return
+ return cqlruleset.dequote_name(namestr)
+
+ def cql_unprotect_value(self, valstr):
+ if valstr is not None:
+ return cqlruleset.dequote_value(valstr)
+
+ def _columnize_unicode(self, name_list):
+ """
+ Used when columnizing identifiers that may contain unicode
+ """
+ names = [n for n in name_list]
+ cmd.Cmd.columnize(self, names)
+ print('')
+
+ def do_describe(self, parsed):
+
+ """
+ DESCRIBE [cqlsh only]
+
+ (DESC may be used as a shorthand.)
+
+ Outputs information about the connected Cassandra cluster, or about
+ the data objects stored in the cluster. Use in one of the following ways:
+
+ DESCRIBE KEYSPACES
+
+ Output the names of all keyspaces.
+
+ DESCRIBE KEYSPACE [<keyspacename>]
+
+ Output CQL commands that could be used to recreate the given keyspace,
+ and the objects in it (such as tables, types, functions, etc.).
+ In some cases, as the CQL interface matures, there will be some metadata
+ about a keyspace that is not representable with CQL. That metadata will not be shown.
+ The '<keyspacename>' argument may be omitted, in which case the current
+ keyspace will be described.
+
+ DESCRIBE TABLES
+
+ Output the names of all tables in the current keyspace, or in all
+ keyspaces if there is no current keyspace.
+
+ DESCRIBE TABLE [<keyspace>.]<tablename>
+
+ Output CQL commands that could be used to recreate the given table.
+ In some cases, as above, there may be table metadata which is not
+ representable and which will not be shown.
+
+ DESCRIBE INDEX <indexname>
+
+ Output the CQL command that could be used to recreate the given index.
+ In some cases, there may be index metadata which is not representable
+ and which will not be shown.
+
+ DESCRIBE MATERIALIZED VIEW <viewname>
+
+ Output the CQL command that could be used to recreate the given materialized view.
+ In some cases, there may be materialized view metadata which is not representable
+ and which will not be shown.
+
+ DESCRIBE CLUSTER
+
+ Output information about the connected Cassandra cluster, such as the
+ cluster name, and the partitioner and snitch in use. When you are
+ connected to a non-system keyspace, also shows endpoint-range
+ ownership information for the Cassandra ring.
+
+ DESCRIBE [FULL] SCHEMA
+
+ Output CQL commands that could be used to recreate the entire (non-system) schema.
+ Works as though "DESCRIBE KEYSPACE k" was invoked for each non-system keyspace
+ k. Use DESCRIBE FULL SCHEMA to include the system keyspaces.
+
+ DESCRIBE TYPES
+
+ Output the names of all user-defined-types in the current keyspace, or in all
+ keyspaces if there is no current keyspace.
+
+ DESCRIBE TYPE [<keyspace>.]<type>
+
+ Output the CQL command that could be used to recreate the given user-defined-type.
+
+ DESCRIBE FUNCTIONS
+
+ Output the names of all user-defined-functions in the current keyspace, or in all
+ keyspaces if there is no current keyspace.
+
+ DESCRIBE FUNCTION [<keyspace>.]<function>
+
+ Output the CQL command that could be used to recreate the given user-defined-function.
+
+ DESCRIBE AGGREGATES
+
+ Output the names of all user-defined-aggregates in the current keyspace, or in all
+ keyspaces if there is no current keyspace.
+
+ DESCRIBE AGGREGATE [<keyspace>.]<aggregate>
+
+ Output the CQL command that could be used to recreate the given user-defined-aggregate.
+
+ DESCRIBE <objname>
+
+ Output CQL commands that could be used to recreate the entire object schema,
+ where object can be either a keyspace or a table or an index or a materialized
+ view (in this order).
+ """
+ stmt = SimpleStatement(parsed.extract_orig(), consistency_level=cassandra.ConsistencyLevel.LOCAL_ONE, fetch_size=self.page_size if self.use_paging else None)
+ future = self.session.execute_async(stmt)
+
+ if self.connection_versions['build'][0] < '4':
+ print('\nWARN: DESCRIBE|DESC was moved to server side in Cassandra 4.0. As a consequence DESRIBE|DESC '
+ 'will not work in cqlsh %r connected to Cassandra %r, the version that you are connected to. '
+ 'DESCRIBE does not exist server side prior Cassandra 4.0.'
+ % (version, self.connection_versions['build']))
+ else:
+ try:
+ result = future.result()
+
+ what = parsed.matched[1][1].lower()
+
+ if what in ('columnfamilies', 'tables', 'types', 'functions', 'aggregates'):
+ self.describe_list(result)
+ elif what == 'keyspaces':
+ self.describe_keyspaces(result)
+ elif what == 'cluster':
+ self.describe_cluster(result)
+ elif what:
+ self.describe_element(result)
+
+ except CQL_ERRORS as err:
+ err_msg = err.message if hasattr(err, 'message') else str(err)
+ self.printerr(err_msg.partition("message=")[2].strip('"'))
+ except Exception:
+ import traceback
+ self.printerr(traceback.format_exc())
+
+ if future:
+ if future.warnings:
+ self.print_warnings(future.warnings)
+
+ do_desc = do_describe
+
+ def describe_keyspaces(self, rows):
+ """
+ Print the output for a DESCRIBE KEYSPACES query
+ """
+ names = [r['name'] for r in rows]
+
+ print('')
+ cmd.Cmd.columnize(self, names)
+ print('')
+
+ def describe_list(self, rows):
+ """
+ Print the output for all the DESCRIBE queries for element names (e.g DESCRIBE TABLES, DESCRIBE FUNCTIONS ...)
+ """
+ keyspace = None
+ names = list()
+ for row in rows:
+ if row['keyspace_name'] != keyspace:
+ if keyspace is not None:
+ self.print_keyspace_element_names(keyspace, names)
+
+ keyspace = row['keyspace_name']
+ names = list()
+
+ names.append(str(row['name']))
+
+ if keyspace is not None:
+ self.print_keyspace_element_names(keyspace, names)
+ print('')
+
+ def print_keyspace_element_names(self, keyspace, names):
+ print('')
+ if self.current_keyspace is None:
+ print('Keyspace %s' % (keyspace))
+ print('---------%s' % ('-' * len(keyspace)))
+ cmd.Cmd.columnize(self, names)
+
+ def describe_element(self, rows):
+ """
+ Print the output for all the DESCRIBE queries where an element name as been specified (e.g DESCRIBE TABLE, DESCRIBE INDEX ...)
+ """
+ for row in rows:
+ print('')
+ self.query_out.write(row['create_statement'])
+ print('')
+
+ def describe_cluster(self, rows):
+ """
+ Print the output for a DESCRIBE CLUSTER query.
+
+ If a specified keyspace was in use the returned ResultSet will contains a 'range_ownership' column,
+ otherwise not.
+ """
+ for row in rows:
+ print('\nCluster: %s' % row['cluster'])
+ print('Partitioner: %s' % row['partitioner'])
+ print('Snitch: %s\n' % row['snitch'])
+ if 'range_ownership' in row:
+ print("Range ownership:")
+ for entry in list(row['range_ownership'].items()):
+ print(' %39s [%s]' % (entry[0], ', '.join([host for host in entry[1]])))
+ print('')
+
+ def do_copy(self, parsed):
+ r"""
+ COPY [cqlsh only]
+
+ COPY x FROM: Imports CSV data into a Cassandra table
+ COPY x TO: Exports data from a Cassandra table in CSV format.
+
+ COPY <table_name> [ ( column [, ...] ) ]
+ FROM ( '<file_pattern_1, file_pattern_2, ... file_pattern_n>' | STDIN )
+ [ WITH <option>='value' [AND ...] ];
+
+ File patterns are either file names or valid python glob expressions, e.g. *.csv or folder/*.csv.
+
+ COPY <table_name> [ ( column [, ...] ) ]
+ TO ( '<filename>' | STDOUT )
+ [ WITH <option>='value' [AND ...] ];
+
+ Available common COPY options and defaults:
+
+ DELIMITER=',' - character that appears between records
+ QUOTE='"' - quoting character to be used to quote fields
+ ESCAPE='\' - character to appear before the QUOTE char when quoted
+ HEADER=false - whether to ignore the first line
+ NULL='' - string that represents a null value
+ DATETIMEFORMAT= - timestamp strftime format
+ '%Y-%m-%d %H:%M:%S%z' defaults to time_format value in cqlshrc
+ MAXATTEMPTS=5 - the maximum number of attempts per batch or range
+ REPORTFREQUENCY=0.25 - the frequency with which we display status updates in seconds
+ DECIMALSEP='.' - the separator for decimal values
+ THOUSANDSSEP='' - the separator for thousands digit groups
+ BOOLSTYLE='True,False' - the representation for booleans, case insensitive, specify true followed by false,
+ for example yes,no or 1,0
+ NUMPROCESSES=n - the number of worker processes, by default the number of cores minus one
+ capped at 16
+ CONFIGFILE='' - a configuration file with the same format as .cqlshrc (see the Python ConfigParser
+ documentation) where you can specify WITH options under the following optional
+ sections: [copy], [copy-to], [copy-from], [copy:ks.table], [copy-to:ks.table],
+ [copy-from:ks.table], where <ks> is your keyspace name and <table> is your table
+ name. Options are read from these sections, in the order specified
+ above, and command line options always override options in configuration files.
+ Depending on the COPY direction, only the relevant copy-from or copy-to sections
+ are used. If no configfile is specified then .cqlshrc is searched instead.
+ RATEFILE='' - an optional file where to print the output statistics
+
+ Available COPY FROM options and defaults:
+
+ CHUNKSIZE=5000 - the size of chunks passed to worker processes
+ INGESTRATE=100000 - an approximate ingest rate in rows per second
+ MINBATCHSIZE=10 - the minimum size of an import batch
+ MAXBATCHSIZE=20 - the maximum size of an import batch
+ MAXROWS=-1 - the maximum number of rows, -1 means no maximum
+ SKIPROWS=0 - the number of rows to skip
+ SKIPCOLS='' - a comma separated list of column names to skip
+ MAXPARSEERRORS=-1 - the maximum global number of parsing errors, -1 means no maximum
+ MAXINSERTERRORS=1000 - the maximum global number of insert errors, -1 means no maximum
+ ERRFILE='' - a file where to store all rows that could not be imported, by default this is
+ import_ks_table.err where <ks> is your keyspace and <table> is your table name.
+ PREPAREDSTATEMENTS=True - whether to use prepared statements when importing, by default True. Set this to
+ False if you don't mind shifting data parsing to the cluster. The cluster will also
+ have to compile every batch statement. For large and oversized clusters
+ this will result in a faster import but for smaller clusters it may generate
+ timeouts.
+ TTL=3600 - the time to live in seconds, by default data will not expire
+
+ Available COPY TO options and defaults:
+
+ ENCODING='utf8' - encoding for CSV output
+ PAGESIZE='1000' - the page size for fetching results
+ PAGETIMEOUT=10 - the page timeout in seconds for fetching results
+ BEGINTOKEN='' - the minimum token string to consider when exporting data
+ ENDTOKEN='' - the maximum token string to consider when exporting data
+ MAXREQUESTS=6 - the maximum number of requests each worker process can work on in parallel
+ MAXOUTPUTSIZE='-1' - the maximum size of the output file measured in number of lines,
+ beyond this maximum the output file will be split into segments,
+ -1 means unlimited.
+ FLOATPRECISION=5 - the number of digits displayed after the decimal point for cql float values
+ DOUBLEPRECISION=12 - the number of digits displayed after the decimal point for cql double values
+
+ When entering CSV data on STDIN, you can use the sequence "\."
+ on a line by itself to end the data input.
+ """
+
+ ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
+ if ks is None:
+ ks = self.current_keyspace
+ if ks is None:
+ raise NoKeyspaceError("Not in any keyspace.")
+ table = self.cql_unprotect_name(parsed.get_binding('cfname'))
+ columns = parsed.get_binding('colnames', None)
+ if columns is not None:
+ columns = list(map(self.cql_unprotect_name, columns))
+ else:
+ # default to all known columns
+ columns = self.get_column_names(ks, table)
+
+ fname = parsed.get_binding('fname', None)
+ if fname is not None:
+ fname = self.cql_unprotect_value(fname)
+
+ copyoptnames = list(map(str.lower, parsed.get_binding('optnames', ())))
+ copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ())))
+ opts = dict(list(zip(copyoptnames, copyoptvals)))
+
+ direction = parsed.get_binding('dir').upper()
+ if direction == 'FROM':
+ task = ImportTask(self, ks, table, columns, fname, opts, self.conn.protocol_version, CONFIG_FILE)
+ elif direction == 'TO':
+ task = ExportTask(self, ks, table, columns, fname, opts, self.conn.protocol_version, CONFIG_FILE)
+ else:
+ raise SyntaxError("Unknown direction %s" % direction)
+
+ task.run()
+
+ def do_show(self, parsed):
+ """
+ SHOW [cqlsh only]
+
+ Displays information about the current cqlsh session. Can be called in
+ the following ways:
+
+ SHOW VERSION
+
+ Shows the version and build of the connected Cassandra instance, as
+ well as the version of the CQL spec that the connected Cassandra
+ instance understands.
+
+ SHOW HOST
+
+ Shows where cqlsh is currently connected.
+
+ SHOW SESSION <sessionid>
+
+ Pretty-prints the requested tracing session.
+
+ SHOW REPLICAS <token> (<keyspace>)
+
+ Lists the replica nodes by IP address for the given token. The current
+ keyspace is used if one is not specified.
+ """
+ showwhat = parsed.get_binding('what').lower()
+ if showwhat == 'version':
+ self.get_connection_versions()
+ self.show_version()
+ elif showwhat == 'host':
+ self.show_host()
+ elif showwhat.startswith('session'):
+ session_id = parsed.get_binding('sessionid').lower()
+ self.show_session(UUID(session_id))
+ elif showwhat.startswith('replicas'):
+ token_id = parsed.get_binding('token')
+ keyspace = parsed.get_binding('keyspace')
+ self.show_replicas(token_id, keyspace)
+ else:
+ self.printerr('Wait, how do I show %r?' % (showwhat,))
+
+ def do_source(self, parsed):
+ """
+ SOURCE [cqlsh only]
+
+ Executes a file containing CQL statements. Gives the output for each
+ statement in turn, if any, or any errors that occur along the way.
+
+ Errors do NOT abort execution of the CQL source file.
+
+ Usage:
+
+ SOURCE '<file>';
+
+ That is, the path to the file to be executed must be given inside a
+ string literal. The path is interpreted relative to the current working
+ directory. The tilde shorthand notation ('~/mydir') is supported for
+ referring to $HOME.
+
+ See also the --file option to cqlsh.
+ """
+ fname = parsed.get_binding('fname')
+ fname = os.path.expanduser(self.cql_unprotect_value(fname))
+ try:
+ encoding, bom_size = get_file_encoding_bomsize(fname)
+ f = codecs.open(fname, 'r', encoding)
+ f.seek(bom_size)
+ except IOError as e:
+ self.printerr('Could not open %r: %s' % (fname, e))
+ return
+ subshell = Shell(self.hostname, self.port, color=self.color,
+ username=self.username,
+ encoding=self.encoding, stdin=f, tty=False, use_conn=self.conn,
+ cqlver=self.cql_version, keyspace=self.current_keyspace,
+ tracing_enabled=self.tracing_enabled,
+ display_nanotime_format=self.display_nanotime_format,
+ display_timestamp_format=self.display_timestamp_format,
+ display_date_format=self.display_date_format,
+ display_float_precision=self.display_float_precision,
+ display_double_precision=self.display_double_precision,
+ display_timezone=self.display_timezone,
+ max_trace_wait=self.max_trace_wait, ssl=self.ssl,
+ request_timeout=self.session.default_timeout,
+ connect_timeout=self.conn.connect_timeout,
+ is_subshell=True,
+ auth_provider=self.auth_provider)
+ # duplicate coverage related settings in subshell
+ if self.coverage:
+ subshell.coverage = True
+ subshell.coveragerc_path = self.coveragerc_path
+ subshell.cmdloop()
+ f.close()
+
+ def do_capture(self, parsed):
+ """
+ CAPTURE [cqlsh only]
+
+ Begins capturing command output and appending it to a specified file.
+ Output will not be shown at the console while it is captured.
+
+ Usage:
+
+ CAPTURE '<file>';
+ CAPTURE OFF;
+ CAPTURE;
+
+ That is, the path to the file to be appended to must be given inside a
+ string literal. The path is interpreted relative to the current working
+ directory. The tilde shorthand notation ('~/mydir') is supported for
+ referring to $HOME.
+
+ Only query result output is captured. Errors and output from cqlsh-only
+ commands will still be shown in the cqlsh session.
+
+ To stop capturing output and show it in the cqlsh session again, use
+ CAPTURE OFF.
+
+ To inspect the current capture configuration, use CAPTURE with no
+ arguments.
+ """
+ fname = parsed.get_binding('fname')
+ if fname is None:
+ if self.shunted_query_out is not None:
+ print("Currently capturing query output to %r." % (self.query_out.name,))
+ else:
+ print("Currently not capturing query output.")
+ return
+
+ if fname.upper() == 'OFF':
+ if self.shunted_query_out is None:
+ self.printerr('Not currently capturing output.')
+ return
+ self.query_out.close()
+ self.query_out = self.shunted_query_out
+ self.color = self.shunted_color
+ self.shunted_query_out = None
+ del self.shunted_color
+ return
+
+ if self.shunted_query_out is not None:
+ self.printerr('Already capturing output to %s. Use CAPTURE OFF'
+ ' to disable.' % (self.query_out.name,))
+ return
+
+ fname = os.path.expanduser(self.cql_unprotect_value(fname))
+ try:
+ f = open(fname, 'a')
+ except IOError as e:
+ self.printerr('Could not open %r for append: %s' % (fname, e))
+ return
+ self.shunted_query_out = self.query_out
+ self.shunted_color = self.color
+ self.query_out = f
+ self.color = False
+ print('Now capturing query output to %r.' % (fname,))
+
+ def do_tracing(self, parsed):
+ """
+ TRACING [cqlsh]
+
+ Enables or disables request tracing.
+
+ TRACING ON
+
+ Enables tracing for all further requests.
+
+ TRACING OFF
+
+ Disables tracing.
+
+ TRACING
+
+ TRACING with no arguments shows the current tracing status.
+ """
+ self.tracing_enabled = SwitchCommand("TRACING", "Tracing").execute(self.tracing_enabled, parsed, self.printerr)
+
+ def do_expand(self, parsed):
+ """
+ EXPAND [cqlsh]
+
+ Enables or disables expanded (vertical) output.
+
+ EXPAND ON
+
+ Enables expanded (vertical) output.
+
+ EXPAND OFF
+
+ Disables expanded (vertical) output.
+
+ EXPAND
+
+ EXPAND with no arguments shows the current value of expand setting.
+ """
+ self.expand_enabled = SwitchCommand("EXPAND", "Expanded output").execute(self.expand_enabled, parsed, self.printerr)
+
+ def do_consistency(self, parsed):
+ """
+ CONSISTENCY [cqlsh only]
+
+ Overrides default consistency level (default level is ONE).
+
+ CONSISTENCY <level>
+
+ Sets consistency level for future requests.
+
+ Valid consistency levels:
+
+ ANY, ONE, TWO, THREE, QUORUM, ALL, LOCAL_ONE, LOCAL_QUORUM, EACH_QUORUM, SERIAL and LOCAL_SERIAL.
+
+ SERIAL and LOCAL_SERIAL may be used only for SELECTs; will be rejected with updates.
+
+ CONSISTENCY
+
+ CONSISTENCY with no arguments shows the current consistency level.
+ """
+ level = parsed.get_binding('level')
+ if level is None:
+ print('Current consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.consistency_level]))
+ return
+
+ self.consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()]
+ print('Consistency level set to %s.' % (level.upper(),))
+
+ def do_serial(self, parsed):
+ """
+ SERIAL CONSISTENCY [cqlsh only]
+
+ Overrides serial consistency level (default level is SERIAL).
+
+ SERIAL CONSISTENCY <level>
+
+ Sets consistency level for future conditional updates.
+
+ Valid consistency levels:
+
+ SERIAL, LOCAL_SERIAL.
+
+ SERIAL CONSISTENCY
+
+ SERIAL CONSISTENCY with no arguments shows the current consistency level.
+ """
+ level = parsed.get_binding('level')
+ if level is None:
+ print('Current serial consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.serial_consistency_level]))
+ return
+
+ self.serial_consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()]
+ print('Serial consistency level set to %s.' % (level.upper(),))
+
+ def do_login(self, parsed):
+ """
+ LOGIN [cqlsh only]
+
+ Changes login information without requiring restart.
+
+ LOGIN <username> (<password>)
+
+ Login using the specified username. If password is specified, it will be used
+ otherwise, you will be prompted to enter.
+ """
+ username = parsed.get_binding('username')
+ password = parsed.get_binding('password')
+ if password is None:
+ password = getpass.getpass()
+ else:
+ password = password[1:-1]
+
+ auth_provider = PlainTextAuthProvider(username=username, password=password)
+
+ conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=self.conn.cql_version,
+ protocol_version=self.conn.protocol_version,
+ auth_provider=auth_provider,
+ ssl_options=self.conn.ssl_options,
+ load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
+ control_connection_timeout=self.conn.connect_timeout,
+ connect_timeout=self.conn.connect_timeout)
+
+ if self.current_keyspace:
+ session = conn.connect(self.current_keyspace)
+ else:
+ session = conn.connect()
+
+ # Copy session properties
+ session.default_timeout = self.session.default_timeout
+ session.row_factory = self.session.row_factory
+ session.default_consistency_level = self.session.default_consistency_level
+ session.max_trace_wait = self.session.max_trace_wait
+
+ # Update after we've connected in case we fail to authenticate
+ self.conn = conn
+ self.auth_provider = auth_provider
+ self.username = username
+ self.session = session
+
+ def do_exit(self, parsed=None):
+ """
+ EXIT/QUIT [cqlsh only]
+
+ Exits cqlsh.
+ """
+ self.stop = True
+ if self.owns_connection:
+ self.conn.shutdown()
+ do_quit = do_exit
+
+ def do_clear(self, parsed):
+ """
+ CLEAR/CLS [cqlsh only]
+
+ Clears the console.
+ """
+ subprocess.call('clear', shell=True)
+ do_cls = do_clear
+
+ def do_debug(self, parsed):
+ import pdb
+ pdb.set_trace()
+
+ def get_help_topics(self):
+ topics = [t[3:] for t in dir(self) if t.startswith('do_') and getattr(self, t, None).__doc__]
+ for hide_from_help in ('quit',):
+ topics.remove(hide_from_help)
+ return topics
+
+ def columnize(self, slist, *a, **kw):
+ return cmd.Cmd.columnize(self, sorted([u.upper() for u in slist]), *a, **kw)
+
+ def do_help(self, parsed):
+ """
+ HELP [cqlsh only]
+
+ Gives information about cqlsh commands. To see available topics,
+ enter "HELP" without any arguments. To see help on a topic,
+ use "HELP <topic>".
+ """
+ topics = parsed.get_binding('topic', ())
+ if not topics:
+ shell_topics = [t.upper() for t in self.get_help_topics()]
+ self.print_topics("\nDocumented shell commands:", shell_topics, 15, 80)
+ cql_topics = [t.upper() for t in cqldocs.get_help_topics()]
+ self.print_topics("CQL help topics:", cql_topics, 15, 80)
+ return
+ for t in topics:
+ if t.lower() in self.get_help_topics():
+ doc = getattr(self, 'do_' + t.lower()).__doc__
+ self.stdout.write(doc + "\n")
+ elif t.lower() in cqldocs.get_help_topics():
+ urlpart = cqldocs.get_help_topic(t)
+ if urlpart is not None:
+ url = "%s#%s" % (CASSANDRA_CQL_HTML, urlpart)
+ if self.browser is not None:
+ opened = webbrowser.get(self.browser).open_new_tab(url)
+ else:
+ opened = webbrowser.open_new_tab(url)
+ if not opened:
+ self.printerr("*** No browser to display CQL help. URL for help topic %s : %s" % (t, url))
+ else:
+ self.printerr("*** No help on %s" % (t,))
+
+ def do_history(self, parsed):
+ """
+ HISTORY [cqlsh only]
+
+ Displays the most recent commands executed in cqlsh
+
+ HISTORY (<n>)
+
+ If n is specified, the history display length is set to n for this session
+ """
+
+ history_length = readline.get_current_history_length()
+
+ n = parsed.get_binding('n')
+ if (n is not None):
+ self.max_history_length_shown = int(n)
+
+ for index in range(max(1, history_length - self.max_history_length_shown), history_length):
+ print(readline.get_history_item(index))
+
+ def do_unicode(self, parsed):
+ """
+ Textual input/output
+
+ When control characters, or other characters which can't be encoded
+ in your current locale, are found in values of 'text' or 'ascii'
+ types, it will be shown as a backslash escape. If color is enabled,
+ any such backslash escapes will be shown in a different color from
+ the surrounding text.
+
+ Unicode code points in your data will be output intact, if the
+ encoding for your locale is capable of decoding them. If you prefer
+ that non-ascii characters be shown with Python-style "\\uABCD"
+ escape sequences, invoke cqlsh with an ASCII locale (for example,
+ by setting the $LANG environment variable to "C").
+ """
+
+ def do_paging(self, parsed):
+ """
+ PAGING [cqlsh]
+
+ Enables or disables query paging.
+
+ PAGING ON
+
+ Enables query paging for all further queries.
+
+ PAGING OFF
+
+ Disables paging.
+
+ PAGING
+
+ PAGING with no arguments shows the current query paging status.
+ """
+ (self.use_paging, requested_page_size) = SwitchCommandWithValue(
+ "PAGING", "Query paging", value_type=int).execute(self.use_paging, parsed, self.printerr)
+ if self.use_paging and requested_page_size is not None:
+ self.page_size = requested_page_size
+ if self.use_paging:
+ print(("Page size: {}".format(self.page_size)))
+ else:
+ self.page_size = self.default_page_size
+
+ def applycolor(self, text, color=None):
+ if not color or not self.color:
+ return text
+ return color + text + ANSI_RESET
+
+ def writeresult(self, text, color=None, newline=True, out=None):
+ if out is None:
+ out = self.query_out
+
+ # convert Exceptions, etc to text
+ if not isinstance(text, str):
+ text = str(text)
+
+ to_write = self.applycolor(text, color) + ('\n' if newline else '')
+ out.write(to_write)
+
+ def flush_output(self):
+ self.query_out.flush()
+
+ def printerr(self, text, color=RED, newline=True, shownum=None):
+ self.statement_error = True
+ if shownum is None:
+ shownum = self.show_line_nums
+ if shownum:
+ text = '%s:%d:%s' % (self.stdin.name, self.lineno, text)
+ self.writeresult(text, color, newline=newline, out=sys.stderr)
+
+ def stop_coverage(self):
+ if self.coverage and self.cov is not None:
+ self.cov.stop()
+ self.cov.save()
+ self.cov = None
+
+ def init_history(self):
+ if readline is not None:
+ try:
+ readline.read_history_file(HISTORY)
+ except IOError:
+ pass
+ delims = readline.get_completer_delims()
+ delims.replace("'", "")
+ delims += '.'
+ readline.set_completer_delims(delims)
+
+ # configure length of history shown
+ self.max_history_length_shown = 50
+
+ def save_history(self):
+ if readline is not None:
+ try:
+ readline.write_history_file(HISTORY)
+ except IOError:
+ pass
+
+
+class SwitchCommand(object):
+ command = None
+ description = None
+
+ def __init__(self, command, desc):
+ self.command = command
+ self.description = desc
+
+ def execute(self, state, parsed, printerr):
+ switch = parsed.get_binding('switch')
+ if switch is None:
+ if state:
+ print("%s is currently enabled. Use %s OFF to disable"
+ % (self.description, self.command))
+ else:
+ print("%s is currently disabled. Use %s ON to enable."
+ % (self.description, self.command))
+ return state
+
+ if switch.upper() == 'ON':
+ if state:
+ printerr('%s is already enabled. Use %s OFF to disable.'
+ % (self.description, self.command))
+ return state
+ print('Now %s is enabled' % (self.description,))
+ return True
+
+ if switch.upper() == 'OFF':
+ if not state:
+ printerr('%s is not enabled.' % (self.description,))
+ return state
+ print('Disabled %s.' % (self.description,))
+ return False
+
+
+class SwitchCommandWithValue(SwitchCommand):
+ """The same as SwitchCommand except it also accepts a value in place of ON.
+
+ This returns a tuple of the form: (SWITCH_VALUE, PASSED_VALUE)
+ eg: PAGING 50 returns (True, 50)
+ PAGING OFF returns (False, None)
+ PAGING ON returns (True, None)
+
+ The value_type must match for the PASSED_VALUE, otherwise it will return None.
+ """
+ def __init__(self, command, desc, value_type=int):
+ SwitchCommand.__init__(self, command, desc)
+ self.value_type = value_type
+
+ def execute(self, state, parsed, printerr):
+ binary_switch_value = SwitchCommand.execute(self, state, parsed, printerr)
+ switch = parsed.get_binding('switch')
+ try:
+ value = self.value_type(switch)
+ binary_switch_value = True
+ except (ValueError, TypeError):
+ value = None
+ return binary_switch_value, value
+
+
+def option_with_default(cparser_getter, section, option, default=None):
+ try:
+ return cparser_getter(section, option)
+ except configparser.Error:
+ return default
+
+
+def raw_option_with_default(configs, section, option, default=None):
+ """
+ Same (almost) as option_with_default() but won't do any string interpolation.
+ Useful for config values that include '%' symbol, e.g. time format string.
+ """
+ try:
+ return configs.get(section, option, raw=True)
+ except configparser.Error:
+ return default
+
+
+def should_use_color():
+ if not sys.stdout.isatty():
+ return False
+ if os.environ.get('TERM', '') in ('dumb', ''):
+ return False
+ try:
+ p = subprocess.Popen(['tput', 'colors'], stdout=subprocess.PIPE)
+ stdout, _ = p.communicate()
+ if int(stdout.strip()) < 8:
+ return False
+ except (OSError, ImportError, ValueError):
+ # oh well, we tried. at least we know there's a $TERM and it's
+ # not "dumb".
+ pass
+ return True
+
+
+def read_options(cmdlineargs, environment=os.environ):
+ configs = configparser.ConfigParser()
+ configs.read(CONFIG_FILE)
+
+ rawconfigs = configparser.RawConfigParser()
+ rawconfigs.read(CONFIG_FILE)
+
+ username_from_cqlshrc = option_with_default(configs.get, 'authentication', 'username')
+ password_from_cqlshrc = option_with_default(rawconfigs.get, 'authentication', 'password')
+ if username_from_cqlshrc or password_from_cqlshrc:
+ if password_from_cqlshrc and not is_file_secure(os.path.expanduser(CONFIG_FILE)):
+ print("\nWarning: Password is found in an insecure cqlshrc file. The file is owned or readable by other users on the system.",
+ end='', file=sys.stderr)
+ print("\nNotice: Credentials in the cqlshrc file is deprecated and will be ignored in the future."
+ "\nPlease use a credentials file to specify the username and password.\n", file=sys.stderr)
+
+ optvalues = optparse.Values()
+
+ optvalues.username = None
+ optvalues.password = None
+ optvalues.credentials = os.path.expanduser(option_with_default(configs.get, 'authentication', 'credentials',
+ os.path.join(CQL_DIR, 'credentials')))
+ optvalues.keyspace = option_with_default(configs.get, 'authentication', 'keyspace')
+ optvalues.browser = option_with_default(configs.get, 'ui', 'browser', None)
+ optvalues.completekey = option_with_default(configs.get, 'ui', 'completekey',
+ DEFAULT_COMPLETEKEY)
+ optvalues.color = option_with_default(configs.getboolean, 'ui', 'color')
+ optvalues.time_format = raw_option_with_default(configs, 'ui', 'time_format',
+ DEFAULT_TIMESTAMP_FORMAT)
+ optvalues.nanotime_format = raw_option_with_default(configs, 'ui', 'nanotime_format',
+ DEFAULT_NANOTIME_FORMAT)
+ optvalues.date_format = raw_option_with_default(configs, 'ui', 'date_format',
+ DEFAULT_DATE_FORMAT)
+ optvalues.float_precision = option_with_default(configs.getint, 'ui', 'float_precision',
+ DEFAULT_FLOAT_PRECISION)
+ optvalues.double_precision = option_with_default(configs.getint, 'ui', 'double_precision',
+ DEFAULT_DOUBLE_PRECISION)
+ optvalues.field_size_limit = option_with_default(configs.getint, 'csv', 'field_size_limit', csv.field_size_limit())
+ optvalues.max_trace_wait = option_with_default(configs.getfloat, 'tracing', 'max_trace_wait',
+ DEFAULT_MAX_TRACE_WAIT)
+ optvalues.timezone = option_with_default(configs.get, 'ui', 'timezone', None)
+
+ optvalues.debug = False
+
+ optvalues.coverage = False
+ if 'CQLSH_COVERAGE' in environment.keys():
+ optvalues.coverage = True
+
+ optvalues.file = None
+ optvalues.ssl = option_with_default(configs.getboolean, 'connection', 'ssl', DEFAULT_SSL)
+ optvalues.encoding = option_with_default(configs.get, 'ui', 'encoding', UTF8)
+
+ optvalues.tty = option_with_default(configs.getboolean, 'ui', 'tty', sys.stdin.isatty())
+ optvalues.protocol_version = option_with_default(configs.getint, 'protocol', 'version', None)
+ optvalues.cqlversion = option_with_default(configs.get, 'cql', 'version', None)
+ optvalues.connect_timeout = option_with_default(configs.getint, 'connection', 'timeout', DEFAULT_CONNECT_TIMEOUT_SECONDS)
+ optvalues.request_timeout = option_with_default(configs.getint, 'connection', 'request_timeout', DEFAULT_REQUEST_TIMEOUT_SECONDS)
+ optvalues.execute = None
+ optvalues.insecure_password_without_warning = False
+
+ (options, arguments) = parser.parse_args(cmdlineargs, values=optvalues)
+
+ # Credentials from cqlshrc will be expanded,
+ # credentials from the command line are also expanded if there is a space...
+ # we need the following so that these two scenarios will work
+ # cqlsh --credentials=~/.cassandra/creds
+ # cqlsh --credentials ~/.cassandra/creds
+ options.credentials = os.path.expanduser(options.credentials)
+
+ if not is_file_secure(options.credentials):
+ print("\nWarning: Credentials file '{0}' exists but is not used, because:"
+ "\n a. the file owner is not the current user; or"
+ "\n b. the file is readable by group or other."
+ "\nPlease ensure the file is owned by the current user and is not readable by group or other."
+ "\nOn a Linux or UNIX-like system, you often can do this by using the `chown` and `chmod` commands:"
+ "\n chown YOUR_USERNAME credentials"
+ "\n chmod 600 credentials\n".format(options.credentials),
+ file=sys.stderr)
+ options.credentials = '' # ConfigParser.read() will ignore unreadable files
+
+ if not options.username:
+ credentials = configparser.ConfigParser()
+ credentials.read(options.credentials)
+
+ # use the username from credentials file but fallback to cqlshrc if username is absent from the command line parameters
+ options.username = username_from_cqlshrc
+
+ if not options.password:
+ rawcredentials = configparser.RawConfigParser()
+ rawcredentials.read(options.credentials)
+
+ # handling password in the same way as username, priority cli > credentials > cqlshrc
+ options.password = option_with_default(rawcredentials.get, 'plain_text_auth', 'password', password_from_cqlshrc)
+ options.password = password_from_cqlshrc
+ elif not options.insecure_password_without_warning:
+ print("\nWarning: Using a password on the command line interface can be insecure."
+ "\nRecommendation: use the credentials file to securely provide the password.\n", file=sys.stderr)
+
+ # Make sure some user values read from the command line are in unicode
+ options.execute = maybe_ensure_text(options.execute)
+ options.username = maybe_ensure_text(options.username)
+ options.password = maybe_ensure_text(options.password)
+ options.keyspace = maybe_ensure_text(options.keyspace)
+
+ hostname = option_with_default(configs.get, 'connection', 'hostname', DEFAULT_HOST)
+ port = option_with_default(configs.get, 'connection', 'port', DEFAULT_PORT)
+
+ try:
+ options.connect_timeout = int(options.connect_timeout)
+ except ValueError:
+ parser.error('"%s" is not a valid connect timeout.' % (options.connect_timeout,))
+ options.connect_timeout = DEFAULT_CONNECT_TIMEOUT_SECONDS
+
+ try:
+ options.request_timeout = int(options.request_timeout)
+ except ValueError:
+ parser.error('"%s" is not a valid request timeout.' % (options.request_timeout,))
+ options.request_timeout = DEFAULT_REQUEST_TIMEOUT_SECONDS
+
+ hostname = environment.get('CQLSH_HOST', hostname)
+ port = environment.get('CQLSH_PORT', port)
+
+ if len(arguments) > 0:
+ hostname = arguments[0]
+ if len(arguments) > 1:
+ port = arguments[1]
+
+ if options.file or options.execute:
+ options.tty = False
+
+ if options.execute and not options.execute.endswith(';'):
+ options.execute += ';'
+
+ if optvalues.color in (True, False):
+ options.color = optvalues.color
+ else:
+ if options.file is not None:
+ options.color = False
+ else:
+ options.color = should_use_color()
+
+ if options.cqlversion is not None:
+ options.cqlversion, cqlvertup = full_cql_version(options.cqlversion)
+ if cqlvertup[0] < 3:
+ parser.error('%r is not a supported CQL version.' % options.cqlversion)
+ options.cqlmodule = cql3handling
+
+ try:
+ port = int(port)
+ except ValueError:
+ parser.error('%r is not a valid port number.' % port)
+ return options, hostname, port
+
+
+def setup_cqlruleset(cqlmodule):
+ global cqlruleset
+ cqlruleset = cqlmodule.CqlRuleSet
+ cqlruleset.append_rules(cqlshhandling.cqlsh_extra_syntax_rules)
+ for rulename, termname, func in cqlshhandling.cqlsh_syntax_completers:
+ cqlruleset.completer_for(rulename, termname)(func)
+ cqlruleset.commands_end_with_newline.update(cqlshhandling.my_commands_ending_with_newline)
+
+
+def setup_cqldocs(cqlmodule):
+ global cqldocs
+ cqldocs = cqlmodule.cqldocs
+
+
+def setup_docspath(path):
+ global CASSANDRA_CQL_HTML
+ CASSANDRA_CQL_HTML_FALLBACK = 'https://cassandra.apache.org/doc/latest/cql/index.html'
+ #
+ # default location of local CQL.html
+ if os.path.exists(path + '/doc/cql3/CQL.html'):
+ # default location of local CQL.html
+ CASSANDRA_CQL_HTML = 'file://' + path + '/doc/cql3/CQL.html'
+ elif os.path.exists('/usr/share/doc/cassandra/CQL.html'):
+ # fallback to package file
+ CASSANDRA_CQL_HTML = 'file:///usr/share/doc/cassandra/CQL.html'
+ else:
+ # fallback to online version
+ CASSANDRA_CQL_HTML = CASSANDRA_CQL_HTML_FALLBACK
+
+
+def insert_driver_hooks():
+
+ class DateOverFlowWarning(RuntimeWarning):
+ pass
+
+ # Display milliseconds when datetime overflows (CASSANDRA-10625), E.g., the year 10000.
+ # Native datetime types blow up outside of datetime.[MIN|MAX]_YEAR. We will fall back to an int timestamp
+ def deserialize_date_fallback_int(byts, protocol_version):
+ timestamp_ms = int64_unpack(byts)
+ try:
+ return datetime_from_timestamp(timestamp_ms / 1000.0)
+ except OverflowError:
+ warnings.warn(DateOverFlowWarning("Some timestamps are larger than Python datetime can represent. "
+ "Timestamps are displayed in milliseconds from epoch."))
+ return timestamp_ms
+
+ cassandra.cqltypes.DateType.deserialize = staticmethod(deserialize_date_fallback_int)
+
+ if hasattr(cassandra, 'deserializers'):
+ del cassandra.deserializers.DesDateType
+
+ # Return cassandra.cqltypes.EMPTY instead of None for empty values
+ cassandra.cqltypes.CassandraType.support_empty_values = True
+
+
+def main(cmdline, pkgpath):
+ insert_driver_hooks()
+ (options, hostname, port) = read_options(cmdline)
+
+ setup_docspath(pkgpath)
+ setup_cqlruleset(options.cqlmodule)
+ setup_cqldocs(options.cqlmodule)
+ csv.field_size_limit(options.field_size_limit)
+
+ if options.file is None:
+ stdin = None
+ else:
+ try:
+ encoding, bom_size = get_file_encoding_bomsize(options.file)
+ stdin = codecs.open(options.file, 'r', encoding)
+ stdin.seek(bom_size)
+ except IOError as e:
+ sys.exit("Can't open %r: %s" % (options.file, e))
+
+ if options.debug:
+ sys.stderr.write("Using CQL driver: %s\n" % (cassandra,))
+ sys.stderr.write("Using connect timeout: %s seconds\n" % (options.connect_timeout,))
+ sys.stderr.write("Using '%s' encoding\n" % (options.encoding,))
+ sys.stderr.write("Using ssl: %s\n" % (options.ssl,))
+
+ # create timezone based on settings, environment or auto-detection
+ timezone = None
+ if options.timezone or 'TZ' in os.environ:
+ try:
+ import pytz
+ if options.timezone:
+ try:
+ timezone = pytz.timezone(options.timezone)
+ except Exception:
+ sys.stderr.write("Warning: could not recognize timezone '%s' specified in cqlshrc\n\n" % (options.timezone))
+ if 'TZ' in os.environ:
+ try:
+ timezone = pytz.timezone(os.environ['TZ'])
+ except Exception:
+ sys.stderr.write("Warning: could not recognize timezone '%s' from environment value TZ\n\n" % (os.environ['TZ']))
+ except ImportError:
+ sys.stderr.write("Warning: Timezone defined and 'pytz' module for timezone conversion not installed. Timestamps will be displayed in UTC timezone.\n\n")
+
+ # try auto-detect timezone if tzlocal is installed
+ if not timezone:
+ try:
+ from tzlocal import get_localzone
+ timezone = get_localzone()
+ except ImportError:
+ # we silently ignore and fallback to UTC unless a custom timestamp format (which likely
+ # does contain a TZ part) was specified
+ if options.time_format != DEFAULT_TIMESTAMP_FORMAT:
+ sys.stderr.write("Warning: custom timestamp format specified in cqlshrc, "
+ + "but local timezone could not be detected.\n"
+ + "Either install Python 'tzlocal' module for auto-detection "
+ + "or specify client timezone in your cqlshrc.\n\n")
+
+ try:
+ shell = Shell(hostname,
+ port,
+ color=options.color,
+ username=options.username,
+ stdin=stdin,
+ tty=options.tty,
+ completekey=options.completekey,
+ browser=options.browser,
+ protocol_version=options.protocol_version,
+ cqlver=options.cqlversion,
+ keyspace=options.keyspace,
+ display_timestamp_format=options.time_format,
+ display_nanotime_format=options.nanotime_format,
+ display_date_format=options.date_format,
+ display_float_precision=options.float_precision,
+ display_double_precision=options.double_precision,
+ display_timezone=timezone,
+ max_trace_wait=options.max_trace_wait,
+ ssl=options.ssl,
+ single_statement=options.execute,
+ request_timeout=options.request_timeout,
+ connect_timeout=options.connect_timeout,
+ encoding=options.encoding,
+ auth_provider=authproviderhandling.load_auth_provider(
+ config_file=CONFIG_FILE,
+ cred_file=options.credentials,
+ username=options.username,
+ password=options.password))
+ except KeyboardInterrupt:
+ sys.exit('Connection aborted.')
+ except CQL_ERRORS as e:
+ sys.exit('Connection error: %s' % (e,))
+ except VersionNotSupported as e:
+ sys.exit('Unsupported CQL version: %s' % (e,))
+ if options.debug:
+ shell.debug = True
+ if options.coverage:
+ shell.coverage = True
+ import signal
+
+ def handle_sighup():
+ shell.stop_coverage()
+ shell.do_exit()
+
+ signal.signal(signal.SIGHUP, handle_sighup)
+
+ shell.init_history()
+ shell.cmdloop()
+ shell.save_history()
+
+ if shell.batch_mode and shell.statement_error:
+ sys.exit(2)
+
+
+# vim: set ft=python et ts=4 sw=4 :
diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py
index ebf9fc7..4eb3658 100644
--- a/pylib/cqlshlib/formatting.py
+++ b/pylib/cqlshlib/formatting.py
@@ -20,12 +20,12 @@
import os
import re
import sys
+import wcwidth
from collections import defaultdict
from cassandra.cqltypes import EMPTY
from cassandra.util import datetime_from_timestamp
-from . import wcwidth
from .displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP
from .util import UTC
@@ -326,19 +326,9 @@
return colorme(bval, colormap, 'int')
-# We can get rid of this in cassandra-2.2
-if sys.version_info >= (2, 7):
- def format_integer_with_thousands_sep(val, thousands_sep=','):
- return "{:,.0f}".format(val).replace(',', thousands_sep)
-else:
- def format_integer_with_thousands_sep(val, thousands_sep=','):
- if val < 0:
- return '-' + format_integer_with_thousands_sep(-val, thousands_sep)
- result = ''
- while val >= 1000:
- val, r = divmod(val, 1000)
- result = "%s%03d%s" % (thousands_sep, r, result)
- return "%d%s" % (val, result)
+def format_integer_with_thousands_sep(val, thousands_sep=','):
+ return "{:,.0f}".format(val).replace(',', thousands_sep)
+
formatter_for('long')(format_integer_type)
formatter_for('int')(format_integer_type)
diff --git a/pylib/cqlshlib/pylexotron.py b/pylib/cqlshlib/pylexotron.py
index 69f31dc..c1fd55e 100644
--- a/pylib/cqlshlib/pylexotron.py
+++ b/pylib/cqlshlib/pylexotron.py
@@ -14,7 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Pylexotron uses Python's re.Scanner module as a simple regex-based tokenizer for BNF production rules"""
+
import re
+import inspect
+import sys
+from typing import Union
from cqlshlib.saferscanner import SaferScanner
@@ -56,8 +61,8 @@
return '%s(%r)' % (self.__class__, self.text)
-def is_hint(x):
- return isinstance(x, Hint)
+def is_hint(obj):
+ return isinstance(obj, Hint)
class ParseContext:
@@ -115,7 +120,7 @@
% (self.__class__.__name__, self.matched, self.remainder, self.productionname, self.bindings)
-class matcher:
+class Matcher:
def __init__(self, arg):
self.arg = arg
@@ -155,38 +160,38 @@
return '%s(%r)' % (self.__class__.__name__, self.arg)
-class choice(matcher):
+class Choice(Matcher):
def match(self, ctxt, completions):
foundctxts = []
- for a in self.arg:
- subctxts = a.match(ctxt, completions)
+ for each in self.arg:
+ subctxts = each.match(ctxt, completions)
foundctxts.extend(subctxts)
return foundctxts
-class one_or_none(matcher):
+class OneOrNone(Matcher):
def match(self, ctxt, completions):
return [ctxt] + list(self.arg.match(ctxt, completions))
-class repeat(matcher):
+class Repeat(Matcher):
def match(self, ctxt, completions):
found = [ctxt]
ctxts = [ctxt]
while True:
new_ctxts = []
- for c in ctxts:
- new_ctxts.extend(self.arg.match(c, completions))
+ for each in ctxts:
+ new_ctxts.extend(self.arg.match(each, completions))
if not new_ctxts:
return found
found.extend(new_ctxts)
ctxts = new_ctxts
-class rule_reference(matcher):
+class RuleReference(Matcher):
def match(self, ctxt, completions):
prevname = ctxt.productionname
@@ -198,24 +203,24 @@
return [c.with_production_named(prevname) for c in output]
-class rule_series(matcher):
+class RuleSeries(Matcher):
def match(self, ctxt, completions):
ctxts = [ctxt]
for patpiece in self.arg:
new_ctxts = []
- for c in ctxts:
- new_ctxts.extend(patpiece.match(c, completions))
+ for each in ctxts:
+ new_ctxts.extend(patpiece.match(each, completions))
if not new_ctxts:
return ()
ctxts = new_ctxts
return ctxts
-class named_symbol(matcher):
+class NamedSymbol(Matcher):
def __init__(self, name, arg):
- matcher.__init__(self, arg)
+ Matcher.__init__(self, arg)
self.name = name
def match(self, ctxt, completions):
@@ -224,13 +229,14 @@
# don't collect other completions under this; use a dummy
pass_in_compls = set()
results = self.arg.match_with_results(ctxt, pass_in_compls)
- return [c.with_binding(self.name, ctxt.extract_orig(matchtoks)) for (c, matchtoks) in results]
+ return [c.with_binding(self.name, ctxt.extract_orig(matchtoks))
+ for (c, matchtoks) in results]
def __repr__(self):
return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.arg)
-class named_collector(named_symbol):
+class NamedCollector(NamedSymbol):
def match(self, ctxt, completions):
pass_in_compls = completions
@@ -244,18 +250,21 @@
return output
-class terminal_matcher(matcher):
+class TerminalMatcher(Matcher):
+
+ def match(self, ctxt, completions):
+ raise NotImplementedError
def pattern(self):
raise NotImplementedError
-class regex_rule(terminal_matcher):
+class RegexRule(TerminalMatcher):
def __init__(self, pat):
- terminal_matcher.__init__(self, pat)
+ TerminalMatcher.__init__(self, pat)
self.regex = pat
- self.re = re.compile(pat + '$', re.I | re.S)
+ self.re = re.compile(pat + '$', re.IGNORECASE | re.DOTALL)
def match(self, ctxt, completions):
if ctxt.remainder:
@@ -269,12 +278,12 @@
return self.regex
-class text_match(terminal_matcher):
+class TextMatch(TerminalMatcher):
alpha_re = re.compile(r'[a-zA-Z]')
def __init__(self, text):
try:
- terminal_matcher.__init__(self, eval(text))
+ TerminalMatcher.__init__(self, eval(text))
except SyntaxError:
print("bad syntax %r" % (text,))
@@ -289,12 +298,13 @@
def pattern(self):
# can't use (?i) here- Scanner component regex flags won't be applied
def ignorecaseify(matchobj):
- c = matchobj.group(0)
- return '[%s%s]' % (c.upper(), c.lower())
+ val = matchobj.group(0)
+ return '[%s%s]' % (val.upper(), val.lower())
+
return self.alpha_re.sub(ignorecaseify, re.escape(self.arg))
-class case_match(text_match):
+class CaseMatch(TextMatch):
def match(self, ctxt, completions):
if ctxt.remainder:
@@ -308,22 +318,22 @@
return re.escape(self.arg)
-class word_match(text_match):
+class WordMatch(TextMatch):
def pattern(self):
- return r'\b' + text_match.pattern(self) + r'\b'
+ return r'\b' + TextMatch.pattern(self) + r'\b'
-class case_word_match(case_match):
+class CaseWordMatch(CaseMatch):
def pattern(self):
- return r'\b' + case_match.pattern(self) + r'\b'
+ return r'\b' + CaseMatch.pattern(self) + r'\b'
-class terminal_type_matcher(matcher):
+class TerminalTypeMatcher(Matcher):
def __init__(self, tokentype, submatcher):
- matcher.__init__(self, tokentype)
+ Matcher.__init__(self, tokentype)
self.tokentype = tokentype
self.submatcher = submatcher
@@ -340,18 +350,24 @@
class ParsingRuleSet:
+ """Define the BNF tokenization rules for cql3handling.syntax_rules. Backus-Naur Form consists of
+ - Production rules in the form: Left-Hand-Side ::= Right-Hand-Side. The LHS is a non-terminal.
+ - Productions or non-terminal symbols
+ - Terminal symbols. Every terminal is a single token.
+ """
+
RuleSpecScanner = SaferScanner([
- (r'::=', lambda s, t: t),
+ (r'::=', lambda s, t: t), # BNF rule definition
(r'\[[a-z0-9_]+\]=', lambda s, t: ('named_collector', t[1:-2])),
(r'[a-z0-9_]+=', lambda s, t: ('named_symbol', t[:-1])),
(r'/(\[\^?.[^]]*\]|[^/]|\\.)*/', lambda s, t: ('regex', t[1:-1].replace(r'\/', '/'))),
- (r'"([^"]|\\.)*"', lambda s, t: ('litstring', t)),
+ (r'"([^"]|\\.)*"', lambda s, t: ('string_literal', t)),
(r'<[^>]*>', lambda s, t: ('reference', t[1:-1])),
(r'\bJUNK\b', lambda s, t: ('junk', t)),
(r'[@()|?*;]', lambda s, t: t),
- (r'\s+', None),
+ (r'\s+', None), # whitespace
(r'#[^\n]*', None),
- ], re.I | re.S | re.U)
+ ], re.IGNORECASE | re.DOTALL | re.UNICODE)
def __init__(self):
self.ruleset = {}
@@ -368,7 +384,7 @@
def parse_rules(cls, rulestr):
tokens, unmatched = cls.RuleSpecScanner.scan(rulestr)
if unmatched:
- raise LexingError.from_text(rulestr, unmatched, msg="Syntax rules unparseable")
+ raise LexingError.from_text(rulestr, unmatched, msg="Syntax rules are unparseable")
rules = {}
terminals = []
tokeniter = iter(tokens)
@@ -379,9 +395,9 @@
raise ValueError('Unexpected token %r; expected "::="' % (assign,))
name = t[1]
production = cls.read_rule_tokens_until(';', tokeniter)
- if isinstance(production, terminal_matcher):
+ if isinstance(production, TerminalMatcher):
terminals.append((name, production))
- production = terminal_type_matcher(name, production)
+ production = TerminalTypeMatcher(name, production)
rules[name] = production
else:
raise ValueError('Unexpected token %r; expected name' % (t,))
@@ -392,11 +408,11 @@
if isinstance(pieces, (tuple, list)):
if len(pieces) == 1:
return pieces[0]
- return rule_series(pieces)
+ return RuleSeries(pieces)
return pieces
@classmethod
- def read_rule_tokens_until(cls, endtoks, tokeniter):
+ def read_rule_tokens_until(cls, endtoks: Union[str, int], tokeniter):
if isinstance(endtoks, str):
endtoks = (endtoks,)
counttarget = None
@@ -411,32 +427,32 @@
if t in endtoks:
if len(mybranches) == 1:
return cls.mkrule(mybranches[0])
- return choice(list(map(cls.mkrule, mybranches)))
+ return Choice(list(map(cls.mkrule, mybranches)))
if isinstance(t, tuple):
if t[0] == 'reference':
- t = rule_reference(t[1])
- elif t[0] == 'litstring':
+ t = RuleReference(t[1])
+ elif t[0] == 'string_literal':
if t[1][1].isalnum() or t[1][1] == '_':
- t = word_match(t[1])
+ t = WordMatch(t[1])
else:
- t = text_match(t[1])
+ t = TextMatch(t[1])
elif t[0] == 'regex':
- t = regex_rule(t[1])
+ t = RegexRule(t[1])
elif t[0] == 'named_collector':
- t = named_collector(t[1], cls.read_rule_tokens_until(1, tokeniter))
+ t = NamedCollector(t[1], cls.read_rule_tokens_until(1, tokeniter))
elif t[0] == 'named_symbol':
- t = named_symbol(t[1], cls.read_rule_tokens_until(1, tokeniter))
+ t = NamedSymbol(t[1], cls.read_rule_tokens_until(1, tokeniter))
elif t == '(':
t = cls.read_rule_tokens_until(')', tokeniter)
elif t == '?':
- t = one_or_none(myrules.pop(-1))
+ t = OneOrNone(myrules.pop(-1))
elif t == '*':
- t = repeat(myrules.pop(-1))
+ t = Repeat(myrules.pop(-1))
elif t == '@':
- x = next(tokeniter)
- if not isinstance(x, tuple) or x[0] != 'litstring':
- raise ValueError("Unexpected token %r following '@'" % (x,))
- t = case_match(x[1])
+ val = next(tokeniter)
+ if not isinstance(val, tuple) or val[0] != 'string_literal':
+ raise ValueError("Unexpected token %r following '@'" % (val,))
+ t = CaseMatch(val[1])
elif t == '|':
myrules = []
mybranches.append(myrules)
@@ -447,7 +463,7 @@
if countsofar == counttarget:
if len(mybranches) == 1:
return cls.mkrule(mybranches[0])
- return choice(list(map(cls.mkrule, mybranches)))
+ return Choice(list(map(cls.mkrule, mybranches)))
raise ValueError('Unexpected end of rule tokens')
def append_rules(self, rulestr):
@@ -465,8 +481,9 @@
if name == 'JUNK':
return None
return lambda s, t: (name, t, s.match.span())
+
regexes = [(p.pattern(), make_handler(name)) for (name, p) in self.terminals]
- return SaferScanner(regexes, re.I | re.S | re.U).scan
+ return SaferScanner(regexes, re.IGNORECASE | re.DOTALL | re.UNICODE).scan
def lex(self, text):
if self.scanner is None:
@@ -487,9 +504,9 @@
bindings = {}
if srcstr is not None:
bindings['*SRC*'] = srcstr
- for c in self.parse(startsymbol, tokens, init_bindings=bindings):
- if not c.remainder:
- return c
+ for val in self.parse(startsymbol, tokens, init_bindings=bindings):
+ if not val.remainder:
+ return val
def lex_and_parse(self, text, startsymbol='Start'):
return self.parse(startsymbol, self.lex(text), init_bindings={'*SRC*': text})
@@ -511,9 +528,6 @@
return completions
-import sys
-
-
class Debugotron(set):
depth = 10
@@ -525,9 +539,9 @@
self._note_addition(item)
set.add(self, item)
- def _note_addition(self, foo):
- self.stream.write("\nitem %r added by:\n" % (foo,))
- frame = sys._getframe().f_back.f_back
+ def _note_addition(self, item):
+ self.stream.write("\nitem %r added by:\n" % (item,))
+ frame = inspect.currentframe().f_back.f_back
for i in range(self.depth):
name = frame.f_code.co_name
filename = frame.f_code.co_filename
diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py
index af9d05e..b0f0b87 100644
--- a/pylib/cqlshlib/test/test_cqlsh_completion.py
+++ b/pylib/cqlshlib/test/test_cqlsh_completion.py
@@ -128,7 +128,7 @@
split_completed_lines=split_completed_lines)
if immediate:
- msg = 'cqlsh completed %r, but we expected %r' % (completed, immediate)
+ msg = 'cqlsh completed %r (%d), but we expected %r (%d)' % (completed, len(completed), immediate, len(immediate))
self.assertEqual(completed, immediate, msg=msg)
return
@@ -164,7 +164,7 @@
'COPY', 'CREATE', 'DEBUG', 'DELETE', 'DESC', 'DESCRIBE',
'DROP', 'GRANT', 'HELP', 'INSERT', 'LIST', 'LOGIN', 'PAGING', 'REVOKE',
'SELECT', 'SHOW', 'SOURCE', 'TRACING', 'EXPAND', 'SERIAL', 'TRUNCATE',
- 'UPDATE', 'USE', 'exit', 'quit', 'CLEAR', 'CLS'))
+ 'UPDATE', 'USE', 'exit', 'quit', 'CLEAR', 'CLS', 'history'))
def test_complete_command_words(self):
self.trycompletions('alt', '\b\b\bALTER ')
@@ -250,7 +250,7 @@
'CREATE', 'DEBUG', 'DELETE', 'DESC', 'DESCRIBE', 'DROP',
'EXPAND', 'GRANT', 'HELP', 'INSERT', 'LIST', 'LOGIN', 'PAGING',
'REVOKE', 'SELECT', 'SHOW', 'SOURCE', 'SERIAL', 'TRACING',
- 'TRUNCATE', 'UPDATE', 'USE', 'exit', 'quit',
+ 'TRUNCATE', 'UPDATE', 'USE', 'exit', 'history', 'quit',
'CLEAR', 'CLS'])
self.trycompletions(
@@ -489,8 +489,10 @@
"b = 'eggs'"),
choices=['AND', 'IF', ';'])
- def test_complete_in_batch(self):
- pass
+ def test_complete_in_begin_batch(self):
+ self.trycompletions('BEGIN ', choices=['BATCH', 'COUNTER', 'UNLOGGED'])
+ self.trycompletions('BEGIN BATCH ', choices=['DELETE', 'INSERT', 'UPDATE', 'USING'])
+ self.trycompletions('BEGIN BATCH INSERT ', immediate='INTO ' )
def test_complete_in_create_keyspace(self):
self.trycompletions('create keyspace ', '', choices=('<identifier>', '<quotedName>', 'IF'))
@@ -566,6 +568,23 @@
self.trycompletions('DROP KEYSPACE I',
immediate='F EXISTS ' + self.cqlsh.keyspace + ' ;')
+ def test_complete_in_create_type(self):
+ self.trycompletions('CREATE TYPE foo ', choices=['(', '.'])
+
+ def test_complete_in_drop_type(self):
+ self.trycompletions('DROP TYPE ', choices=['IF', 'system_views.',
+ 'tags', 'system_traces.', 'system_distributed.',
+ 'phone_number', 'quote_udt', 'band_info_type', 'address', 'system.', 'system_schema.',
+ 'system_auth.', 'system_virtual_schema.', self.cqlsh.keyspace + '.'
+ ])
+
+ def test_complete_in_create_trigger(self):
+ self.trycompletions('CREATE TRIGGER ', choices=['<identifier>', '<quotedName>', 'IF' ])
+ self.trycompletions('CREATE TRIGGER foo ', immediate='ON ' )
+ self.trycompletions('CREATE TRIGGER foo ON ', choices=['system.', 'system_auth.',
+ 'system_distributed.', 'system_schema.', 'system_traces.', 'system_views.',
+ 'system_virtual_schema.' ], other_choices_ok=True)
+
def create_columnfamily_table_template(self, name):
"""Parameterized test for CREATE COLUMNFAMILY and CREATE TABLE. Since
they're synonyms, they should have the same completion behavior, so this
@@ -601,7 +620,12 @@
self.trycompletions(prefix + ' new_table (col_a ine',
immediate='t ')
self.trycompletions(prefix + ' new_table (col_a int ',
- choices=[',', 'PRIMARY'])
+ choices=[',', 'MASKED', 'PRIMARY'])
+ self.trycompletions(prefix + ' new_table (col_a int M',
+ immediate='ASKED WITH ')
+ self.trycompletions(prefix + ' new_table (col_a int MASKED WITH ',
+ choices=['DEFAULT', self.cqlsh.keyspace + '.', 'system.'],
+ other_choices_ok=True)
self.trycompletions(prefix + ' new_table (col_a int P',
immediate='RIMARY KEY ')
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY ',
@@ -616,19 +640,25 @@
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) W',
immediate='ITH ')
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ',
- choices=['bloom_filter_fp_chance', 'compaction',
+ choices=['allow_auto_snapshot',
+ 'bloom_filter_fp_chance', 'compaction',
'compression',
'default_time_to_live', 'gc_grace_seconds',
+ 'incremental_backups',
'max_index_interval',
+ 'memtable',
'memtable_flush_period_in_ms',
'CLUSTERING',
'COMPACT', 'caching', 'comment',
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ',
- choices=['bloom_filter_fp_chance', 'compaction',
+ choices=['allow_auto_snapshot',
+ 'bloom_filter_fp_chance', 'compaction',
'compression',
'default_time_to_live', 'gc_grace_seconds',
+ 'incremental_backups',
'max_index_interval',
+ 'memtable',
'memtable_flush_period_in_ms',
'CLUSTERING',
'COMPACT', 'caching', 'comment',
@@ -644,7 +674,6 @@
+ "{'class': '",
choices=['SizeTieredCompactionStrategy',
'LeveledCompactionStrategy',
- 'DateTieredCompactionStrategy',
'TimeWindowCompactionStrategy'])
self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = "
+ "{'class': 'S",
@@ -673,23 +702,17 @@
choices=[';', 'AND'])
self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = "
+ "{'class': 'SizeTieredCompactionStrategy'} AND ",
- choices=['bloom_filter_fp_chance', 'compaction',
+ choices=['allow_auto_snapshot', 'bloom_filter_fp_chance', 'compaction',
'compression',
'default_time_to_live', 'gc_grace_seconds',
+ 'incremental_backups',
'max_index_interval',
+ 'memtable',
'memtable_flush_period_in_ms',
'CLUSTERING',
'COMPACT', 'caching', 'comment',
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = "
- + "{'class': 'DateTieredCompactionStrategy', '",
- choices=['base_time_seconds', 'max_sstable_age_days',
- 'timestamp_resolution', 'min_threshold', 'class', 'max_threshold',
- 'tombstone_compaction_interval', 'tombstone_threshold',
- 'enabled', 'unchecked_tombstone_compaction',
- 'max_window_size_seconds',
- 'only_purge_repaired_tombstones', 'provide_overlapping_tombstones'])
- self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = "
+ "{'class': 'TimeWindowCompactionStrategy', '",
choices=['compaction_window_unit', 'compaction_window_size',
'timestamp_resolution', 'min_threshold', 'class', 'max_threshold',
@@ -822,24 +845,28 @@
'aggmax'],
other_choices_ok=True)
- # TODO: CASSANDRA-16640
- # def test_complete_in_drop_columnfamily(self):
- # pass
- #
- # def test_complete_in_truncate(self):
- # pass
- #
- # def test_complete_in_alter_columnfamily(self):
- # pass
- #
- # def test_complete_in_use(self):
- # pass
- #
- # def test_complete_in_create_index(self):
- # pass
- #
- # def test_complete_in_drop_index(self):
- # pass
+ def test_complete_in_drop_table(self):
+ self.trycompletions('DROP T', choices=['TABLE', 'TRIGGER', 'TYPE'])
+ self.trycompletions('DROP TA', immediate='BLE ')
+
+ def test_complete_in_truncate(self):
+ self.trycompletions('TR', choices=['TRACING', 'TRUNCATE'])
+ self.trycompletions('TRU', immediate='NCATE ')
+ self.trycompletions('TRUNCATE T', choices=['TABLE', 'twenty_rows_composite_table', 'twenty_rows_table'])
+
+ def test_complete_in_use(self):
+ self.trycompletions('US', immediate='E ')
+ self.trycompletions('USE ', choices=[self.cqlsh.keyspace, 'system', 'system_auth',
+ 'system_distributed', 'system_schema', 'system_traces', 'system_views',
+ 'system_virtual_schema' ])
+
+ def test_complete_in_create_index(self):
+ self.trycompletions('CREATE I', immediate='NDEX ')
+ self.trycompletions('CREATE INDEX ', choices=['<new_index_name>', 'IF', 'ON'])
+ self.trycompletions('CREATE INDEX example ', immediate='ON ')
+
+ def test_complete_in_drop_index(self):
+ self.trycompletions('DROP I', immediate='NDEX ')
def test_complete_in_alter_keyspace(self):
self.trycompletions('ALTER KEY', 'SPACE ')
@@ -854,7 +881,7 @@
self.trycompletions("GR",
immediate='ANT ')
self.trycompletions("GRANT ",
- choices=['ALL', 'ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'MODIFY', 'SELECT'],
+ choices=['ALL', 'ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'MODIFY', 'SELECT', 'UNMASK', 'SELECT_MASKED'],
other_choices_ok=True)
self.trycompletions("GRANT MODIFY ",
choices=[',', 'ON', 'PERMISSION'])
@@ -863,7 +890,7 @@
self.trycompletions("GRANT MODIFY PERMISSION ",
choices=[',', 'ON'])
self.trycompletions("GRANT MODIFY PERMISSION, ",
- choices=['ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'SELECT'])
+ choices=['ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'SELECT', 'UNMASK', 'SELECT_MASKED'])
self.trycompletions("GRANT MODIFY PERMISSION, D",
choices=['DESCRIBE', 'DROP'])
self.trycompletions("GRANT MODIFY PERMISSION, DR",
@@ -885,7 +912,7 @@
self.trycompletions("RE",
immediate='VOKE ')
self.trycompletions("REVOKE ",
- choices=['ALL', 'ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'MODIFY', 'SELECT'],
+ choices=['ALL', 'ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'MODIFY', 'SELECT', 'UNMASK', 'SELECT_MASKED'],
other_choices_ok=True)
self.trycompletions("REVOKE MODIFY ",
choices=[',', 'ON', 'PERMISSION'])
@@ -894,7 +921,7 @@
self.trycompletions("REVOKE MODIFY PERMISSION ",
choices=[',', 'ON'])
self.trycompletions("REVOKE MODIFY PERMISSION, ",
- choices=['ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'SELECT'])
+ choices=['ALTER', 'AUTHORIZE', 'CREATE', 'DESCRIBE', 'DROP', 'EXECUTE', 'SELECT', 'UNMASK', 'SELECT_MASKED'])
self.trycompletions("REVOKE MODIFY PERMISSION, D",
choices=['DESCRIBE', 'DROP'])
self.trycompletions("REVOKE MODIFY PERMISSION, DR",
@@ -920,7 +947,7 @@
self.trycompletions('ALTER TABLE ', choices=['IF', 'twenty_rows_table',
'ascii_with_special_chars', 'users',
'has_all_types', 'system.',
- 'empty_composite_table', 'empty_table',
+ 'empty_composite_table', 'escape_quotes', 'empty_table',
'system_auth.', 'undefined_values_table',
'dynamic_columns',
'twenty_rows_composite_table',
@@ -932,15 +959,29 @@
self.trycompletions('ALTER TABLE IF EXISTS new_table ADD ', choices=['<new_column_name>', 'IF'])
self.trycompletions('ALTER TABLE IF EXISTS new_table ADD IF NOT EXISTS ', choices=['<new_column_name>'])
self.trycompletions('ALTER TABLE new_table ADD IF NOT EXISTS ', choices=['<new_column_name>'])
+ self.trycompletions('ALTER TABLE new_table ADD col int ', choices=[';', 'MASKED', 'static'])
+ self.trycompletions('ALTER TABLE new_table ADD col int M', immediate='ASKED WITH ')
+ self.trycompletions('ALTER TABLE new_table ADD col int MASKED WITH ',
+ choices=['DEFAULT', self.cqlsh.keyspace + '.', 'system.'],
+ other_choices_ok=True)
self.trycompletions('ALTER TABLE IF EXISTS new_table RENAME ', choices=['IF', '<quotedName>', '<identifier>'])
self.trycompletions('ALTER TABLE new_table RENAME ', choices=['IF', '<quotedName>', '<identifier>'])
self.trycompletions('ALTER TABLE IF EXISTS new_table DROP ', choices=['IF', '<quotedName>', '<identifier>'])
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER ', choices=['IF', '<quotedName>', '<identifier>'])
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER IF E', immediate='XISTS ')
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER IF EXISTS col ', choices=['MASKED', 'DROP'])
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER IF EXISTS col M', immediate='ASKED WITH ')
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER IF EXISTS col MASKED WITH ',
+ choices=['DEFAULT', self.cqlsh.keyspace + '.', 'system.'],
+ other_choices_ok=True)
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER IF EXISTS col D', immediate='ROP MASKED ;')
+ self.trycompletions('ALTER TABLE IF EXISTS new_table ALTER IF EXISTS col DROP M', immediate='ASKED ;')
def test_complete_in_alter_type(self):
self.trycompletions('ALTER TYPE I', immediate='F EXISTS ')
self.trycompletions('ALTER TYPE ', choices=['IF', 'system_views.',
'tags', 'system_traces.', 'system_distributed.',
- 'phone_number', 'band_info_type', 'address', 'system.', 'system_schema.',
+ 'phone_number', 'quote_udt', 'band_info_type', 'address', 'system.', 'system_schema.',
'system_auth.', 'system_virtual_schema.', self.cqlsh.keyspace + '.'
])
self.trycompletions('ALTER TYPE IF EXISTS new_type ADD ', choices=['<new_field_name>', 'IF'])
@@ -950,5 +991,44 @@
def test_complete_in_alter_user(self):
self.trycompletions('ALTER USER ', choices=['<identifier>', 'IF', '<pgStringLiteral>', '<quotedStringLiteral>'])
+ def test_complete_in_create_role(self):
+ self.trycompletions('CREATE ROLE ', choices=['<identifier>', 'IF', '<quotedName>'])
+ self.trycompletions('CREATE ROLE IF ', immediate='NOT EXISTS ');
+ self.trycompletions('CREATE ROLE foo WITH ', choices=['ACCESS', 'HASHED', 'LOGIN', 'OPTIONS', 'PASSWORD', 'SUPERUSER'])
+ self.trycompletions('CREATE ROLE foo WITH HASHED ', immediate='PASSWORD = ');
+ self.trycompletions('CREATE ROLE foo WITH ACCESS TO ', choices=['ALL', 'DATACENTERS'])
+ self.trycompletions('CREATE ROLE foo WITH ACCESS TO ALL ', immediate='DATACENTERS ')
+
def test_complete_in_alter_role(self):
self.trycompletions('ALTER ROLE ', choices=['<identifier>', 'IF', '<quotedName>'])
+ self.trycompletions('ALTER ROLE foo ', immediate='WITH ')
+ self.trycompletions('ALTER ROLE foo WITH ', choices=['ACCESS', 'HASHED', 'LOGIN', 'OPTIONS', 'PASSWORD', 'SUPERUSER'])
+ self.trycompletions('ALTER ROLE foo WITH ACCESS TO ', choices=['ALL', 'DATACENTERS'])
+
+ def test_complete_in_drop_role(self):
+ self.trycompletions('DROP ROLE ', choices=['<identifier>', 'IF', '<quotedName>'])
+
+
+ def test_complete_in_list(self):
+ self.trycompletions('LIST ', choices=['ALL', 'AUTHORIZE', 'DESCRIBE', 'EXECUTE', 'ROLES', 'USERS', 'ALTER', 'CREATE', 'DROP', 'MODIFY', 'SELECT', 'UNMASK', 'SELECT_MASKED'])
+
+
+ # Non-CQL Shell Commands
+
+ def test_complete_in_capture(self):
+ self.trycompletions('CAPTURE ', choices=['OFF', ';', '<enter>'], other_choices_ok=True)
+
+ def test_complete_in_paging(self):
+ self.trycompletions('PAGING ', choices=['ON', 'OFF', ';', '<enter>', '<wholenumber>' ] )
+ self.trycompletions('PAGING 50 ', choices=[';', '<enter>' ] )
+
+ def test_complete_in_serial(self):
+ self.trycompletions('SERIAL CONSISTENCY ', choices=[';', '<enter>', 'LOCAL_SERIAL', 'SERIAL'])
+
+ def test_complete_in_show(self):
+ self.trycompletions('SHOW ', choices=['HOST', 'REPLICAS', 'SESSION', 'VERSION'])
+ self.trycompletions('SHOW SESSION ', choices=['<uuid>'])
+ self.trycompletions('SHOW REPLICAS ', choices=['-', '<wholenumber>'])
+
+ def test_complete_in_tracing(self):
+ self.trycompletions('TRACING ', choices=[';', '<enter>', 'OFF', 'ON'])
diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py
index 52f564a..fe08c8d 100644
--- a/pylib/cqlshlib/test/test_cqlsh_output.py
+++ b/pylib/cqlshlib/test/test_cqlsh_output.py
@@ -675,6 +675,7 @@
varcharcol text,
varintcol varint
) WITH additional_write_policy = '99p'
+ AND allow_auto_snapshot = true
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND cdc = false
@@ -686,6 +687,7 @@
AND default_time_to_live = 0
AND extensions = {}
AND gc_grace_seconds = 864000
+ AND incremental_backups = true
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
@@ -936,3 +938,51 @@
row_headers = [s for s in output.splitlines() if "@ Row" in s]
row_ids = [int(s.split(' ')[2]) for s in row_headers]
self.assertEqual([i for i in range(1, 21)], row_ids)
+
+ def test_quoted_output_text_in_map(self):
+ ks = get_keyspace()
+
+ query = "SELECT text_data FROM " + ks + ".escape_quotes;"
+ output, result = testcall_cqlsh(prompt=None, env=self.default_env,
+ tty=False, input=query)
+ self.assertEqual(0, result)
+ self.assertEqual(output.splitlines()[3].strip(), "I'm newb")
+
+ query = "SELECT map_data FROM " + ks + ".escape_quotes;"
+ output, result = testcall_cqlsh(prompt=None, env=self.default_env,
+ tty=False, input=query)
+ self.assertEqual(0, result)
+ self.assertEqual(output.splitlines()[3].strip(), "{1: 'I''m newb'}")
+
+ def test_quoted_output_text_in_simple_collections(self):
+ ks = get_keyspace()
+
+ # Sets
+ query = "SELECT set_data FROM " + ks + ".escape_quotes;"
+ output, result = testcall_cqlsh(prompt=None, env=self.default_env,
+ tty=False, input=query)
+ self.assertEqual(0, result)
+ self.assertEqual(output.splitlines()[3].strip(), "{'I''m newb'}")
+
+ # Lists
+ query = "SELECT list_data FROM " + ks + ".escape_quotes;"
+ output, result = testcall_cqlsh(prompt=None, env=self.default_env,
+ tty=False, input=query)
+ self.assertEqual(0, result)
+ self.assertEqual(output.splitlines()[3].strip(), "['I''m newb']")
+
+ # Tuples
+ query = "SELECT tuple_data FROM " + ks + ".escape_quotes;"
+ output, result = testcall_cqlsh(prompt=None, env=self.default_env,
+ tty=False, input=query)
+ self.assertEqual(0, result)
+ self.assertEqual(output.splitlines()[3].strip(), "(1, 'I''m newb')")
+
+ def test_quoted_output_text_in_udts(self):
+ ks = get_keyspace()
+
+ query = "SELECT udt_data FROM " + ks + ".escape_quotes;"
+ output, result = testcall_cqlsh(prompt=None, env=self.default_env,
+ tty=False, input=query)
+ self.assertEqual(0, result)
+ self.assertEqual(output.splitlines()[3].strip(), "{data: 'I''m newb'}")
\ No newline at end of file
diff --git a/pylib/cqlshlib/test/test_keyspace_init.cql b/pylib/cqlshlib/test/test_keyspace_init.cql
index 96bf9ea..5ff4108 100644
--- a/pylib/cqlshlib/test/test_keyspace_init.cql
+++ b/pylib/cqlshlib/test/test_keyspace_init.cql
@@ -69,9 +69,9 @@
INSERT INTO has_all_types (num, intcol, asciicol, bigintcol, blobcol, booleancol,
decimalcol, doublecol, floatcol, smallintcol, textcol,
timestampcol, tinyintcol, uuidcol, varcharcol, varintcol)
-VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x),
- blobAsDecimal(0x), blobAsDouble(0x), blobAsFloat(0x), blobAsSmallInt(0x0000), '',
- blobAsTimestamp(0x), blobAsTinyInt(0x00), blobAsUuid(0x), '', blobAsVarint(0x));
+VALUES (4, blob_as_int(0x), '', blob_as_bigint(0x), 0x, blob_as_boolean(0x),
+ blob_as_decimal(0x), blob_as_double(0x), blob_as_float(0x), blob_as_smallInt(0x0000), '',
+ blob_as_timestamp(0x), blob_as_tinyint(0x00), blob_as_uuid(0x), '', blob_as_varint(0x));
@@ -141,13 +141,13 @@
);
-- "newline:\n"
-INSERT INTO ascii_with_special_chars (k, val) VALUES (0, blobAsAscii(0x6e65776c696e653a0a));
+INSERT INTO ascii_with_special_chars (k, val) VALUES (0, blob_as_ascii(0x6e65776c696e653a0a));
-- "return\rand null\0!"
-INSERT INTO ascii_with_special_chars (k, val) VALUES (1, blobAsAscii(0x72657475726e0d616e64206e756c6c0021));
+INSERT INTO ascii_with_special_chars (k, val) VALUES (1, blob_as_ascii(0x72657475726e0d616e64206e756c6c0021));
-- "\x00\x01\x02\x03\x04\x05control chars\x06\x07"
-INSERT INTO ascii_with_special_chars (k, val) VALUES (2, blobAsAscii(0x000102030405636f6e74726f6c2063686172730607));
+INSERT INTO ascii_with_special_chars (k, val) VALUES (2, blob_as_ascii(0x000102030405636f6e74726f6c2063686172730607));
-- "fake special chars\\x00\\n"
-INSERT INTO ascii_with_special_chars (k, val) VALUES (3, blobAsAscii(0x66616b65207370656369616c2063686172735c7830305c6e));
+INSERT INTO ascii_with_special_chars (k, val) VALUES (3, blob_as_ascii(0x66616b65207370656369616c2063686172735c7830305c6e));
@@ -280,26 +280,26 @@
CREATE FUNCTION fBestband ( input double )
RETURNS NULL ON NULL INPUT
- RETURNS text
+ RETURNS text
LANGUAGE java
AS 'return "Iron Maiden";';
CREATE FUNCTION fBestsong ( input double )
RETURNS NULL ON NULL INPUT
- RETURNS text
+ RETURNS text
LANGUAGE java
AS 'return "Revelations";';
CREATE FUNCTION fMax(current int, candidate int)
CALLED ON NULL INPUT
- RETURNS int
- LANGUAGE java
+ RETURNS int
+ LANGUAGE java
AS 'if (current == null) return candidate; else return Math.max(current, candidate);' ;
CREATE FUNCTION fMin(current int, candidate int)
CALLED ON NULL INPUT
RETURNS int
- LANGUAGE java
+ LANGUAGE java
AS 'if (current == null) return candidate; else return Math.min(current, candidate);' ;
CREATE AGGREGATE aggMax(int)
@@ -312,3 +312,18 @@
STYPE int
INITCOND null;
+CREATE TYPE quote_udt (
+ data text
+);
+
+CREATE TABLE escape_quotes (
+ id int PRIMARY KEY,
+ text_data text,
+ map_data map<int, text>,
+ set_data set<text>,
+ list_data list<text>,
+ tuple_data tuple<int, text>,
+ udt_data frozen<quote_udt>
+);
+
+INSERT INTO escape_quotes (id, text_data, map_data, set_data, list_data, tuple_data, udt_data) values(1, 'I''m newb', {1:'I''m newb'}, {'I''m newb'}, ['I''m newb'], (1, 'I''m newb'), {data: 'I''m newb'});
diff --git a/pylib/cqlshlib/wcwidth.py b/pylib/cqlshlib/wcwidth.py
deleted file mode 100644
index 0be3af2..0000000
--- a/pylib/cqlshlib/wcwidth.py
+++ /dev/null
@@ -1,379 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# adapted from http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
-# -thepaul
-
-# This is an implementation of wcwidth() and wcswidth() (defined in
-# IEEE Std 1002.1-2001) for Unicode.
-#
-# http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
-# http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
-#
-# In fixed-width output devices, Latin characters all occupy a single
-# "cell" position of equal width, whereas ideographic CJK characters
-# occupy two such cells. Interoperability between terminal-line
-# applications and (teletype-style) character terminals using the
-# UTF-8 encoding requires agreement on which character should advance
-# the cursor by how many cell positions. No established formal
-# standards exist at present on which Unicode character shall occupy
-# how many cell positions on character terminals. These routines are
-# a first attempt of defining such behavior based on simple rules
-# applied to data provided by the Unicode Consortium.
-#
-# For some graphical characters, the Unicode standard explicitly
-# defines a character-cell width via the definition of the East Asian
-# FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
-# In all these cases, there is no ambiguity about which width a
-# terminal shall use. For characters in the East Asian Ambiguous (A)
-# class, the width choice depends purely on a preference of backward
-# compatibility with either historic CJK or Western practice.
-# Choosing single-width for these characters is easy to justify as
-# the appropriate long-term solution, as the CJK practice of
-# displaying these characters as double-width comes from historic
-# implementation simplicity (8-bit encoded characters were displayed
-# single-width and 16-bit ones double-width, even for Greek,
-# Cyrillic, etc.) and not any typographic considerations.
-#
-# Much less clear is the choice of width for the Not East Asian
-# (Neutral) class. Existing practice does not dictate a width for any
-# of these characters. It would nevertheless make sense
-# typographically to allocate two character cells to characters such
-# as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
-# represented adequately with a single-width glyph. The following
-# routines at present merely assign a single-cell width to all
-# neutral characters, in the interest of simplicity. This is not
-# entirely satisfactory and should be reconsidered before
-# establishing a formal standard in this area. At the moment, the
-# decision which Not East Asian (Neutral) characters should be
-# represented by double-width glyphs cannot yet be answered by
-# applying a simple rule from the Unicode database content. Setting
-# up a proper standard for the behavior of UTF-8 character terminals
-# will require a careful analysis not only of each Unicode character,
-# but also of each presentation form, something the author of these
-# routines has avoided to do so far.
-#
-# http://www.unicode.org/unicode/reports/tr11/
-#
-# Markus Kuhn -- 2007-05-26 (Unicode 5.0)
-#
-# Permission to use, copy, modify, and distribute this software
-# for any purpose and without fee is hereby granted. The author
-# disclaims all warranties with regard to this software.
-#
-# Latest C version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
-
-# auxiliary function for binary search in interval table
-
-
-def bisearch(ucs, table):
- min = 0
- max = len(table) - 1
- if ucs < table[0][0] or ucs > table[max][1]:
- return 0
- while max >= min:
- mid = int((min + max) / 2)
- if ucs > table[mid][1]:
- min = mid + 1
- elif ucs < table[mid][0]:
- max = mid - 1
- else:
- return 1
- return 0
-
-
-# The following two functions define the column width of an ISO 10646
-# character as follows:
-#
-# - The null character (U+0000) has a column width of 0.
-#
-# - Other C0/C1 control characters and DEL will lead to a return
-# value of -1.
-#
-# - Non-spacing and enclosing combining characters (general
-# category code Mn or Me in the Unicode database) have a
-# column width of 0.
-#
-# - SOFT HYPHEN (U+00AD) has a column width of 1.
-#
-# - Other format characters (general category code Cf in the Unicode
-# database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
-#
-# - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
-# have a column width of 0.
-#
-# - Spacing characters in the East Asian Wide (W) or East Asian
-# Full-width (F) category as defined in Unicode Technical
-# Report #11 have a column width of 2.
-#
-# - All remaining characters (including all printable
-# ISO 8859-1 and WGL4 characters, Unicode control characters,
-# etc.) have a column width of 1.
-#
-# This implementation assumes that wchar_t characters are encoded
-# in ISO 10646.
-
-# sorted list of non-overlapping intervals of non-spacing characters
-# generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c"
-combining = (
- (0x0300, 0x036F), (0x0483, 0x0486), (0x0488, 0x0489),
- (0x0591, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
- (0x05C4, 0x05C5), (0x05C7, 0x05C7), (0x0600, 0x0603),
- (0x0610, 0x0615), (0x064B, 0x065E), (0x0670, 0x0670),
- (0x06D6, 0x06E4), (0x06E7, 0x06E8), (0x06EA, 0x06ED),
- (0x070F, 0x070F), (0x0711, 0x0711), (0x0730, 0x074A),
- (0x07A6, 0x07B0), (0x07EB, 0x07F3), (0x0901, 0x0902),
- (0x093C, 0x093C), (0x0941, 0x0948), (0x094D, 0x094D),
- (0x0951, 0x0954), (0x0962, 0x0963), (0x0981, 0x0981),
- (0x09BC, 0x09BC), (0x09C1, 0x09C4), (0x09CD, 0x09CD),
- (0x09E2, 0x09E3), (0x0A01, 0x0A02), (0x0A3C, 0x0A3C),
- (0x0A41, 0x0A42), (0x0A47, 0x0A48), (0x0A4B, 0x0A4D),
- (0x0A70, 0x0A71), (0x0A81, 0x0A82), (0x0ABC, 0x0ABC),
- (0x0AC1, 0x0AC5), (0x0AC7, 0x0AC8), (0x0ACD, 0x0ACD),
- (0x0AE2, 0x0AE3), (0x0B01, 0x0B01), (0x0B3C, 0x0B3C),
- (0x0B3F, 0x0B3F), (0x0B41, 0x0B43), (0x0B4D, 0x0B4D),
- (0x0B56, 0x0B56), (0x0B82, 0x0B82), (0x0BC0, 0x0BC0),
- (0x0BCD, 0x0BCD), (0x0C3E, 0x0C40), (0x0C46, 0x0C48),
- (0x0C4A, 0x0C4D), (0x0C55, 0x0C56), (0x0CBC, 0x0CBC),
- (0x0CBF, 0x0CBF), (0x0CC6, 0x0CC6), (0x0CCC, 0x0CCD),
- (0x0CE2, 0x0CE3), (0x0D41, 0x0D43), (0x0D4D, 0x0D4D),
- (0x0DCA, 0x0DCA), (0x0DD2, 0x0DD4), (0x0DD6, 0x0DD6),
- (0x0E31, 0x0E31), (0x0E34, 0x0E3A), (0x0E47, 0x0E4E),
- (0x0EB1, 0x0EB1), (0x0EB4, 0x0EB9), (0x0EBB, 0x0EBC),
- (0x0EC8, 0x0ECD), (0x0F18, 0x0F19), (0x0F35, 0x0F35),
- (0x0F37, 0x0F37), (0x0F39, 0x0F39), (0x0F71, 0x0F7E),
- (0x0F80, 0x0F84), (0x0F86, 0x0F87), (0x0F90, 0x0F97),
- (0x0F99, 0x0FBC), (0x0FC6, 0x0FC6), (0x102D, 0x1030),
- (0x1032, 0x1032), (0x1036, 0x1037), (0x1039, 0x1039),
- (0x1058, 0x1059), (0x1160, 0x11FF), (0x135F, 0x135F),
- (0x1712, 0x1714), (0x1732, 0x1734), (0x1752, 0x1753),
- (0x1772, 0x1773), (0x17B4, 0x17B5), (0x17B7, 0x17BD),
- (0x17C6, 0x17C6), (0x17C9, 0x17D3), (0x17DD, 0x17DD),
- (0x180B, 0x180D), (0x18A9, 0x18A9), (0x1920, 0x1922),
- (0x1927, 0x1928), (0x1932, 0x1932), (0x1939, 0x193B),
- (0x1A17, 0x1A18), (0x1B00, 0x1B03), (0x1B34, 0x1B34),
- (0x1B36, 0x1B3A), (0x1B3C, 0x1B3C), (0x1B42, 0x1B42),
- (0x1B6B, 0x1B73), (0x1DC0, 0x1DCA), (0x1DFE, 0x1DFF),
- (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x2063),
- (0x206A, 0x206F), (0x20D0, 0x20EF), (0x302A, 0x302F),
- (0x3099, 0x309A), (0xA806, 0xA806), (0xA80B, 0xA80B),
- (0xA825, 0xA826), (0xFB1E, 0xFB1E), (0xFE00, 0xFE0F),
- (0xFE20, 0xFE23), (0xFEFF, 0xFEFF), (0xFFF9, 0xFFFB),
- (0x10A01, 0x10A03), (0x10A05, 0x10A06), (0x10A0C, 0x10A0F),
- (0x10A38, 0x10A3A), (0x10A3F, 0x10A3F), (0x1D167, 0x1D169),
- (0x1D173, 0x1D182), (0x1D185, 0x1D18B), (0x1D1AA, 0x1D1AD),
- (0x1D242, 0x1D244), (0xE0001, 0xE0001), (0xE0020, 0xE007F),
- (0xE0100, 0xE01EF)
-)
-
-
-# sorted list of non-overlapping intervals of East Asian Ambiguous
-# characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c"
-ambiguous = (
- (0x00A1, 0x00A1), (0x00A4, 0x00A4), (0x00A7, 0x00A8),
- (0x00AA, 0x00AA), (0x00AE, 0x00AE), (0x00B0, 0x00B4),
- (0x00B6, 0x00BA), (0x00BC, 0x00BF), (0x00C6, 0x00C6),
- (0x00D0, 0x00D0), (0x00D7, 0x00D8), (0x00DE, 0x00E1),
- (0x00E6, 0x00E6), (0x00E8, 0x00EA), (0x00EC, 0x00ED),
- (0x00F0, 0x00F0), (0x00F2, 0x00F3), (0x00F7, 0x00FA),
- (0x00FC, 0x00FC), (0x00FE, 0x00FE), (0x0101, 0x0101),
- (0x0111, 0x0111), (0x0113, 0x0113), (0x011B, 0x011B),
- (0x0126, 0x0127), (0x012B, 0x012B), (0x0131, 0x0133),
- (0x0138, 0x0138), (0x013F, 0x0142), (0x0144, 0x0144),
- (0x0148, 0x014B), (0x014D, 0x014D), (0x0152, 0x0153),
- (0x0166, 0x0167), (0x016B, 0x016B), (0x01CE, 0x01CE),
- (0x01D0, 0x01D0), (0x01D2, 0x01D2), (0x01D4, 0x01D4),
- (0x01D6, 0x01D6), (0x01D8, 0x01D8), (0x01DA, 0x01DA),
- (0x01DC, 0x01DC), (0x0251, 0x0251), (0x0261, 0x0261),
- (0x02C4, 0x02C4), (0x02C7, 0x02C7), (0x02C9, 0x02CB),
- (0x02CD, 0x02CD), (0x02D0, 0x02D0), (0x02D8, 0x02DB),
- (0x02DD, 0x02DD), (0x02DF, 0x02DF), (0x0391, 0x03A1),
- (0x03A3, 0x03A9), (0x03B1, 0x03C1), (0x03C3, 0x03C9),
- (0x0401, 0x0401), (0x0410, 0x044F), (0x0451, 0x0451),
- (0x2010, 0x2010), (0x2013, 0x2016), (0x2018, 0x2019),
- (0x201C, 0x201D), (0x2020, 0x2022), (0x2024, 0x2027),
- (0x2030, 0x2030), (0x2032, 0x2033), (0x2035, 0x2035),
- (0x203B, 0x203B), (0x203E, 0x203E), (0x2074, 0x2074),
- (0x207F, 0x207F), (0x2081, 0x2084), (0x20AC, 0x20AC),
- (0x2103, 0x2103), (0x2105, 0x2105), (0x2109, 0x2109),
- (0x2113, 0x2113), (0x2116, 0x2116), (0x2121, 0x2122),
- (0x2126, 0x2126), (0x212B, 0x212B), (0x2153, 0x2154),
- (0x215B, 0x215E), (0x2160, 0x216B), (0x2170, 0x2179),
- (0x2190, 0x2199), (0x21B8, 0x21B9), (0x21D2, 0x21D2),
- (0x21D4, 0x21D4), (0x21E7, 0x21E7), (0x2200, 0x2200),
- (0x2202, 0x2203), (0x2207, 0x2208), (0x220B, 0x220B),
- (0x220F, 0x220F), (0x2211, 0x2211), (0x2215, 0x2215),
- (0x221A, 0x221A), (0x221D, 0x2220), (0x2223, 0x2223),
- (0x2225, 0x2225), (0x2227, 0x222C), (0x222E, 0x222E),
- (0x2234, 0x2237), (0x223C, 0x223D), (0x2248, 0x2248),
- (0x224C, 0x224C), (0x2252, 0x2252), (0x2260, 0x2261),
- (0x2264, 0x2267), (0x226A, 0x226B), (0x226E, 0x226F),
- (0x2282, 0x2283), (0x2286, 0x2287), (0x2295, 0x2295),
- (0x2299, 0x2299), (0x22A5, 0x22A5), (0x22BF, 0x22BF),
- (0x2312, 0x2312), (0x2460, 0x24E9), (0x24EB, 0x254B),
- (0x2550, 0x2573), (0x2580, 0x258F), (0x2592, 0x2595),
- (0x25A0, 0x25A1), (0x25A3, 0x25A9), (0x25B2, 0x25B3),
- (0x25B6, 0x25B7), (0x25BC, 0x25BD), (0x25C0, 0x25C1),
- (0x25C6, 0x25C8), (0x25CB, 0x25CB), (0x25CE, 0x25D1),
- (0x25E2, 0x25E5), (0x25EF, 0x25EF), (0x2605, 0x2606),
- (0x2609, 0x2609), (0x260E, 0x260F), (0x2614, 0x2615),
- (0x261C, 0x261C), (0x261E, 0x261E), (0x2640, 0x2640),
- (0x2642, 0x2642), (0x2660, 0x2661), (0x2663, 0x2665),
- (0x2667, 0x266A), (0x266C, 0x266D), (0x266F, 0x266F),
- (0x273D, 0x273D), (0x2776, 0x277F), (0xE000, 0xF8FF),
- (0xFFFD, 0xFFFD), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)
-)
-
-
-def mk_wcwidth(ucs):
- # test for 8-bit control characters
- if ucs == 0:
- return 0
- if ucs < 32 or (ucs >= 0x7f and ucs < 0xa0):
- return -1
-
- # binary search in table of non-spacing characters
- if bisearch(ucs, combining):
- return 0
-
- # if we arrive here, ucs is not a combining or C0/C1 control character
-
- return 1 + int(
- ucs >= 0x1100
- and (ucs <= 0x115f # Hangul Jamo init. consonants
- or ucs == 0x2329 or ucs == 0x232a
- or (ucs >= 0x2e80 and ucs <= 0xa4cf
- and ucs != 0x303f) # CJK ... Yi
- or (ucs >= 0xac00 and ucs <= 0xd7a3) # Hangul Syllables
- or (ucs >= 0xf900 and ucs <= 0xfaff) # CJK Compatibility Ideographs
- or (ucs >= 0xfe10 and ucs <= 0xfe19) # Vertical forms
- or (ucs >= 0xfe30 and ucs <= 0xfe6f) # CJK Compatibility Forms
- or (ucs >= 0xff00 and ucs <= 0xff60) # Fullwidth Forms
- or (ucs >= 0xffe0 and ucs <= 0xffe6)
- or (ucs >= 0x20000 and ucs <= 0x2fffd)
- or (ucs >= 0x30000 and ucs <= 0x3fffd))
- )
-
-
-def mk_wcswidth(pwcs):
- width = 0
- for c in pwcs:
- w = mk_wcwidth(c)
- if w < 0:
- return -1
- else:
- width += w
-
- return width
-
-
-# The following functions are the same as mk_wcwidth() and
-# mk_wcswidth(), except that spacing characters in the East Asian
-# Ambiguous (A) category as defined in Unicode Technical Report #11
-# have a column width of 2. This variant might be useful for users of
-# CJK legacy encodings who want to migrate to UCS without changing
-# the traditional terminal character-width behaviour. It is not
-# otherwise recommended for general use.
-def mk_wcwidth_cjk(ucs):
- # binary search in table of non-spacing characters
- if bisearch(ucs, ambiguous):
- return 2
-
- return mk_wcwidth(ucs)
-
-
-def mk_wcswidth_cjk(pwcs):
- width = 0
-
- for c in pwcs:
- w = mk_wcwidth_cjk(c)
- if w < 0:
- return -1
- width += w
-
- return width
-
-# python-y versions, dealing with unicode objects
-
-
-def wcwidth(c):
- return mk_wcwidth(ord(c))
-
-
-def wcswidth(s):
- return mk_wcswidth(list(map(ord, s)))
-
-
-def wcwidth_cjk(c):
- return mk_wcwidth_cjk(ord(c))
-
-
-def wcswidth_cjk(s):
- return mk_wcswidth_cjk(list(map(ord, s)))
-
-
-if __name__ == "__main__":
- samples = (
- ('MUSIC SHARP SIGN', 1),
- ('FULLWIDTH POUND SIGN', 2),
- ('FULLWIDTH LATIN CAPITAL LETTER P', 2),
- ('CJK RADICAL BOLT OF CLOTH', 2),
- ('LATIN SMALL LETTER A', 1),
- ('LATIN SMALL LETTER AE', 1),
- ('SPACE', 1),
- ('NO-BREAK SPACE', 1),
- ('CJK COMPATIBILITY IDEOGRAPH-F920', 2),
- ('MALAYALAM VOWEL SIGN UU', 0),
- ('ZERO WIDTH SPACE', 0),
- ('ZERO WIDTH NO-BREAK SPACE', 0),
- ('COMBINING PALATALIZED HOOK BELOW', 0),
- ('COMBINING GRAVE ACCENT', 0),
- )
- nonprinting = '\r\n\t\a\b\f\v\x7f'
-
- import unicodedata
-
- for name, printwidth in samples:
- uchr = unicodedata.lookup(name)
- calculatedwidth = wcwidth(uchr)
- assert calculatedwidth == printwidth, \
- 'width for %r should be %d, but is %d?' % (uchr, printwidth, calculatedwidth)
-
- for c in nonprinting:
- calculatedwidth = wcwidth(c)
- assert calculatedwidth < 0, \
- '%r is a control character, but wcwidth gives %d' % (c, calculatedwidth)
-
- assert wcwidth('\0') == 0 # special case
-
- # depending on how python is compiled, code points above U+FFFF may not be
- # treated as single characters, so ord() won't work. test a few of these
- # manually.
-
- assert mk_wcwidth(0xe01ef) == 0
- assert mk_wcwidth(0x10ffff) == 1
- assert mk_wcwidth(0x3fffd) == 2
-
- teststr = 'B\0ig br\u00f8wn moose\ub143\u200b'
- calculatedwidth = wcswidth(teststr)
- assert calculatedwidth == 17, 'expected 17, got %d' % calculatedwidth
-
- calculatedwidth = wcswidth_cjk(teststr)
- assert calculatedwidth == 18, 'expected 18, got %d' % calculatedwidth
-
- assert wcswidth('foobar\u200b\a') < 0
-
- print('tests pass.')
diff --git a/pylib/requirements.txt b/pylib/requirements.txt
index c030460..379f6ae 100644
--- a/pylib/requirements.txt
+++ b/pylib/requirements.txt
@@ -3,3 +3,4 @@
-e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm
coverage
pytest
+wcwidth
diff --git a/pylib/setup.py b/pylib/setup.py
index a9f654a..f5fd184 100755
--- a/pylib/setup.py
+++ b/pylib/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
diff --git a/redhat/cassandra.spec b/redhat/cassandra.spec
index 7431c1c..ae5ad4c 100644
--- a/redhat/cassandra.spec
+++ b/redhat/cassandra.spec
@@ -17,8 +17,8 @@
#
%define __jar_repack %{nil}
-# Turn off the brp-python-bytecompile script
-%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+# Turn off the brp-python-bytecompile script and mangling shebangs for Python scripts
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g' -e 's!/usr/lib[^[:space:]]*/brp-mangle-shebangs[[:space:]].*$!!g')
# rpmbuild should not barf when it spots we ship
# binary executable files in our 'noarch' package
@@ -161,9 +161,9 @@
%{_sysconfdir}/security/limits.d/%{username}.conf
/usr/share/%{username}*
%config(noreplace) /%{_sysconfdir}/%{username}
-%attr(755,%{username},%{username}) %config(noreplace) /var/lib/%{username}/*
-%attr(755,%{username},%{username}) /var/log/%{username}*
-%attr(755,%{username},%{username}) /var/run/%{username}*
+%attr(750,%{username},%{username}) %config(noreplace) /var/lib/%{username}/*
+%attr(750,%{username},%{username}) /var/log/%{username}*
+%attr(750,%{username},%{username}) /var/run/%{username}*
%{python_sitelib}/cqlshlib/
%{python_sitelib}/cassandra_pylib*.egg-info
@@ -198,6 +198,7 @@
%attr(755,root,root) %{_bindir}/sstableofflinerelevel
%attr(755,root,root) %{_bindir}/sstablerepairedset
%attr(755,root,root) %{_bindir}/sstablesplit
+%attr(755,root,root) %{_bindir}/sstablepartitions
%attr(755,root,root) %{_bindir}/auditlogviewer
%attr(755,root,root) %{_bindir}/jmxtool
%attr(755,root,root) %{_bindir}/fqltool
diff --git a/redhat/noboolean/cassandra.spec b/redhat/noboolean/cassandra.spec
index a3abaa6..2b6cdd4 100644
--- a/redhat/noboolean/cassandra.spec
+++ b/redhat/noboolean/cassandra.spec
@@ -17,8 +17,8 @@
#
%define __jar_repack %{nil}
-# Turn off the brp-python-bytecompile script
-%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+# Turn off the brp-python-bytecompile script and mangling shebangs for Python scripts
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g' -e 's!/usr/lib[^[:space:]]*/brp-mangle-shebangs[[:space:]].*$!!g')
# rpmbuild should not barf when it spots we ship
# binary executable files in our 'noarch' package
@@ -161,9 +161,9 @@
%{_sysconfdir}/security/limits.d/%{username}.conf
/usr/share/%{username}*
%config(noreplace) /%{_sysconfdir}/%{username}
-%attr(755,%{username},%{username}) %config(noreplace) /var/lib/%{username}/*
-%attr(755,%{username},%{username}) /var/log/%{username}*
-%attr(755,%{username},%{username}) /var/run/%{username}*
+%attr(750,%{username},%{username}) %config(noreplace) /var/lib/%{username}/*
+%attr(750,%{username},%{username}) /var/log/%{username}*
+%attr(750,%{username},%{username}) /var/run/%{username}*
%{python_sitelib}/cqlshlib/
%{python_sitelib}/cassandra_pylib*.egg-info
@@ -190,6 +190,7 @@
This package contains extra tools for working with Cassandra clusters.
%files tools
+%attr(755,root,root) %{_bindir}/sstablepartitions
%attr(755,root,root) %{_bindir}/sstabledump
%attr(755,root,root) %{_bindir}/compaction-stress
%attr(755,root,root) %{_bindir}/sstableexpiredblockers
diff --git a/relocate-dependencies.pom b/relocate-dependencies.pom
index 07728dd..96d8d16 100644
--- a/relocate-dependencies.pom
+++ b/relocate-dependencies.pom
@@ -67,7 +67,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.2.1</version>
+ <version>3.4.1</version>
<configuration>
<createSourcesJar>false</createSourcesJar>
diff --git a/src/antlr/Cql.g b/src/antlr/Cql.g
index 272c63b..0eb2ac5 100644
--- a/src/antlr/Cql.g
+++ b/src/antlr/Cql.g
@@ -39,6 +39,7 @@
import org.apache.cassandra.auth.*;
import org.apache.cassandra.cql3.conditions.*;
import org.apache.cassandra.cql3.functions.*;
+ import org.apache.cassandra.cql3.functions.masking.*;
import org.apache.cassandra.cql3.restrictions.CustomIndexExpression;
import org.apache.cassandra.cql3.selection.*;
import org.apache.cassandra.cql3.statements.*;
diff --git a/src/antlr/Lexer.g b/src/antlr/Lexer.g
index 72ab7db..512a82d 100644
--- a/src/antlr/Lexer.g
+++ b/src/antlr/Lexer.g
@@ -178,6 +178,7 @@
K_TIMEUUID: T I M E U U I D;
K_TOKEN: T O K E N;
K_WRITETIME: W R I T E T I M E;
+K_MAXWRITETIME:M A X W R I T E T I M E;
K_DATE: D A T E;
K_TIME: T I M E;
@@ -217,6 +218,10 @@
K_UNSET: U N S E T;
K_LIKE: L I K E;
+K_MASKED: M A S K E D;
+K_UNMASK: U N M A S K;
+K_SELECT_MASKED: S E L E C T '_' M A S K E D;
+
// Case-insensitive alpha characters
fragment A: ('a'|'A');
fragment B: ('b'|'B');
diff --git a/src/antlr/Parser.g b/src/antlr/Parser.g
index d061ee4..029d7fc 100644
--- a/src/antlr/Parser.g
+++ b/src/antlr/Parser.g
@@ -414,11 +414,12 @@
;
selectionFunction returns [Selectable.Raw s]
- : K_COUNT '(' '\*' ')' { $s = Selectable.WithFunction.Raw.newCountRowsFunction(); }
- | K_WRITETIME '(' c=sident ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, true); }
- | K_TTL '(' c=sident ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, false); }
- | K_CAST '(' sn=unaliasedSelector K_AS t=native_type ')' {$s = new Selectable.WithCast.Raw(sn, t);}
- | f=functionName args=selectionFunctionArgs { $s = new Selectable.WithFunction.Raw(f, args); }
+ : K_COUNT '(' '\*' ')' { $s = Selectable.WithFunction.Raw.newCountRowsFunction(); }
+ | K_MAXWRITETIME '(' c=sident m=selectorModifier[c] ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, m, Selectable.WritetimeOrTTL.Kind.MAX_WRITE_TIME); }
+ | K_WRITETIME '(' c=sident m=selectorModifier[c] ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, m, Selectable.WritetimeOrTTL.Kind.WRITE_TIME); }
+ | K_TTL '(' c=sident m=selectorModifier[c] ')' { $s = new Selectable.WritetimeOrTTL.Raw(c, m, Selectable.WritetimeOrTTL.Kind.TTL); }
+ | K_CAST '(' sn=unaliasedSelector K_AS t=native_type ')' { $s = new Selectable.WithCast.Raw(sn, t);}
+ | f=functionName args=selectionFunctionArgs { $s = new Selectable.WithFunction.Raw(f, args); }
;
selectionLiteral returns [Term.Raw value]
@@ -781,11 +782,21 @@
tableColumns[CreateTableStatement.Raw stmt]
@init { boolean isStatic = false; }
- : k=ident v=comparatorType (K_STATIC { isStatic = true; })? { $stmt.addColumn(k, v, isStatic); }
+ : k=ident v=comparatorType (K_STATIC { isStatic = true; })? (mask=columnMask)? { $stmt.addColumn(k, v, isStatic, mask); }
(K_PRIMARY K_KEY { $stmt.setPartitionKeyColumn(k); })?
| K_PRIMARY K_KEY '(' tablePartitionKey[stmt] (',' c=ident { $stmt.markClusteringColumn(c); } )* ')'
;
+columnMask returns [ColumnMask.Raw mask]
+ @init { List<Term.Raw> arguments = new ArrayList<>(); }
+ : K_MASKED K_WITH name=functionName columnMaskArguments[arguments] { $mask = new ColumnMask.Raw(name, arguments); }
+ | K_MASKED K_WITH K_DEFAULT { $mask = new ColumnMask.Raw(FunctionName.nativeFunction("mask_default"), arguments); }
+ ;
+
+columnMaskArguments[List<Term.Raw> arguments]
+ : '(' ')' | '(' c=term { arguments.add(c); } (',' cn=term { arguments.add(cn); })* ')'
+ ;
+
tablePartitionKey[CreateTableStatement.Raw stmt]
@init {List<ColumnIdentifier> l = new ArrayList<ColumnIdentifier>();}
@after{ $stmt.setPartitionKeyColumns(l); }
@@ -928,7 +939,9 @@
/**
* ALTER TABLE <table> ALTER <column> TYPE <newtype>;
- * ALTER TABLE [IF EXISTS] <table> ADD [IF NOT EXISTS] <column> <newtype>; | ALTER TABLE [IF EXISTS] <table> ADD [IF NOT EXISTS] (<column> <newtype>,<column1> <newtype1>..... <column n> <newtype n>)
+ * ALTER TABLE [IF EXISTS] <table> ALTER [IF EXISTS] <column> MASKED WITH <maskFunction>);
+ * ALTER TABLE [IF EXISTS] <table> ALTER [IF EXISTS] <column> DROP MASKED;
+ * ALTER TABLE [IF EXISTS] <table> ADD [IF NOT EXISTS] <column> <newtype> <maskFunction>; | ALTER TABLE [IF EXISTS] <table> ADD [IF NOT EXISTS] (<column> <newtype> <maskFunction>, <column1> <newtype1> <maskFunction1>..... <column n> <newtype n> <maskFunction n>)
* ALTER TABLE [IF EXISTS] <table> DROP [IF EXISTS] <column>; | ALTER TABLE [IF EXISTS] <table> DROP [IF EXISTS] ( <column>,<column1>.....<column n>)
* ALTER TABLE [IF EXISTS] <table> RENAME [IF EXISTS] <column> TO <column>;
* ALTER TABLE [IF EXISTS] <table> WITH <property> = <value>;
@@ -940,10 +953,14 @@
(
K_ALTER id=cident K_TYPE v=comparatorType { $stmt.alter(id, v); }
+ | K_ALTER ( K_IF K_EXISTS { $stmt.ifColumnExists(true); } )? id=cident
+ ( mask=columnMask { $stmt.mask(id, mask); }
+ | K_DROP K_MASKED { $stmt.mask(id, null); } )
+
| K_ADD ( K_IF K_NOT K_EXISTS { $stmt.ifColumnNotExists(true); } )?
- ( id=ident v=comparatorType b=isStaticColumn { $stmt.add(id, v, b); }
- | ('(' id1=ident v1=comparatorType b1=isStaticColumn { $stmt.add(id1, v1, b1); }
- ( ',' idn=ident vn=comparatorType bn=isStaticColumn { $stmt.add(idn, vn, bn); } )* ')') )
+ ( id=ident v=comparatorType b=isStaticColumn (m=columnMask)? { $stmt.add(id, v, b, m); }
+ | ('(' id1=ident v1=comparatorType b1=isStaticColumn (m1=columnMask)? { $stmt.add(id1, v1, b1, m1); }
+ ( ',' idn=ident vn=comparatorType bn=isStaticColumn (mn=columnMask)? { $stmt.add(idn, vn, bn, mn); mn=null; } )* ')') )
| K_DROP ( K_IF K_EXISTS { $stmt.ifColumnExists(true); } )?
( id=ident { $stmt.drop(id); }
@@ -1111,7 +1128,7 @@
;
permission returns [Permission perm]
- : p=(K_CREATE | K_ALTER | K_DROP | K_SELECT | K_MODIFY | K_AUTHORIZE | K_DESCRIBE | K_EXECUTE)
+ : p=(K_CREATE | K_ALTER | K_DROP | K_SELECT | K_MODIFY | K_AUTHORIZE | K_DESCRIBE | K_EXECUTE | K_UNMASK | K_SELECT_MASKED)
{ $perm = Permission.valueOf($p.text.toUpperCase()); }
;
@@ -1870,7 +1887,7 @@
unreserved_keyword returns [String str]
: u=unreserved_function_keyword { $str = u; }
- | k=(K_TTL | K_COUNT | K_WRITETIME | K_KEY | K_CAST | K_JSON | K_DISTINCT) { $str = $k.text; }
+ | k=(K_TTL | K_COUNT | K_WRITETIME | K_MAXWRITETIME | K_KEY | K_CAST | K_JSON | K_DISTINCT) { $str = $k.text; }
;
unreserved_function_keyword returns [String str]
@@ -1939,5 +1956,8 @@
| K_MBEANS
| K_REPLACE
| K_UNSET
+ | K_MASKED
+ | K_UNMASK
+ | K_SELECT_MASKED
) { $str = $k.text; }
;
diff --git a/src/java/org/apache/cassandra/auth/AllowAllInternodeAuthenticator.java b/src/java/org/apache/cassandra/auth/AllowAllInternodeAuthenticator.java
index d0d2d74..ac62bfa 100644
--- a/src/java/org/apache/cassandra/auth/AllowAllInternodeAuthenticator.java
+++ b/src/java/org/apache/cassandra/auth/AllowAllInternodeAuthenticator.java
@@ -20,12 +20,14 @@
package org.apache.cassandra.auth;
import java.net.InetAddress;
+import java.security.cert.Certificate;
import org.apache.cassandra.exceptions.ConfigurationException;
public class AllowAllInternodeAuthenticator implements IInternodeAuthenticator
{
- public boolean authenticate(InetAddress remoteAddress, int remotePort)
+ public boolean authenticate(InetAddress remoteAddress, int remotePort,
+ Certificate[] certificates, InternodeConnectionDirection connectionType)
{
return true;
}
diff --git a/src/java/org/apache/cassandra/auth/AuthCache.java b/src/java/org/apache/cassandra/auth/AuthCache.java
index 66a2a4f..7d3a164 100644
--- a/src/java/org/apache/cassandra/auth/AuthCache.java
+++ b/src/java/org/apache/cassandra/auth/AuthCache.java
@@ -48,6 +48,9 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_CACHE_WARMING_MAX_RETRIES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_CACHE_WARMING_RETRY_INTERVAL_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION;
public class AuthCache<K, V> implements AuthCacheMBean, Shutdownable
{
@@ -55,11 +58,6 @@
public static final String MBEAN_NAME_BASE = "org.apache.cassandra.auth:type=";
- // We expect default values on cache retries and interval to be sufficient for everyone but have this escape hatch
- // just in case.
- static final String CACHE_LOAD_RETRIES_PROPERTY = "cassandra.auth_cache.warming.max_retries";
- static final String CACHE_LOAD_RETRY_INTERVAL_PROPERTY = "cassandra.auth_cache.warming.retry_interval_ms";
-
private volatile ScheduledFuture cacheRefresher = null;
// Keep a handle on created instances so their executors can be terminated cleanly
@@ -254,7 +252,7 @@
*/
public synchronized void setValidity(int validityPeriod)
{
- if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
+ if (DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION.getBoolean())
throw new UnsupportedOperationException("Remote configuration of auth caches is disabled");
setValidityDelegate.accept(validityPeriod);
@@ -272,7 +270,7 @@
*/
public synchronized void setUpdateInterval(int updateInterval)
{
- if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
+ if (DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION.getBoolean())
throw new UnsupportedOperationException("Remote configuration of auth caches is disabled");
setUpdateIntervalDelegate.accept(updateInterval);
@@ -290,7 +288,7 @@
*/
public synchronized void setMaxEntries(int maxEntries)
{
- if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
+ if (DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION.getBoolean())
throw new UnsupportedOperationException("Remote configuration of auth caches is disabled");
setMaxEntriesDelegate.accept(maxEntries);
@@ -309,7 +307,7 @@
public synchronized void setActiveUpdate(boolean update)
{
- if (Boolean.getBoolean("cassandra.disable_auth_caches_remote_configuration"))
+ if (DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION.getBoolean())
throw new UnsupportedOperationException("Remote configuration of auth caches is disabled");
setActiveUpdate.accept(update);
@@ -409,8 +407,8 @@
return;
}
- int retries = Integer.getInteger(CACHE_LOAD_RETRIES_PROPERTY, 10);
- long retryInterval = Long.getLong(CACHE_LOAD_RETRY_INTERVAL_PROPERTY, 1000);
+ int retries = AUTH_CACHE_WARMING_MAX_RETRIES.getInt(10);
+ long retryInterval = AUTH_CACHE_WARMING_RETRY_INTERVAL_MS.getLong(1000);
while (retries-- > 0)
{
diff --git a/src/java/org/apache/cassandra/auth/AuthKeyspace.java b/src/java/org/apache/cassandra/auth/AuthKeyspace.java
index 67fc9c1..f0a3da8 100644
--- a/src/java/org/apache/cassandra/auth/AuthKeyspace.java
+++ b/src/java/org/apache/cassandra/auth/AuthKeyspace.java
@@ -30,6 +30,7 @@
import org.apache.cassandra.schema.Tables;
import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS;
public final class AuthKeyspace
{
@@ -55,7 +56,7 @@
public static final String RESOURCE_ROLE_INDEX = "resource_role_permissons_index";
public static final String NETWORK_PERMISSIONS = "network_permissions";
- public static final long SUPERUSER_SETUP_DELAY = Long.getLong("cassandra.superuser_setup_delay_ms", 10000);
+ public static final long SUPERUSER_SETUP_DELAY = SUPERUSER_SETUP_DELAY_MS.getLong();
private static final TableMetadata Roles =
parse(ROLES,
diff --git a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
index 733e9da..37bda4e 100644
--- a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
+++ b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
@@ -34,7 +34,6 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.*;
@@ -43,11 +42,13 @@
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.mindrot.jbcrypt.BCrypt;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_BCRYPT_GENSALT_LOG2_ROUNDS;
import static org.apache.cassandra.service.QueryState.forInternalCalls;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@@ -81,7 +82,7 @@
private static final Logger logger = LoggerFactory.getLogger(CassandraRoleManager.class);
public static final String DEFAULT_SUPERUSER_NAME = "cassandra";
- static final String DEFAULT_SUPERUSER_PASSWORD = "cassandra";
+ public static final String DEFAULT_SUPERUSER_PASSWORD = "cassandra";
/**
* We need to treat the default superuser as a special case since during initial node startup, we may end up with
@@ -112,19 +113,15 @@
}
};
- // 2 ** GENSALT_LOG2_ROUNDS rounds of hashing will be performed.
- @VisibleForTesting
- public static final String GENSALT_LOG2_ROUNDS_PROPERTY = Config.PROPERTY_PREFIX + "auth_bcrypt_gensalt_log2_rounds";
private static final int GENSALT_LOG2_ROUNDS = getGensaltLogRounds();
static int getGensaltLogRounds()
{
- int rounds = Integer.getInteger(GENSALT_LOG2_ROUNDS_PROPERTY, 10);
- if (rounds < 4 || rounds > 30)
- throw new ConfigurationException(String.format("Bad value for system property -D%s." +
- "Please use a value between 4 and 30 inclusively",
- GENSALT_LOG2_ROUNDS_PROPERTY));
- return rounds;
+ int rounds = AUTH_BCRYPT_GENSALT_LOG2_ROUNDS.getInt(10);
+ if (rounds < 4 || rounds > 30)
+ throw new ConfigurationException(String.format("Bad value for system property %s." +
+ "Please use a value between 4 and 30 inclusively", AUTH_BCRYPT_GENSALT_LOG2_ROUNDS.getKey()));
+ return rounds;
}
private SelectStatement loadRoleStatement;
@@ -386,6 +383,12 @@
{
// The delay is to give the node a chance to see its peers before attempting the operation
ScheduledExecutors.optionalTasks.scheduleSelfRecurring(() -> {
+ if (!StorageProxy.isSafeToPerformRead())
+ {
+ logger.trace("Setup task may not run due to it not being safe to perform reads... rescheduling");
+ scheduleSetupTask(setupTask);
+ return;
+ }
try
{
setupTask.call();
diff --git a/src/java/org/apache/cassandra/auth/DataResource.java b/src/java/org/apache/cassandra/auth/DataResource.java
index 2421930..4923a0b 100644
--- a/src/java/org/apache/cassandra/auth/DataResource.java
+++ b/src/java/org/apache/cassandra/auth/DataResource.java
@@ -46,7 +46,9 @@
Permission.DROP,
Permission.SELECT,
Permission.MODIFY,
- Permission.AUTHORIZE);
+ Permission.AUTHORIZE,
+ Permission.UNMASK,
+ Permission.SELECT_MASKED);
// permissions which may be granted on all tables of a given keyspace
private static final Set<Permission> ALL_TABLES_LEVEL_PERMISSIONS = Sets.immutableEnumSet(Permission.CREATE,
@@ -54,7 +56,9 @@
Permission.DROP,
Permission.SELECT,
Permission.MODIFY,
- Permission.AUTHORIZE);
+ Permission.AUTHORIZE,
+ Permission.UNMASK,
+ Permission.SELECT_MASKED);
// permissions which may be granted on one or all keyspaces
private static final Set<Permission> KEYSPACE_LEVEL_PERMISSIONS = Sets.immutableEnumSet(Permission.CREATE,
@@ -62,7 +66,9 @@
Permission.DROP,
Permission.SELECT,
Permission.MODIFY,
- Permission.AUTHORIZE);
+ Permission.AUTHORIZE,
+ Permission.UNMASK,
+ Permission.SELECT_MASKED);
private static final String ROOT_NAME = "data";
private static final DataResource ROOT_RESOURCE = new DataResource(Level.ROOT, null, null);
diff --git a/src/java/org/apache/cassandra/auth/FunctionResource.java b/src/java/org/apache/cassandra/auth/FunctionResource.java
index a67ef1a..86b1592 100644
--- a/src/java/org/apache/cassandra/auth/FunctionResource.java
+++ b/src/java/org/apache/cassandra/auth/FunctionResource.java
@@ -29,13 +29,14 @@
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.SchemaConstants;
/**
* IResource implementation representing functions.
@@ -135,7 +136,7 @@
return new FunctionResource(keyspace, name, argTypes);
}
- public static FunctionResource function(Function function)
+ public static FunctionResource function(UserFunction function)
{
return new FunctionResource(function.name().keyspace, function.name().name, function.argTypes());
}
@@ -271,6 +272,7 @@
public boolean exists()
{
+ validate();
switch (level)
{
case ROOT:
@@ -278,13 +280,14 @@
case KEYSPACE:
return Schema.instance.getKeyspaces().contains(keyspace);
case FUNCTION:
- return Schema.instance.findFunction(getFunctionName(), argTypes).isPresent();
+ return Schema.instance.findUserFunction(getFunctionName(), argTypes).isPresent();
}
throw new AssertionError();
}
public Set<Permission> applicablePermissions()
{
+ validate();
switch (level)
{
case ROOT:
@@ -292,7 +295,7 @@
return COLLECTION_LEVEL_PERMISSIONS;
case FUNCTION:
{
- Optional<Function> function = Schema.instance.findFunction(getFunctionName(), argTypes);
+ Optional<UserFunction> function = Schema.instance.findUserFunction(getFunctionName(), argTypes);
assert function.isPresent() : "Unable to find function object for resource " + toString();
return function.get().isAggregate() ? AGGREGATE_FUNCTION_PERMISSIONS : SCALAR_FUNCTION_PERMISSIONS;
}
@@ -300,6 +303,12 @@
throw new AssertionError();
}
+ private void validate()
+ {
+ if (SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(keyspace))
+ throw new InvalidRequestException("Altering permissions on builtin functions is not supported");
+ }
+
public int compareTo(FunctionResource o)
{
return this.name.compareTo(o.name);
diff --git a/src/java/org/apache/cassandra/auth/IAuthenticator.java b/src/java/org/apache/cassandra/auth/IAuthenticator.java
index 80ea719..9963e4e 100644
--- a/src/java/org/apache/cassandra/auth/IAuthenticator.java
+++ b/src/java/org/apache/cassandra/auth/IAuthenticator.java
@@ -18,11 +18,10 @@
package org.apache.cassandra.auth;
import java.net.InetAddress;
+import java.security.cert.Certificate;
import java.util.Map;
import java.util.Set;
-import javax.security.cert.X509Certificate;
-
import org.apache.cassandra.exceptions.AuthenticationException;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -73,11 +72,14 @@
* override this method to gain access to client's certificate chain, if present.
* @param clientAddress the IP address of the client whom we wish to authenticate, or null
* if an internal client (one not connected over the remote transport).
- * @param certificates the peer's X509 Certificate chain, if present.
+ * @param certificates the peer's Certificate chain, if present.
+ * It is expected that these will all be instances of {@link java.security.cert.X509Certificate},
+ * but we pass them as the base {@link Certificate} in case future implementations leverage
+ * other certificate types.
* @return org.apache.cassandra.auth.IAuthenticator.SaslNegotiator implementation
* (see {@link org.apache.cassandra.auth.PasswordAuthenticator.PlainTextSaslAuthenticator})
*/
- default SaslNegotiator newSaslNegotiator(InetAddress clientAddress, X509Certificate[] certificates)
+ default SaslNegotiator newSaslNegotiator(InetAddress clientAddress, Certificate[] certificates)
{
return newSaslNegotiator(clientAddress);
}
diff --git a/src/java/org/apache/cassandra/auth/IInternodeAuthenticator.java b/src/java/org/apache/cassandra/auth/IInternodeAuthenticator.java
index 8e09b90..e5038c0 100644
--- a/src/java/org/apache/cassandra/auth/IInternodeAuthenticator.java
+++ b/src/java/org/apache/cassandra/auth/IInternodeAuthenticator.java
@@ -20,6 +20,7 @@
package org.apache.cassandra.auth;
import java.net.InetAddress;
+import java.security.cert.Certificate;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -33,7 +34,35 @@
* @param remotePort port of the connecting node.
* @return true if the connection should be accepted, false otherwise.
*/
- boolean authenticate(InetAddress remoteAddress, int remotePort);
+ @Deprecated
+ default boolean authenticate(InetAddress remoteAddress, int remotePort)
+ {
+ return false;
+ }
+
+ /**
+ * Decides whether a peer is allowed to connect to this node.
+ * If this method returns false, the socket will be immediately closed.
+ * <p>
+ * Default implementation calls authenticate method by IP and port method
+ * <p>
+ * 1. If it is IP based authentication ignore the certificates & connectionType parameters in the implementation
+ * of this method.
+ * 2. For certificate based authentication like mTLS, server's identity for outbound connections is verified by the
+ * trusted root certificates in the outbound_keystore. In such cases this method may be overridden to return true
+ * when certificateType is OUTBOUND, as the authentication of the server happens during SSL Handshake.
+ *
+ * @param remoteAddress ip address of the connecting node.
+ * @param remotePort port of the connecting node.
+ * @param certificates peer certificates
+ * @param connectionType If the connection is inbound/outbound connection.
+ * @return true if the connection should be accepted, false otherwise.
+ */
+ default boolean authenticate(InetAddress remoteAddress, int remotePort,
+ Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ return authenticate(remoteAddress, remotePort);
+ }
/**
* Validates configuration of IInternodeAuthenticator implementation (if configurable).
@@ -41,4 +70,30 @@
* @throws ConfigurationException when there is a configuration error.
*/
void validateConfiguration() throws ConfigurationException;
+
+ /**
+ * Setup is called once upon system startup to initialize the IAuthenticator.
+ *
+ * For example, use this method to create any required keyspaces/column families.
+ */
+ default void setupInternode()
+ {
+
+ }
+
+ /**
+ * Enum that represents connection type of internode connection.
+ *
+ * INBOUND - called after connection established, with certificate available if present.
+ * OUTBOUND - called after connection established, with certificate available if present.
+ * OUTBOUND_PRECONNECT - called before initiating a connection, without certificate available.
+ * The outbound connection will be authenticated with the certificate once a redirected connection is established.
+ * This is an extra check that can be used to detect misconfiguration before reconnection, or ignored by returning true.
+ */
+ enum InternodeConnectionDirection
+ {
+ INBOUND,
+ OUTBOUND,
+ OUTBOUND_PRECONNECT
+ }
}
diff --git a/src/java/org/apache/cassandra/auth/Permission.java b/src/java/org/apache/cassandra/auth/Permission.java
index d552280..d325652 100644
--- a/src/java/org/apache/cassandra/auth/Permission.java
+++ b/src/java/org/apache/cassandra/auth/Permission.java
@@ -61,9 +61,13 @@
DESCRIBE, // required on the root-level RoleResource to list all Roles
// UDF permissions
- EXECUTE; // required to invoke any user defined function or aggregate
+ EXECUTE, // required to invoke any user defined function or aggregate
+
+ UNMASK, // required to see masked data
+
+ SELECT_MASKED; // required for SELECT on a table with restictions on masked columns
public static final Set<Permission> ALL =
- Sets.immutableEnumSet(EnumSet.range(Permission.CREATE, Permission.EXECUTE));
+ Sets.immutableEnumSet(EnumSet.range(Permission.CREATE, Permission.SELECT_MASKED));
public static final Set<Permission> NONE = ImmutableSet.of();
}
diff --git a/src/java/org/apache/cassandra/batchlog/Batch.java b/src/java/org/apache/cassandra/batchlog/Batch.java
index b5a6288..667263b 100644
--- a/src/java/org/apache/cassandra/batchlog/Batch.java
+++ b/src/java/org/apache/cassandra/batchlog/Batch.java
@@ -126,10 +126,10 @@
batch.id.serialize(out);
out.writeLong(batch.creationTime);
- out.writeUnsignedVInt(batch.decodedMutations.size());
+ out.writeUnsignedVInt32(batch.decodedMutations.size());
for (Mutation mutation : batch.decodedMutations)
{
- out.writeUnsignedVInt(mutation.serializedSize(version));
+ out.writeUnsignedVInt32(mutation.serializedSize(version));
Mutation.serializer.serialize(mutation, out, version);
}
}
@@ -150,7 +150,7 @@
private static Collection<ByteBuffer> readEncodedMutations(DataInputPlus in) throws IOException
{
- int count = (int) in.readUnsignedVInt();
+ int count = in.readUnsignedVInt32();
ArrayList<ByteBuffer> mutations = new ArrayList<>(count);
for (int i = 0; i < count; i++)
@@ -161,12 +161,12 @@
private static Collection<Mutation> decodeMutations(DataInputPlus in, int version) throws IOException
{
- int count = (int) in.readUnsignedVInt();
+ int count = in.readUnsignedVInt32();
ArrayList<Mutation> mutations = new ArrayList<>(count);
for (int i = 0; i < count; i++)
{
- in.readUnsignedVInt(); // skip mutation size
+ in.readUnsignedVInt32(); // skip mutation size
mutations.add(Mutation.serializer.deserialize(in, version));
}
diff --git a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
index 6d102b0..b7b25cb 100644
--- a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
+++ b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java
@@ -77,6 +77,7 @@
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BATCHLOG_REPLAY_TIMEOUT_IN_MS;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithPaging;
import static org.apache.cassandra.net.Verb.MUTATION_REQ;
@@ -91,7 +92,7 @@
private static final Logger logger = LoggerFactory.getLogger(BatchlogManager.class);
public static final BatchlogManager instance = new BatchlogManager();
- public static final long BATCHLOG_REPLAY_TIMEOUT = Long.getLong("cassandra.batchlog.replay_timeout_in_ms", DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS) * 2);
+ public static final long BATCHLOG_REPLAY_TIMEOUT = BATCHLOG_REPLAY_TIMEOUT_IN_MS.getLong(DatabaseDescriptor.getWriteRpcTimeout(MILLISECONDS) * 2);
private volatile long totalBatchesReplayed = 0; // no concurrency protection necessary as only written by replay thread.
private volatile TimeUUID lastReplayedUuid = TimeUUID.minAtUnixMillis(0);
diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
index 1f383ec..dde36f1 100644
--- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java
+++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
@@ -17,15 +17,17 @@
*/
package org.apache.cassandra.cache;
-import java.io.BufferedInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.nio.file.NoSuchFileException;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import javax.annotation.concurrent.NotThreadSafe;
import org.cliffc.high_scale_lib.NonBlockingHashSet;
import org.slf4j.Logger;
@@ -33,19 +35,30 @@
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.util.ChecksummedRandomAccessReader;
+import org.apache.cassandra.io.util.ChecksummedSequentialWriter;
+import org.apache.cassandra.io.util.CorruptFileException;
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Pair;
@@ -59,8 +72,9 @@
{
public interface IStreamFactory
{
- InputStream getInputStream(File dataPath, File crcPath) throws IOException;
- OutputStream getOutputStream(File dataPath, File crcPath);
+ DataInputStreamPlus getInputStream(File dataPath, File crcPath) throws IOException;
+
+ DataOutputStreamPlus getOutputStream(File dataPath, File crcPath);
}
private static final Logger logger = LoggerFactory.getLogger(AutoSavingCache.class);
@@ -85,8 +99,10 @@
* "e" introduced with CASSANDRA-11206, omits IndexInfo from key-cache, stores offset into index-file
*
* "f" introduced with CASSANDRA-9425, changes "keyspace.table.index" in cache keys to TableMetadata.id+TableMetadata.indexName
+ *
+ * "g" introduced an explicit sstable format type ordinal number so that the entry can be skipped regardless of the actual implementation and used serializer
*/
- private static final String CURRENT_VERSION = "f";
+ private static final String CURRENT_VERSION = "g";
private static volatile IStreamFactory streamFactory = new IStreamFactory()
{
@@ -95,12 +111,12 @@
.trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024)
.finishOnClose(true).build();
- public InputStream getInputStream(File dataPath, File crcPath) throws IOException
+ public DataInputStreamPlus getInputStream(File dataPath, File crcPath) throws IOException
{
return ChecksummedRandomAccessReader.open(dataPath, crcPath);
}
- public OutputStream getOutputStream(File dataPath, File crcPath)
+ public DataOutputStreamPlus getOutputStream(File dataPath, File crcPath)
{
return new ChecksummedSequentialWriter(dataPath, crcPath, null, writerOption);
}
@@ -121,12 +137,17 @@
public File getCacheDataPath(String version)
{
- return DatabaseDescriptor.getSerializedCachePath( cacheType, version, "db");
+ return DatabaseDescriptor.getSerializedCachePath(cacheType, version, "db");
}
public File getCacheCrcPath(String version)
{
- return DatabaseDescriptor.getSerializedCachePath( cacheType, version, "crc");
+ return DatabaseDescriptor.getSerializedCachePath(cacheType, version, "crc");
+ }
+
+ public File getCacheMetadataPath(String version)
+ {
+ return DatabaseDescriptor.getSerializedCachePath(cacheType, version, "metadata");
}
public Writer getWriter(int keysToSave)
@@ -175,6 +196,7 @@
return cacheLoad;
}
+ @SuppressWarnings("resource")
public int loadSaved()
{
int count = 0;
@@ -183,39 +205,33 @@
// modern format, allows both key and value (so key cache load can be purely sequential)
File dataPath = getCacheDataPath(CURRENT_VERSION);
File crcPath = getCacheCrcPath(CURRENT_VERSION);
- if (dataPath.exists() && crcPath.exists())
+ File metadataPath = getCacheMetadataPath(CURRENT_VERSION);
+ if (dataPath.exists() && crcPath.exists() && metadataPath.exists())
{
DataInputStreamPlus in = null;
try
{
- logger.info("reading saved cache {}", dataPath);
- in = new DataInputStreamPlus(new LengthAvailableInputStream(new BufferedInputStream(streamFactory.getInputStream(dataPath, crcPath)), dataPath.length()));
+ logger.info("Reading saved cache: {}, {}, {}", dataPath, crcPath, metadataPath);
+ try (FileInputStreamPlus metadataIn = metadataPath.newInputStream())
+ {
+ cacheLoader.deserializeMetadata(metadataIn);
+ }
+
+ in = streamFactory.getInputStream(dataPath, crcPath);
//Check the schema has not changed since CFs are looked up by name which is ambiguous
UUID schemaVersion = new UUID(in.readLong(), in.readLong());
if (!schemaVersion.equals(Schema.instance.getVersion()))
throw new RuntimeException("Cache schema version "
- + schemaVersion
- + " does not match current schema version "
- + Schema.instance.getVersion());
+ + schemaVersion
+ + " does not match current schema version "
+ + Schema.instance.getVersion());
ArrayDeque<Future<Pair<K, V>>> futures = new ArrayDeque<>();
long loadByNanos = start + TimeUnit.SECONDS.toNanos(DatabaseDescriptor.getCacheLoadTimeout());
while (nanoTime() < loadByNanos && in.available() > 0)
{
- //tableId and indexName are serialized by the serializers in CacheService
- //That is delegated there because there are serializer specific conditions
- //where a cache key is skipped and not written
- TableId tableId = TableId.deserialize(in);
- String indexName = in.readUTF();
- if (indexName.isEmpty())
- indexName = null;
-
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
- if (indexName != null && cfs != null)
- cfs = cfs.indexManager.getIndexByName(indexName).getBackingTable().orElse(null);
-
- Future<Pair<K, V>> entryFuture = cacheLoader.deserialize(in, cfs);
+ Future<Pair<K, V>> entryFuture = cacheLoader.deserialize(in);
// Key cache entry can return null, if the SSTable doesn't exist.
if (entryFuture == null)
continue;
@@ -268,7 +284,7 @@
}
if (logger.isTraceEnabled())
logger.trace("completed reading ({} ms; {} keys) saved cache {}",
- TimeUnit.NANOSECONDS.toMillis(nanoTime() - start), count, dataPath);
+ TimeUnit.NANOSECONDS.toMillis(nanoTime() - start), count, dataPath);
return count;
}
@@ -313,7 +329,8 @@
0,
keysEstimate,
Unit.KEYS,
- nextTimeUUID());
+ nextTimeUUID(),
+ getCacheDataPath(CURRENT_VERSION).toPath().toString());
}
public CacheService.CacheType cacheType()
@@ -341,8 +358,12 @@
long start = nanoTime();
- Pair<File, File> cacheFilePaths = tempCacheFiles();
- try (WrappedDataOutputStreamPlus writer = new WrappedDataOutputStreamPlus(streamFactory.getOutputStream(cacheFilePaths.left, cacheFilePaths.right)))
+ File dataTmpFile = getTempCacheFile(getCacheDataPath(CURRENT_VERSION));
+ File crcTmpFile = getTempCacheFile(getCacheCrcPath(CURRENT_VERSION));
+ File metadataTmpFile = getTempCacheFile(getCacheMetadataPath(CURRENT_VERSION));
+
+ try (WrappedDataOutputStreamPlus writer = new WrappedDataOutputStreamPlus(streamFactory.getOutputStream(dataTmpFile, crcTmpFile));
+ FileOutputStreamPlus metadataWriter = metadataTmpFile.newOutputStream(File.WriteMode.OVERWRITE))
{
//Need to be able to check schema version because CF names are ambiguous
@@ -366,6 +387,9 @@
if (keysWritten >= keysEstimate)
break;
}
+
+ cacheLoader.serializeMetadata(metadataWriter);
+ metadataWriter.sync();
}
catch (FileNotFoundException | NoSuchFileException e)
{
@@ -373,30 +397,36 @@
}
catch (IOException e)
{
- throw new FSWriteError(e, cacheFilePaths.left);
+ throw new FSWriteError(e, dataTmpFile);
+ }
+ finally
+ {
+ cacheLoader.cleanupAfterSerialize();
}
- File cacheFile = getCacheDataPath(CURRENT_VERSION);
+ File dataFile = getCacheDataPath(CURRENT_VERSION);
File crcFile = getCacheCrcPath(CURRENT_VERSION);
+ File metadataFile = getCacheMetadataPath(CURRENT_VERSION);
- cacheFile.tryDelete(); // ignore error if it didn't exist
+ dataFile.tryDelete(); // ignore error if it didn't exist
crcFile.tryDelete();
+ metadataFile.tryDelete();
- if (!cacheFilePaths.left.tryMove(cacheFile))
- logger.error("Unable to rename {} to {}", cacheFilePaths.left, cacheFile);
+ if (!dataTmpFile.tryMove(dataFile))
+ logger.error("Unable to rename {} to {}", dataTmpFile, dataFile);
- if (!cacheFilePaths.right.tryMove(crcFile))
- logger.error("Unable to rename {} to {}", cacheFilePaths.right, crcFile);
+ if (!crcTmpFile.tryMove(crcFile))
+ logger.error("Unable to rename {} to {}", crcTmpFile, crcFile);
- logger.info("Saved {} ({} items) in {} ms", cacheType, keysWritten, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
+ if (!metadataTmpFile.tryMove(metadataFile))
+ logger.error("Unable to rename {} to {}", metadataTmpFile, metadataFile);
+
+ logger.info("Saved {} ({} items) in {} ms to {} : {} MB", cacheType, keysWritten, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start), dataFile.toPath(), dataFile.length() / (1 << 20));
}
- private Pair<File, File> tempCacheFiles()
+ private File getTempCacheFile(File cacheFile)
{
- File dataPath = getCacheDataPath(CURRENT_VERSION);
- File crcPath = getCacheCrcPath(CURRENT_VERSION);
- return Pair.create(FileUtils.createTempFile(dataPath.name(), null, dataPath.parent()),
- FileUtils.createTempFile(crcPath.name(), null, crcPath.parent()));
+ return FileUtils.createTempFile(cacheFile.name(), null, cacheFile.parent());
}
private void deleteOldCacheFiles()
@@ -432,12 +462,91 @@
}
}
- public interface CacheSerializer<K extends CacheKey, V>
+ /**
+ * A base cache serializer that is used to serialize/deserialize a cache to/from disk.
+ * <p>
+ * It expects the following lifecycle:
+ * Serializations:
+ * 1. {@link #serialize(CacheKey, DataOutputPlus, ColumnFamilyStore)} is called for each key in the cache.
+ * 2. {@link #serializeMetadata(DataOutputPlus)} is called to serialize any metadata.
+ * 3. {@link #cleanupAfterSerialize()} is called to clean up any resources allocated for serialization.
+ * <p>
+ * Deserializations:
+ * 1. {@link #deserializeMetadata(DataInputPlus)} is called to deserialize any metadata.
+ * 2. {@link #deserialize(DataInputPlus)} is called for each key in the cache.
+ * 3. {@link #cleanupAfterDeserialize()} is called to clean up any resources allocated for deserialization.
+ * <p>
+ * This abstract class provides the default implementation for the metadata serialization/deserialization.
+ * The metadata includes a dictionary of column family stores collected during serialization whenever
+ * {@link #writeCFS(DataOutputPlus, ColumnFamilyStore)} or {@link #getOrCreateCFSOrdinal(ColumnFamilyStore)}
+ * are called. When such metadata is deserialized, the implementation of {@link #deserialize(DataInputPlus)} may
+ * use {@link #readCFS(DataInputPlus)} method to read the ColumnFamilyStore stored with
+ * {@link #writeCFS(DataOutputPlus, ColumnFamilyStore)}.
+ */
+ @NotThreadSafe
+ public static abstract class CacheSerializer<K extends CacheKey, V>
{
- void serialize(K key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException;
+ private ColumnFamilyStore[] cfStores;
- Future<Pair<K, V>> deserialize(DataInputPlus in, ColumnFamilyStore cfs) throws IOException;
+ private final LinkedHashMap<Pair<TableId, String>, Integer> cfsOrdinals = new LinkedHashMap<>();
- default void cleanupAfterDeserialize() { }
+ protected final int getOrCreateCFSOrdinal(ColumnFamilyStore cfs)
+ {
+ Integer ordinal = cfsOrdinals.putIfAbsent(Pair.create(cfs.metadata().id, cfs.metadata().indexName().orElse("")), cfsOrdinals.size());
+ if (ordinal == null)
+ ordinal = cfsOrdinals.size() - 1;
+ return ordinal;
+ }
+
+ protected ColumnFamilyStore readCFS(DataInputPlus in) throws IOException
+ {
+ return cfStores[in.readUnsignedVInt32()];
+ }
+
+ protected void writeCFS(DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
+ {
+ out.writeUnsignedVInt32(getOrCreateCFSOrdinal(cfs));
+ }
+
+ public void serializeMetadata(DataOutputPlus out) throws IOException
+ {
+ // write the table ids
+ out.writeUnsignedVInt32(cfsOrdinals.size());
+ for (Pair<TableId, String> tableAndIndex : cfsOrdinals.keySet())
+ {
+ tableAndIndex.left.serialize(out);
+ out.writeUTF(tableAndIndex.right);
+ }
+ }
+
+ public void deserializeMetadata(DataInputPlus in) throws IOException
+ {
+ int tableEntries = in.readUnsignedVInt32();
+ if (tableEntries == 0)
+ return;
+ cfStores = new ColumnFamilyStore[tableEntries];
+ for (int i = 0; i < tableEntries; i++)
+ {
+ TableId tableId = TableId.deserialize(in);
+ String indexName = in.readUTF();
+ cfStores[i] = Schema.instance.getColumnFamilyStoreInstance(tableId);
+ if (cfStores[i] != null && !indexName.isEmpty())
+ cfStores[i] = cfStores[i].indexManager.getIndexByName(indexName).getBackingTable().orElse(null);
+ }
+ }
+
+ public abstract void serialize(K key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException;
+
+ public abstract Future<Pair<K, V>> deserialize(DataInputPlus in) throws IOException;
+
+ public void cleanupAfterSerialize()
+ {
+ cfsOrdinals.clear();
+ }
+
+ public void cleanupAfterDeserialize()
+ {
+ cfStores = null;
+ }
}
}
diff --git a/src/java/org/apache/cassandra/cache/ChunkCache.java b/src/java/org/apache/cassandra/cache/ChunkCache.java
index 51dbdc6..8f22ee8 100644
--- a/src/java/org/apache/cassandra/cache/ChunkCache.java
+++ b/src/java/org/apache/cassandra/cache/ChunkCache.java
@@ -23,15 +23,22 @@
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
-import com.github.benmanes.caffeine.cache.*;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.RemovalListener;
import org.apache.cassandra.concurrent.ImmediateExecutor;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.ChunkReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.RebuffererFactory;
import org.apache.cassandra.metrics.ChunkCacheMetrics;
import org.apache.cassandra.utils.memory.BufferPool;
import org.apache.cassandra.utils.memory.BufferPools;
@@ -170,7 +177,7 @@
cache.invalidateAll();
}
- private RebuffererFactory wrap(ChunkReader file)
+ public RebuffererFactory wrap(ChunkReader file)
{
return new CachingRebufferer(file);
}
@@ -196,14 +203,6 @@
cache.invalidateAll(Iterables.filter(cache.asMap().keySet(), x -> x.path.equals(fileName)));
}
- @VisibleForTesting
- public void enable(boolean enabled)
- {
- ChunkCache.enabled = enabled;
- cache.invalidateAll();
- metrics.reset();
- }
-
// TODO: Invalidate caches for obsoleted/MOVED_START tables?
/**
@@ -238,8 +237,10 @@
}
catch (Throwable t)
{
- Throwables.propagateIfInstanceOf(t.getCause(), CorruptSSTableException.class);
- throw Throwables.propagate(t);
+ if (t.getCause() instanceof CorruptSSTableException)
+ throw (CorruptSSTableException)t.getCause();
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
}
@@ -317,4 +318,4 @@
.map(policy -> policy.weightedSize().orElseGet(cache::estimatedSize))
.orElseGet(cache::estimatedSize);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/concurrent/DebuggableTask.java b/src/java/org/apache/cassandra/concurrent/DebuggableTask.java
new file mode 100644
index 0000000..ac04eb4
--- /dev/null
+++ b/src/java/org/apache/cassandra/concurrent/DebuggableTask.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.concurrent;
+
+import org.apache.cassandra.utils.Shared;
+
+import static org.apache.cassandra.utils.Shared.Recursive.INTERFACES;
+import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
+
+/**
+ * Interface to include on a Runnable or Callable submitted to the {@link SharedExecutorPool} to provide more
+ * detailed diagnostics.
+ */
+@Shared(scope = SIMULATION, inner = INTERFACES)
+public interface DebuggableTask
+{
+ public long creationTimeNanos();
+
+ public long startTimeNanos();
+
+ public String description();
+
+ interface RunnableDebuggableTask extends Runnable, DebuggableTask {}
+
+ /**
+ * Wraps a {@link DebuggableTask} to include the name of the thread running it.
+ */
+ public static class RunningDebuggableTask implements DebuggableTask
+ {
+ private final DebuggableTask task;
+ private final String threadId;
+
+ public RunningDebuggableTask(String threadId, DebuggableTask task)
+ {
+ this.task = task;
+ this.threadId = threadId;
+ }
+
+ public String threadId()
+ {
+ return threadId;
+ }
+
+ public boolean hasTask()
+ {
+ return task != null;
+ }
+
+ @Override
+ public long creationTimeNanos()
+ {
+ assert hasTask();
+ return task.creationTimeNanos();
+ }
+
+ @Override
+ public long startTimeNanos()
+ {
+ assert hasTask();
+ return task.startTimeNanos();
+ }
+
+ @Override
+ public String description()
+ {
+ assert hasTask();
+ return task.description();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java b/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java
index 7fa7dcb..27ab885 100644
--- a/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java
+++ b/src/java/org/apache/cassandra/concurrent/ExecutionFailure.java
@@ -21,6 +21,7 @@
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
+import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,6 +107,14 @@
}
/**
+ * @see #suppressing(WithResources, Runnable)
+ */
+ static RunnableDebuggableTask suppressingDebuggable(WithResources withResources, RunnableDebuggableTask debuggable)
+ {
+ return enforceOptionsDebuggable(withResources, debuggable, false);
+ }
+
+ /**
* Encapsulate the execution, propagating or suppressing any exceptions as requested.
*
* note that if {@code wrap} is a {@link java.util.concurrent.Future} its exceptions may not be captured,
@@ -119,7 +128,7 @@
@Override
public void run()
{
- try (Closeable close = withResources.get())
+ try (@SuppressWarnings("unused") Closeable close = withResources.get())
{
wrap.run();
}
@@ -140,6 +149,54 @@
}
/**
+ * @see #enforceOptions(WithResources, Runnable, boolean)
+ */
+ private static RunnableDebuggableTask enforceOptionsDebuggable(WithResources withResources, RunnableDebuggableTask debuggable, boolean propagate)
+ {
+ return new RunnableDebuggableTask()
+ {
+ @Override
+ public void run()
+ {
+ try (@SuppressWarnings("unused") Closeable close = withResources.get())
+ {
+ debuggable.run();
+ }
+ catch (Throwable t)
+ {
+ handle(t);
+ if (propagate)
+ throw t;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return debuggable.toString();
+ }
+
+ @Override
+ public long creationTimeNanos()
+ {
+ return debuggable.creationTimeNanos();
+ }
+
+ @Override
+ public long startTimeNanos()
+ {
+ return debuggable.startTimeNanos();
+ }
+
+ @Override
+ public String description()
+ {
+ return debuggable.description();
+ }
+ };
+ }
+
+ /**
* See {@link #enforceOptions(WithResources, Callable)}
*/
static <V> Callable<V> propagating(Callable<V> wrap)
@@ -158,7 +215,7 @@
@Override
public V call() throws Exception
{
- try (Closeable close = withResources.get())
+ try (@SuppressWarnings("unused") Closeable close = withResources.get())
{
return wrap.call();
}
diff --git a/src/java/org/apache/cassandra/concurrent/FutureTask.java b/src/java/org/apache/cassandra/concurrent/FutureTask.java
index 2348ff6..763884a 100644
--- a/src/java/org/apache/cassandra/concurrent/FutureTask.java
+++ b/src/java/org/apache/cassandra/concurrent/FutureTask.java
@@ -20,9 +20,10 @@
import java.util.concurrent.Callable;
-import org.apache.cassandra.utils.concurrent.RunnableFuture;
+import javax.annotation.Nullable;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
+import org.apache.cassandra.utils.concurrent.RunnableFuture;
/**
* A FutureTask that utilises Cassandra's {@link AsyncFuture}, making it compatible with {@link ExecutorPlus}.
@@ -31,15 +32,28 @@
public class FutureTask<V> extends AsyncFuture<V> implements RunnableFuture<V>
{
private Callable<? extends V> call;
+ private volatile DebuggableTask debuggable;
public FutureTask(Callable<? extends V> call)
{
- this.call = call;
+ this(call, call instanceof DebuggableTask ? (DebuggableTask) call : null);
}
public FutureTask(Runnable run)
{
- this.call = callable(run);
+ this(callable(run), run instanceof DebuggableTask ? (DebuggableTask) run : null);
+ }
+
+ private FutureTask(Callable<? extends V> call, DebuggableTask debuggable)
+ {
+ this.call = call;
+ this.debuggable = debuggable;
+ }
+
+ @Nullable
+ DebuggableTask debuggableTask()
+ {
+ return debuggable;
}
V call() throws Exception
@@ -63,6 +77,7 @@
finally
{
call = null;
+ debuggable = null;
}
}
diff --git a/src/java/org/apache/cassandra/concurrent/SEPExecutor.java b/src/java/org/apache/cassandra/concurrent/SEPExecutor.java
index 05b59c6..f4b74c0 100644
--- a/src/java/org/apache/cassandra/concurrent/SEPExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/SEPExecutor.java
@@ -70,7 +70,7 @@
SEPExecutor(SharedExecutorPool pool, int maximumPoolSize, MaximumPoolSizeListener maximumPoolSizeListener, String jmxPath, String name)
{
this.pool = pool;
- this.name = name;
+ this.name = NamedThreadFactory.globalPrefix() + name;
this.mbeanName = "org.apache.cassandra." + jmxPath + ":type=" + name;
this.maximumPoolSize = new AtomicInteger(maximumPoolSize);
this.maximumPoolSizeListener = maximumPoolSizeListener;
diff --git a/src/java/org/apache/cassandra/concurrent/SEPWorker.java b/src/java/org/apache/cassandra/concurrent/SEPWorker.java
index c7b9abf..93c01fa 100644
--- a/src/java/org/apache/cassandra/concurrent/SEPWorker.java
+++ b/src/java/org/apache/cassandra/concurrent/SEPWorker.java
@@ -30,12 +30,13 @@
import static org.apache.cassandra.concurrent.SEPExecutor.TakeTaskPermitResult.RETURNED_WORK_PERMIT;
import static org.apache.cassandra.concurrent.SEPExecutor.TakeTaskPermitResult.TOOK_PERMIT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SET_SEP_THREAD_NAME;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
final class SEPWorker extends AtomicReference<SEPWorker.Work> implements Runnable
{
private static final Logger logger = LoggerFactory.getLogger(SEPWorker.class);
- private static final boolean SET_THREAD_NAME = Boolean.parseBoolean(System.getProperty("cassandra.set_sep_thread_name", "true"));
+ private static final boolean SET_THREAD_NAME = SET_SEP_THREAD_NAME.getBoolean();
final Long workerId;
final Thread thread;
@@ -48,6 +49,8 @@
long prevStopCheck = 0;
long soleSpinnerSpinTime = 0;
+ private final AtomicReference<Runnable> currentTask = new AtomicReference<>();
+
SEPWorker(ThreadGroup threadGroup, Long workerId, Work initialState, SharedExecutorPool pool)
{
this.pool = pool;
@@ -58,9 +61,27 @@
thread.start();
}
+ /**
+ * @return the current {@link DebuggableTask}, if one exists
+ */
+ public DebuggableTask currentDebuggableTask()
+ {
+ // can change after null check so go off local reference
+ Runnable task = currentTask.get();
+
+ // Local read and mutation Runnables are themselves debuggable
+ if (task instanceof DebuggableTask)
+ return (DebuggableTask) task;
+
+ if (task instanceof FutureTask)
+ return ((FutureTask<?>) task).debuggableTask();
+
+ return null;
+ }
+
public void run()
{
- /**
+ /*
* we maintain two important invariants:
* 1) after exiting spinning phase, we ensure at least one more task on _each_ queue will be processed
* promptly after we begin, assuming any are outstanding on any pools. this is to permit producers to
@@ -101,8 +122,10 @@
if (assigned == null)
continue;
if (SET_THREAD_NAME)
- Thread.currentThread().setName(assigned.name + "-" + workerId);
+ Thread.currentThread().setName(assigned.name + '-' + workerId);
+
task = assigned.tasks.poll();
+ currentTask.lazySet(task);
// if we do have tasks assigned, nobody will change our state so we can simply set it to WORKING
// (which is also a state that will never be interrupted externally)
@@ -128,9 +151,12 @@
break;
task = assigned.tasks.poll();
+ currentTask.lazySet(task);
}
// return our work permit, and maybe signal shutdown
+ currentTask.lazySet(null);
+
if (status != RETURNED_WORK_PERMIT)
assigned.returnWorkPermit();
@@ -173,6 +199,11 @@
logger.error("Unexpected exception killed worker", t);
}
}
+ finally
+ {
+ currentTask.lazySet(null);
+ pool.workerEnded(this);
+ }
}
// try to assign this worker the provided work
@@ -420,4 +451,22 @@
return assigned != null;
}
}
+
+ @Override
+ public String toString()
+ {
+ return thread.getName();
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return workerId.intValue();
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ return obj == this;
+ }
}
diff --git a/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java b/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java
index f74854f..0631ec6 100644
--- a/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java
+++ b/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java
@@ -17,8 +17,11 @@
*/
package org.apache.cassandra.concurrent;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
@@ -26,6 +29,9 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.concurrent.DebuggableTask.RunningDebuggableTask;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.concurrent.SEPWorker.Work;
@@ -77,6 +83,8 @@
final ConcurrentSkipListMap<Long, SEPWorker> spinning = new ConcurrentSkipListMap<>();
// the collection of threads that have been asked to stop/deschedule - new workers are scheduled from here last
final ConcurrentSkipListMap<Long, SEPWorker> descheduled = new ConcurrentSkipListMap<>();
+ // All SEPWorkers that are currently running
+ private final Set<SEPWorker> allWorkers = Collections.newSetFromMap(new ConcurrentHashMap<>());
volatile boolean shuttingDown = false;
@@ -102,7 +110,23 @@
return;
if (!work.isStop())
- new SEPWorker(threadGroup, workerId.incrementAndGet(), work, this);
+ {
+ SEPWorker worker = new SEPWorker(threadGroup, workerId.incrementAndGet(), work, this);
+ allWorkers.add(worker);
+ }
+ }
+
+ void workerEnded(SEPWorker worker)
+ {
+ allWorkers.remove(worker);
+ }
+
+ public List<RunningDebuggableTask> runningTasks()
+ {
+ return allWorkers.stream()
+ .map(worker -> new RunningDebuggableTask(worker.toString(), worker.currentDebuggableTask()))
+ .filter(RunningDebuggableTask::hasTask)
+ .collect(Collectors.toList());
}
void maybeStartSpinningWorker()
diff --git a/src/java/org/apache/cassandra/concurrent/TaskFactory.java b/src/java/org/apache/cassandra/concurrent/TaskFactory.java
index 56087d9..faeabe6 100644
--- a/src/java/org/apache/cassandra/concurrent/TaskFactory.java
+++ b/src/java/org/apache/cassandra/concurrent/TaskFactory.java
@@ -20,6 +20,7 @@
import java.util.concurrent.Callable;
+import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.apache.cassandra.utils.Shared;
import org.apache.cassandra.utils.WithResources;
import org.apache.cassandra.utils.concurrent.RunnableFuture;
@@ -127,6 +128,9 @@
@Override
public Runnable toExecute(Runnable runnable)
{
+ if (runnable instanceof RunnableDebuggableTask)
+ return ExecutionFailure.suppressingDebuggable(ExecutorLocals.propagate(), (RunnableDebuggableTask) runnable);
+
// no reason to propagate exception when it is inaccessible to caller
return ExecutionFailure.suppressing(ExecutorLocals.propagate(), runnable);
}
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java b/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
index 4960374..0fdcd6c 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
@@ -18,13 +18,19 @@
package org.apache.cassandra.config;
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
public enum CassandraRelevantEnv
{
/**
* Searching in the JAVA_HOME is safer than searching into System.getProperty("java.home") as the Oracle
* JVM might use the JRE which do not contains jmap.
*/
- JAVA_HOME ("JAVA_HOME");
+ JAVA_HOME ("JAVA_HOME"),
+ CIRCLECI("CIRCLECI"),
+ CASSANDRA_SKIP_SYNC("CASSANDRA_SKIP_SYNC")
+
+ ;
CassandraRelevantEnv(String key)
{
@@ -38,6 +44,15 @@
return System.getenv(key);
}
+ /**
+ * Gets the value of a system env as a boolean.
+ * @return System env boolean value if it exists, false otherwise.
+ */
+ public boolean getBoolean()
+ {
+ return Boolean.parseBoolean(System.getenv(key));
+ }
+
public String getKey() {
return key;
}
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 3e45ebc..78e1992 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -18,302 +18,493 @@
package org.apache.cassandra.config;
+import java.util.HashSet;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.db.virtual.LogMessagesTable;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.service.FileSystemOwnershipCheck;
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
/** A class that extracts system properties for the cassandra node it runs within. */
public enum CassandraRelevantProperties
{
- //base JVM properties
- JAVA_HOME("java.home"),
- CASSANDRA_PID_FILE ("cassandra-pidfile"),
-
+ ACQUIRE_RETRY_SECONDS("cassandra.acquire_retry_seconds", "60"),
+ ACQUIRE_SLEEP_MS("cassandra.acquire_sleep_ms", "1000"),
+ ALLOCATE_TOKENS_FOR_KEYSPACE("cassandra.allocate_tokens_for_keyspace"),
+ ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT("cassandra.allow_alter_rf_during_range_movement"),
+ /** If we should allow having duplicate keys in the config file, default to true for legacy reasons */
+ ALLOW_DUPLICATE_CONFIG_KEYS("cassandra.allow_duplicate_config_keys", "true"),
+ /** If we should allow having both new (post CASSANDRA-15234) and old config keys for the same config item in the yaml */
+ ALLOW_NEW_OLD_CONFIG_KEYS("cassandra.allow_new_old_config_keys"),
+ ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS("cassandra.allow_unlimited_concurrent_validations"),
+ ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION("cassandra.allow_unsafe_aggressive_sstable_expiration"),
+ ALLOW_UNSAFE_JOIN("cassandra.allow_unsafe_join"),
+ ALLOW_UNSAFE_REPLACE("cassandra.allow_unsafe_replace"),
+ ALLOW_UNSAFE_TRANSIENT_CHANGES("cassandra.allow_unsafe_transient_changes"),
+ APPROXIMATE_TIME_PRECISION_MS("cassandra.approximate_time_precision_ms", "2"),
+ /** 2 ** GENSALT_LOG2_ROUNDS rounds of hashing will be performed. */
+ AUTH_BCRYPT_GENSALT_LOG2_ROUNDS("cassandra.auth_bcrypt_gensalt_log2_rounds"),
+ /** We expect default values on cache retries and interval to be sufficient for everyone but have this escape hatch just in case. */
+ AUTH_CACHE_WARMING_MAX_RETRIES("cassandra.auth_cache.warming.max_retries"),
+ AUTH_CACHE_WARMING_RETRY_INTERVAL_MS("cassandra.auth_cache.warming.retry_interval_ms"),
+ AUTOCOMPACTION_ON_STARTUP_ENABLED("cassandra.autocompaction_on_startup_enabled", "true"),
+ AUTO_BOOTSTRAP("cassandra.auto_bootstrap"),
+ AUTO_REPAIR_FREQUENCY_SECONDS("cassandra.auto_repair_frequency_seconds", convertToString(TimeUnit.MINUTES.toSeconds(5))),
+ BATCHLOG_REPLAY_TIMEOUT_IN_MS("cassandra.batchlog.replay_timeout_in_ms"),
+ BATCH_COMMIT_LOG_SYNC_INTERVAL("cassandra.batch_commitlog_sync_interval_millis", "1000"),
/**
- * Indicates the temporary directory used by the Java Virtual Machine (JVM)
- * to create and store temporary files.
+ * When bootstraping how long to wait for schema versions to be seen.
*/
- JAVA_IO_TMPDIR ("java.io.tmpdir"),
-
+ BOOTSTRAP_SCHEMA_DELAY_MS("cassandra.schema_delay_ms"),
/**
- * Path from which to load native libraries.
- * Default is absolute path to lib directory.
+ * When bootstraping we wait for all schema versions found in gossip to be seen, and if not seen in time we fail
+ * the bootstrap; this property will avoid failing and allow bootstrap to continue if set to true.
*/
- JAVA_LIBRARY_PATH ("java.library.path"),
-
- JAVA_SECURITY_EGD ("java.security.egd"),
-
- /** Java Runtime Environment version */
- JAVA_VERSION ("java.version"),
-
- /** Java Virtual Machine implementation name */
- JAVA_VM_NAME ("java.vm.name"),
-
- /** Line separator ("\n" on UNIX). */
- LINE_SEPARATOR ("line.separator"),
-
- /** Java class path. */
- JAVA_CLASS_PATH ("java.class.path"),
-
- /** Operating system architecture. */
- OS_ARCH ("os.arch"),
-
- /** Operating system name. */
- OS_NAME ("os.name"),
-
- /** User's home directory. */
- USER_HOME ("user.home"),
-
- /** Platform word size sun.arch.data.model. Examples: "32", "64", "unknown"*/
- SUN_ARCH_DATA_MODEL ("sun.arch.data.model"),
-
- //JMX properties
+ BOOTSTRAP_SKIP_SCHEMA_CHECK("cassandra.skip_schema_check"),
+ BROADCAST_INTERVAL_MS("cassandra.broadcast_interval_ms", "60000"),
+ BTREE_BRANCH_SHIFT("cassandra.btree.branchshift", "5"),
+ BTREE_FAN_FACTOR("cassandra.btree.fanfactor"),
+ /** Represents the maximum size (in bytes) of a serialized mutation that can be cached **/
+ CACHEABLE_MUTATION_SIZE_LIMIT("cassandra.cacheable_mutation_size_limit_bytes", convertToString(1_000_000)),
+ CASSANDRA_ALLOW_SIMPLE_STRATEGY("cassandra.allow_simplestrategy"),
+ CASSANDRA_AVAILABLE_PROCESSORS("cassandra.available_processors"),
+ /** The classpath storage configuration file. */
+ CASSANDRA_CONFIG("cassandra.config", "cassandra.yaml"),
/**
- * The value of this property represents the host name string
- * that should be associated with remote stubs for locally created remote objects,
- * in order to allow clients to invoke methods on the remote object.
+ * The cassandra-foreground option will tell CassandraDaemon whether
+ * to close stdout/stderr, but it's up to us not to background.
+ * yes/null
*/
- JAVA_RMI_SERVER_HOSTNAME ("java.rmi.server.hostname"),
-
+ CASSANDRA_FOREGROUND("cassandra-foreground"),
+ CASSANDRA_JMX_AUTHORIZER("cassandra.jmx.authorizer"),
+ CASSANDRA_JMX_LOCAL_PORT("cassandra.jmx.local.port"),
+ CASSANDRA_JMX_REMOTE_LOGIN_CONFIG("cassandra.jmx.remote.login.config"),
+ /** Cassandra jmx remote and local port */
+ CASSANDRA_JMX_REMOTE_PORT("cassandra.jmx.remote.port"),
+ CASSANDRA_MAX_HINT_TTL("cassandra.maxHintTTL", convertToString(Integer.MAX_VALUE)),
+ CASSANDRA_MINIMUM_REPLICATION_FACTOR("cassandra.minimum_replication_factor"),
+ CASSANDRA_NETTY_USE_HEAP_ALLOCATOR("cassandra.netty_use_heap_allocator"),
+ CASSANDRA_PID_FILE("cassandra-pidfile"),
+ CASSANDRA_RACKDC_PROPERTIES("cassandra-rackdc.properties"),
+ CASSANDRA_SKIP_AUTOMATIC_UDT_FIX("cassandra.skipautomaticudtfix"),
+ CASSANDRA_STREAMING_DEBUG_STACKTRACE_LIMIT("cassandra.streaming.debug_stacktrace_limit", "2"),
+ CASSANDRA_UNSAFE_TIME_UUID_NODE("cassandra.unsafe.timeuuidnode"),
+ CASSANDRA_VERSION("cassandra.version"),
+ /** default heartbeating period is 1 minute */
+ CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD("check_data_resurrection_heartbeat_period_milli", "60000"),
+ CHRONICLE_ANNOUNCER_DISABLE("chronicle.announcer.disable"),
+ CLOCK_GLOBAL("cassandra.clock"),
+ CLOCK_MONOTONIC_APPROX("cassandra.monotonic_clock.approx"),
+ CLOCK_MONOTONIC_PRECISE("cassandra.monotonic_clock.precise"),
+ COMMITLOG_ALLOW_IGNORE_SYNC_CRC("cassandra.commitlog.allow_ignore_sync_crc"),
+ COMMITLOG_IGNORE_REPLAY_ERRORS("cassandra.commitlog.ignorereplayerrors"),
+ COMMITLOG_MAX_OUTSTANDING_REPLAY_BYTES("cassandra.commitlog_max_outstanding_replay_bytes", convertToString(1024 * 1024 * 64)),
+ COMMITLOG_MAX_OUTSTANDING_REPLAY_COUNT("cassandra.commitlog_max_outstanding_replay_count", "1024"),
+ COMMITLOG_STOP_ON_ERRORS("cassandra.commitlog.stop_on_errors"),
/**
- * If this value is true, object identifiers for remote objects exported by this VM will be generated by using
- * a cryptographically secure random number generator. The default value is false.
+ * Entities to replay mutations for upon commit log replay, property is meant to contain
+ * comma-separated entities which are either names of keyspaces or keyspaces and tables or their mix.
+ * Examples:
+ * just keyspaces
+ * -Dcassandra.replayList=ks1,ks2,ks3
+ * specific tables
+ * -Dcassandra.replayList=ks1.tb1,ks2.tb2
+ * mix of tables and keyspaces
+ * -Dcassandra.replayList=ks1.tb1,ks2
+ *
+ * If only keyspaces are specified, mutations for all tables in such keyspace will be replayed
+ * */
+ COMMIT_LOG_REPLAY_LIST("cassandra.replayList"),
+ /**
+ * This property indicates the location for the access file. If com.sun.management.jmxremote.authenticate is false,
+ * then this property and the password and access files, are ignored. Otherwise, the access file must exist and
+ * be in the valid format. If the access file is empty or nonexistent, then no access is allowed.
*/
- JAVA_RMI_SERVER_RANDOM_ID ("java.rmi.server.randomIDs"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_ACCESS_FILE("com.sun.management.jmxremote.access.file"),
/**
* This property indicates whether password authentication for remote monitoring is
* enabled. By default it is disabled - com.sun.management.jmxremote.authenticate
*/
- COM_SUN_MANAGEMENT_JMXREMOTE_AUTHENTICATE ("com.sun.management.jmxremote.authenticate"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_AUTHENTICATE("com.sun.management.jmxremote.authenticate"),
+ /** This property indicates the path to the password file - com.sun.management.jmxremote.password.file */
+ COM_SUN_MANAGEMENT_JMXREMOTE_PASSWORD_FILE("com.sun.management.jmxremote.password.file"),
+ /** Port number to enable JMX RMI connections - com.sun.management.jmxremote.port */
+ COM_SUN_MANAGEMENT_JMXREMOTE_PORT("com.sun.management.jmxremote.port"),
/**
* The port number to which the RMI connector will be bound - com.sun.management.jmxremote.rmi.port.
* An Integer object that represents the value of the second argument is returned
* if there is no port specified, if the port does not have the correct numeric format,
* or if the specified name is empty or null.
*/
- COM_SUN_MANAGEMENT_JMXREMOTE_RMI_PORT ("com.sun.management.jmxremote.rmi.port", "0"),
-
- /** Cassandra jmx remote and local port */
- CASSANDRA_JMX_REMOTE_PORT("cassandra.jmx.remote.port"),
- CASSANDRA_JMX_LOCAL_PORT("cassandra.jmx.local.port"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_RMI_PORT("com.sun.management.jmxremote.rmi.port", "0"),
/** This property indicates whether SSL is enabled for monitoring remotely. Default is set to false. */
- COM_SUN_MANAGEMENT_JMXREMOTE_SSL ("com.sun.management.jmxremote.ssl"),
-
- /**
- * This property indicates whether SSL client authentication is enabled - com.sun.management.jmxremote.ssl.need.client.auth.
- * Default is set to false.
- */
- COM_SUN_MANAGEMENT_JMXREMOTE_SSL_NEED_CLIENT_AUTH ("com.sun.management.jmxremote.ssl.need.client.auth"),
-
- /**
- * This property indicates the location for the access file. If com.sun.management.jmxremote.authenticate is false,
- * then this property and the password and access files, are ignored. Otherwise, the access file must exist and
- * be in the valid format. If the access file is empty or nonexistent, then no access is allowed.
- */
- COM_SUN_MANAGEMENT_JMXREMOTE_ACCESS_FILE ("com.sun.management.jmxremote.access.file"),
-
- /** This property indicates the path to the password file - com.sun.management.jmxremote.password.file */
- COM_SUN_MANAGEMENT_JMXREMOTE_PASSWORD_FILE ("com.sun.management.jmxremote.password.file"),
-
- /** Port number to enable JMX RMI connections - com.sun.management.jmxremote.port */
- COM_SUN_MANAGEMENT_JMXREMOTE_PORT ("com.sun.management.jmxremote.port"),
-
- /**
- * A comma-delimited list of SSL/TLS protocol versions to enable.
- * Used in conjunction with com.sun.management.jmxremote.ssl - com.sun.management.jmxremote.ssl.enabled.protocols
- */
- COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_PROTOCOLS ("com.sun.management.jmxremote.ssl.enabled.protocols"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_SSL("com.sun.management.jmxremote.ssl"),
/**
* A comma-delimited list of SSL/TLS cipher suites to enable.
* Used in conjunction with com.sun.management.jmxremote.ssl - com.sun.management.jmxremote.ssl.enabled.cipher.suites
*/
- COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_CIPHER_SUITES ("com.sun.management.jmxremote.ssl.enabled.cipher.suites"),
-
- /** mx4jaddress */
- MX4JADDRESS ("mx4jaddress"),
-
- /** mx4jport */
- MX4JPORT ("mx4jport"),
-
- RING_DELAY("cassandra.ring_delay_ms"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_CIPHER_SUITES("com.sun.management.jmxremote.ssl.enabled.cipher.suites"),
/**
- * When bootstraping we wait for all schema versions found in gossip to be seen, and if not seen in time we fail
- * the bootstrap; this property will avoid failing and allow bootstrap to continue if set to true.
+ * A comma-delimited list of SSL/TLS protocol versions to enable.
+ * Used in conjunction with com.sun.management.jmxremote.ssl - com.sun.management.jmxremote.ssl.enabled.protocols
*/
- BOOTSTRAP_SKIP_SCHEMA_CHECK("cassandra.skip_schema_check"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_PROTOCOLS("com.sun.management.jmxremote.ssl.enabled.protocols"),
/**
- * When bootstraping how long to wait for schema versions to be seen.
+ * This property indicates whether SSL client authentication is enabled - com.sun.management.jmxremote.ssl.need.client.auth.
+ * Default is set to false.
*/
- BOOTSTRAP_SCHEMA_DELAY_MS("cassandra.schema_delay_ms"),
-
+ COM_SUN_MANAGEMENT_JMXREMOTE_SSL_NEED_CLIENT_AUTH("com.sun.management.jmxremote.ssl.need.client.auth"),
+ /** Defaults to false for 4.1 but plan to switch to true in a later release the thinking is that environments
+ * may not work right off the bat so safer to add this feature disabled by default */
+ CONFIG_ALLOW_SYSTEM_PROPERTIES("cassandra.config.allow_system_properties"),
+ CONFIG_LOADER("cassandra.config.loader"),
+ CONSISTENT_DIRECTORY_LISTINGS("cassandra.consistent_directory_listings"),
+ CONSISTENT_RANGE_MOVEMENT("cassandra.consistent.rangemovement", "true"),
+ CONSISTENT_SIMULTANEOUS_MOVES_ALLOW("cassandra.consistent.simultaneousmoves.allow"),
+ CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS("cassandra.custom_guardrails_config_provider_class"),
+ CUSTOM_QUERY_HANDLER_CLASS("cassandra.custom_query_handler_class"),
+ CUSTOM_TRACING_CLASS("cassandra.custom_tracing_class"),
+ /** Controls the type of bufffer (heap/direct) used for shared scratch buffers */
+ DATA_OUTPUT_BUFFER_ALLOCATE_TYPE("cassandra.dob.allocate_type"),
+ DATA_OUTPUT_STREAM_PLUS_TEMP_BUFFER_SIZE("cassandra.data_output_stream_plus_temp_buffer_size", "8192"),
+ DECAYING_ESTIMATED_HISTOGRAM_RESERVOIR_STRIPE_COUNT("cassandra.dehr_stripe_count", "2"),
+ DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES("default.provide.overlapping.tombstones"),
+ /** determinism properties for testing */
+ DETERMINISM_SSTABLE_COMPRESSION_DEFAULT("cassandra.sstable_compression_default", "true"),
+ DETERMINISM_UNSAFE_UUID_NODE("cassandra.unsafe.deterministicuuidnode"),
+ DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS("cassandra.diagnostic_snapshot_interval_nanos", "60000000000"),
+ DISABLE_AUTH_CACHES_REMOTE_CONFIGURATION("cassandra.disable_auth_caches_remote_configuration"),
+ /** properties to disable certain behaviours for testing */
+ DISABLE_GOSSIP_ENDPOINT_REMOVAL("cassandra.gossip.disable_endpoint_removal"),
+ DISABLE_PAXOS_AUTO_REPAIRS("cassandra.disable_paxos_auto_repairs"),
+ DISABLE_PAXOS_STATE_FLUSH("cassandra.disable_paxos_state_flush"),
+ DISABLE_SSTABLE_ACTIVITY_TRACKING("cassandra.sstable_activity_tracking", "true"),
+ DISABLE_STCS_IN_L0("cassandra.disable_stcs_in_l0"),
+ DISABLE_TCACTIVE_OPENSSL("cassandra.disable_tcactive_openssl"),
+ /** property for the rate of the scheduled task that monitors disk usage */
+ DISK_USAGE_MONITOR_INTERVAL_MS("cassandra.disk_usage.monitor_interval_ms", convertToString(TimeUnit.SECONDS.toMillis(30))),
+ /** property for the interval on which the repeated client warnings and diagnostic events about disk usage are ignored */
+ DISK_USAGE_NOTIFY_INTERVAL_MS("cassandra.disk_usage.notify_interval_ms", convertToString(TimeUnit.MINUTES.toMillis(30))),
+ DOB_DOUBLING_THRESHOLD_MB("cassandra.DOB_DOUBLING_THRESHOLD_MB", "64"),
+ DOB_MAX_RECYCLE_BYTES("cassandra.dob_max_recycle_bytes", convertToString(1024 * 1024)),
/**
* When draining, how long to wait for mutating executors to shutdown.
*/
- DRAIN_EXECUTOR_TIMEOUT_MS("cassandra.drain_executor_timeout_ms", String.valueOf(TimeUnit.MINUTES.toMillis(5))),
-
+ DRAIN_EXECUTOR_TIMEOUT_MS("cassandra.drain_executor_timeout_ms", convertToString(TimeUnit.MINUTES.toMillis(5))),
+ DROP_OVERSIZED_READ_REPAIR_MUTATIONS("cassandra.drop_oversized_readrepair_mutations"),
+ DTEST_API_LOG_TOPOLOGY("cassandra.dtest.api.log.topology"),
+ ENABLE_DC_LOCAL_COMMIT("cassandra.enable_dc_local_commit", "true"),
+ /**
+ * Whether {@link org.apache.cassandra.db.ConsistencyLevel#NODE_LOCAL} should be allowed.
+ */
+ ENABLE_NODELOCAL_QUERIES("cassandra.enable_nodelocal_queries"),
+ EXPIRATION_DATE_OVERFLOW_POLICY("cassandra.expiration_date_overflow_policy"),
+ EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES("cassandra.expiration_overflow_warning_interval_minutes", "5"),
+ FAILURE_LOGGING_INTERVAL_SECONDS("cassandra.request_failure_log_interval_seconds", "60"),
+ FD_INITIAL_VALUE_MS("cassandra.fd_initial_value_ms"),
+ FD_MAX_INTERVAL_MS("cassandra.fd_max_interval_ms"),
+ FILE_CACHE_ENABLED("cassandra.file_cache_enabled"),
+ /** @deprecated should be removed in favor of enable flag of relevant startup check (FileSystemOwnershipCheck) */
+ @Deprecated
+ FILE_SYSTEM_CHECK_ENABLE("cassandra.enable_fs_ownership_check"),
+ /** @deprecated should be removed in favor of flags in relevant startup check (FileSystemOwnershipCheck) */
+ @Deprecated
+ FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME("cassandra.fs_ownership_filename", FileSystemOwnershipCheck.DEFAULT_FS_OWNERSHIP_FILENAME),
+ /** @deprecated should be removed in favor of flags in relevant startup check (FileSystemOwnershipCheck) */
+ @Deprecated
+ FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN(FileSystemOwnershipCheck.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN),
+ FORCE_DEFAULT_INDEXING_PAGE_SIZE("cassandra.force_default_indexing_page_size"),
+ /** Used when running in Client mode and the system and schema keyspaces need to be initialized outside of their normal initialization path **/
+ FORCE_LOAD_LOCAL_KEYSPACES("cassandra.schema.force_load_local_keyspaces"),
+ FORCE_PAXOS_STATE_REBUILD("cassandra.force_paxos_state_rebuild"),
+ GIT_SHA("cassandra.gitSHA"),
/**
* Gossip quarantine delay is used while evaluating membership changes and should only be changed with extreme care.
*/
GOSSIPER_QUARANTINE_DELAY("cassandra.gossip_quarantine_delay_ms"),
-
GOSSIPER_SKIP_WAITING_TO_SETTLE("cassandra.skip_wait_for_gossip_to_settle", "-1"),
-
- IGNORED_SCHEMA_CHECK_VERSIONS("cassandra.skip_schema_check_for_versions"),
-
+ GOSSIP_DISABLE_THREAD_VALIDATION("cassandra.gossip.disable_thread_validation"),
IGNORED_SCHEMA_CHECK_ENDPOINTS("cassandra.skip_schema_check_for_endpoints"),
-
- SHUTDOWN_ANNOUNCE_DELAY_IN_MS("cassandra.shutdown_announce_in_ms", "2000"),
-
+ IGNORED_SCHEMA_CHECK_VERSIONS("cassandra.skip_schema_check_for_versions"),
+ IGNORE_CORRUPTED_SCHEMA_TABLES("cassandra.ignore_corrupted_schema_tables"),
+ /** @deprecated should be removed in favor of enable flag of relevant startup check (checkDatacenter) */
+ @Deprecated
+ IGNORE_DC("cassandra.ignore_dc"),
+ IGNORE_DYNAMIC_SNITCH_SEVERITY("cassandra.ignore_dynamic_snitch_severity"),
+ IGNORE_MISSING_NATIVE_FILE_HINTS("cassandra.require_native_file_hints"),
+ /** @deprecated should be removed in favor of enable flag of relevant startup check (checkRack) */
+ @Deprecated
+ IGNORE_RACK("cassandra.ignore_rack"),
+ INDEX_SUMMARY_EXPECTED_KEY_SIZE("cassandra.index_summary_expected_key_size", "64"),
+ INITIAL_TOKEN("cassandra.initial_token"),
+ INTERNODE_EVENT_THREADS("cassandra.internode-event-threads"),
+ IO_NETTY_EVENTLOOP_THREADS("io.netty.eventLoopThreads"),
+ IO_NETTY_TRANSPORT_ESTIMATE_SIZE_ON_SUBMIT("io.netty.transport.estimateSizeOnSubmit"),
+ JAVAX_RMI_SSL_CLIENT_ENABLED_CIPHER_SUITES("javax.rmi.ssl.client.enabledCipherSuites"),
+ JAVAX_RMI_SSL_CLIENT_ENABLED_PROTOCOLS("javax.rmi.ssl.client.enabledProtocols"),
+ /** Java class path. */
+ JAVA_CLASS_PATH("java.class.path"),
+ JAVA_HOME("java.home"),
+ /**
+ * Indicates the temporary directory used by the Java Virtual Machine (JVM)
+ * to create and store temporary files.
+ */
+ JAVA_IO_TMPDIR("java.io.tmpdir"),
+ /**
+ * Path from which to load native libraries.
+ * Default is absolute path to lib directory.
+ */
+ JAVA_LIBRARY_PATH("java.library.path"),
+ /**
+ * The value of this property represents the host name string
+ * that should be associated with remote stubs for locally created remote objects,
+ * in order to allow clients to invoke methods on the remote object.
+ */
+ JAVA_RMI_SERVER_HOSTNAME("java.rmi.server.hostname"),
+ /**
+ * If this value is true, object identifiers for remote objects exported by this VM will be generated by using
+ * a cryptographically secure random number generator. The default value is false.
+ */
+ JAVA_RMI_SERVER_RANDOM_ID("java.rmi.server.randomIDs"),
+ JAVA_SECURITY_AUTH_LOGIN_CONFIG("java.security.auth.login.config"),
+ JAVA_SECURITY_EGD("java.security.egd"),
+ /** Java Runtime Environment version */
+ JAVA_VERSION("java.version"),
+ /** Java Virtual Machine implementation name */
+ JAVA_VM_NAME("java.vm.name"),
+ JOIN_RING("cassandra.join_ring", "true"),
+ /** startup checks properties */
+ LIBJEMALLOC("cassandra.libjemalloc"),
+ /** Line separator ("\n" on UNIX). */
+ LINE_SEPARATOR("line.separator"),
+ /** Load persistence ring state. Default value is {@code true}. */
+ LOAD_RING_STATE("cassandra.load_ring_state", "true"),
+ LOG4J2_DISABLE_JMX("log4j2.disableJmx"),
+ LOG4J2_DISABLE_JMX_LEGACY("log4j2.disable.jmx"),
+ LOG4J_SHUTDOWN_HOOK_ENABLED("log4j.shutdownHookEnabled"),
+ LOGBACK_CONFIGURATION_FILE("logback.configurationFile"),
+ /** Maximum number of rows in system_views.logs table */
+ LOGS_VIRTUAL_TABLE_MAX_ROWS("cassandra.virtual.logs.max.rows", convertToString(LogMessagesTable.LOGS_VIRTUAL_TABLE_DEFAULT_ROWS)),
+ /**
+ * Directory where Cassandra puts its logs, defaults to "." which is current directory.
+ */
+ LOG_DIR("cassandra.logdir", "."),
+ /**
+ * Directory where Cassandra persists logs from audit logging. If this property is not set, the audit log framework
+ * will set it automatically to {@link CassandraRelevantProperties#LOG_DIR} + "/audit".
+ */
+ LOG_DIR_AUDIT("cassandra.logdir.audit"),
+ /** Loosen the definition of "empty" for gossip state, for use during host replacements if things go awry */
+ LOOSE_DEF_OF_EMPTY_ENABLED(Config.PROPERTY_PREFIX + "gossiper.loose_empty_enabled"),
+ MAX_CONCURRENT_RANGE_REQUESTS("cassandra.max_concurrent_range_requests"),
+ MAX_HINT_BUFFERS("cassandra.MAX_HINT_BUFFERS", "3"),
+ MAX_LOCAL_PAUSE_IN_MS("cassandra.max_local_pause_in_ms", "5000"),
+ /** what class to use for mbean registeration */
+ MBEAN_REGISTRATION_CLASS("org.apache.cassandra.mbean_registration_class"),
+ MEMTABLE_OVERHEAD_COMPUTE_STEPS("cassandra.memtable_row_overhead_computation_step", "100000"),
+ MEMTABLE_OVERHEAD_SIZE("cassandra.memtable.row_overhead_size", "-1"),
+ MEMTABLE_SHARD_COUNT("cassandra.memtable.shard.count"),
+ MEMTABLE_TRIE_SIZE_LIMIT("cassandra.trie_size_limit_mb"),
+ METRICS_REPORTER_CONFIG_FILE("cassandra.metricsReporterConfigFile"),
+ MIGRATION_DELAY("cassandra.migration_delay_ms", "60000"),
+ /** Defines the maximum number of unique timed out queries that will be reported in the logs. Use a negative number to remove any limit. */
+ MONITORING_MAX_OPERATIONS("cassandra.monitoring_max_operations", "50"),
+ /** Defines the interval for reporting any operations that have timed out. */
+ MONITORING_REPORT_INTERVAL_MS("cassandra.monitoring_report_interval_ms", "5000"),
+ MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE("cassandra.mv.allow_filtering_nonkey_columns_unsafe"),
+ MV_ENABLE_COORDINATOR_BATCHLOG("cassandra.mv_enable_coordinator_batchlog"),
+ /** mx4jaddress */
+ MX4JADDRESS("mx4jaddress"),
+ /** mx4jport */
+ MX4JPORT("mx4jport"),
+ NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL("cassandra.NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL", "10000"),
+ NATIVE_EPOLL_ENABLED("cassandra.native.epoll.enabled", "true"),
+ /** This is the port used with RPC address for the native protocol to communicate with clients. Now that thrift RPC is no longer in use there is no RPC port. */
+ NATIVE_TRANSPORT_PORT("cassandra.native_transport_port"),
+ NEVER_PURGE_TOMBSTONES("cassandra.never_purge_tombstones"),
+ NIO_DATA_OUTPUT_STREAM_PLUS_BUFFER_SIZE("cassandra.nio_data_output_stream_plus_buffer_size", convertToString(32 * 1024)),
+ NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS("cassandra.nodetool.jmx_notification_poll_interval_seconds", convertToString(TimeUnit.SECONDS.convert(5, TimeUnit.MINUTES))),
+ /** If set, {@link org.apache.cassandra.net.MessagingService} is shutdown abrtuptly without waiting for anything.
+ * This is an optimization used in unit tests becuase we never restart a node there. The only node is stopoped
+ * when the JVM terminates. Therefore, we can use such optimization and not wait unnecessarily. */
+ NON_GRACEFUL_SHUTDOWN("cassandra.test.messagingService.nonGracefulShutdown"),
+ /** for specific tests */
+ /** This property indicates whether disable_mbean_registration is true */
+ ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION("org.apache.cassandra.disable_mbean_registration"),
+ /** Operating system architecture. */
+ OS_ARCH("os.arch"),
+ /** Operating system name. */
+ OS_NAME("os.name"),
+ OTCP_LARGE_MESSAGE_THRESHOLD("cassandra.otcp_large_message_threshold", convertToString(1024 * 64)),
+ /** Enabled/disable TCP_NODELAY for intradc connections. Defaults is enabled. */
+ OTC_INTRADC_TCP_NODELAY("cassandra.otc_intradc_tcp_nodelay", "true"),
+ OVERRIDE_DECOMMISSION("cassandra.override_decommission"),
+ PARENT_REPAIR_STATUS_CACHE_SIZE("cassandra.parent_repair_status_cache_size", "100000"),
+ PARENT_REPAIR_STATUS_EXPIRY_SECONDS("cassandra.parent_repair_status_expiry_seconds", convertToString(TimeUnit.SECONDS.convert(1, TimeUnit.DAYS))),
+ PARTITIONER("cassandra.partitioner"),
+ PAXOS_CLEANUP_SESSION_TIMEOUT_SECONDS("cassandra.paxos_cleanup_session_timeout_seconds", convertToString(TimeUnit.HOURS.toSeconds(2))),
+ PAXOS_DISABLE_COORDINATOR_LOCKING("cassandra.paxos.disable_coordinator_locking"),
+ PAXOS_LOG_TTL_LINEARIZABILITY_VIOLATIONS("cassandra.paxos.log_ttl_linearizability_violations", "true"),
+ PAXOS_MODERN_RELEASE("cassandra.paxos.modern_release", "4.1"),
+ PAXOS_REPAIR_ALLOW_MULTIPLE_PENDING_UNSAFE("cassandra.paxos_repair_allow_multiple_pending_unsafe"),
+ PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRIES("cassandra.paxos_repair_on_topology_change_retries", "10"),
+ PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRY_DELAY_SECONDS("cassandra.paxos_repair_on_topology_change_retry_delay_seconds", "10"),
+ PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS("cassandra.paxos_repair_retry_timeout_millis", "60000"),
+ PAXOS_USE_SELF_EXECUTION("cassandra.paxos.use_self_execution", "true"),
+ PRINT_HEAP_HISTOGRAM_ON_OUT_OF_MEMORY_ERROR("cassandra.printHeapHistogramOnOutOfMemoryError"),
+ READS_THRESHOLDS_COORDINATOR_DEFENSIVE_CHECKS_ENABLED("cassandra.reads.thresholds.coordinator.defensive_checks_enabled"),
+ RELEASE_VERSION("cassandra.releaseVersion"),
+ RELOCATED_SHADED_IO_NETTY_TRANSPORT_NONATIVE("relocated.shaded.io.netty.transport.noNative"),
+ REPAIR_CLEANUP_INTERVAL_SECONDS("cassandra.repair_cleanup_interval_seconds", convertToString(Ints.checkedCast(TimeUnit.MINUTES.toSeconds(10)))),
+ REPAIR_DELETE_TIMEOUT_SECONDS("cassandra.repair_delete_timeout_seconds", convertToString(Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)))),
+ REPAIR_FAIL_TIMEOUT_SECONDS("cassandra.repair_fail_timeout_seconds", convertToString(Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)))),
+ REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH("cassandra.repair.mutation_repair_rows_per_batch", "100"),
+ REPAIR_STATUS_CHECK_TIMEOUT_SECONDS("cassandra.repair_status_check_timeout_seconds", convertToString(Ints.checkedCast(TimeUnit.HOURS.toSeconds(1)))),
/**
* When doing a host replacement its possible that the gossip state is "empty" meaning that the endpoint is known
* but the current state isn't known. If the host replacement is needed to repair this state, this property must
* be true.
*/
REPLACEMENT_ALLOW_EMPTY("cassandra.allow_empty_replace_address", "true"),
-
+ REPLACE_ADDRESS("cassandra.replace_address"),
+ REPLACE_ADDRESS_FIRST_BOOT("cassandra.replace_address_first_boot"),
+ REPLACE_NODE("cassandra.replace_node"),
+ REPLACE_TOKEN("cassandra.replace_token"),
/**
- * Directory where Cassandra puts its logs, defaults to "." which is current directory.
+ * Whether we reset any found data from previously run bootstraps.
*/
- LOG_DIR("cassandra.logdir", "."),
-
- /**
- * Directory where Cassandra persists logs from audit logging. If this property is not set, the audit log framework
- * will set it automatically to {@link CassandraRelevantProperties#LOG_DIR} + "/audit".
- */
- LOG_DIR_AUDIT("cassandra.logdir.audit"),
-
- CONSISTENT_DIRECTORY_LISTINGS("cassandra.consistent_directory_listings", "false"),
- CLOCK_GLOBAL("cassandra.clock", null),
- CLOCK_MONOTONIC_APPROX("cassandra.monotonic_clock.approx", null),
- CLOCK_MONOTONIC_PRECISE("cassandra.monotonic_clock.precise", null),
-
- /*
- * Whether {@link org.apache.cassandra.db.ConsistencyLevel#NODE_LOCAL} should be allowed.
- */
- ENABLE_NODELOCAL_QUERIES("cassandra.enable_nodelocal_queries", "false"),
-
- //cassandra properties (without the "cassandra." prefix)
-
- /**
- * The cassandra-foreground option will tell CassandraDaemon whether
- * to close stdout/stderr, but it's up to us not to background.
- * yes/null
- */
- CASSANDRA_FOREGROUND ("cassandra-foreground"),
-
- DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES ("default.provide.overlapping.tombstones"),
- ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION ("org.apache.cassandra.disable_mbean_registration"),
-
- /** This property indicates whether disable_mbean_registration is true */
- IS_DISABLED_MBEAN_REGISTRATION("org.apache.cassandra.disable_mbean_registration"),
-
- /** snapshots ttl cleanup period in seconds */
- SNAPSHOT_CLEANUP_PERIOD_SECONDS("cassandra.snapshot.ttl_cleanup_period_seconds", "60"),
-
- /** snapshots ttl cleanup initial delay in seconds */
- SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS("cassandra.snapshot.ttl_cleanup_initial_delay_seconds", "5"),
-
- /** minimum allowed TTL for snapshots */
- SNAPSHOT_MIN_ALLOWED_TTL_SECONDS("cassandra.snapshot.min_allowed_ttl_seconds", "60"),
-
- /** what class to use for mbean registeration */
- MBEAN_REGISTRATION_CLASS("org.apache.cassandra.mbean_registration_class"),
-
- BATCH_COMMIT_LOG_SYNC_INTERVAL("cassandra.batch_commitlog_sync_interval_millis", "1000"),
-
- SYSTEM_AUTH_DEFAULT_RF("cassandra.system_auth.default_rf", "1"),
- SYSTEM_TRACES_DEFAULT_RF("cassandra.system_traces.default_rf", "2"),
- SYSTEM_DISTRIBUTED_DEFAULT_RF("cassandra.system_distributed.default_rf", "3"),
-
- MEMTABLE_OVERHEAD_SIZE("cassandra.memtable.row_overhead_size", "-1"),
- MEMTABLE_OVERHEAD_COMPUTE_STEPS("cassandra.memtable_row_overhead_computation_step", "100000"),
- MIGRATION_DELAY("cassandra.migration_delay_ms", "60000"),
+ RESET_BOOTSTRAP_PROGRESS("cassandra.reset_bootstrap_progress"),
+ RING_DELAY("cassandra.ring_delay_ms"),
/** Defines how often schema definitions are pulled from the other nodes */
SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
-
- PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS("cassandra.paxos_repair_retry_timeout_millis", "60000"),
-
- /** If we should allow having duplicate keys in the config file, default to true for legacy reasons */
- ALLOW_DUPLICATE_CONFIG_KEYS("cassandra.allow_duplicate_config_keys", "true"),
- /** If we should allow having both new (post CASSANDRA-15234) and old config keys for the same config item in the yaml */
- ALLOW_NEW_OLD_CONFIG_KEYS("cassandra.allow_new_old_config_keys", "false"),
-
- // startup checks properties
- LIBJEMALLOC("cassandra.libjemalloc"),
- @Deprecated // should be removed in favor of enable flag of relevant startup check (checkDatacenter)
- IGNORE_DC("cassandra.ignore_dc"),
- @Deprecated // should be removed in favor of enable flag of relevant startup check (checkRack)
- IGNORE_RACK("cassandra.ignore_rack"),
- @Deprecated // should be removed in favor of enable flag of relevant startup check (FileSystemOwnershipCheck)
- FILE_SYSTEM_CHECK_ENABLE("cassandra.enable_fs_ownership_check"),
- @Deprecated // should be removed in favor of flags in relevant startup check (FileSystemOwnershipCheck)
- FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME("cassandra.fs_ownership_filename", FileSystemOwnershipCheck.DEFAULT_FS_OWNERSHIP_FILENAME),
- @Deprecated // should be removed in favor of flags in relevant startup check (FileSystemOwnershipCheck)
- FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN(FileSystemOwnershipCheck.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN),
- // default heartbeating period is 1 minute
- CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD("check_data_resurrection_heartbeat_period_milli", "60000"),
-
- // defaults to false for 4.1 but plan to switch to true in a later release
- // the thinking is that environments may not work right off the bat so safer to add this feature disabled by default
- CONFIG_ALLOW_SYSTEM_PROPERTIES("cassandra.config.allow_system_properties", "false"),
-
- // properties for debugging simulator ASM output
- TEST_SIMULATOR_PRINT_ASM("cassandra.test.simulator.print_asm", "none"),
- TEST_SIMULATOR_PRINT_ASM_TYPES("cassandra.test.simulator.print_asm_types", ""),
- TEST_SIMULATOR_LIVENESS_CHECK("cassandra.test.simulator.livenesscheck", "true"),
- TEST_SIMULATOR_DEBUG("cassandra.test.simulator.debug", "false"),
- TEST_SIMULATOR_DETERMINISM_CHECK("cassandra.test.simulator.determinismcheck", "none"),
- TEST_JVM_DTEST_DISABLE_SSL("cassandra.test.disable_ssl", "false"),
-
- // determinism properties for testing
- DETERMINISM_SSTABLE_COMPRESSION_DEFAULT("cassandra.sstable_compression_default", "true"),
- DETERMINISM_CONSISTENT_DIRECTORY_LISTINGS("cassandra.consistent_directory_listings", "false"),
- DETERMINISM_UNSAFE_UUID_NODE("cassandra.unsafe.deterministicuuidnode", "false"),
- FAILURE_LOGGING_INTERVAL_SECONDS("cassandra.request_failure_log_interval_seconds", "60"),
-
- // properties to disable certain behaviours for testing
- DISABLE_GOSSIP_ENDPOINT_REMOVAL("cassandra.gossip.disable_endpoint_removal"),
- IGNORE_MISSING_NATIVE_FILE_HINTS("cassandra.require_native_file_hints", "false"),
- DISABLE_SSTABLE_ACTIVITY_TRACKING("cassandra.sstable_activity_tracking", "true"),
- TEST_IGNORE_SIGAR("cassandra.test.ignore_sigar", "false"),
- PAXOS_EXECUTE_ON_SELF("cassandra.paxos.use_self_execution", "true"),
-
- /** property for the rate of the scheduled task that monitors disk usage */
- DISK_USAGE_MONITOR_INTERVAL_MS("cassandra.disk_usage.monitor_interval_ms", Long.toString(TimeUnit.SECONDS.toMillis(30))),
-
- /** property for the interval on which the repeated client warnings and diagnostic events about disk usage are ignored */
- DISK_USAGE_NOTIFY_INTERVAL_MS("cassandra.disk_usage.notify_interval_ms", Long.toString(TimeUnit.MINUTES.toMillis(30))),
-
- // for specific tests
- ORG_APACHE_CASSANDRA_CONF_CASSANDRA_RELEVANT_PROPERTIES_TEST("org.apache.cassandra.conf.CassandraRelevantPropertiesTest"),
- ORG_APACHE_CASSANDRA_DB_VIRTUAL_SYSTEM_PROPERTIES_TABLE_TEST("org.apache.cassandra.db.virtual.SystemPropertiesTableTest"),
-
- /** Used when running in Client mode and the system and schema keyspaces need to be initialized outside of their normal initialization path **/
- FORCE_LOAD_LOCAL_KEYSPACES("cassandra.schema.force_load_local_keyspaces"),
-
- /** When enabled, recursive directory deletion will be executed using a unix command `rm -rf` instead of traversing
- * and removing individual files. This is now used only tests, but eventually we will make it true by default.*/
- USE_NIX_RECURSIVE_DELETE("cassandra.use_nix_recursive_delete"),
-
- /** If set, {@link org.apache.cassandra.net.MessagingService} is shutdown abrtuptly without waiting for anything.
- * This is an optimization used in unit tests becuase we never restart a node there. The only node is stopoped
- * when the JVM terminates. Therefore, we can use such optimization and not wait unnecessarily. */
- NON_GRACEFUL_SHUTDOWN("cassandra.test.messagingService.nonGracefulShutdown"),
-
+ SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
+ SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
+ /**
+ * The maximum number of seeds returned by a seed provider before emmitting a warning.
+ * A large seed list may impact effectiveness of the third gossip round.
+ * The default used in SimpleSeedProvider is 20.
+ */
+ SEED_COUNT_WARN_THRESHOLD("cassandra.seed_count_warn_threshold"),
+ SERIALIZATION_EMPTY_TYPE_NONEMPTY_BEHAVIOR("cassandra.serialization.emptytype.nonempty_behavior"),
+ SET_SEP_THREAD_NAME("cassandra.set_sep_thread_name", "true"),
+ SHUTDOWN_ANNOUNCE_DELAY_IN_MS("cassandra.shutdown_announce_in_ms", "2000"),
+ SIZE_RECORDER_INTERVAL("cassandra.size_recorder_interval", "300"),
+ SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE("cassandra.skip_paxos_repair_on_topology_change"),
+ /** If necessary for operational purposes, permit certain keyspaces to be ignored for paxos topology repairs. */
+ SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES("cassandra.skip_paxos_repair_on_topology_change_keyspaces"),
+ SKIP_PAXOS_REPAIR_VERSION_VALIDATION("cassandra.skip_paxos_repair_version_validation"),
+ SKIP_PAXOS_STATE_REBUILD("cassandra.skip_paxos_state_rebuild"),
+ /** snapshots ttl cleanup initial delay in seconds */
+ SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS("cassandra.snapshot.ttl_cleanup_initial_delay_seconds", "5"),
+ /** snapshots ttl cleanup period in seconds */
+ SNAPSHOT_CLEANUP_PERIOD_SECONDS("cassandra.snapshot.ttl_cleanup_period_seconds", "60"),
+ /** minimum allowed TTL for snapshots */
+ SNAPSHOT_MIN_ALLOWED_TTL_SECONDS("cassandra.snapshot.min_allowed_ttl_seconds", "60"),
+ SSL_ENABLE("ssl.enable"),
+ SSL_STORAGE_PORT("cassandra.ssl_storage_port"),
+ SSTABLE_FORMAT_DEFAULT("cassandra.sstable.format.default"),
+ START_GOSSIP("cassandra.start_gossip", "true"),
+ START_NATIVE_TRANSPORT("cassandra.start_native_transport"),
+ STORAGE_DIR("cassandra.storagedir"),
+ STORAGE_HOOK("cassandra.storage_hook"),
+ STORAGE_PORT("cassandra.storage_port"),
+ STREAMING_HISTOGRAM_ROUND_SECONDS("cassandra.streaminghistogram.roundseconds", "60"),
+ STREAMING_SESSION_PARALLELTRANSFERS("cassandra.streaming.session.parallelTransfers"),
+ STREAM_HOOK("cassandra.stream_hook"),
+ /** Platform word size sun.arch.data.model. Examples: "32", "64", "unknown"*/
+ SUN_ARCH_DATA_MODEL("sun.arch.data.model"),
+ SUN_JAVA_COMMAND("sun.java.command", ""),
+ SUN_STDERR_ENCODING("sun.stderr.encoding"),
+ SUN_STDOUT_ENCODING("sun.stdout.encoding"),
+ SUPERUSER_SETUP_DELAY_MS("cassandra.superuser_setup_delay_ms", "10000"),
+ SYSTEM_AUTH_DEFAULT_RF("cassandra.system_auth.default_rf", "1"),
+ SYSTEM_DISTRIBUTED_DEFAULT_RF("cassandra.system_distributed.default_rf", "3"),
+ SYSTEM_TRACES_DEFAULT_RF("cassandra.system_traces.default_rf", "2"),
+ TEST_BBFAILHELPER_ENABLED("test.bbfailhelper.enabled"),
+ TEST_BLOB_SHARED_SEED("cassandra.test.blob.shared.seed"),
+ TEST_BYTEMAN_TRANSFORMATIONS_DEBUG("cassandra.test.byteman.transformations.debug"),
+ TEST_CASSANDRA_KEEPBRIEFBRIEF("cassandra.keepBriefBrief"),
+ TEST_CASSANDRA_RELEVANT_PROPERTIES("org.apache.cassandra.conf.CassandraRelevantPropertiesTest"),
+ /** A property for various mechanisms for syncing files that makes it possible it intercept and skip syncing. */
+ TEST_CASSANDRA_SKIP_SYNC("cassandra.skip_sync"),
+ TEST_CASSANDRA_SUITENAME("suitename", "suitename_IS_UNDEFINED"),
+ TEST_CASSANDRA_TESTTAG("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED"),
+ TEST_COMPRESSION("cassandra.test.compression"),
+ TEST_COMPRESSION_ALGO("cassandra.test.compression.algo", "lz4"),
+ TEST_DEBUG_REF_COUNT("cassandra.debugrefcount"),
+ TEST_DRIVER_CONNECTION_TIMEOUT_MS("cassandra.test.driver.connection_timeout_ms", "5000"),
+ TEST_DRIVER_READ_TIMEOUT_MS("cassandra.test.driver.read_timeout_ms", "12000"),
+ TEST_FAIL_MV_LOCKS_COUNT("cassandra.test.fail_mv_locks_count", "0"),
+ TEST_FAIL_WRITES_KS("cassandra.test.fail_writes_ks", ""),
/** Flush changes of {@link org.apache.cassandra.schema.SchemaKeyspace} after each schema modification. In production,
* we always do that. However, tests which do not restart nodes may disable this functionality in order to run
* faster. Note that this is disabled for unit tests but if an individual test requires schema to be flushed, it
* can be also done manually for that particular case: {@code flush(SchemaConstants.SCHEMA_KEYSPACE_NAME);}. */
- FLUSH_LOCAL_SCHEMA_CHANGES("cassandra.test.flush_local_schema_changes", "true"),
+ TEST_FLUSH_LOCAL_SCHEMA_CHANGES("cassandra.test.flush_local_schema_changes", "true"),
+ TEST_IGNORE_SIGAR("cassandra.test.ignore_sigar"),
+ TEST_INVALID_LEGACY_SSTABLE_ROOT("invalid-legacy-sstable-root"),
+ TEST_JVM_DTEST_DISABLE_SSL("cassandra.test.disable_ssl"),
+ TEST_LEGACY_SSTABLE_ROOT("legacy-sstable-root"),
+ TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT("org.caffinitas.ohc.segmentCount"),
+ TEST_READ_ITERATION_DELAY_MS("cassandra.test.read_iteration_delay_ms", "0"),
+ TEST_REUSE_PREPARED("cassandra.test.reuse_prepared", "true"),
+ TEST_ROW_CACHE_SIZE("cassandra.test.row_cache_size"),
+ TEST_SERIALIZATION_WRITES("cassandra.test-serialization-writes"),
+ TEST_SIMULATOR_DEBUG("cassandra.test.simulator.debug"),
+ TEST_SIMULATOR_DETERMINISM_CHECK("cassandra.test.simulator.determinismcheck", "none"),
+ TEST_SIMULATOR_LIVENESS_CHECK("cassandra.test.simulator.livenesscheck", "true"),
+ /** properties for debugging simulator ASM output */
+ TEST_SIMULATOR_PRINT_ASM("cassandra.test.simulator.print_asm", "none"),
+ TEST_SIMULATOR_PRINT_ASM_CLASSES("cassandra.test.simulator.print_asm_classes", ""),
+ TEST_SIMULATOR_PRINT_ASM_OPTS("cassandra.test.simulator.print_asm_opts", ""),
+ TEST_SIMULATOR_PRINT_ASM_TYPES("cassandra.test.simulator.print_asm_types", ""),
+ TEST_SSTABLE_FORMAT_DEVELOPMENT("cassandra.test.sstableformatdevelopment"),
+ TEST_STRICT_LCS_CHECKS("cassandra.test.strict_lcs_checks"),
+ /** Turns some warnings into exceptions for testing. */
+ TEST_STRICT_RUNTIME_CHECKS("cassandra.strict.runtime.checks"),
+ /** Not to be used in production, this causes a Netty logging handler to be added to the pipeline, which will throttle a system under any normal load. */
+ TEST_UNSAFE_VERBOSE_DEBUG_CLIENT_PROTOCOL("cassandra.unsafe_verbose_debug_client_protocol"),
+ TEST_USE_PREPARED("cassandra.test.use_prepared", "true"),
+ TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST("org.apache.cassandra.tools.UtilALLOW_TOOL_REINIT_FOR_TEST"),
+ /** Activate write survey mode. The node not becoming an active ring member, and you must use JMX StorageService->joinRing() to finalize the ring joining. */
+ TEST_WRITE_SURVEY("cassandra.write_survey"),
+ TOLERATE_SSTABLE_SIZE("cassandra.tolerate_sstable_size"),
+ TRIGGERS_DIR("cassandra.triggers_dir"),
+ TRUNCATE_BALLOT_METADATA("cassandra.truncate_ballot_metadata"),
+ TYPE_UDT_CONFLICT_BEHAVIOR("cassandra.type.udt.conflict_behavior"),
+ UDF_EXECUTOR_THREAD_KEEPALIVE_MS("cassandra.udf_executor_thread_keepalive_ms", "30000"),
+ UNSAFE_SYSTEM("cassandra.unsafesystem"),
+ /** User's home directory. */
+ USER_HOME("user.home"),
+ /** When enabled, recursive directory deletion will be executed using a unix command `rm -rf` instead of traversing
+ * and removing individual files. This is now used only tests, but eventually we will make it true by default.*/
+ USE_NIX_RECURSIVE_DELETE("cassandra.use_nix_recursive_delete"),
+ /** Gossiper compute expiration timeout. Default value 3 days. */
+ VERY_LONG_TIME_MS("cassandra.very_long_time_ms", "259200000"),
+ WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS("cassandra.wait_for_tracing_events_timeout_secs", "0");
- ;
+ static
+ {
+ CassandraRelevantProperties[] values = CassandraRelevantProperties.values();
+ Set<String> visited = new HashSet<>(values.length);
+ CassandraRelevantProperties prev = null;
+ for (CassandraRelevantProperties next : values)
+ {
+ if (!visited.add(next.getKey()))
+ throw new IllegalStateException("System properties have duplicate key: " + next.getKey());
+ if (prev != null && next.name().compareTo(prev.name()) < 0)
+ throw new IllegalStateException("Enum constants are not in alphabetical order: " + prev.name() + " should come before " + next.name());
+ else
+ prev = next;
+ }
+ }
CassandraRelevantProperties(String key, String defaultVal)
{
@@ -389,13 +580,32 @@
return converter.convert(value);
}
+ public static String convertToString(@Nullable Object value)
+ {
+ if (value == null)
+ return null;
+ if (value instanceof String)
+ return (String) value;
+ if (value instanceof Boolean)
+ return Boolean.toString((Boolean) value);
+ if (value instanceof Long)
+ return Long.toString((Long) value);
+ if (value instanceof Integer)
+ return Integer.toString((Integer) value);
+ if (value instanceof Double)
+ return Double.toString((Double) value);
+ if (value instanceof Float)
+ return Float.toString((Float) value);
+ throw new IllegalArgumentException("Unknown type " + value.getClass());
+ }
+
/**
* Sets the value into system properties.
* @param value to set
*/
- public void setString(String value)
+ public String setString(String value)
{
- System.setProperty(key, value);
+ return System.setProperty(key, value);
}
/**
@@ -425,36 +635,61 @@
/**
* Sets the value into system properties.
* @param value to set
+ * @return Previous value if it exists.
*/
- public void setBoolean(boolean value)
+ public Boolean setBoolean(boolean value)
{
- System.setProperty(key, Boolean.toString(value));
+ String prev = System.setProperty(key, convertToString(value));
+ return prev == null ? null : BOOLEAN_CONVERTER.convert(prev);
+ }
+
+ /**
+ * Clears the value set in the system property.
+ */
+ public void clearValue()
+ {
+ System.clearProperty(key);
}
/**
* Gets the value of a system property as a int.
- * @return system property int value if it exists, defaultValue otherwise.
+ * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
*/
public int getInt()
{
String value = System.getProperty(key);
-
+ if (value == null && defaultVal == null)
+ throw new ConfigurationException("Missing property value or default value is not set: " + key);
return INTEGER_CONVERTER.convert(value == null ? defaultVal : value);
}
/**
- * Gets the value of a system property as a int.
- * @return system property int value if it exists, defaultValue otherwise.
+ * Gets the value of a system property as a long.
+ * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
*/
public long getLong()
{
String value = System.getProperty(key);
-
+ if (value == null && defaultVal == null)
+ throw new ConfigurationException("Missing property value or default value is not set: " + key);
return LONG_CONVERTER.convert(value == null ? defaultVal : value);
}
/**
- * Gets the value of a system property as a int.
+ * Gets the value of a system property as a long.
+ * @return system property long value if it exists, defaultValue otherwise.
+ */
+ public long getLong(long overrideDefaultValue)
+ {
+ String value = System.getProperty(key);
+ if (value == null)
+ return overrideDefaultValue;
+
+ return LONG_CONVERTER.convert(value);
+ }
+
+ /**
+ * Gets the value of a system property as an int.
* @return system property int value if it exists, overrideDefaultValue otherwise.
*/
public int getInt(int overrideDefaultValue)
@@ -469,19 +704,75 @@
/**
* Sets the value into system properties.
* @param value to set
+ * @return Previous value or null if it did not have one.
*/
- public void setInt(int value)
+ public Integer setInt(int value)
{
- System.setProperty(key, Integer.toString(value));
+ String prev = System.setProperty(key, convertToString(value));
+ return prev == null ? null : INTEGER_CONVERTER.convert(prev);
+ }
+
+ /**
+ * Sets the value into system properties.
+ * @param value to set
+ * @return Previous value or null if it did not have one.
+ */
+ public Long setLong(long value)
+ {
+ String prev = System.setProperty(key, convertToString(value));
+ return prev == null ? null : LONG_CONVERTER.convert(prev);
+ }
+
+ /**
+ * Gets the value of a system property as a enum, calling {@link String#toUpperCase()} first.
+ *
+ * @param defaultValue to return when not defined
+ * @param <T> type
+ * @return enum value
+ */
+ public <T extends Enum<T>> T getEnum(T defaultValue)
+ {
+ return getEnum(true, defaultValue);
+ }
+
+ /**
+ * Gets the value of a system property as a enum, optionally calling {@link String#toUpperCase()} first.
+ *
+ * @param toUppercase before converting to enum
+ * @param defaultValue to return when not defined
+ * @param <T> type
+ * @return enum value
+ */
+ public <T extends Enum<T>> T getEnum(boolean toUppercase, T defaultValue)
+ {
+ String value = System.getProperty(key);
+ if (value == null)
+ return defaultValue;
+ return Enum.valueOf(defaultValue.getDeclaringClass(), toUppercase ? value.toUpperCase() : value);
+ }
+
+ /**
+ * Gets the value of a system property as an enum, optionally calling {@link String#toUpperCase()} first.
+ * If the value is missing, the default value for this property is used
+ *
+ * @param toUppercase before converting to enum
+ * @param enumClass enumeration class
+ * @param <T> type
+ * @return enum value
+ */
+ public <T extends Enum<T>> T getEnum(boolean toUppercase, Class<T> enumClass)
+ {
+ String value = System.getProperty(key, defaultVal);
+ return Enum.valueOf(enumClass, toUppercase ? value.toUpperCase() : value);
}
/**
* Sets the value into system properties.
* @param value to set
*/
- public void setLong(long value)
+ public void setEnum(Enum<?> value)
{
- System.setProperty(key, Long.toString(value));
+ System.setProperty(key, value.name());
}
public interface PropertyConverter<T>
@@ -515,7 +806,7 @@
catch (NumberFormatException e)
{
throw new ConfigurationException(String.format("Invalid value for system property: " +
- "expected integer value but got '%s'", value));
+ "expected long value but got '%s'", value));
}
};
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 8a59ca2..6e5f873 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -21,27 +21,31 @@
import java.lang.reflect.Modifier;
import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Supplier;
-
import javax.annotation.Nullable;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.audit.AuditLogOptions;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.fql.FullQueryLoggerOptions;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.service.StartupChecks.StartupCheckType;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTOCOMPACTION_ON_STARTUP_ENABLED;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FILE_CACHE_ENABLED;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES;
+
/**
* A class that contains configuration properties for the cassandra node it runs within.
*
@@ -160,6 +164,8 @@
@Replaces(oldName = "slow_query_log_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
public volatile DurationSpec.LongMillisecondsBound slow_query_log_timeout = new DurationSpec.LongMillisecondsBound("500ms");
+ public volatile DurationSpec.LongMillisecondsBound stream_transfer_task_timeout = new DurationSpec.LongMillisecondsBound("12h");
+
public volatile double phi_convict_threshold = 8.0;
public int concurrent_reads = 32;
@@ -267,6 +273,8 @@
public int native_transport_max_threads = 128;
@Replaces(oldName = "native_transport_max_frame_size_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
public DataStorageSpec.IntMebibytesBound native_transport_max_frame_size = new DataStorageSpec.IntMebibytesBound("16MiB");
+ /** do bcrypt hashing in a limited pool to prevent cpu load spikes; note: any value < 1 will be set to 1 on init **/
+ public int native_transport_max_auth_threads = 4;
public volatile long native_transport_max_concurrent_connections = -1L;
public volatile long native_transport_max_concurrent_connections_per_ip = -1L;
public boolean native_transport_flush_in_batches_legacy = false;
@@ -307,7 +315,7 @@
/* if the size of columns or super-columns are more than this, indexing will kick in */
@Replaces(oldName = "column_index_size_in_kb", converter = Converters.KIBIBYTES_DATASTORAGE, deprecated = true)
- public volatile DataStorageSpec.IntKibibytesBound column_index_size = new DataStorageSpec.IntKibibytesBound("64KiB");
+ public volatile DataStorageSpec.IntKibibytesBound column_index_size;
@Replaces(oldName = "column_index_cache_size_in_kb", converter = Converters.KIBIBYTES_DATASTORAGE, deprecated = true)
public volatile DataStorageSpec.IntKibibytesBound column_index_cache_size = new DataStorageSpec.IntKibibytesBound("2KiB");
@Replaces(oldName = "batch_size_warn_threshold_in_kb", converter = Converters.KIBIBYTES_DATASTORAGE, deprecated = true)
@@ -319,15 +327,22 @@
public volatile Integer concurrent_compactors;
@Replaces(oldName = "compaction_throughput_mb_per_sec", converter = Converters.MEBIBYTES_PER_SECOND_DATA_RATE, deprecated = true)
public volatile DataRateSpec.LongBytesPerSecondBound compaction_throughput = new DataRateSpec.LongBytesPerSecondBound("64MiB/s");
+ @Deprecated
@Replaces(oldName = "compaction_large_partition_warning_threshold_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
public volatile DataStorageSpec.IntMebibytesBound compaction_large_partition_warning_threshold = new DataStorageSpec.IntMebibytesBound("100MiB");
@Replaces(oldName = "min_free_space_per_drive_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
public DataStorageSpec.IntMebibytesBound min_free_space_per_drive = new DataStorageSpec.IntMebibytesBound("50MiB");
public volatile Integer compaction_tombstone_warning_threshold = 100000;
+ // fraction of free disk space available for compaction after min free space is subtracted
+ public volatile Double max_space_usable_for_compactions_in_percentage = .95;
+
public volatile int concurrent_materialized_view_builders = 1;
public volatile int reject_repair_compaction_threshold = Integer.MAX_VALUE;
+ // The number of executors to use for building secondary indexes
+ public int concurrent_index_builders = 2;
+
/**
* @deprecated retry support removed on CASSANDRA-10992
*/
@@ -344,6 +359,14 @@
public String[] data_file_directories = new String[0];
+ public static class SSTableConfig
+ {
+ public String selected_format = BigFormat.NAME;
+ public Map<String, Map<String, String>> format = new HashMap<>();
+ }
+
+ public final SSTableConfig sstable = new SSTableConfig();
+
/**
* The directory to use for storing the system keyspaces data.
* If unspecified the data will be stored in the first of the data_file_directories.
@@ -383,6 +406,9 @@
// When true, new CDC mutations are rejected/blocked when reaching max CDC storage.
// When false, new CDC mutations can always be added. But it will remove the oldest CDC commit log segment on full.
public volatile boolean cdc_block_writes = true;
+ // When true, CDC data in SSTable go through commit logs during internodes streaming, e.g. repair
+ // When false, it behaves the same as normal streaming.
+ public volatile boolean cdc_on_repair_enabled = true;
public String cdc_raw_directory;
@Replaces(oldName = "cdc_total_space_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
public DataStorageSpec.IntMebibytesBound cdc_total_space = new DataStorageSpec.IntMebibytesBound("0MiB");
@@ -420,6 +446,7 @@
public ParameterizedClass hints_compression;
public volatile boolean auto_hints_cleanup_enabled = false;
+ public volatile boolean transfer_hints_on_decommission = true;
public volatile boolean incremental_backups = false;
public boolean trickle_fsync = false;
@@ -464,7 +491,7 @@
@Replaces(oldName = "file_cache_size_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true)
public DataStorageSpec.IntMebibytesBound file_cache_size;
- public boolean file_cache_enabled = Boolean.getBoolean("cassandra.file_cache_enabled");
+ public boolean file_cache_enabled = FILE_CACHE_ENABLED.getBoolean();
/**
* Because of the current {@link org.apache.cassandra.utils.memory.BufferPool} slab sizes of 64 KiB, we
@@ -552,6 +579,8 @@
@Replaces(oldName = "enable_user_defined_functions", converter = Converters.IDENTITY, deprecated = true)
public boolean user_defined_functions_enabled = false;
+
+ @Deprecated
@Replaces(oldName = "enable_scripted_user_defined_functions", converter = Converters.IDENTITY, deprecated = true)
public boolean scripted_user_defined_functions_enabled = false;
@@ -595,6 +624,8 @@
*/
public boolean allow_extra_insecure_udfs = false;
+ public boolean dynamic_data_masking_enabled = false;
+
/**
* Time in milliseconds after a warning will be emitted to the log and to the client that a UDF runs too long.
* (Only valid, if user_defined_functions_threads_enabled==true)
@@ -653,6 +684,8 @@
public volatile int max_concurrent_automatic_sstable_upgrades = 1;
public boolean stream_entire_sstables = true;
+ public volatile boolean skip_stream_disk_space_check = false;
+
public volatile AuditLogOptions audit_logging_options = new AuditLogOptions();
public volatile FullQueryLoggerOptions full_query_logging_options = new FullQueryLoggerOptions();
@@ -774,7 +807,7 @@
public volatile boolean check_for_duplicate_rows_during_reads = true;
public volatile boolean check_for_duplicate_rows_during_compaction = true;
- public boolean autocompaction_on_startup_enabled = Boolean.parseBoolean(System.getProperty("cassandra.autocompaction_on_startup_enabled", "true"));
+ public boolean autocompaction_on_startup_enabled = AUTOCOMPACTION_ON_STARTUP_ENABLED.getBoolean();
// see CASSANDRA-3200 / CASSANDRA-16274
public volatile boolean auto_optimise_inc_repair_streams = false;
@@ -830,13 +863,20 @@
public volatile Set<ConsistencyLevel> write_consistency_levels_warned = Collections.emptySet();
public volatile Set<ConsistencyLevel> write_consistency_levels_disallowed = Collections.emptySet();
public volatile boolean user_timestamps_enabled = true;
+ public volatile boolean alter_table_enabled = true;
public volatile boolean group_by_enabled = true;
public volatile boolean drop_truncate_table_enabled = true;
+ public volatile boolean drop_keyspace_enabled = true;
public volatile boolean secondary_indexes_enabled = true;
public volatile boolean uncompressed_tables_enabled = true;
public volatile boolean compact_tables_enabled = true;
public volatile boolean read_before_write_list_operations_enabled = true;
public volatile boolean allow_filtering_enabled = true;
+ public volatile boolean simplestrategy_enabled = true;
+ public volatile DataStorageSpec.LongBytesBound partition_size_warn_threshold = null;
+ public volatile DataStorageSpec.LongBytesBound partition_size_fail_threshold = null;
+ public volatile DataStorageSpec.LongBytesBound column_value_size_warn_threshold = null;
+ public volatile DataStorageSpec.LongBytesBound column_value_size_fail_threshold = null;
public volatile DataStorageSpec.LongBytesBound collection_size_warn_threshold = null;
public volatile DataStorageSpec.LongBytesBound collection_size_fail_threshold = null;
public volatile int items_per_collection_warn_threshold = -1;
@@ -848,6 +888,10 @@
public volatile DataStorageSpec.LongBytesBound data_disk_usage_max_disk_size = null;
public volatile int minimum_replication_factor_warn_threshold = -1;
public volatile int minimum_replication_factor_fail_threshold = -1;
+ public volatile int maximum_replication_factor_warn_threshold = -1;
+ public volatile int maximum_replication_factor_fail_threshold = -1;
+ public volatile boolean zero_ttl_on_twcs_warned = true;
+ public volatile boolean zero_ttl_on_twcs_enabled = true;
public volatile DurationSpec.LongNanosecondsBound streaming_state_expires = new DurationSpec.LongNanosecondsBound("3d");
public volatile DataStorageSpec.LongBytesBound streaming_state_size = new DataStorageSpec.LongBytesBound("40MiB");
@@ -861,6 +905,12 @@
public volatile DurationSpec.LongNanosecondsBound repair_state_expires = new DurationSpec.LongNanosecondsBound("3d");
public volatile int repair_state_size = 100_000;
+ /** The configuration of timestamp bounds */
+ public volatile DurationSpec.LongMicrosecondsBound maximum_timestamp_warn_threshold = null;
+ public volatile DurationSpec.LongMicrosecondsBound maximum_timestamp_fail_threshold = null;
+ public volatile DurationSpec.LongMicrosecondsBound minimum_timestamp_warn_threshold = null;
+ public volatile DurationSpec.LongMicrosecondsBound minimum_timestamp_fail_threshold = null;
+
/**
* The variants of paxos implementation and semantics supported by Cassandra.
*/
@@ -951,7 +1001,7 @@
* rare operation circumstances e.g. where for some reason the repair is impossible to perform (e.g. too few replicas)
* and an unsafe topology change must be made
*/
- public volatile boolean skip_paxos_repair_on_topology_change = Boolean.getBoolean("cassandra.skip_paxos_repair_on_topology_change");
+ public volatile boolean skip_paxos_repair_on_topology_change = SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE.getBoolean();
/**
* A safety margin when purging paxos state information that has been safely replicated to a quorum.
@@ -1012,7 +1062,7 @@
/**
* If necessary for operational purposes, permit certain keyspaces to be ignored for paxos topology repairs
*/
- public volatile Set<String> skip_paxos_repair_on_topology_change_keyspaces = splitCommaDelimited(System.getProperty("cassandra.skip_paxos_repair_on_topology_change_keyspaces"));
+ public volatile Set<String> skip_paxos_repair_on_topology_change_keyspaces = splitCommaDelimited(SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES.getString());
/**
* See {@link org.apache.cassandra.service.paxos.ContentionStrategy}
@@ -1039,6 +1089,10 @@
*/
public volatile int paxos_repair_parallelism = -1;
+ public volatile boolean sstable_read_rate_persistence_enabled = false;
+
+ public volatile boolean client_request_size_metrics_enabled = true;
+
public volatile int max_top_size_partition_count = 10;
public volatile int max_top_tombstone_partition_count = 10;
public volatile DataStorageSpec.LongBytesBound min_tracked_partition_size = new DataStorageSpec.LongBytesBound("1MiB");
@@ -1170,4 +1224,9 @@
logger.info("Node configuration:[{}]", Joiner.on("; ").join(configMap.entrySet()));
}
+
+ public volatile boolean dump_heap_on_uncaught_exception = false;
+ public String heap_dump_path = "heapdump";
+
+ public double severity_during_decommission = 0;
}
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index a04e85c..2531e5c 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -18,6 +18,8 @@
package org.apache.cassandra.config;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
@@ -25,24 +27,33 @@
import java.net.SocketException;
import java.net.UnknownHostException;
import java.nio.file.FileStore;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.Enumeration;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.ServiceLoader;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.Supplier;
-
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import com.google.common.util.concurrent.RateLimiter;
@@ -51,6 +62,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.googlecode.concurrenttrees.common.Iterables;
import org.apache.cassandra.audit.AuditLogOptions;
import org.apache.cassandra.auth.AllowAllInternodeAuthenticator;
import org.apache.cassandra.auth.AuthConfig;
@@ -72,6 +84,8 @@
import org.apache.cassandra.fql.FullQueryLoggerOptions;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DiskOptimizationStrategy;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
@@ -90,9 +104,29 @@
import org.apache.cassandra.service.paxos.Paxos;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOCATE_TOKENS_FOR_KEYSPACE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTO_BOOTSTRAP;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_LOADER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_STCS_IN_L0;
+import static org.apache.cassandra.config.CassandraRelevantProperties.INITIAL_TOKEN;
+import static org.apache.cassandra.config.CassandraRelevantProperties.IO_NETTY_TRANSPORT_ESTIMATE_SIZE_ON_SUBMIT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NATIVE_TRANSPORT_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.OS_ARCH;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PARTITIONER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS_FIRST_BOOT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_NODE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_TOKEN;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SEARCH_CONCURRENCY_FACTOR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SSL_STORAGE_PORT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.STORAGE_DIR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.STORAGE_PORT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_STRICT_RUNTIME_CHECKS;
import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_ARCH_DATA_MODEL;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_FAIL_MV_LOCKS_COUNT;
import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYSTEM;
import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.BYTES_PER_SECOND;
import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.MEBIBYTES_PER_SECOND;
import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.MEBIBYTES;
@@ -106,7 +140,7 @@
{
// This static block covers most usages
FBUtilities.preventIllegalAccessWarnings();
- System.setProperty("io.netty.transport.estimateSizeOnSubmit", "false");
+ IO_NETTY_TRANSPORT_ESTIMATE_SIZE_ON_SUBMIT.setBoolean(false);
}
private static final Logger logger = LoggerFactory.getLogger(DatabaseDescriptor.class);
@@ -164,24 +198,29 @@
private static boolean toolInitialized;
private static boolean daemonInitialized;
- private static final int searchConcurrencyFactor = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "search_concurrency_factor", "1"));
+ private static final int searchConcurrencyFactor = SEARCH_CONCURRENCY_FACTOR.getInt();
private static DurationSpec.IntSecondsBound autoSnapshoTtl;
- private static volatile boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0");
- private static final boolean unsafeSystem = Boolean.getBoolean(Config.PROPERTY_PREFIX + "unsafesystem");
+ private static volatile boolean disableSTCSInL0 = DISABLE_STCS_IN_L0.getBoolean();
+ private static final boolean unsafeSystem = UNSAFE_SYSTEM.getBoolean();
// turns some warnings into exceptions for testing
- private static final boolean strictRuntimeChecks = Boolean.getBoolean("cassandra.strict.runtime.checks");
+ private static final boolean strictRuntimeChecks = TEST_STRICT_RUNTIME_CHECKS.getBoolean();
- public static volatile boolean allowUnlimitedConcurrentValidations = Boolean.getBoolean("cassandra.allow_unlimited_concurrent_validations");
+ public static volatile boolean allowUnlimitedConcurrentValidations = ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS.getBoolean();
- /** The configuration for guardrails. */
+ /**
+ * The configuration for guardrails.
+ */
private static GuardrailsOptions guardrails;
private static StartupChecksOptions startupChecksOptions;
+ private static ImmutableMap<String, SSTableFormat<?, ?>> sstableFormats;
+ private static SSTableFormat<?, ?> selectedSSTableFormat;
+
private static Function<CommitLog, AbstractCommitLogSegmentManager> commitLogSegmentMgrProvider = c -> DatabaseDescriptor.isCDCEnabled()
- ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation())
- : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation());
+ ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation())
+ : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation());
public static void daemonInitialization() throws ConfigurationException
{
@@ -241,6 +280,8 @@
setConfig(loadConfig());
+ applySSTableFormats();
+
applySimpleConfig();
applyPartitioner();
@@ -259,6 +300,14 @@
}
/**
+ * Equivalent to {@link #clientInitialization(boolean) clientInitialization(true, Config::new)}.
+ */
+ public static void clientInitialization(boolean failIfDaemonOrTool)
+ {
+ clientInitialization(failIfDaemonOrTool, Config::new);
+ }
+
+ /**
* Initializes this class as a client, which means that just an empty configuration will
* be used.
*
@@ -266,7 +315,7 @@
* {@link #toolInitialization()} has been performed before, an
* {@link AssertionError} will be thrown.
*/
- public static void clientInitialization(boolean failIfDaemonOrTool)
+ public static void clientInitialization(boolean failIfDaemonOrTool, Supplier<Config> configSupplier)
{
if (!failIfDaemonOrTool && (daemonInitialized || toolInitialized))
{
@@ -285,8 +334,9 @@
clientInitialized = true;
setDefaultFailureDetector();
Config.setClientMode(true);
- conf = new Config();
+ conf = configSupplier.get();
diskOptimizationStrategy = new SpinningDiskOptimizationStrategy();
+ applySSTableFormats();
}
public static boolean isClientInitialized()
@@ -320,7 +370,7 @@
if (Config.getOverrideLoadConfig() != null)
return Config.getOverrideLoadConfig().get();
- String loaderClass = System.getProperty(Config.PROPERTY_PREFIX + "config.loader");
+ String loaderClass = CONFIG_LOADER.getString();
ConfigurationLoader loader = loaderClass == null
? new YamlConfigurationLoader()
: FBUtilities.construct(loaderClass, "configuration loading");
@@ -373,6 +423,8 @@
private static void applyAll() throws ConfigurationException
{
//InetAddressAndPort cares that applySimpleConfig runs first
+ applySSTableFormats();
+
applySimpleConfig();
applyPartitioner();
@@ -483,7 +535,7 @@
throw new ConfigurationException("concurrent_reads must be at least 2, but was " + conf.concurrent_reads, false);
}
- if (conf.concurrent_writes < 2 && System.getProperty("cassandra.test.fail_mv_locks_count", "").isEmpty())
+ if (conf.concurrent_writes < 2 && TEST_FAIL_MV_LOCKS_COUNT.getString("").isEmpty())
{
throw new ConfigurationException("concurrent_writes must be at least 2, but was " + conf.concurrent_writes, false);
}
@@ -549,7 +601,8 @@
false);
}
- checkValidForByteConversion(conf.column_index_size, "column_index_size");
+ if (conf.column_index_size != null)
+ checkValidForByteConversion(conf.column_index_size, "column_index_size");
checkValidForByteConversion(conf.column_index_cache_size, "column_index_cache_size");
checkValidForByteConversion(conf.batch_size_warn_threshold, "batch_size_warn_threshold");
@@ -808,7 +861,8 @@
logger.warn("Allowing java.lang.System.* access in UDFs is dangerous and not recommended. Set allow_extra_insecure_udfs: false to disable.");
if(conf.scripted_user_defined_functions_enabled)
- logger.warn("JavaScript user-defined functions have been deprecated. You can still use them but the plan is to remove them in the next major version. For more information - CASSANDRA-17280");
+ throw new ConfigurationException("JavaScript user-defined functions were removed in CASSANDRA-18252. " +
+ "Hooks are planned to be introduced as part of CASSANDRA-17280");
if (conf.commitlog_segment_size.toMebibytes() == 0)
throw new ConfigurationException("commitlog_segment_size must be positive, but was "
@@ -906,6 +960,12 @@
conf.paxos_state_purging = PaxosStatePurging.legacy;
logInitializationOutcome(logger);
+
+ if (conf.max_space_usable_for_compactions_in_percentage < 0 || conf.max_space_usable_for_compactions_in_percentage > 1)
+ throw new ConfigurationException("max_space_usable_for_compactions_in_percentage must be between 0 and 1", false);
+
+ if (conf.dump_heap_on_uncaught_exception && DatabaseDescriptor.getHeapDumpPath() == null)
+ throw new ConfigurationException(String.format("Invalid configuration. Heap dump is enabled but cannot create heap dump output path: %s.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
}
@VisibleForTesting
@@ -948,7 +1008,7 @@
else if (config.concurrent_validations > config.concurrent_compactors && !allowUnlimitedConcurrentValidations)
{
throw new ConfigurationException("To set concurrent_validations > concurrent_compactors, " +
- "set the system property cassandra.allow_unlimited_concurrent_validations=true");
+ "set the system property -D" + ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS.getKey() + "=true");
}
}
@@ -1009,9 +1069,9 @@
private static String storagedir(String errMsgType)
{
- String storagedir = System.getProperty(Config.PROPERTY_PREFIX + "storagedir", null);
+ String storagedir = STORAGE_DIR.getString();
if (storagedir == null)
- throw new ConfigurationException(errMsgType + " is missing and -Dcassandra.storagedir is not set", false);
+ throw new ConfigurationException(errMsgType + " is missing and " + STORAGE_DIR.getKey() + " system property is not set", false);
return storagedir;
}
@@ -1306,7 +1366,7 @@
String name = conf.partitioner;
try
{
- name = System.getProperty(Config.PROPERTY_PREFIX + "partitioner", conf.partitioner);
+ name = PARTITIONER.getString(conf.partitioner);
partitioner = FBUtilities.newPartitioner(name);
}
catch (Exception e)
@@ -1317,10 +1377,95 @@
paritionerName = partitioner.getClass().getCanonicalName();
}
+ private static void validateSSTableFormatFactories(Iterable<SSTableFormat.Factory> factories)
+ {
+ Map<String, SSTableFormat.Factory> factoryByName = new HashMap<>();
+ for (SSTableFormat.Factory factory : factories)
+ {
+ if (factory.name() == null)
+ throw new ConfigurationException(String.format("SSTable format name in %s cannot be null", factory.getClass().getCanonicalName()));
+
+ if (!factory.name().matches("^[a-z]+$"))
+ throw new ConfigurationException(String.format("SSTable format name for %s must be non-empty, lower-case letters only string", factory.getClass().getCanonicalName()));
+
+ SSTableFormat.Factory prev = factoryByName.put(factory.name(), factory);
+ if (prev != null)
+ throw new ConfigurationException(String.format("Multiple sstable format implementations with the same name %s: %s and %s", factory.name(), factory.getClass().getCanonicalName(), prev.getClass().getCanonicalName()));
+ }
+ }
+
+ private static ImmutableMap<String, Supplier<SSTableFormat<?, ?>>> validateAndMatchSSTableFormatOptions(Iterable<SSTableFormat.Factory> factories, Map<String, Map<String, String>> options)
+ {
+ ImmutableMap.Builder<String, Supplier<SSTableFormat<?, ?>>> providersBuilder = ImmutableMap.builder();
+ if (options == null)
+ options = ImmutableMap.of();
+ for (SSTableFormat.Factory factory : factories)
+ {
+ Map<String, String> formatOptions = options.getOrDefault(factory.name(), ImmutableMap.of());
+ providersBuilder.put(factory.name(), () -> factory.getInstance(ImmutableMap.copyOf(formatOptions)));
+ }
+ ImmutableMap<String, Supplier<SSTableFormat<?, ?>>> providers = providersBuilder.build();
+ if (options != null)
+ {
+ Sets.SetView<String> unknownFormatNames = Sets.difference(options.keySet(), providers.keySet());
+ if (!unknownFormatNames.isEmpty())
+ throw new ConfigurationException(String.format("Configuration contains options of unknown sstable formats: %s", unknownFormatNames));
+ }
+ return providers;
+ }
+
+ private static SSTableFormat<?, ?> getAndValidateWriteFormat(Map<String, SSTableFormat<?, ?>> sstableFormats, String selectedFormatName)
+ {
+ SSTableFormat<?, ?> selectedFormat;
+ if (StringUtils.isBlank(selectedFormatName))
+ selectedFormatName = BigFormat.NAME;
+ selectedFormat = sstableFormats.get(selectedFormatName);
+ if (selectedFormat == null)
+ throw new ConfigurationException(String.format("Selected sstable format '%s' is not available.", selectedFormatName));
+
+ return selectedFormat;
+ }
+
+ private static void applySSTableFormats()
+ {
+ ServiceLoader<SSTableFormat.Factory> loader = ServiceLoader.load(SSTableFormat.Factory.class, DatabaseDescriptor.class.getClassLoader());
+ List<SSTableFormat.Factory> factories = Iterables.toList(loader);
+ if (factories.isEmpty())
+ factories = ImmutableList.of(new BigFormat.BigFormatFactory());
+ applySSTableFormats(factories, conf.sstable);
+ }
+
+ private static void applySSTableFormats(Iterable<SSTableFormat.Factory> factories, Config.SSTableConfig sstableFormatsConfig)
+ {
+ if (sstableFormats != null)
+ return;
+
+ validateSSTableFormatFactories(factories);
+ ImmutableMap<String, Supplier<SSTableFormat<?, ?>>> providers = validateAndMatchSSTableFormatOptions(factories, sstableFormatsConfig.format);
+
+ ImmutableMap.Builder<String, SSTableFormat<?, ?>> sstableFormatsBuilder = ImmutableMap.builder();
+ providers.forEach((name, provider) -> {
+ try
+ {
+ sstableFormatsBuilder.put(name, provider.get());
+ }
+ catch (RuntimeException | Error ex)
+ {
+ throw new ConfigurationException(String.format("Failed to instantiate sstable format '%s'", name), ex);
+ }
+ });
+ sstableFormats = sstableFormatsBuilder.build();
+
+ selectedSSTableFormat = getAndValidateWriteFormat(sstableFormats, sstableFormatsConfig.selected_format);
+
+ sstableFormats.values().forEach(SSTableFormat::allComponents); // make sure to reach all supported components for a type so that we know all of them are registered
+ logger.info("Supported sstable formats are: {}", sstableFormats.values().stream().map(f -> f.name() + " -> " + f.getClass().getName() + " with singleton components: " + f.allComponents()).collect(Collectors.joining(", ")));
+ }
+
/**
* Computes the sum of the 2 specified positive values returning {@code Long.MAX_VALUE} if the sum overflow.
*
- * @param left the left operand
+ * @param left the left operand
* @param right the right operand
* @return the sum of the 2 specified positive values of {@code Long.MAX_VALUE} if the sum overflow.
*/
@@ -1582,6 +1727,13 @@
throw new ConfigurationException("cdc_raw_directory must be specified", false);
FileUtils.createDirectory(conf.cdc_raw_directory);
}
+
+ boolean created = maybeCreateHeapDumpPath();
+ if (!created && conf.dump_heap_on_uncaught_exception)
+ {
+ logger.error(String.format("cassandra.yaml:dump_heap_on_uncaught_exception is enabled but unable to create heap dump path %s. Disabling.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
+ conf.dump_heap_on_uncaught_exception = false;
+ }
}
catch (ConfigurationException e)
{
@@ -1630,19 +1782,19 @@
newFailureDetector = () -> createFailureDetector("FailureDetector");
}
- public static int getColumnIndexSize()
+ public static int getColumnIndexSize(int defaultValue)
{
- return conf.column_index_size.toBytes();
+ return conf.column_index_size != null ? conf.column_index_size.toBytes() : defaultValue;
}
public static int getColumnIndexSizeInKiB()
{
- return conf.column_index_size.toKibibytes();
+ return conf.column_index_size != null ? conf.column_index_size.toKibibytes() : -1;
}
- public static void setColumnIndexSize(int val)
+ public static void setColumnIndexSizeInKiB(int val)
{
- conf.column_index_size = createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val,"column_index_size");
+ conf.column_index_size = val != -1 ? createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val,"column_index_size") : null;
}
public static int getColumnIndexCacheSize()
@@ -1697,12 +1849,12 @@
public static Collection<String> getInitialTokens()
{
- return tokensFromString(System.getProperty(Config.PROPERTY_PREFIX + "initial_token", conf.initial_token));
+ return tokensFromString(INITIAL_TOKEN.getString(conf.initial_token));
}
public static String getAllocateTokensForKeyspace()
{
- return System.getProperty(Config.PROPERTY_PREFIX + "allocate_tokens_for_keyspace", conf.allocate_tokens_for_keyspace);
+ return ALLOCATE_TOKENS_FOR_KEYSPACE.getString(conf.allocate_tokens_for_keyspace);
}
public static Integer getAllocateTokensForLocalRf()
@@ -1728,10 +1880,14 @@
{
try
{
- if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null) != null)
- return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null));
- else if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null) != null)
- return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null));
+ String replaceAddress = REPLACE_ADDRESS.getString();
+ if (replaceAddress != null)
+ return InetAddressAndPort.getByName(replaceAddress);
+
+ String replaceAddressFirsstBoot = REPLACE_ADDRESS_FIRST_BOOT.getString();
+ if (replaceAddressFirsstBoot != null)
+ return InetAddressAndPort.getByName(replaceAddressFirsstBoot);
+
return null;
}
catch (UnknownHostException e)
@@ -1742,14 +1898,14 @@
public static Collection<String> getReplaceTokens()
{
- return tokensFromString(System.getProperty(Config.PROPERTY_PREFIX + "replace_token", null));
+ return tokensFromString(REPLACE_TOKEN.getString());
}
public static UUID getReplaceNode()
{
try
{
- return UUID.fromString(System.getProperty(Config.PROPERTY_PREFIX + "replace_node", null));
+ return UUID.fromString(REPLACE_NODE.getString());
} catch (NullPointerException e)
{
return null;
@@ -1763,12 +1919,12 @@
public static int getStoragePort()
{
- return Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "storage_port", Integer.toString(conf.storage_port)));
+ return STORAGE_PORT.getInt(conf.storage_port);
}
public static int getSSLStoragePort()
{
- return Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "ssl_storage_port", Integer.toString(conf.ssl_storage_port)));
+ return SSL_STORAGE_PORT.getInt(conf.ssl_storage_port);
}
public static long nativeTransportIdleTimeout()
@@ -2016,7 +2172,11 @@
conf.compaction_throughput = new DataRateSpec.LongBytesPerSecondBound(value, MEBIBYTES_PER_SECOND);
}
- public static long getCompactionLargePartitionWarningThreshold() { return conf.compaction_large_partition_warning_threshold.toBytesInLong(); }
+ @Deprecated
+ public static long getCompactionLargePartitionWarningThreshold()
+ {
+ return conf.compaction_large_partition_warning_threshold.toBytesInLong();
+ }
public static int getCompactionTombstoneWarningThreshold()
{
@@ -2033,6 +2193,11 @@
return conf.concurrent_validations;
}
+ public static int getConcurrentIndexBuilders()
+ {
+ return conf.concurrent_index_builders;
+ }
+
public static void setConcurrentValidations(int value)
{
value = value > 0 ? value : Integer.MAX_VALUE;
@@ -2049,11 +2214,33 @@
conf.concurrent_materialized_view_builders = value;
}
+ public static long getMinFreeSpacePerDriveInMebibytes()
+ {
+ return conf.min_free_space_per_drive.toMebibytes();
+ }
+
public static long getMinFreeSpacePerDriveInBytes()
{
return conf.min_free_space_per_drive.toBytesInLong();
}
+ @VisibleForTesting
+ public static long setMinFreeSpacePerDriveInMebibytes(long mebiBytes)
+ {
+ conf.min_free_space_per_drive = new DataStorageSpec.IntMebibytesBound(mebiBytes);
+ return getMinFreeSpacePerDriveInBytes();
+ }
+
+ public static double getMaxSpaceForCompactionsPerDrive()
+ {
+ return conf.max_space_usable_for_compactions_in_percentage;
+ }
+
+ public static void setMaxSpaceForCompactionsPerDrive(double percentage)
+ {
+ conf.max_space_usable_for_compactions_in_percentage = percentage;
+ }
+
public static boolean getDisableSTCSInL0()
{
return disableSTCSInL0;
@@ -2548,7 +2735,7 @@
*/
public static int getNativeTransportPort()
{
- return Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "native_transport_port", Integer.toString(conf.native_transport_port)));
+ return NATIVE_TRANSPORT_PORT.getInt(conf.native_transport_port);
}
@VisibleForTesting
@@ -2578,6 +2765,22 @@
conf.native_transport_max_threads = max_threads;
}
+ public static Integer getNativeTransportMaxAuthThreads()
+ {
+ return conf.native_transport_max_auth_threads;
+ }
+
+ /**
+ * If this value is set to <= 0 it will move auth requests to the standard request pool regardless of the current
+ * size of the {@link org.apache.cassandra.transport.Dispatcher#authExecutor}'s active size.
+ *
+ * see {@link org.apache.cassandra.transport.Dispatcher#dispatch} for executor selection
+ */
+ public static void setNativeTransportMaxAuthThreads(int threads)
+ {
+ conf.native_transport_max_auth_threads = threads;
+ }
+
public static int getNativeTransportMaxFrameSize()
{
return conf.native_transport_max_frame_size.toBytes();
@@ -2935,7 +3138,7 @@
public static boolean isAutoBootstrap()
{
- return Boolean.parseBoolean(System.getProperty(Config.PROPERTY_PREFIX + "auto_bootstrap", Boolean.toString(conf.auto_bootstrap)));
+ return AUTO_BOOTSTRAP.getBoolean(conf.auto_bootstrap);
}
public static void setHintedHandoffEnabled(boolean hintedHandoffEnabled)
@@ -3120,6 +3323,16 @@
conf.auto_hints_cleanup_enabled = value;
}
+ public static boolean getTransferHintsOnDecommission()
+ {
+ return conf.transfer_hints_on_decommission;
+ }
+
+ public static void setTransferHintsOnDecommission(boolean enabled)
+ {
+ conf.transfer_hints_on_decommission = enabled;
+ }
+
public static boolean isIncrementalBackupsEnabled()
{
return conf.incremental_backups;
@@ -3135,6 +3348,12 @@
return conf.file_cache_enabled;
}
+ @VisibleForTesting
+ public static void setFileCacheEnabled(boolean enabled)
+ {
+ conf.file_cache_enabled = enabled;
+ }
+
public static int getFileCacheSizeInMiB()
{
if (conf.file_cache_size == null)
@@ -3344,6 +3563,21 @@
return conf.stream_entire_sstables;
}
+ public static DurationSpec.LongMillisecondsBound getStreamTransferTaskTimeout()
+ {
+ return conf.stream_transfer_task_timeout;
+ }
+
+ public static boolean getSkipStreamDiskSpaceCheck()
+ {
+ return conf.skip_stream_disk_space_check;
+ }
+
+ public static void setSkipStreamDiskSpaceCheck(boolean value)
+ {
+ conf.skip_stream_disk_space_check = value;
+ }
+
public static String getLocalDataCenter()
{
return localDC;
@@ -3497,11 +3731,6 @@
return conf.scripted_user_defined_functions_enabled;
}
- public static void enableScriptedUserDefinedFunctions(boolean enableScriptedUserDefinedFunctions)
- {
- conf.scripted_user_defined_functions_enabled = enableScriptedUserDefinedFunctions;
- }
-
public static boolean enableUserDefinedFunctionsThreads()
{
return conf.user_defined_functions_threads_enabled;
@@ -3634,6 +3863,16 @@
conf.cdc_block_writes = val;
}
+ public static boolean isCDCOnRepairEnabled()
+ {
+ return conf.cdc_on_repair_enabled;
+ }
+
+ public static void setCDCOnRepairEnabled(boolean val)
+ {
+ conf.cdc_on_repair_enabled = val;
+ }
+
public static String getCDCLogLocation()
{
return conf.cdc_raw_directory;
@@ -4239,6 +4478,11 @@
throw new IllegalArgumentException(String.format("default_keyspace_rf to be set (%d) cannot be less than minimum_replication_factor_fail_threshold (%d)", value, guardrails.getMinimumReplicationFactorFailThreshold()));
}
+ if (guardrails.getMaximumReplicationFactorFailThreshold() != -1 && value > guardrails.getMaximumReplicationFactorFailThreshold())
+ {
+ throw new IllegalArgumentException(String.format("default_keyspace_rf to be set (%d) cannot be greater than maximum_replication_factor_fail_threshold (%d)", value, guardrails.getMaximumReplicationFactorFailThreshold()));
+ }
+
conf.default_keyspace_rf = value;
}
@@ -4403,4 +4647,130 @@
{
conf.min_tracked_partition_tombstone_count = value;
}
+
+ public static boolean getDumpHeapOnUncaughtException()
+ {
+ return conf.dump_heap_on_uncaught_exception;
+ }
+
+ /**
+ * @return Whether the path exists (be it created now or already prior)
+ */
+ private static boolean maybeCreateHeapDumpPath()
+ {
+ if (!conf.dump_heap_on_uncaught_exception)
+ return false;
+
+ Path heap_dump_path = getHeapDumpPath();
+ if (heap_dump_path == null)
+ {
+ logger.warn("Neither -XX:HeapDumpPath nor cassandra.yaml:heap_dump_path are set; unable to create a directory to hold the output.");
+ return false;
+ }
+ if (PathUtils.exists(File.getPath(conf.heap_dump_path)))
+ return true;
+ return PathUtils.createDirectoryIfNotExists(File.getPath(conf.heap_dump_path));
+ }
+
+ /**
+ * As this is at its heart a debug operation (getting a one-shot heapdump from an uncaught exception), we support
+ * both the more evolved cassandra.yaml approach but also the -XX param to override it on a one-off basis so you don't
+ * have to change the full config of a node or a cluster in order to get a heap dump from a single node that's
+ * misbehaving.
+ * @return the absolute path of the -XX param if provided, else the heap_dump_path in cassandra.yaml
+ */
+ public static Path getHeapDumpPath()
+ {
+ RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
+ Optional<String> pathArg = runtimeMxBean.getInputArguments().stream().filter(s -> s.startsWith("-XX:HeapDumpPath=")).findFirst();
+
+ if (pathArg.isPresent())
+ {
+ Pattern HEAP_DUMP_PATH_SPLITTER = Pattern.compile("HeapDumpPath=");
+ String fullHeapPathString = HEAP_DUMP_PATH_SPLITTER.split(pathArg.get())[1];
+ Path absolutePath = File.getPath(fullHeapPathString).toAbsolutePath();
+ Path basePath = fullHeapPathString.endsWith(".hprof") ? absolutePath.subpath(0, absolutePath.getNameCount() - 1) : absolutePath;
+ return File.getPath("/").resolve(basePath);
+ }
+ if (conf.heap_dump_path == null)
+ throw new ConfigurationException("Attempted to get heap dump path without -XX:HeapDumpPath or cassandra.yaml:heap_dump_path set.");
+ return File.getPath(conf.heap_dump_path);
+ }
+
+ public static void setDumpHeapOnUncaughtException(boolean enabled)
+ {
+ conf.dump_heap_on_uncaught_exception = enabled;
+ boolean pathExists = maybeCreateHeapDumpPath();
+
+ if (enabled && !pathExists)
+ {
+ logger.error("Attempted to enable heap dump but cannot create the requested path. Disabling.");
+ conf.dump_heap_on_uncaught_exception = false;
+ }
+ else
+ logger.info("Setting dump_heap_on_uncaught_exception to {}", enabled);
+ }
+
+ public static boolean getSStableReadRatePersistenceEnabled()
+ {
+ return conf.sstable_read_rate_persistence_enabled;
+ }
+
+ public static void setSStableReadRatePersistenceEnabled(boolean enabled)
+ {
+ if (enabled != conf.sstable_read_rate_persistence_enabled)
+ {
+ logger.info("Setting sstable_read_rate_persistence_enabled to {}", enabled);
+ conf.sstable_read_rate_persistence_enabled = enabled;
+ }
+ }
+
+ public static boolean getClientRequestSizeMetricsEnabled()
+ {
+ return conf.client_request_size_metrics_enabled;
+ }
+
+ public static void setClientRequestSizeMetricsEnabled(boolean enabled)
+ {
+ conf.client_request_size_metrics_enabled = enabled;
+ }
+
+ @VisibleForTesting
+ public static void resetSSTableFormats(Iterable<SSTableFormat.Factory> factories, Config.SSTableConfig config)
+ {
+ sstableFormats = null;
+ selectedSSTableFormat = null;
+ applySSTableFormats(factories, config);
+ }
+
+ public static ImmutableMap<String, SSTableFormat<?, ?>> getSSTableFormats()
+ {
+ return Objects.requireNonNull(sstableFormats, "Forgot to initialize DatabaseDescriptor?");
+ }
+
+ public static SSTableFormat<?, ?> getSelectedSSTableFormat()
+ {
+ return Objects.requireNonNull(selectedSSTableFormat, "Forgot to initialize DatabaseDescriptor?");
+ }
+
+ public static boolean getDynamicDataMaskingEnabled()
+ {
+ return conf.dynamic_data_masking_enabled;
+ }
+
+ public static void setDynamicDataMaskingEnabled(boolean enabled)
+ {
+ if (enabled != conf.dynamic_data_masking_enabled)
+ {
+ logger.info("Setting dynamic_data_masking_enabled to {}", enabled);
+ conf.dynamic_data_masking_enabled = enabled;
+ }
+ }
+
+ public static OptionalDouble getSeverityDuringDecommission()
+ {
+ return conf.severity_during_decommission > 0 ?
+ OptionalDouble.of(conf.severity_during_decommission) :
+ OptionalDouble.empty();
+ }
}
diff --git a/src/java/org/apache/cassandra/config/DurationSpec.java b/src/java/org/apache/cassandra/config/DurationSpec.java
index 10d56c2..2522d86 100644
--- a/src/java/org/apache/cassandra/config/DurationSpec.java
+++ b/src/java/org/apache/cassandra/config/DurationSpec.java
@@ -273,6 +273,64 @@
}
/**
+ * Represents a duration used for Cassandra configuration. The bound is [0, Long.MAX_VALUE) in microseconds.
+ * If the user sets a different unit - we still validate that converted to microseconds the quantity will not exceed
+ * that upper bound. (CASSANDRA-17571)
+ */
+ public final static class LongMicrosecondsBound extends DurationSpec
+ {
+ /**
+ * Creates a {@code DurationSpec.LongMicrosecondsBound} of the specified amount.
+ * The bound is [0, Long.MAX_VALUE) in microseconds.
+ *
+ * @param value the duration
+ */
+ public LongMicrosecondsBound(String value)
+ {
+ super(value, MICROSECONDS, Long.MAX_VALUE);
+ }
+
+ /**
+ * Creates a {@code DurationSpec.LongMicrosecondsBound} of the specified amount in the specified unit.
+ * The bound is [0, Long.MAX_VALUE) in milliseconds.
+ *
+ * @param quantity where quantity shouldn't be bigger than Long.MAX_VALUE - 1 in microseconds
+ * @param unit in which the provided quantity is
+ */
+ public LongMicrosecondsBound(long quantity, TimeUnit unit)
+ {
+ super(quantity, unit, MICROSECONDS, Long.MAX_VALUE);
+ }
+
+ /**
+ * Creates a {@code DurationSpec.LongMicrosecondsBound} of the specified amount in microseconds.
+ * The bound is [0, Long.MAX_VALUE) in microseconds.
+ *
+ * @param microseconds where milliseconds shouldn't be bigger than Long.MAX_VALUE-1
+ */
+ public LongMicrosecondsBound(long microseconds)
+ {
+ this(microseconds, MICROSECONDS);
+ }
+
+ /**
+ * @return this duration in number of milliseconds
+ */
+ public long toMicroseconds()
+ {
+ return unit().toMicros(quantity());
+ }
+
+ /**
+ * @return this duration in number of seconds
+ */
+ public long toSeconds()
+ {
+ return unit().toSeconds(quantity());
+ }
+ }
+
+ /**
* Represents a duration used for Cassandra configuration. The bound is [0, Long.MAX_VALUE) in milliseconds.
* If the user sets a different unit - we still validate that converted to milliseconds the quantity will not exceed
* that upper bound. (CASSANDRA-17571)
diff --git a/src/java/org/apache/cassandra/config/EncryptionOptions.java b/src/java/org/apache/cassandra/config/EncryptionOptions.java
index 2610ff6..afa0d66 100644
--- a/src/java/org/apache/cassandra/config/EncryptionOptions.java
+++ b/src/java/org/apache/cassandra/config/EncryptionOptions.java
@@ -33,7 +33,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.security.DisableSslContextFactory;
@@ -110,6 +109,8 @@
{
KEYSTORE("keystore"),
KEYSTORE_PASSWORD("keystore_password"),
+ OUTBOUND_KEYSTORE("outbound_keystore"),
+ OUTBOUND_KEYSTORE_PASSWORD("outbound_keystore_password"),
TRUSTSTORE("truststore"),
TRUSTSTORE_PASSWORD("truststore_password"),
CIPHER_SUITES("cipher_suites"),
@@ -262,11 +263,8 @@
}
}
- private void initializeSslContextFactory()
+ protected void fillSslContextParams(Map<String, Object> sslContextFactoryParameters)
{
- Map<String,Object> sslContextFactoryParameters = new HashMap<>();
- prepareSslContextFactoryParameterizedKeys(sslContextFactoryParameters);
-
/*
* Copy all configs to the Map to pass it on to the ISslContextFactory's implementation
*/
@@ -283,6 +281,13 @@
putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.REQUIRE_ENDPOINT_VERIFICATION, this.require_endpoint_verification);
putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.ENABLED, this.enabled);
putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.OPTIONAL, this.optional);
+ }
+
+ private void initializeSslContextFactory()
+ {
+ Map<String, Object> sslContextFactoryParameters = new HashMap<>();
+ prepareSslContextFactoryParameterizedKeys(sslContextFactoryParameters);
+ fillSslContextParams(sslContextFactoryParameters);
if (CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL.getBoolean())
{
@@ -295,8 +300,7 @@
}
}
- private void putSslContextFactoryParameter(Map<String,Object> existingParameters, ConfigKey configKey,
- Object value)
+ protected static void putSslContextFactoryParameter(Map<String, Object> existingParameters, ConfigKey configKey, Object value)
{
if (value != null) {
existingParameters.put(configKey.getKeyName(), value);
@@ -607,15 +611,21 @@
public final InternodeEncryption internode_encryption;
@Replaces(oldName = "enable_legacy_ssl_storage_port", deprecated = true)
public final boolean legacy_ssl_storage_port_enabled;
+ public final String outbound_keystore;
+ @Nullable
+ public final String outbound_keystore_password;
public ServerEncryptionOptions()
{
this.internode_encryption = InternodeEncryption.none;
this.legacy_ssl_storage_port_enabled = false;
+ this.outbound_keystore = null;
+ this.outbound_keystore_password = null;
}
public ServerEncryptionOptions(ParameterizedClass sslContextFactoryClass, String keystore,
- String keystore_password, String truststore, String truststore_password,
+ String keystore_password,String outbound_keystore,
+ String outbound_keystore_password, String truststore, String truststore_password,
List<String> cipher_suites, String protocol, List<String> accepted_protocols,
String algorithm, String store_type, boolean require_client_auth,
boolean require_endpoint_verification, Boolean optional,
@@ -626,6 +636,8 @@
null, optional);
this.internode_encryption = internode_encryption;
this.legacy_ssl_storage_port_enabled = legacy_ssl_storage_port_enabled;
+ this.outbound_keystore = outbound_keystore;
+ this.outbound_keystore_password = outbound_keystore_password;
}
public ServerEncryptionOptions(ServerEncryptionOptions options)
@@ -633,6 +645,16 @@
super(options);
this.internode_encryption = options.internode_encryption;
this.legacy_ssl_storage_port_enabled = options.legacy_ssl_storage_port_enabled;
+ this.outbound_keystore = options.outbound_keystore;
+ this.outbound_keystore_password = options.outbound_keystore_password;
+ }
+
+ @Override
+ protected void fillSslContextParams(Map<String, Object> sslContextFactoryParameters)
+ {
+ super.fillSslContextParams(sslContextFactoryParameters);
+ putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.OUTBOUND_KEYSTORE, this.outbound_keystore);
+ putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.OUTBOUND_KEYSTORE_PASSWORD, this.outbound_keystore_password);
}
@Override
@@ -696,7 +718,6 @@
* values of "dc" and "all". This method returns the explicit, raw value of {@link #optional}
* as set by the user (if set at all).
*/
- @JsonIgnore
public boolean isExplicitlyOptional()
{
return optional != null && optional;
@@ -704,7 +725,8 @@
public ServerEncryptionOptions withSslContextFactory(ParameterizedClass sslContextFactoryClass)
{
- return new ServerEncryptionOptions(sslContextFactoryClass, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(sslContextFactoryClass, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -713,7 +735,8 @@
public ServerEncryptionOptions withKeyStore(String keystore)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -722,7 +745,8 @@
public ServerEncryptionOptions withKeyStorePassword(String keystore_password)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -731,7 +755,8 @@
public ServerEncryptionOptions withTrustStore(String truststore)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -740,7 +765,8 @@
public ServerEncryptionOptions withTrustStorePassword(String truststore_password)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -749,16 +775,18 @@
public ServerEncryptionOptions withCipherSuites(List<String> cipher_suites)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
legacy_ssl_storage_port_enabled).applyConfigInternal();
}
- public ServerEncryptionOptions withCipherSuites(String ... cipher_suites)
+ public ServerEncryptionOptions withCipherSuites(String... cipher_suites)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, Arrays.asList(cipher_suites), protocol,
accepted_protocols, algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -767,7 +795,8 @@
public ServerEncryptionOptions withProtocol(String protocol)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -776,7 +805,8 @@
public ServerEncryptionOptions withAcceptedProtocols(List<String> accepted_protocols)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -785,7 +815,8 @@
public ServerEncryptionOptions withAlgorithm(String algorithm)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -794,7 +825,8 @@
public ServerEncryptionOptions withStoreType(String store_type)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -803,7 +835,8 @@
public ServerEncryptionOptions withRequireClientAuth(boolean require_client_auth)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -812,7 +845,8 @@
public ServerEncryptionOptions withRequireEndpointVerification(boolean require_endpoint_verification)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -821,7 +855,8 @@
public ServerEncryptionOptions withOptional(boolean optional)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -830,7 +865,8 @@
public ServerEncryptionOptions withInternodeEncryption(InternodeEncryption internode_encryption)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
@@ -839,12 +875,32 @@
public ServerEncryptionOptions withLegacySslStoragePort(boolean enable_legacy_ssl_storage_port)
{
- return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password, truststore,
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outbound_keystore_password, truststore,
truststore_password, cipher_suites, protocol, accepted_protocols,
algorithm, store_type, require_client_auth,
require_endpoint_verification, optional, internode_encryption,
enable_legacy_ssl_storage_port).applyConfigInternal();
}
+ public ServerEncryptionOptions withOutboundKeystore(String outboundKeystore)
+ {
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outboundKeystore, outbound_keystore_password, truststore,
+ truststore_password, cipher_suites, protocol, accepted_protocols,
+ algorithm, store_type, require_client_auth,
+ require_endpoint_verification, optional, internode_encryption,
+ legacy_ssl_storage_port_enabled).applyConfigInternal();
+ }
+
+ public ServerEncryptionOptions withOutboundKeystorePassword(String outboundKeystorePassword)
+ {
+ return new ServerEncryptionOptions(ssl_context_factory, keystore, keystore_password,
+ outbound_keystore, outboundKeystorePassword, truststore,
+ truststore_password, cipher_suites, protocol, accepted_protocols,
+ algorithm, store_type, require_client_auth,
+ require_endpoint_verification, optional, internode_encryption,
+ legacy_ssl_storage_port_enabled).applyConfigInternal();
+ }
}
}
diff --git a/src/java/org/apache/cassandra/config/GuardrailsOptions.java b/src/java/org/apache/cassandra/config/GuardrailsOptions.java
index e4694b9..7affe6f 100644
--- a/src/java/org/apache/cassandra/config/GuardrailsOptions.java
+++ b/src/java/org/apache/cassandra/config/GuardrailsOptions.java
@@ -76,12 +76,17 @@
config.read_consistency_levels_disallowed = validateConsistencyLevels(config.read_consistency_levels_disallowed, "read_consistency_levels_disallowed");
config.write_consistency_levels_warned = validateConsistencyLevels(config.write_consistency_levels_warned, "write_consistency_levels_warned");
config.write_consistency_levels_disallowed = validateConsistencyLevels(config.write_consistency_levels_disallowed, "write_consistency_levels_disallowed");
+ validateSizeThreshold(config.partition_size_warn_threshold, config.partition_size_fail_threshold, false, "partition_size");
+ validateSizeThreshold(config.column_value_size_warn_threshold, config.column_value_size_fail_threshold, false, "column_value_size");
validateSizeThreshold(config.collection_size_warn_threshold, config.collection_size_fail_threshold, false, "collection_size");
validateMaxIntThreshold(config.items_per_collection_warn_threshold, config.items_per_collection_fail_threshold, "items_per_collection");
validateMaxIntThreshold(config.fields_per_udt_warn_threshold, config.fields_per_udt_fail_threshold, "fields_per_udt");
validatePercentageThreshold(config.data_disk_usage_percentage_warn_threshold, config.data_disk_usage_percentage_fail_threshold, "data_disk_usage_percentage");
validateDataDiskUsageMaxDiskSize(config.data_disk_usage_max_disk_size);
- validateMinRFThreshold(config.minimum_replication_factor_warn_threshold, config.minimum_replication_factor_fail_threshold, "minimum_replication_factor");
+ validateMinRFThreshold(config.minimum_replication_factor_warn_threshold, config.minimum_replication_factor_fail_threshold);
+ validateMaxRFThreshold(config.maximum_replication_factor_warn_threshold, config.maximum_replication_factor_fail_threshold);
+ validateTimestampThreshold(config.maximum_timestamp_warn_threshold, config.maximum_timestamp_fail_threshold, "maximum_timestamp");
+ validateTimestampThreshold(config.minimum_timestamp_warn_threshold, config.minimum_timestamp_fail_threshold, "minimum_timestamp");
}
@Override
@@ -344,6 +349,20 @@
}
@Override
+ public boolean getDropKeyspaceEnabled()
+ {
+ return config.drop_keyspace_enabled;
+ }
+
+ public void setDropKeyspaceEnabled(boolean enabled)
+ {
+ updatePropertyWithLogging("drop_keyspace_enabled",
+ enabled,
+ () -> config.drop_keyspace_enabled,
+ x -> config.drop_keyspace_enabled = x);
+ }
+
+ @Override
public boolean getSecondaryIndexesEnabled()
{
return config.secondary_indexes_enabled;
@@ -386,6 +405,20 @@
}
@Override
+ public boolean getAlterTableEnabled()
+ {
+ return config.alter_table_enabled;
+ }
+
+ public void setAlterTableEnabled(boolean enabled)
+ {
+ updatePropertyWithLogging("alter_table_enabled",
+ enabled,
+ () -> config.alter_table_enabled,
+ x -> config.alter_table_enabled = x);
+ }
+
+ @Override
public boolean getReadBeforeWriteListOperationsEnabled()
{
return config.read_before_write_list_operations_enabled;
@@ -414,6 +447,20 @@
}
@Override
+ public boolean getSimpleStrategyEnabled()
+ {
+ return config.simplestrategy_enabled;
+ }
+
+ public void setSimpleStrategyEnabled(boolean enabled)
+ {
+ updatePropertyWithLogging("simplestrategy_enabled",
+ enabled,
+ () -> config.simplestrategy_enabled,
+ x -> config.simplestrategy_enabled = x);
+ }
+
+ @Override
public int getInSelectCartesianProductWarnThreshold()
{
return config.in_select_cartesian_product_warn_threshold;
@@ -495,6 +542,60 @@
@Override
@Nullable
+ public DataStorageSpec.LongBytesBound getPartitionSizeWarnThreshold()
+ {
+ return config.partition_size_warn_threshold;
+ }
+
+ @Override
+ @Nullable
+ public DataStorageSpec.LongBytesBound getPartitionSizeFailThreshold()
+ {
+ return config.partition_size_fail_threshold;
+ }
+
+ public void setPartitionSizeThreshold(@Nullable DataStorageSpec.LongBytesBound warn, @Nullable DataStorageSpec.LongBytesBound fail)
+ {
+ validateSizeThreshold(warn, fail, false, "partition_size");
+ updatePropertyWithLogging("partition_size_warn_threshold",
+ warn,
+ () -> config.partition_size_warn_threshold,
+ x -> config.partition_size_warn_threshold = x);
+ updatePropertyWithLogging("partition_size_fail_threshold",
+ fail,
+ () -> config.partition_size_fail_threshold,
+ x -> config.partition_size_fail_threshold = x);
+ }
+
+ @Override
+ @Nullable
+ public DataStorageSpec.LongBytesBound getColumnValueSizeWarnThreshold()
+ {
+ return config.column_value_size_warn_threshold;
+ }
+
+ @Override
+ @Nullable
+ public DataStorageSpec.LongBytesBound getColumnValueSizeFailThreshold()
+ {
+ return config.column_value_size_fail_threshold;
+ }
+
+ public void setColumnValueSizeThreshold(@Nullable DataStorageSpec.LongBytesBound warn, @Nullable DataStorageSpec.LongBytesBound fail)
+ {
+ validateSizeThreshold(warn, fail, false, "column_value_size");
+ updatePropertyWithLogging("column_value_size_warn_threshold",
+ warn,
+ () -> config.column_value_size_warn_threshold,
+ x -> config.column_value_size_warn_threshold = x);
+ updatePropertyWithLogging("column_value_size_fail_threshold",
+ fail,
+ () -> config.column_value_size_fail_threshold,
+ x -> config.column_value_size_fail_threshold = x);
+ }
+
+ @Override
+ @Nullable
public DataStorageSpec.LongBytesBound getCollectionSizeWarnThreshold()
{
return config.collection_size_warn_threshold;
@@ -623,7 +724,7 @@
public void setMinimumReplicationFactorThreshold(int warn, int fail)
{
- validateMinRFThreshold(warn, fail, "minimum_replication_factor");
+ validateMinRFThreshold(warn, fail);
updatePropertyWithLogging("minimum_replication_factor_warn_threshold",
warn,
() -> config.minimum_replication_factor_warn_threshold,
@@ -634,6 +735,119 @@
x -> config.minimum_replication_factor_fail_threshold = x);
}
+ @Override
+ public int getMaximumReplicationFactorWarnThreshold()
+ {
+ return config.maximum_replication_factor_warn_threshold;
+ }
+
+ @Override
+ public int getMaximumReplicationFactorFailThreshold()
+ {
+ return config.maximum_replication_factor_fail_threshold;
+ }
+
+ public void setMaximumReplicationFactorThreshold(int warn, int fail)
+ {
+ validateMaxRFThreshold(warn, fail);
+ updatePropertyWithLogging("maximum_replication_factor_warn_threshold",
+ warn,
+ () -> config.maximum_replication_factor_warn_threshold,
+ x -> config.maximum_replication_factor_warn_threshold = x);
+ updatePropertyWithLogging("maximum_replication_factor_fail_threshold",
+ fail,
+ () -> config.maximum_replication_factor_fail_threshold,
+ x -> config.maximum_replication_factor_fail_threshold = x);
+ }
+
+ @Override
+ public boolean getZeroTTLOnTWCSWarned()
+ {
+ return config.zero_ttl_on_twcs_warned;
+ }
+
+ @Override
+ public void setZeroTTLOnTWCSWarned(boolean value)
+ {
+ updatePropertyWithLogging("zero_ttl_on_twcs_warned",
+ value,
+ () -> config.zero_ttl_on_twcs_warned,
+ x -> config.zero_ttl_on_twcs_warned = x);
+ }
+
+ @Override
+ public boolean getZeroTTLOnTWCSEnabled()
+ {
+ return config.zero_ttl_on_twcs_enabled;
+ }
+
+ @Override
+ public void setZeroTTLOnTWCSEnabled(boolean value)
+ {
+ updatePropertyWithLogging("zero_ttl_on_twcs_enabled",
+ value,
+ () -> config.zero_ttl_on_twcs_enabled,
+ x -> config.zero_ttl_on_twcs_enabled = x);
+ }
+
+ @Override
+ public DurationSpec.LongMicrosecondsBound getMaximumTimestampWarnThreshold()
+ {
+ return config.maximum_timestamp_warn_threshold;
+ }
+
+ @Override
+ public DurationSpec.LongMicrosecondsBound getMaximumTimestampFailThreshold()
+ {
+ return config.maximum_timestamp_fail_threshold;
+ }
+
+ @Override
+ public void setMaximumTimestampThreshold(@Nullable DurationSpec.LongMicrosecondsBound warn,
+ @Nullable DurationSpec.LongMicrosecondsBound fail)
+ {
+ validateTimestampThreshold(warn, fail, "maximum_timestamp");
+
+ updatePropertyWithLogging("maximum_timestamp_warn_threshold",
+ warn,
+ () -> config.maximum_timestamp_warn_threshold,
+ x -> config.maximum_timestamp_warn_threshold = x);
+
+ updatePropertyWithLogging("maximum_timestamp_fail_threshold",
+ fail,
+ () -> config.maximum_timestamp_fail_threshold,
+ x -> config.maximum_timestamp_fail_threshold = x);
+ }
+
+ @Override
+ public DurationSpec.LongMicrosecondsBound getMinimumTimestampWarnThreshold()
+ {
+ return config.minimum_timestamp_warn_threshold;
+ }
+
+ @Override
+ public DurationSpec.LongMicrosecondsBound getMinimumTimestampFailThreshold()
+ {
+ return config.minimum_timestamp_fail_threshold;
+ }
+
+ @Override
+ public void setMinimumTimestampThreshold(@Nullable DurationSpec.LongMicrosecondsBound warn,
+ @Nullable DurationSpec.LongMicrosecondsBound fail)
+ {
+ validateTimestampThreshold(warn, fail, "minimum_timestamp");
+
+ updatePropertyWithLogging("minimum_timestamp_warn_threshold",
+ warn,
+ () -> config.minimum_timestamp_warn_threshold,
+ x -> config.minimum_timestamp_warn_threshold = x);
+
+ updatePropertyWithLogging("minimum_timestamp_fail_threshold",
+ fail,
+ () -> config.minimum_timestamp_fail_threshold,
+ x -> config.minimum_timestamp_fail_threshold = x);
+ }
+
private static <T> void updatePropertyWithLogging(String propertyName, T newValue, Supplier<T> getter, Consumer<T> setter)
{
T oldValue = getter.get();
@@ -646,6 +860,11 @@
private static void validatePositiveNumeric(long value, long maxValue, String name)
{
+ validatePositiveNumeric(value, maxValue, name, false);
+ }
+
+ private static void validatePositiveNumeric(long value, long maxValue, String name, boolean allowZero)
+ {
if (value == -1)
return;
@@ -653,12 +872,12 @@
throw new IllegalArgumentException(format("Invalid value %d for %s: maximum allowed value is %d",
value, name, maxValue));
- if (value == 0)
+ if (!allowZero && value == 0)
throw new IllegalArgumentException(format("Invalid value for %s: 0 is not allowed; " +
"if attempting to disable use -1", name));
// We allow -1 as a general "disabling" flag. But reject anything lower to avoid mistakes.
- if (value <= 0)
+ if (value < 0)
throw new IllegalArgumentException(format("Invalid value %d for %s: negative values are not allowed, " +
"outside of -1 which disables the guardrail", value, name));
}
@@ -682,6 +901,13 @@
validateWarnLowerThanFail(warn, fail, name);
}
+ private static void validateMaxLongThreshold(long warn, long fail, String name, boolean allowZero)
+ {
+ validatePositiveNumeric(warn, Long.MAX_VALUE, name + "_warn_threshold", allowZero);
+ validatePositiveNumeric(fail, Long.MAX_VALUE, name + "_fail_threshold", allowZero);
+ validateWarnLowerThanFail(warn, fail, name);
+ }
+
private static void validateMinIntThreshold(int warn, int fail, String name)
{
validatePositiveNumeric(warn, Integer.MAX_VALUE, name + "_warn_threshold");
@@ -689,10 +915,36 @@
validateWarnGreaterThanFail(warn, fail, name);
}
- private static void validateMinRFThreshold(int warn, int fail, String name)
+ private static void validateMinRFThreshold(int warn, int fail)
{
- validateMinIntThreshold(warn, fail, name);
- validateMinRFVersusDefaultRF(fail, name);
+ validateMinIntThreshold(warn, fail, "minimum_replication_factor");
+
+ if (fail > DatabaseDescriptor.getDefaultKeyspaceRF())
+ throw new IllegalArgumentException(format("minimum_replication_factor_fail_threshold to be set (%d) " +
+ "cannot be greater than default_keyspace_rf (%d)",
+ fail, DatabaseDescriptor.getDefaultKeyspaceRF()));
+ }
+
+ private static void validateMaxRFThreshold(int warn, int fail)
+ {
+ validateMaxIntThreshold(warn, fail, "maximum_replication_factor");
+
+ if (fail != -1 && fail < DatabaseDescriptor.getDefaultKeyspaceRF())
+ throw new IllegalArgumentException(format("maximum_replication_factor_fail_threshold to be set (%d) " +
+ "cannot be lesser than default_keyspace_rf (%d)",
+ fail, DatabaseDescriptor.getDefaultKeyspaceRF()));
+ }
+
+ public static void validateTimestampThreshold(DurationSpec.LongMicrosecondsBound warn,
+ DurationSpec.LongMicrosecondsBound fail,
+ String name)
+ {
+ // this function is used for both upper and lower thresholds because lower threshold is relative
+ // despite using MinThreshold we still want the warn threshold to be less than or equal to
+ // the fail threshold.
+ validateMaxLongThreshold(warn == null ? -1 : warn.toMicroseconds(),
+ fail == null ? -1 : fail.toMicroseconds(),
+ name, true);
}
private static void validateWarnLowerThanFail(long warn, long fail, String name)
@@ -715,15 +967,6 @@
"than the fail threshold %d", warn, name, fail));
}
- private static void validateMinRFVersusDefaultRF(int fail, String name) throws IllegalArgumentException
- {
- if (fail > DatabaseDescriptor.getDefaultKeyspaceRF())
- {
- throw new IllegalArgumentException(String.format("%s_fail_threshold to be set (%d) cannot be greater than default_keyspace_rf (%d)",
- name, fail, DatabaseDescriptor.getDefaultKeyspaceRF()));
- }
- }
-
private static void validateSize(DataStorageSpec.LongBytesBound size, boolean allowZero, String name)
{
if (size == null)
diff --git a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
index 528accd..edff7be 100644
--- a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
+++ b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
@@ -29,7 +29,6 @@
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
@@ -46,8 +45,8 @@
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.composer.Composer;
-import org.yaml.snakeyaml.constructor.SafeConstructor;
import org.yaml.snakeyaml.constructor.CustomClassLoaderConstructor;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
import org.yaml.snakeyaml.error.YAMLException;
import org.yaml.snakeyaml.introspector.MissingProperty;
import org.yaml.snakeyaml.introspector.Property;
@@ -56,13 +55,13 @@
import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_DUPLICATE_CONFIG_KEYS;
import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_NEW_OLD_CONFIG_KEYS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
import static org.apache.cassandra.config.Replacements.getNameReplacements;
public class YamlConfigurationLoader implements ConfigurationLoader
{
private static final Logger logger = LoggerFactory.getLogger(YamlConfigurationLoader.class);
- private final static String DEFAULT_CONFIGURATION = "cassandra.yaml";
/**
* This is related to {@link Config#PROPERTY_PREFIX} but is different to make sure Config properties updated via
* system properties do not conflict with other system properties; the name "settings" matches system_views.settings.
@@ -74,9 +73,7 @@
*/
private static URL getStorageConfigURL() throws ConfigurationException
{
- String configUrl = System.getProperty("cassandra.config");
- if (configUrl == null)
- configUrl = DEFAULT_CONFIGURATION;
+ String configUrl = CASSANDRA_CONFIG.getString();
URL url;
try
@@ -203,6 +200,8 @@
Yaml rawYaml = new Yaml(loaderOptions);
Map<String, Object> rawConfig = rawYaml.load(new ByteArrayInputStream(configBytes));
+ if (rawConfig == null)
+ rawConfig = new HashMap<>();
verifyReplacements(replacements, rawConfig);
}
diff --git a/src/java/org/apache/cassandra/cql3/AssignmentTestable.java b/src/java/org/apache/cassandra/cql3/AssignmentTestable.java
index 41b80eb..fdf21b9 100644
--- a/src/java/org/apache/cassandra/cql3/AssignmentTestable.java
+++ b/src/java/org/apache/cassandra/cql3/AssignmentTestable.java
@@ -19,6 +19,10 @@
import java.util.Collection;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+
public interface AssignmentTestable
{
/**
@@ -32,6 +36,25 @@
*/
public TestResult testAssignment(String keyspace, ColumnSpecification receiver);
+ /**
+ * @return A data type that can represent this, or {@code null} if we can't determine that type. The returned type
+ * won't necessarely be the exact type, but one that is compatible with it.
+ */
+ @Nullable
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace);
+
+ /**
+ * @return A data type that can represent all the specified types, or {@code null} if there isn't one.
+ */
+ @Nullable
+ public static AbstractType<?> getCompatibleTypeIfKnown(Collection<AbstractType<?>> types)
+ {
+ return types.stream()
+ .filter(type -> types.stream().allMatch(t -> t.testAssignment(type).isAssignable()))
+ .findFirst()
+ .orElse(null);
+ }
+
public enum TestResult
{
EXACT_MATCH, WEAKLY_ASSIGNABLE, NOT_ASSIGNABLE;
diff --git a/src/java/org/apache/cassandra/cql3/Attributes.java b/src/java/org/apache/cassandra/cql3/Attributes.java
index e841828..559882f 100644
--- a/src/java/org/apache/cassandra/cql3/Attributes.java
+++ b/src/java/org/apache/cassandra/cql3/Attributes.java
@@ -117,6 +117,11 @@
if (tval == ByteBufferUtil.UNSET_BYTE_BUFFER)
return metadata.params.defaultTimeToLive;
+ // byte[0] and null are the same for Int32Type. UNSET_BYTE_BUFFER is also byte[0] but we rely on pointer
+ // identity, so need to check this after checking that
+ if (ByteBufferUtil.EMPTY_BYTE_BUFFER.equals(tval))
+ return 0;
+
try
{
Int32Type.instance.validate(tval);
diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java
index 1d792b2..1dd641a 100644
--- a/src/java/org/apache/cassandra/cql3/CQL3Type.java
+++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java
@@ -24,13 +24,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.marshal.CollectionType.Kind;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
@@ -54,7 +54,6 @@
}
public AbstractType<?> getType();
- default public AbstractType<?> getUDFType() { return getType(); }
/**
* Generates CQL literal from a binary value of this type.
@@ -101,11 +100,6 @@
return type;
}
- public AbstractType<?> getUDFType()
- {
- return this == TIMEUUID ? UUID.type : type;
- }
-
/**
* Delegate to
* {@link org.apache.cassandra.serializers.TypeSerializer#toCQLLiteral(ByteBuffer)}
@@ -176,14 +170,14 @@
public static class Collection implements CQL3Type
{
- private final CollectionType type;
+ private final CollectionType<?> type;
- public Collection(CollectionType type)
+ public Collection(CollectionType<?> type)
{
this.type = type;
}
- public AbstractType<?> getType()
+ public CollectionType<?> getType()
{
return type;
}
@@ -201,19 +195,19 @@
StringBuilder target = new StringBuilder();
buffer = buffer.duplicate();
- int size = CollectionSerializer.readCollectionSize(buffer, version);
- buffer.position(buffer.position() + CollectionSerializer.sizeOfCollectionSize(size, version));
+ int size = CollectionSerializer.readCollectionSize(buffer, ByteBufferAccessor.instance);
+ buffer.position(buffer.position() + CollectionSerializer.sizeOfCollectionSize());
switch (type.kind)
{
case LIST:
- CQL3Type elements = ((ListType) type).getElementsType().asCQL3Type();
+ CQL3Type elements = ((ListType<?>) type).getElementsType().asCQL3Type();
target.append('[');
generateSetOrListCQLLiteral(buffer, version, target, size, elements);
target.append(']');
break;
case SET:
- elements = ((SetType) type).getElementsType().asCQL3Type();
+ elements = ((SetType<?>) type).getElementsType().asCQL3Type();
target.append('{');
generateSetOrListCQLLiteral(buffer, version, target, size, elements);
target.append('}');
@@ -229,19 +223,19 @@
private void generateMapCQLLiteral(ByteBuffer buffer, ProtocolVersion version, StringBuilder target, int size)
{
- CQL3Type keys = ((MapType) type).getKeysType().asCQL3Type();
- CQL3Type values = ((MapType) type).getValuesType().asCQL3Type();
+ CQL3Type keys = ((MapType<?, ?>) type).getKeysType().asCQL3Type();
+ CQL3Type values = ((MapType<?, ?>) type).getValuesType().asCQL3Type();
int offset = 0;
for (int i = 0; i < size; i++)
{
if (i > 0)
target.append(", ");
- ByteBuffer element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset, version);
- offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance, version);
+ ByteBuffer element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance);
target.append(keys.toCQLLiteral(element, version));
target.append(": ");
- element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset, version);
- offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance, version);
+ element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance);
target.append(values.toCQLLiteral(element, version));
}
}
@@ -253,8 +247,8 @@
{
if (i > 0)
target.append(", ");
- ByteBuffer element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset, version);
- offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance, version);
+ ByteBuffer element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance);
target.append(elements.toCQLLiteral(element, version));
}
}
@@ -283,16 +277,16 @@
switch (type.kind)
{
case LIST:
- AbstractType<?> listType = ((ListType)type).getElementsType();
+ AbstractType<?> listType = ((ListType<?>) type).getElementsType();
sb.append("list<").append(listType.asCQL3Type());
break;
case SET:
- AbstractType<?> setType = ((SetType)type).getElementsType();
+ AbstractType<?> setType = ((SetType<?>) type).getElementsType();
sb.append("set<").append(setType.asCQL3Type());
break;
case MAP:
- AbstractType<?> keysType = ((MapType)type).getKeysType();
- AbstractType<?> valuesType = ((MapType)type).getValuesType();
+ AbstractType<?> keysType = ((MapType<?, ?>) type).getKeysType();
+ AbstractType<?> valuesType = ((MapType<?, ?>) type).getValuesType();
sb.append("map<").append(keysType.asCQL3Type()).append(", ").append(valuesType.asCQL3Type());
break;
default:
@@ -416,7 +410,7 @@
return new Tuple(type);
}
- public AbstractType<?> getType()
+ public TupleType getType()
{
return type;
}
diff --git a/src/java/org/apache/cassandra/cql3/Constants.java b/src/java/org/apache/cassandra/cql3/Constants.java
index 64d9d69..f4418fd 100644
--- a/src/java/org/apache/cassandra/cql3/Constants.java
+++ b/src/java/org/apache/cassandra/cql3/Constants.java
@@ -20,17 +20,15 @@
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -41,15 +39,14 @@
*/
public abstract class Constants
{
- private static final Logger logger = LoggerFactory.getLogger(Constants.class);
-
public enum Type
{
STRING
{
+ @Override
public AbstractType<?> getPreferedTypeFor(String text)
{
- if(Charset.forName("US-ASCII").newEncoder().canEncode(text))
+ if (StandardCharsets.US_ASCII.newEncoder().canEncode(text))
{
return AsciiType.instance;
}
@@ -59,6 +56,7 @@
},
INTEGER
{
+ @Override
public AbstractType<?> getPreferedTypeFor(String text)
{
// We only try to determine the smallest possible type between int, long and BigInteger
@@ -73,9 +71,19 @@
return IntegerType.instance;
}
},
- UUID,
+ UUID
+ {
+ @Override
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ return java.util.UUID.fromString(text).version() == 1
+ ? TimeUUIDType.instance
+ : UUIDType.instance;
+ }
+ },
FLOAT
{
+ @Override
public AbstractType<?> getPreferedTypeFor(String text)
{
if ("NaN".equals(text) || "-NaN".equals(text) || "Infinity".equals(text) || "-Infinity".equals(text))
@@ -90,9 +98,30 @@
return DecimalType.instance;
}
},
- BOOLEAN,
- HEX,
- DURATION;
+ BOOLEAN
+ {
+ @Override
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ return BooleanType.instance;
+ }
+ },
+ HEX
+ {
+ @Override
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ return ByteType.instance;
+ }
+ },
+ DURATION
+ {
+ @Override
+ public AbstractType<?> getPreferedTypeFor(String text)
+ {
+ return DurationType.instance;
+ }
+ };
/**
* Returns the exact type for the specified text
@@ -362,6 +391,12 @@
return null;
}
+ @Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return preferedType;
+ }
+
public String getRawText()
{
return text;
@@ -385,7 +420,7 @@
this.bytes = bytes;
}
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
return bytes;
}
@@ -494,10 +529,9 @@
{
ByteBuffer append = t.bindAndGet(params.options);
ByteBuffer current = getCurrentCellBuffer(partitionKey, params);
- ByteBuffer newValue;
if (current == null)
return;
- newValue = ByteBuffer.allocate(current.remaining() + append.remaining());
+ ByteBuffer newValue = ByteBuffer.allocate(current.remaining() + append.remaining());
FastByteOperations.copy(current, current.position(), newValue, newValue.position(), current.remaining());
FastByteOperations.copy(append, append.position(), newValue, newValue.position() + current.remaining(), append.remaining());
params.addCell(column, newValue);
diff --git a/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java b/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java
index 13aa7f5..af765d0 100644
--- a/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java
+++ b/src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.cql3.statements.BatchStatement;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
@@ -29,7 +30,7 @@
/**
* Custom QueryHandler that sends custom request payloads back with the result.
* Used to facilitate testing.
- * Enabled with system property cassandra.custom_query_handler_class.
+ * Enabled with system property {@link CassandraRelevantProperties#CUSTOM_QUERY_HANDLER_CLASS}.
*/
public class CustomPayloadMirroringQueryHandler implements QueryHandler
{
diff --git a/src/java/org/apache/cassandra/cql3/Json.java b/src/java/org/apache/cassandra/cql3/Json.java
index d05096a..d60eabd 100644
--- a/src/java/org/apache/cassandra/cql3/Json.java
+++ b/src/java/org/apache/cassandra/cql3/Json.java
@@ -18,44 +18,31 @@
package org.apache.cassandra.cql3;
import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import com.fasterxml.jackson.core.util.BufferRecyclers;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.JsonUtils;
-/** Term-related classes for INSERT JSON support. */
-public class Json
+import static java.lang.String.format;
+
+/**
+ * Term-related classes for INSERT JSON support.
+ */
+public final class Json
{
- public static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper();
-
public static final ColumnIdentifier JSON_COLUMN_ID = new ColumnIdentifier("[json]", true);
- /**
- * Quotes string contents using standard JSON quoting.
- */
- public static String quoteAsJsonString(String s)
+ private Json()
{
- // In future should update to directly use `JsonStringEncoder.getInstance()` but for now:
- return new String(BufferRecyclers.getJsonStringEncoder().quoteAsString(s));
- }
-
- public static Object decodeJson(String json)
- {
- try
- {
- return JSON_OBJECT_MAPPER.readValue(json, Object.class);
- }
- catch (IOException exc)
- {
- throw new MarshalException("Error decoding JSON string: " + exc.getMessage());
- }
}
public interface Raw
@@ -131,13 +118,13 @@
{
Term value = columnMap.get(def.name);
return value == null
- ? (defaultUnset ? Constants.UNSET_LITERAL : Constants.NULL_LITERAL)
- : new ColumnValue(value);
+ ? (defaultUnset ? Constants.UNSET_LITERAL : Constants.NULL_LITERAL)
+ : new ColumnValue(value);
}
}
/**
- * A prepared bind marker for a set of JSON values
+ * A prepared bind marker for a set of JSON values
*/
private static class PreparedMarker extends Prepared
{
@@ -158,7 +145,7 @@
/**
* A Terminal for a single column.
- *
+ * <p>
* Note that this is intrinsically an already prepared term, but this still implements Term.Raw so that we can
* easily use it to create raw operations.
*/
@@ -266,8 +253,8 @@
{
Term term = options.getJsonColumnValue(marker.bindIndex, column.name, marker.columns);
return term == null
- ? (defaultUnset ? Constants.UNSET_VALUE : null)
- : term.bind(options);
+ ? (defaultUnset ? Constants.UNSET_VALUE : null)
+ : term.bind(options);
}
@Override
@@ -279,16 +266,16 @@
/**
* Given a JSON string, return a map of columns to their values for the insert.
*/
- public static Map<ColumnIdentifier, Term> parseJson(String jsonString, Collection<ColumnMetadata> expectedReceivers)
+ static Map<ColumnIdentifier, Term> parseJson(String jsonString, Collection<ColumnMetadata> expectedReceivers)
{
try
{
- Map<String, Object> valueMap = JSON_OBJECT_MAPPER.readValue(jsonString, Map.class);
+ Map<String, Object> valueMap = JsonUtils.JSON_OBJECT_MAPPER.readValue(jsonString, Map.class);
if (valueMap == null)
throw new InvalidRequestException("Got null for INSERT JSON values");
- handleCaseSensitivity(valueMap);
+ JsonUtils.handleCaseSensitivity(valueMap);
Map<ColumnIdentifier, Term> columnMap = new HashMap<>(expectedReceivers.size());
for (ColumnSpecification spec : expectedReceivers)
@@ -310,49 +297,28 @@
{
columnMap.put(spec.name, spec.type.fromJSONObject(parsedJsonObject));
}
- catch(MarshalException exc)
+ catch (MarshalException exc)
{
- throw new InvalidRequestException(String.format("Error decoding JSON value for %s: %s", spec.name, exc.getMessage()));
+ throw new InvalidRequestException(format("Error decoding JSON value for %s: %s", spec.name, exc.getMessage()));
}
}
}
if (!valueMap.isEmpty())
{
- throw new InvalidRequestException(String.format(
- "JSON values map contains unrecognized column: %s", valueMap.keySet().iterator().next()));
+ throw new InvalidRequestException(format("JSON values map contains unrecognized column: %s",
+ valueMap.keySet().iterator().next()));
}
return columnMap;
}
catch (IOException exc)
{
- throw new InvalidRequestException(String.format("Could not decode JSON string as a map: %s. (String was: %s)", exc.toString(), jsonString));
+ throw new InvalidRequestException(format("Could not decode JSON string as a map: %s. (String was: %s)", exc.toString(), jsonString));
}
catch (MarshalException exc)
{
throw new InvalidRequestException(exc.getMessage());
}
}
-
- /**
- * Handles unquoting and case-insensitivity in map keys.
- */
- public static void handleCaseSensitivity(Map<String, Object> valueMap)
- {
- for (String mapKey : new ArrayList<>(valueMap.keySet()))
- {
- // if it's surrounded by quotes, remove them and preserve the case
- if (mapKey.startsWith("\"") && mapKey.endsWith("\""))
- {
- valueMap.put(mapKey.substring(1, mapKey.length() - 1), valueMap.remove(mapKey));
- continue;
- }
-
- // otherwise, lowercase it if needed
- String lowered = mapKey.toLowerCase(Locale.US);
- if (!mapKey.equals(lowered))
- valueMap.put(lowered, valueMap.remove(mapKey));
- }
- }
}
diff --git a/src/java/org/apache/cassandra/cql3/Lists.java b/src/java/org/apache/cassandra/cql3/Lists.java
index bdac046..605a8be 100644
--- a/src/java/org/apache/cassandra/cql3/Lists.java
+++ b/src/java/org/apache/cassandra/cql3/Lists.java
@@ -17,15 +17,12 @@
*/
package org.apache.cassandra.cql3;
-import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.TimeUUID.Generator.atUnixMillisAsBytes;
-
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -47,6 +44,10 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import static org.apache.cassandra.utils.TimeUUID.Generator.atUnixMillisAsBytes;
+
/**
* Static helper methods and classes for lists.
*/
@@ -71,7 +72,7 @@
private static AbstractType<?> elementsType(AbstractType<?> type)
{
- return ((ListType) unwrap(type)).getElementsType();
+ return ((ListType<?>) unwrap(type)).getElementsType();
}
/**
@@ -116,7 +117,7 @@
public static <T> String listToString(Iterable<T> items, java.util.function.Function<T, String> mapper)
{
return StreamSupport.stream(items.spliterator(), false)
- .map(e -> mapper.apply(e))
+ .map(mapper)
.collect(Collectors.joining(", ", "[", "]"));
}
@@ -127,13 +128,21 @@
* @param mapper the mapper used to retrieve the element types from the items
* @return the exact ListType from the items if it can be known or <code>null</code>
*/
- public static <T> AbstractType<?> getExactListTypeIfKnown(List<T> items,
- java.util.function.Function<T, AbstractType<?>> mapper)
+ public static <T> ListType<?> getExactListTypeIfKnown(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
{
Optional<AbstractType<?>> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst();
return type.isPresent() ? ListType.getInstance(type.get(), false) : null;
}
+ public static <T> ListType<?> getPreferredCompatibleType(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ Set<AbstractType<?>> types = items.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toSet());
+ AbstractType<?> type = AssignmentTestable.getCompatibleTypeIfKnown(types);
+ return type == null ? null : ListType.getInstance(type, false);
+ }
+
public static class Literal extends Term.Raw
{
private final List<Term.Raw> elements;
@@ -192,6 +201,12 @@
return getExactListTypeIfKnown(elements, p -> p.getExactTypeIfKnown(keyspace));
}
+ @Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return Lists.getPreferredCompatibleType(elements, p -> p.getCompatibleTypeIfKnown(keyspace));
+ }
+
public String getText()
{
return listToString(elements, Term.Raw::getText);
@@ -207,15 +222,15 @@
this.elements = elements;
}
- public static Value fromSerialized(ByteBuffer value, ListType type, ProtocolVersion version) throws InvalidRequestException
+ public static <T> Value fromSerialized(ByteBuffer value, ListType<T> type) throws InvalidRequestException
{
try
{
// Collections have this small hack that validate cannot be called on a serialized object,
// but compose does the validation (so we're fine).
- List<?> l = type.getSerializer().deserializeForNativeProtocol(value, ByteBufferAccessor.instance, version);
+ List<T> l = type.getSerializer().deserialize(value, ByteBufferAccessor.instance);
List<ByteBuffer> elements = new ArrayList<>(l.size());
- for (Object element : l)
+ for (T element : l)
// elements can be null in lists that represent a set of IN values
elements.add(element == null ? null : type.getElementsType().decompose(element));
return new Value(elements);
@@ -226,12 +241,12 @@
}
}
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
- return CollectionSerializer.pack(elements, elements.size(), protocolVersion);
+ return CollectionSerializer.pack(elements, elements.size());
}
- public boolean equals(ListType lt, Value v)
+ public boolean equals(ListType<?> lt, Value v)
{
if (elements.size() != v.elements.size())
return false;
@@ -318,7 +333,7 @@
return null;
if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
return UNSET_VALUE;
- return Value.fromSerialized(value, (ListType)receiver.type, options.getProtocolVersion());
+ return Value.fromSerialized(value, (ListType<?>) receiver.type);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/Maps.java b/src/java/org/apache/cassandra/cql3/Maps.java
index a2d23a6..9040d1a 100644
--- a/src/java/org/apache/cassandra/cql3/Maps.java
+++ b/src/java/org/apache/cassandra/cql3/Maps.java
@@ -17,25 +17,38 @@
*/
package org.apache.cassandra.cql3;
-import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
-
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.stream.Collectors;
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
+import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
+
/**
* Static helper methods and classes for maps.
*/
@@ -128,8 +141,8 @@
* @param mapper the mapper used to retrieve the key and value types from the entries
* @return the exact MapType from the entries if it can be known or <code>null</code>
*/
- public static <T> AbstractType<?> getExactMapTypeIfKnown(List<Pair<T, T>> entries,
- java.util.function.Function<T, AbstractType<?>> mapper)
+ public static <T> MapType<?, ?> getExactMapTypeIfKnown(List<Pair<T, T>> entries,
+ java.util.function.Function<T, AbstractType<?>> mapper)
{
AbstractType<?> keyType = null;
AbstractType<?> valueType = null;
@@ -145,6 +158,22 @@
return null;
}
+ public static <T> MapType<?, ?> getPreferredCompatibleType(List<Pair<T, T>> entries,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ Set<AbstractType<?>> keyTypes = entries.stream().map(Pair::left).map(mapper).filter(Objects::nonNull).collect(Collectors.toSet());
+ AbstractType<?> keyType = AssignmentTestable.getCompatibleTypeIfKnown(keyTypes);
+ if (keyType == null)
+ return null;
+
+ Set<AbstractType<?>> valueTypes = entries.stream().map(Pair::right).map(mapper).filter(Objects::nonNull).collect(Collectors.toSet());
+ AbstractType<?> valueType = AssignmentTestable.getCompatibleTypeIfKnown(valueTypes);
+ if (valueType == null)
+ return null;
+
+ return MapType.getInstance(keyType, valueType, false);
+ }
+
public static class Literal extends Term.Raw
{
public final List<Pair<Term.Raw, Term.Raw>> entries;
@@ -208,6 +237,12 @@
return getExactMapTypeIfKnown(entries, p -> p.getExactTypeIfKnown(keyspace));
}
+ @Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return Maps.getPreferredCompatibleType(entries, p -> p.getCompatibleTypeIfKnown(keyspace));
+ }
+
public String getText()
{
return mapToString(entries, Term.Raw::getText);
@@ -223,16 +258,16 @@
this.map = map;
}
- public static Value fromSerialized(ByteBuffer value, MapType type, ProtocolVersion version) throws InvalidRequestException
+ public static <K, V> Value fromSerialized(ByteBuffer value, MapType<K, V> type) throws InvalidRequestException
{
try
{
// Collections have this small hack that validate cannot be called on a serialized object,
// but compose does the validation (so we're fine).
- Map<?, ?> m = type.getSerializer().deserializeForNativeProtocol(value, ByteBufferAccessor.instance, version);
+ Map<K, V> m = type.getSerializer().deserialize(value, ByteBufferAccessor.instance);
// We depend on Maps to be properly sorted by their keys, so use a sorted map implementation here.
SortedMap<ByteBuffer, ByteBuffer> map = new TreeMap<>(type.getKeysType());
- for (Map.Entry<?, ?> entry : m.entrySet())
+ for (Map.Entry<K, V> entry : m.entrySet())
map.put(type.getKeysType().decompose(entry.getKey()), type.getValuesType().decompose(entry.getValue()));
return new Value(map);
}
@@ -243,7 +278,7 @@
}
@Override
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
List<ByteBuffer> buffers = new ArrayList<>(2 * map.size());
for (Map.Entry<ByteBuffer, ByteBuffer> entry : map.entrySet())
@@ -251,10 +286,10 @@
buffers.add(entry.getKey());
buffers.add(entry.getValue());
}
- return CollectionSerializer.pack(buffers, map.size(), protocolVersion);
+ return CollectionSerializer.pack(buffers, map.size());
}
- public boolean equals(MapType mt, Value v)
+ public boolean equals(MapType<?, ?> mt, Value v)
{
if (map.size() != v.map.size())
return false;
@@ -342,7 +377,7 @@
return null;
if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
return UNSET_VALUE;
- return Value.fromSerialized(value, (MapType)receiver.type, options.getProtocolVersion());
+ return Value.fromSerialized(value, (MapType<?, ?>) receiver.type);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index 1e2d0db..ba72ad4 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -79,7 +79,7 @@
public class QueryProcessor implements QueryHandler
{
- public static final CassandraVersion CQL_VERSION = new CassandraVersion("3.4.6");
+ public static final CassandraVersion CQL_VERSION = new CassandraVersion("3.4.7");
// See comments on QueryProcessor #prepare
public static final CassandraVersion NEW_PREPARED_STATEMENT_BEHAVIOUR_SINCE_30 = new CassandraVersion("3.0.26");
@@ -237,7 +237,7 @@
if (key == ByteBufferUtil.UNSET_BYTE_BUFFER)
throw new InvalidRequestException("Key may not be unset");
- // check that key can be handled by FBUtilities.writeShortByteArray
+ // check that key can be handled by ByteArrayUtil.writeWithShortLength and ByteBufferUtil.writeWithShortLength
if (key.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
{
throw new InvalidRequestException("Key length of " + key.remaining() +
@@ -483,7 +483,7 @@
.map(m -> MessagingService.instance().<ReadResponse>sendWithResult(m, address))
.collect(Collectors.toList()));
- ResultSetBuilder result = new ResultSetBuilder(select.getResultMetadata(), select.getSelection().newSelectors(options), null);
+ ResultSetBuilder result = new ResultSetBuilder(select.getResultMetadata(), select.getSelection().newSelectors(options), false);
return future.map(list -> {
int i = 0;
for (Message<ReadResponse> m : list)
@@ -611,17 +611,19 @@
return select.executeRawInternal(makeInternalOptionsWithNowInSec(prepared.statement, nowInSec, values), internalQueryState().getClientState(), nowInSec);
}
+ @VisibleForTesting
public static UntypedResultSet resultify(String query, RowIterator partition)
{
return resultify(query, PartitionIterators.singletonIterator(partition));
}
+ @VisibleForTesting
public static UntypedResultSet resultify(String query, PartitionIterator partitions)
{
try (PartitionIterator iter = partitions)
{
SelectStatement ss = (SelectStatement) getStatement(query, null);
- ResultSet cqlRows = ss.process(iter, FBUtilities.nowInSeconds());
+ ResultSet cqlRows = ss.process(iter, FBUtilities.nowInSeconds(), true);
return UntypedResultSet.create(cqlRows);
}
}
@@ -1022,7 +1024,7 @@
{
// in case there are other overloads, we have to remove all overloads since argument type
// matching may change (due to type casting)
- if (Schema.instance.getKeyspaceMetadata(ksName).functions.get(new FunctionName(ksName, functionName)).size() > 1)
+ if (Schema.instance.getKeyspaceMetadata(ksName).userFunctions.get(new FunctionName(ksName, functionName)).size() > 1)
removeInvalidPreparedStatementsForFunction(ksName, functionName);
}
diff --git a/src/java/org/apache/cassandra/cql3/Sets.java b/src/java/org/apache/cassandra/cql3/Sets.java
index 104a857..00d6870 100644
--- a/src/java/org/apache/cassandra/cql3/Sets.java
+++ b/src/java/org/apache/cassandra/cql3/Sets.java
@@ -20,17 +20,31 @@
import static org.apache.cassandra.cql3.Constants.UNSET_VALUE;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -55,7 +69,7 @@
private static AbstractType<?> elementsType(AbstractType<?> type)
{
- return ((SetType) unwrap(type)).getElementsType();
+ return ((SetType<?>) unwrap(type)).getElementsType();
}
/**
@@ -106,7 +120,7 @@
public static <T> String setToString(Iterable<T> items, java.util.function.Function<T, String> mapper)
{
return StreamSupport.stream(items.spliterator(), false)
- .map(e -> mapper.apply(e))
+ .map(mapper)
.collect(Collectors.joining(", ", "{", "}"));
}
@@ -117,13 +131,21 @@
* @param mapper the mapper used to retrieve the element types from the items
* @return the exact SetType from the items if it can be known or <code>null</code>
*/
- public static <T> AbstractType<?> getExactSetTypeIfKnown(List<T> items,
- java.util.function.Function<T, AbstractType<?>> mapper)
+ public static <T> SetType<?> getExactSetTypeIfKnown(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
{
Optional<AbstractType<?>> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst();
return type.isPresent() ? SetType.getInstance(type.get(), false) : null;
}
+ public static <T> SetType<?> getPreferredCompatibleType(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
+ {
+ Set<AbstractType<?>> types = items.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toSet());
+ AbstractType<?> type = AssignmentTestable.getCompatibleTypeIfKnown(types);
+ return type == null ? null : SetType.getInstance(type, false);
+ }
+
public static class Literal extends Term.Raw
{
private final List<Term.Raw> elements;
@@ -194,6 +216,12 @@
return getExactSetTypeIfKnown(elements, p -> p.getExactTypeIfKnown(keyspace));
}
+ @Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return Sets.getPreferredCompatibleType(elements, p -> p.getCompatibleTypeIfKnown(keyspace));
+ }
+
public String getText()
{
return setToString(elements, Term.Raw::getText);
@@ -209,15 +237,15 @@
this.elements = elements;
}
- public static Value fromSerialized(ByteBuffer value, SetType type, ProtocolVersion version) throws InvalidRequestException
+ public static <T> Value fromSerialized(ByteBuffer value, SetType<T> type) throws InvalidRequestException
{
try
{
// Collections have this small hack that validate cannot be called on a serialized object,
// but compose does the validation (so we're fine).
- Set<?> s = type.getSerializer().deserializeForNativeProtocol(value, ByteBufferAccessor.instance, version);
+ Set<T> s = type.getSerializer().deserialize(value, ByteBufferAccessor.instance);
SortedSet<ByteBuffer> elements = new TreeSet<>(type.getElementsType());
- for (Object element : s)
+ for (T element : s)
elements.add(type.getElementsType().decomposeUntyped(element));
return new Value(elements);
}
@@ -227,19 +255,19 @@
}
}
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
- return CollectionSerializer.pack(elements, elements.size(), protocolVersion);
+ return CollectionSerializer.pack(elements, elements.size());
}
- public boolean equals(SetType st, Value v)
+ public boolean equals(SetType<?> st, Value v)
{
if (elements.size() != v.elements.size())
return false;
Iterator<ByteBuffer> thisIter = elements.iterator();
Iterator<ByteBuffer> thatIter = v.elements.iterator();
- AbstractType elementsType = st.getElementsType();
+ AbstractType<?> elementsType = st.getElementsType();
while (thisIter.hasNext())
if (elementsType.compare(thisIter.next(), thatIter.next()) != 0)
return false;
@@ -308,7 +336,7 @@
return null;
if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
return UNSET_VALUE;
- return Value.fromSerialized(value, (SetType)receiver.type, options.getProtocolVersion());
+ return Value.fromSerialized(value, (SetType<?>) receiver.type);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/Term.java b/src/java/org/apache/cassandra/cql3/Term.java
index f536baa..c94b614 100644
--- a/src/java/org/apache/cassandra/cql3/Term.java
+++ b/src/java/org/apache/cassandra/cql3/Term.java
@@ -113,13 +113,19 @@
*
* @param keyspace the keyspace on which the statement containing this term is on.
* @return the type of this {@code Term} if inferrable, or {@code null}
- * otherwise (for instance, the type isn't inferable for a bind marker. Even for
+ * otherwise (for instance, the type isn't inferrable for a bind marker. Even for
* literals, the exact type is not inferrable since they are valid for many
* different types and so this will return {@code null} too).
*/
public abstract AbstractType<?> getExactTypeIfKnown(String keyspace);
@Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return getExactTypeIfKnown(keyspace);
+ }
+
+ @Override
public String toString()
{
return getText();
@@ -181,9 +187,8 @@
/**
* @return the serialized value of this terminal.
- * @param protocolVersion
*/
- public abstract ByteBuffer get(ProtocolVersion protocolVersion) throws InvalidRequestException;
+ public abstract ByteBuffer get(ProtocolVersion version) throws InvalidRequestException;
public ByteBuffer bindAndGet(QueryOptions options) throws InvalidRequestException
{
diff --git a/src/java/org/apache/cassandra/cql3/Terms.java b/src/java/org/apache/cassandra/cql3/Terms.java
index 33ce2e9..31b3fd0 100644
--- a/src/java/org/apache/cassandra/cql3/Terms.java
+++ b/src/java/org/apache/cassandra/cql3/Terms.java
@@ -145,11 +145,11 @@
switch (((CollectionType<?>) type).kind)
{
case LIST:
- return e -> Lists.Value.fromSerialized(e, (ListType<?>) type, version);
+ return e -> Lists.Value.fromSerialized(e, (ListType<?>) type);
case SET:
- return e -> Sets.Value.fromSerialized(e, (SetType<?>) type, version);
+ return e -> Sets.Value.fromSerialized(e, (SetType<?>) type);
case MAP:
- return e -> Maps.Value.fromSerialized(e, (MapType<?, ?>) type, version);
+ return e -> Maps.Value.fromSerialized(e, (MapType<?, ?>) type);
}
throw new AssertionError();
}
diff --git a/src/java/org/apache/cassandra/cql3/Tuples.java b/src/java/org/apache/cassandra/cql3/Tuples.java
index b8acd59..60f963c 100644
--- a/src/java/org/apache/cassandra/cql3/Tuples.java
+++ b/src/java/org/apache/cassandra/cql3/Tuples.java
@@ -24,11 +24,12 @@
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -41,8 +42,6 @@
*/
public class Tuples
{
- private static final Logger logger = LoggerFactory.getLogger(Tuples.class);
-
private Tuples() {}
public static ColumnSpecification componentSpecOf(ColumnSpecification column, int component)
@@ -154,17 +153,17 @@
public static Value fromSerialized(ByteBuffer bytes, TupleType type)
{
- ByteBuffer[] values = type.split(bytes);
+ ByteBuffer[] values = type.split(ByteBufferAccessor.instance, bytes);
if (values.length > type.size())
{
throw new InvalidRequestException(String.format(
"Tuple value contained too many fields (expected %s, got %s)", type.size(), values.length));
}
- return new Value(type.split(bytes));
+ return new Value(type.split(ByteBufferAccessor.instance, bytes));
}
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
return TupleType.buildValue(elements);
}
@@ -258,21 +257,21 @@
this.elements = items;
}
- public static InValue fromSerialized(ByteBuffer value, ListType type, QueryOptions options) throws InvalidRequestException
+ public static <T> InValue fromSerialized(ByteBuffer value, ListType<T> type) throws InvalidRequestException
{
try
{
// Collections have this small hack that validate cannot be called on a serialized object,
// but the deserialization does the validation (so we're fine).
- List<?> l = type.getSerializer().deserializeForNativeProtocol(value, ByteBufferAccessor.instance, options.getProtocolVersion());
+ List<T> l = type.getSerializer().deserialize(value, ByteBufferAccessor.instance);
assert type.getElementsType() instanceof TupleType;
TupleType tupleType = Tuples.getTupleType(type.getElementsType());
// type.split(bytes)
List<List<ByteBuffer>> elements = new ArrayList<>(l.size());
- for (Object element : l)
- elements.add(Arrays.asList(tupleType.split(type.getElementsType().decompose(element))));
+ for (T element : l)
+ elements.add(Arrays.asList(tupleType.split(ByteBufferAccessor.instance, type.getElementsType().decompose(element))));
return new InValue(elements);
}
catch (MarshalException e)
@@ -281,7 +280,7 @@
}
}
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
throw new UnsupportedOperationException();
}
@@ -416,7 +415,7 @@
ByteBuffer value = options.getValues().get(bindIndex);
if (value == ByteBufferUtil.UNSET_BYTE_BUFFER)
throw new InvalidRequestException(String.format("Invalid unset value for %s", receiver.name));
- return value == null ? null : InValue.fromSerialized(value, (ListType)receiver.type, options);
+ return value == null ? null : InValue.fromSerialized(value, (ListType<?>) receiver.type);
}
}
@@ -442,7 +441,7 @@
public static <T> String tupleToString(Iterable<T> items, java.util.function.Function<T, String> mapper)
{
return StreamSupport.stream(items.spliterator(), false)
- .map(e -> mapper.apply(e))
+ .map(mapper)
.collect(Collectors.joining(", ", "(", ")"));
}
@@ -453,8 +452,8 @@
* @param mapper the mapper used to retrieve the element types from the items
* @return the exact TupleType from the items if it can be known or <code>null</code>
*/
- public static <T> AbstractType<?> getExactTupleTypeIfKnown(List<T> items,
- java.util.function.Function<T, AbstractType<?>> mapper)
+ public static <T> TupleType getExactTupleTypeIfKnown(List<T> items,
+ java.util.function.Function<T, AbstractType<?>> mapper)
{
List<AbstractType<?>> types = new ArrayList<>(items.size());
for (T item : items)
@@ -520,12 +519,12 @@
public static boolean checkIfTupleType(AbstractType<?> tuple)
{
return (tuple instanceof TupleType) ||
- (tuple instanceof ReversedType && ((ReversedType) tuple).baseType instanceof TupleType);
+ (tuple instanceof ReversedType && ((ReversedType<?>) tuple).baseType instanceof TupleType);
}
public static TupleType getTupleType(AbstractType<?> tuple)
{
- return (tuple instanceof ReversedType ? ((TupleType) ((ReversedType) tuple).baseType) : (TupleType)tuple);
+ return (tuple instanceof ReversedType ? ((TupleType) ((ReversedType<?>) tuple).baseType) : (TupleType)tuple);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
index f00c137..ce2cc0f 100644
--- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
+++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
@@ -20,22 +20,32 @@
import java.net.InetAddress;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
import com.datastax.driver.core.CodecUtils;
import org.apache.cassandra.cql3.functions.types.LocalDate;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.statements.SelectStatement;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadExecutionController;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.pager.QueryPager;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
@@ -110,7 +120,7 @@
{
return new AbstractIterator<Row>()
{
- Iterator<List<ByteBuffer>> iter = cqlRows.rows.iterator();
+ final Iterator<List<ByteBuffer>> iter = cqlRows.rows.iterator();
protected Row computeNext()
{
@@ -152,7 +162,7 @@
{
return new AbstractIterator<Row>()
{
- Iterator<Map<String, ByteBuffer>> iter = cqlRows.iterator();
+ final Iterator<Map<String, ByteBuffer>> iter = cqlRows.iterator();
protected Row computeNext()
{
@@ -211,7 +221,7 @@
try (ReadExecutionController executionController = pager.executionController();
PartitionIterator iter = pager.fetchPageInternal(pageSize, executionController))
{
- currentPage = select.process(iter, nowInSec).rows.iterator();
+ currentPage = select.process(iter, nowInSec, true).rows.iterator();
}
}
return new Row(metadata, currentPage.next());
@@ -276,7 +286,7 @@
try (PartitionIterator iter = pager.fetchPage(pageSize, cl, clientState, nanoTime()))
{
- currentPage = select.process(iter, nowInSec).rows.iterator();
+ currentPage = select.process(iter, nowInSec, true).rows.iterator();
}
}
return new Row(metadata, currentPage.next());
@@ -331,7 +341,7 @@
{
ComplexColumnData complexData = row.getComplexColumnData(def);
if (complexData != null)
- data.put(def.name.toString(), ((CollectionType)def.type).serializeForNativeProtocol(complexData.iterator(), ProtocolVersion.V3));
+ data.put(def.name.toString(), ((CollectionType<?>) def.type).serializeForNativeProtocol(complexData.iterator()));
}
}
@@ -451,11 +461,6 @@
return raw == null ? null : MapType.getInstance(keyType, valueType, true).compose(raw);
}
- public Map<String, String> getTextMap(String column)
- {
- return getMap(column, UTF8Type.instance, UTF8Type.instance);
- }
-
public <T> Set<T> getFrozenSet(String column, AbstractType<T> type)
{
ByteBuffer raw = data.get(column);
diff --git a/src/java/org/apache/cassandra/cql3/UpdateParameters.java b/src/java/org/apache/cassandra/cql3/UpdateParameters.java
index 2d59366..b505480 100644
--- a/src/java/org/apache/cassandra/cql3/UpdateParameters.java
+++ b/src/java/org/apache/cassandra/cql3/UpdateParameters.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
@@ -143,6 +144,15 @@
public void addTombstone(ColumnMetadata column, CellPath path) throws InvalidRequestException
{
+ // Deleting individual elements of non-frozen sets and maps involves creating tombstones that contain the value
+ // of the deleted element, independently on whether the element existed or not. That tombstone value is guarded
+ // by the columnValueSize guardrail, to prevent the insertion of tombstones over the threshold. The downside is
+ // that enabling or raising this threshold can prevent users from deleting set/map elements that were written
+ // when the guardrail was disabled or with a lower value. Deleting the entire column, row or partition is always
+ // allowed, since the tombstones created for those operations don't contain the CQL column values.
+ if (path != null && column.type.isMultiCell())
+ Guardrails.columnValueSize.guard(path.dataSize(), column.name.toString(), false, clientState);
+
builder.addCell(BufferCell.tombstone(column, timestamp, nowInSec, path));
}
@@ -153,6 +163,11 @@
public Cell<?> addCell(ColumnMetadata column, CellPath path, ByteBuffer value) throws InvalidRequestException
{
+ Guardrails.columnValueSize.guard(value.remaining(), column.name.toString(), false, clientState);
+
+ if (path != null && column.type.isMultiCell())
+ Guardrails.columnValueSize.guard(path.dataSize(), column.name.toString(), false, clientState);
+
Cell<?> cell = ttl == LivenessInfo.NO_TTL
? BufferCell.live(column, timestamp, value, path)
: BufferCell.expiring(column, timestamp, ttl, nowInSec, value, path);
diff --git a/src/java/org/apache/cassandra/cql3/UserTypes.java b/src/java/org/apache/cassandra/cql3/UserTypes.java
index b023a8a..76276a7 100644
--- a/src/java/org/apache/cassandra/cql3/UserTypes.java
+++ b/src/java/org/apache/cassandra/cql3/UserTypes.java
@@ -217,10 +217,10 @@
public static Value fromSerialized(ByteBuffer bytes, UserType type)
{
type.validate(bytes);
- return new Value(type, type.split(bytes));
+ return new Value(type, type.split(ByteBufferAccessor.instance, bytes));
}
- public ByteBuffer get(ProtocolVersion protocolVersion)
+ public ByteBuffer get(ProtocolVersion version)
{
return TupleType.buildValue(elements);
}
diff --git a/src/java/org/apache/cassandra/cql3/Validation.java b/src/java/org/apache/cassandra/cql3/Validation.java
index 34a4027..27a1b4e 100644
--- a/src/java/org/apache/cassandra/cql3/Validation.java
+++ b/src/java/org/apache/cassandra/cql3/Validation.java
@@ -47,7 +47,7 @@
if (key == null || key.remaining() == 0)
throw new InvalidRequestException("Key may not be empty");
- // check that key can be handled by FBUtilities.writeShortByteArray
+ // check that key can be handled by ByteArrayUtil.writeWithShortLength and ByteBufferUtil.writeWithShortLength
if (key.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
{
throw new InvalidRequestException("Key length of " + key.remaining() +
diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
index e3f463a..68cf2d3 100644
--- a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
+++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java
@@ -650,8 +650,8 @@
Cell<?> cell = getCell(row, column);
return cell == null
- ? null
- : userType.split(cell.buffer())[userType.fieldPosition(field)];
+ ? null
+ : userType.split(ByteBufferAccessor.instance, cell.buffer())[userType.fieldPosition(field)];
}
private boolean isSatisfiedBy(ByteBuffer rowValue)
diff --git a/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java b/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
index aab2046..c3183f6 100644
--- a/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/AbstractFunction.java
@@ -118,6 +118,12 @@
}
@Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return returnType();
+ }
+
+ @Override
public String toString()
{
return new CqlBuilder().append(name)
@@ -165,4 +171,34 @@
.append(')')
.toString();
}
+
+ /*
+ * We need to compare the CQL3 representation of the type because comparing
+ * the AbstractType will fail for example if a UDT has been changed.
+ * Reason is that UserType.equals() takes the field names and types into account.
+ * Example CQL sequence that would fail when comparing AbstractType:
+ * CREATE TYPE foo ...
+ * CREATE FUNCTION bar ( par foo ) RETURNS foo ...
+ * ALTER TYPE foo ADD ...
+ * or
+ * ALTER TYPE foo ALTER ...
+ * or
+ * ALTER TYPE foo RENAME ...
+ */
+ public boolean typesMatch(List<AbstractType<?>> types)
+ {
+ if (argTypes().size() != types.size())
+ return false;
+
+ for (int i = 0; i < argTypes().size(); i++)
+ if (!typesMatch(argTypes().get(i), types.get(i)))
+ return false;
+
+ return true;
+ }
+
+ private static boolean typesMatch(AbstractType<?> t1, AbstractType<?> t2)
+ {
+ return t1.freeze().asCQL3Type().toString().equals(t2.freeze().asCQL3Type().toString());
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java b/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java
index 5797de4..ce8e8e1 100644
--- a/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/AggregateFcts.java
@@ -21,13 +21,8 @@
import java.math.BigInteger;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -37,10 +32,8 @@
*/
public abstract class AggregateFcts
{
- public static Collection<AggregateFunction> all()
+ public static void addFunctionsTo(NativeFunctions functions)
{
- Collection<AggregateFunction> functions = new ArrayList<>();
-
functions.add(countRowsFunction);
// sum for primitives
@@ -65,71 +58,86 @@
functions.add(avgFunctionForVarint);
functions.add(avgFunctionForCounter);
- // count, max, and min for all standard types
- Set<AbstractType<?>> types = new HashSet<>();
- for (CQL3Type type : CQL3Type.Native.values())
+ // count for all types
+ functions.add(makeCountFunction(BytesType.instance));
+
+ // max for all types
+ functions.add(new FunctionFactory("max", FunctionParameter.anyType(true))
{
- AbstractType<?> udfType = type.getType().udfType();
- if (!types.add(udfType))
- continue;
-
- functions.add(AggregateFcts.makeCountFunction(udfType));
- if (type != CQL3Type.Native.COUNTER)
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
{
- functions.add(AggregateFcts.makeMaxFunction(udfType));
- functions.add(AggregateFcts.makeMinFunction(udfType));
+ AbstractType<?> type = argTypes.get(0);
+ return type.isCounter() ? maxFunctionForCounter : makeMaxFunction(type);
}
- else
- {
- functions.add(AggregateFcts.maxFunctionForCounter);
- functions.add(AggregateFcts.minFunctionForCounter);
- }
- }
+ });
- return functions;
+ // min for all types
+ functions.add(new FunctionFactory("min", FunctionParameter.anyType(true))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ AbstractType<?> type = argTypes.get(0);
+ return type.isCounter() ? minFunctionForCounter : makeMinFunction(type);
+ }
+ });
}
/**
* The function used to count the number of rows of a result set. This function is called when COUNT(*) or COUNT(1)
* is specified.
*/
- public static final AggregateFunction countRowsFunction =
- new NativeAggregateFunction("countRows", LongType.instance)
+ public static final CountRowsFunction countRowsFunction = new CountRowsFunction(false);
+
+ public static class CountRowsFunction extends NativeAggregateFunction
+ {
+ private CountRowsFunction(boolean useLegacyName)
+ {
+ super(useLegacyName ? "countRows" : "count_rows", LongType.instance);
+ }
+
+ @Override
+ public Aggregate newAggregate()
+ {
+ return new Aggregate()
{
- public Aggregate newAggregate()
+ private long count;
+
+ public void reset()
{
- return new Aggregate()
- {
- private long count;
-
- public void reset()
- {
- count = 0;
- }
-
- public ByteBuffer compute(ProtocolVersion protocolVersion)
- {
- return LongType.instance.decompose(count);
- }
-
- public void addInput(ProtocolVersion protocolVersion, List<ByteBuffer> values)
- {
- count++;
- }
- };
+ count = 0;
}
- @Override
- public String columnName(List<String> columnNames)
+ public ByteBuffer compute(ProtocolVersion protocolVersion)
{
- return "count";
+ return LongType.instance.decompose(count);
+ }
+
+ public void addInput(ProtocolVersion protocolVersion, List<ByteBuffer> values)
+ {
+ count++;
}
};
+ }
+
+ @Override
+ public String columnName(List<String> columnNames)
+ {
+ return "count";
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new CountRowsFunction(true);
+ }
+ }
/**
* The SUM function for decimal values.
*/
- public static final AggregateFunction sumFunctionForDecimal =
+ public static final NativeAggregateFunction sumFunctionForDecimal =
new NativeAggregateFunction("sum", DecimalType.instance, DecimalType.instance)
{
@Override
@@ -165,8 +173,10 @@
/**
* The AVG function for decimal values.
+ * </p>
+ * The average of an empty value set returns zero.
*/
- public static final AggregateFunction avgFunctionForDecimal =
+ public static final NativeAggregateFunction avgFunctionForDecimal =
new NativeAggregateFunction("avg", DecimalType.instance, DecimalType.instance)
{
public Aggregate newAggregate()
@@ -209,7 +219,7 @@
/**
* The SUM function for varint values.
*/
- public static final AggregateFunction sumFunctionForVarint =
+ public static final NativeAggregateFunction sumFunctionForVarint =
new NativeAggregateFunction("sum", IntegerType.instance, IntegerType.instance)
{
public Aggregate newAggregate()
@@ -244,8 +254,11 @@
/**
* The AVG function for varint values.
+ * </p>
+ * The average of an empty value set returns zero. The returned value is of the same type as the input values,
+ * so the returned average won't have a decimal part.
*/
- public static final AggregateFunction avgFunctionForVarint =
+ public static final NativeAggregateFunction avgFunctionForVarint =
new NativeAggregateFunction("avg", IntegerType.instance, IntegerType.instance)
{
public Aggregate newAggregate()
@@ -287,8 +300,11 @@
/**
* The SUM function for byte values (tinyint).
+ * </p>
+ * The returned value is of the same type as the input values, so there is a risk of overflow if the sum of the
+ * values exceeds the maximum value that the type can represent.
*/
- public static final AggregateFunction sumFunctionForByte =
+ public static final NativeAggregateFunction sumFunctionForByte =
new NativeAggregateFunction("sum", ByteType.instance, ByteType.instance)
{
public Aggregate newAggregate()
@@ -323,8 +339,11 @@
/**
* AVG function for byte values (tinyint).
+ * </p>
+ * The average of an empty value set returns zero. The returned value is of the same type as the input values,
+ * so the returned average won't have a decimal part.
*/
- public static final AggregateFunction avgFunctionForByte =
+ public static final NativeAggregateFunction avgFunctionForByte =
new NativeAggregateFunction("avg", ByteType.instance, ByteType.instance)
{
public Aggregate newAggregate()
@@ -341,8 +360,11 @@
/**
* The SUM function for short values (smallint).
+ * </p>
+ * The returned value is of the same type as the input values, so there is a risk of overflow if the sum of the
+ * values exceeds the maximum value that the type can represent.
*/
- public static final AggregateFunction sumFunctionForShort =
+ public static final NativeAggregateFunction sumFunctionForShort =
new NativeAggregateFunction("sum", ShortType.instance, ShortType.instance)
{
public Aggregate newAggregate()
@@ -377,8 +399,11 @@
/**
* AVG function for for short values (smallint).
+ * </p>
+ * The average of an empty value set returns zero. The returned value is of the same type as the input values,
+ * so the returned average won't have a decimal part.
*/
- public static final AggregateFunction avgFunctionForShort =
+ public static final NativeAggregateFunction avgFunctionForShort =
new NativeAggregateFunction("avg", ShortType.instance, ShortType.instance)
{
public Aggregate newAggregate()
@@ -395,8 +420,11 @@
/**
* The SUM function for int32 values.
+ * </p>
+ * The returned value is of the same type as the input values, so there is a risk of overflow if the sum of the
+ * values exceeds the maximum value that the type can represent.
*/
- public static final AggregateFunction sumFunctionForInt32 =
+ public static final NativeAggregateFunction sumFunctionForInt32 =
new NativeAggregateFunction("sum", Int32Type.instance, Int32Type.instance)
{
public Aggregate newAggregate()
@@ -431,8 +459,11 @@
/**
* AVG function for int32 values.
+ * </p>
+ * The average of an empty value set returns zero. The returned value is of the same type as the input values,
+ * so the returned average won't have a decimal part.
*/
- public static final AggregateFunction avgFunctionForInt32 =
+ public static final NativeAggregateFunction avgFunctionForInt32 =
new NativeAggregateFunction("avg", Int32Type.instance, Int32Type.instance)
{
public Aggregate newAggregate()
@@ -449,8 +480,11 @@
/**
* The SUM function for long values.
+ * </p>
+ * The returned value is of the same type as the input values, so there is a risk of overflow if the sum of the
+ * values exceeds the maximum value that the type can represent.
*/
- public static final AggregateFunction sumFunctionForLong =
+ public static final NativeAggregateFunction sumFunctionForLong =
new NativeAggregateFunction("sum", LongType.instance, LongType.instance)
{
public Aggregate newAggregate()
@@ -461,8 +495,11 @@
/**
* AVG function for long values.
+ * </p>
+ * The average of an empty value set returns zero. The returned value is of the same type as the input values,
+ * so the returned average won't have a decimal part.
*/
- public static final AggregateFunction avgFunctionForLong =
+ public static final NativeAggregateFunction avgFunctionForLong =
new NativeAggregateFunction("avg", LongType.instance, LongType.instance)
{
public Aggregate newAggregate()
@@ -479,8 +516,11 @@
/**
* The SUM function for float values.
+ * </p>
+ * The returned value is of the same type as the input values, so there is a risk of overflow if the sum of the
+ * values exceeds the maximum value that the type can represent.
*/
- public static final AggregateFunction sumFunctionForFloat =
+ public static final NativeAggregateFunction sumFunctionForFloat =
new NativeAggregateFunction("sum", FloatType.instance, FloatType.instance)
{
public Aggregate newAggregate()
@@ -497,8 +537,10 @@
/**
* AVG function for float values.
+ * </p>
+ * The average of an empty value set returns zero.
*/
- public static final AggregateFunction avgFunctionForFloat =
+ public static final NativeAggregateFunction avgFunctionForFloat =
new NativeAggregateFunction("avg", FloatType.instance, FloatType.instance)
{
public Aggregate newAggregate()
@@ -515,8 +557,11 @@
/**
* The SUM function for double values.
+ * </p>
+ * The returned value is of the same type as the input values, so there is a risk of overflow if the sum of the
+ * values exceeds the maximum value that the type can represent.
*/
- public static final AggregateFunction sumFunctionForDouble =
+ public static final NativeAggregateFunction sumFunctionForDouble =
new NativeAggregateFunction("sum", DoubleType.instance, DoubleType.instance)
{
public Aggregate newAggregate()
@@ -541,9 +586,9 @@
private double compensation;
private double simpleSum;
- private final AbstractType numberType;
+ private final AbstractType<?> numberType;
- public FloatSumAggregate(AbstractType numberType)
+ public FloatSumAggregate(AbstractType<?> numberType)
{
this.numberType = numberType;
}
@@ -599,9 +644,9 @@
private BigDecimal bigSum = null;
private boolean overflow = false;
- private final AbstractType numberType;
+ private final AbstractType<?> numberType;
- public FloatAvgAggregate(AbstractType numberType)
+ public FloatAvgAggregate(AbstractType<?> numberType)
{
this.numberType = numberType;
}
@@ -675,8 +720,10 @@
/**
* AVG function for double values.
+ * </p>
+ * The average of an empty value set returns zero.
*/
- public static final AggregateFunction avgFunctionForDouble =
+ public static final NativeAggregateFunction avgFunctionForDouble =
new NativeAggregateFunction("avg", DoubleType.instance, DoubleType.instance)
{
public Aggregate newAggregate()
@@ -694,7 +741,7 @@
/**
* The SUM function for counter column values.
*/
- public static final AggregateFunction sumFunctionForCounter =
+ public static final NativeAggregateFunction sumFunctionForCounter =
new NativeAggregateFunction("sum", CounterColumnType.instance, CounterColumnType.instance)
{
public Aggregate newAggregate()
@@ -706,7 +753,7 @@
/**
* AVG function for counter column values.
*/
- public static final AggregateFunction avgFunctionForCounter =
+ public static final NativeAggregateFunction avgFunctionForCounter =
new NativeAggregateFunction("avg", CounterColumnType.instance, CounterColumnType.instance)
{
public Aggregate newAggregate()
@@ -724,7 +771,7 @@
/**
* The MIN function for counter column values.
*/
- public static final AggregateFunction minFunctionForCounter =
+ public static final NativeAggregateFunction minFunctionForCounter =
new NativeAggregateFunction("min", CounterColumnType.instance, CounterColumnType.instance)
{
public Aggregate newAggregate()
@@ -762,7 +809,7 @@
/**
* MAX function for counter column values.
*/
- public static final AggregateFunction maxFunctionForCounter =
+ public static final NativeAggregateFunction maxFunctionForCounter =
new NativeAggregateFunction("max", CounterColumnType.instance, CounterColumnType.instance)
{
public Aggregate newAggregate()
@@ -803,7 +850,7 @@
* @param inputType the function input and output type
* @return a MAX function for the specified type.
*/
- public static AggregateFunction makeMaxFunction(final AbstractType<?> inputType)
+ public static NativeAggregateFunction makeMaxFunction(final AbstractType<?> inputType)
{
return new NativeAggregateFunction("max", inputType, inputType)
{
@@ -844,7 +891,7 @@
* @param inputType the function input and output type
* @return a MIN function for the specified type.
*/
- public static AggregateFunction makeMinFunction(final AbstractType<?> inputType)
+ public static NativeAggregateFunction makeMinFunction(final AbstractType<?> inputType)
{
return new NativeAggregateFunction("min", inputType, inputType)
{
@@ -885,7 +932,7 @@
* @param inputType the function input type
* @return a COUNT function for the specified type.
*/
- public static AggregateFunction makeCountFunction(AbstractType<?> inputType)
+ public static NativeAggregateFunction makeCountFunction(AbstractType<?> inputType)
{
return new NativeAggregateFunction("count", LongType.instance, inputType)
{
@@ -957,9 +1004,9 @@
private BigInteger bigSum = null;
private boolean overflow = false;
- private final AbstractType numberType;
+ private final AbstractType<?> numberType;
- public AvgAggregate(AbstractType type)
+ public AvgAggregate(AbstractType<?> type)
{
this.numberType = type;
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/BytesConversionFcts.java b/src/java/org/apache/cassandra/cql3/functions/BytesConversionFcts.java
index 7e9708a..8a8eb9c 100644
--- a/src/java/org/apache/cassandra/cql3/functions/BytesConversionFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/BytesConversionFcts.java
@@ -18,98 +18,104 @@
package org.apache.cassandra.cql3.functions;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
public abstract class BytesConversionFcts
{
- public static Collection<Function> all()
+ public static void addFunctionsTo(NativeFunctions functions)
{
- Collection<Function> functions = new ArrayList<>();
-
- // because text and varchar ends up being synonymous, our automatic makeToBlobFunction doesn't work
- // for varchar, so we special case it below. We also skip blob for obvious reasons.
- Set<AbstractType<?>> types = new HashSet<>();
for (CQL3Type type : CQL3Type.Native.values())
{
if (type == CQL3Type.Native.BLOB)
continue;
- AbstractType<?> udfType = type.getType().udfType();
- if (!types.add(udfType))
- continue;
- functions.add(makeToBlobFunction(type.getType().udfType()));
- functions.add(makeFromBlobFunction(type.getType().udfType()));
+ functions.add(new ToBlobFunction(type));
+ functions.add(new FromBlobFunction(type));
+ }
+ }
+
+ // Most of the X_as_blob and blob_as_X functions are basically no-op since everything is
+ // bytes internally. They only "trick" the type system.
+ public static class ToBlobFunction extends NativeScalarFunction
+ {
+ private final CQL3Type fromType;
+
+ public ToBlobFunction(CQL3Type fromType)
+ {
+ this(fromType, false);
}
- functions.add(VarcharAsBlobFct);
- functions.add(BlobAsVarcharFct);
+ private ToBlobFunction(CQL3Type fromType, boolean useLegacyName)
+ {
+ super(fromType + (useLegacyName ? "asblob" : "_as_blob"),
+ BytesType.instance,
+ fromType.getType().udfType());
+ this.fromType = fromType;
+ }
- return functions;
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ return parameters.get(0);
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new ToBlobFunction(fromType, true);
+ }
}
- // Most of the XAsBlob and blobAsX functions are basically no-op since everything is
- // bytes internally. They only "trick" the type system.
- public static Function makeToBlobFunction(AbstractType<?> fromType)
+ public static class FromBlobFunction extends NativeScalarFunction
{
- String name = fromType.asCQL3Type() + "asblob";
- return new NativeScalarFunction(name, BytesType.instance, fromType)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- return parameters.get(0);
- }
- };
- }
+ private final CQL3Type toType;
- public static Function makeFromBlobFunction(final AbstractType<?> toType)
- {
- final String name = "blobas" + toType.asCQL3Type();
- return new NativeScalarFunction(name, toType, BytesType.instance)
+ public FromBlobFunction(CQL3Type toType)
{
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters) throws InvalidRequestException
+ this(toType, false);
+ }
+
+ private FromBlobFunction(CQL3Type toType, boolean useLegacyName)
+ {
+ super((useLegacyName ? "blobas" : "blob_as_") + toType,
+ toType.getType().udfType(),
+ BytesType.instance);
+ this.toType = toType;
+ }
+
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer val = parameters.get(0);
+
+ if (val != null)
{
- ByteBuffer val = parameters.get(0);
try
{
- if (val != null)
- toType.validate(val);
- return val;
+ toType.getType().validate(val);
}
catch (MarshalException e)
{
- throw new InvalidRequestException(String.format("In call to function %s, value 0x%s is not a valid binary representation for type %s",
- name, ByteBufferUtil.bytesToHex(val), toType.asCQL3Type()));
+ throw new InvalidRequestException(String.format("In call to function %s, value 0x%s is not a " +
+ "valid binary representation for type %s",
+ name, ByteBufferUtil.bytesToHex(val), toType));
}
}
- };
+
+ return val;
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new FromBlobFunction(toType, true);
+ }
}
-
- public static final Function VarcharAsBlobFct = new NativeScalarFunction("varcharasblob", BytesType.instance, UTF8Type.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- return parameters.get(0);
- }
- };
-
- public static final Function BlobAsVarcharFct = new NativeScalarFunction("blobasvarchar", UTF8Type.instance, BytesType.instance)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- return parameters.get(0);
- }
- };
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/CastFcts.java b/src/java/org/apache/cassandra/cql3/functions/CastFcts.java
index 81986c8..1761778 100644
--- a/src/java/org/apache/cassandra/cql3/functions/CastFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/CastFcts.java
@@ -20,8 +20,6 @@
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import org.apache.cassandra.cql3.CQL3Type;
@@ -48,7 +46,7 @@
import static org.apache.cassandra.cql3.functions.TimeFcts.*;
-import org.apache.commons.lang3.text.WordUtils;
+import org.apache.commons.lang3.StringUtils;
/**
* Casting functions
@@ -56,12 +54,14 @@
*/
public final class CastFcts
{
- private static final String FUNCTION_NAME_PREFIX = "castAs";
+ private static final String FUNCTION_NAME_PREFIX = "cast_as_";
- public static Collection<Function> all()
+ // Until 5.0 we have used camel cased names for cast functions. That changed with CASSANDRA-18037, where we decided
+ // to adopt snake case for all native function names. However, we should still support the old came case names.
+ private static final String LEGACY_FUNCTION_NAME_PREFIX = "castAs";
+
+ public static void addFunctionsTo(NativeFunctions functions)
{
- List<Function> functions = new ArrayList<>();
-
@SuppressWarnings("unchecked")
final AbstractType<? extends Number>[] numericTypes = new AbstractType[] {ByteType.instance,
ShortType.instance,
@@ -110,8 +110,6 @@
functions.add(CastAsTextFunction.create(UUIDType.instance, AsciiType.instance));
functions.add(CastAsTextFunction.create(UUIDType.instance, UTF8Type.instance));
-
- return functions;
}
/**
@@ -132,25 +130,38 @@
}
/**
- * Creates the name of the cast function use to cast to the specified type.
+ * Creates the snake-cased name of the cast function used to cast to the specified type.
*
* @param outputType the output type
- * @return the name of the cast function use to cast to the specified type
- */
- public static String getFunctionName(AbstractType<?> outputType)
- {
- return getFunctionName(outputType.asCQL3Type());
- }
-
- /**
- * Creates the name of the cast function use to cast to the specified type.
- *
- * @param outputType the output type
- * @return the name of the cast function use to cast to the specified type
+ * @return the name of the cast function used to cast to the specified type
*/
public static String getFunctionName(CQL3Type outputType)
{
- return FUNCTION_NAME_PREFIX + WordUtils.capitalize(toLowerCaseString(outputType));
+ return FUNCTION_NAME_PREFIX + toLowerCaseString(outputType);
+ }
+
+ /**
+ * Creates the legacy camel-cased name of the cast function used to cast to the specified type.
+ *
+ * @param outputType the output type
+ * @return the legacy camel-cased name of the cast function used to cast to the specified type
+ */
+ private static String getLegacyFunctionName(CQL3Type outputType)
+ {
+ return LEGACY_FUNCTION_NAME_PREFIX + StringUtils.capitalize(toLowerCaseString(outputType));
+ }
+
+ /**
+ * Creates the name of the cast function used to cast to the specified type.
+ *
+ * @param outputType the output type
+ * @param legacy whether to use the old cameCase names, instead of the new snake_case names
+ * @return the name of the cast function used to cast to the specified type
+ */
+ private static String getFunctionName(AbstractType<?> outputType, boolean legacy)
+ {
+ CQL3Type type = outputType.asCQL3Type();
+ return legacy ? getLegacyFunctionName(type) : getFunctionName(type);
}
/**
@@ -161,7 +172,7 @@
* @param outputType the output type
* @param converter the function use to convert the input type into the output type
*/
- private static <I, O> void addFunctionIfNeeded(List<Function> functions,
+ private static <I, O> void addFunctionIfNeeded(NativeFunctions functions,
AbstractType<I> inputType,
AbstractType<O> outputType,
java.util.function.Function<I, O> converter)
@@ -171,9 +182,9 @@
}
@SuppressWarnings("unchecked")
- private static <O, I> Function wrapJavaFunction(AbstractType<I> inputType,
- AbstractType<O> outputType,
- java.util.function.Function<I, O> converter)
+ private static <O, I> CastFunction<?, O> wrapJavaFunction(AbstractType<I> inputType,
+ AbstractType<O> outputType,
+ java.util.function.Function<I, O> converter)
{
return inputType.equals(CounterColumnType.instance)
? JavaCounterFunctionWrapper.create(outputType, (java.util.function.Function<Long, O>) converter)
@@ -193,9 +204,9 @@
*/
private static abstract class CastFunction<I, O> extends NativeScalarFunction
{
- public CastFunction(AbstractType<I> inputType, AbstractType<O> outputType)
+ public CastFunction(AbstractType<I> inputType, AbstractType<O> outputType, boolean useLegacyName)
{
- super(getFunctionName(outputType), outputType, inputType);
+ super(getFunctionName(outputType, useLegacyName), outputType, inputType);
}
@Override
@@ -228,23 +239,30 @@
/**
* The java function used to convert the input type into the output one.
*/
- private final java.util.function.Function<I, O> converter;
+ protected final java.util.function.Function<I, O> converter;
public static <I, O> JavaFunctionWrapper<I, O> create(AbstractType<I> inputType,
AbstractType<O> outputType,
java.util.function.Function<I, O> converter)
{
- return new JavaFunctionWrapper<I, O>(inputType, outputType, converter);
+ return new JavaFunctionWrapper<>(inputType, outputType, converter, false);
}
protected JavaFunctionWrapper(AbstractType<I> inputType,
AbstractType<O> outputType,
- java.util.function.Function<I, O> converter)
+ java.util.function.Function<I, O> converter,
+ boolean useLegacyName)
{
- super(inputType, outputType);
+ super(inputType, outputType, useLegacyName);
this.converter = converter;
}
+ @Override
+ public JavaFunctionWrapper<I, O> withLegacyName()
+ {
+ return new JavaFunctionWrapper<>(inputType(), outputType(), converter, true);
+ }
+
public final ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
ByteBuffer bb = parameters.get(0);
@@ -273,13 +291,20 @@
public static <O> JavaFunctionWrapper<Long, O> create(AbstractType<O> outputType,
java.util.function.Function<Long, O> converter)
{
- return new JavaCounterFunctionWrapper<O>(outputType, converter);
+ return new JavaCounterFunctionWrapper<>(outputType, converter, false);
}
protected JavaCounterFunctionWrapper(AbstractType<O> outputType,
- java.util.function.Function<Long, O> converter)
+ java.util.function.Function<Long, O> converter,
+ boolean useLegacyName)
{
- super(CounterColumnType.instance, outputType, converter);
+ super(CounterColumnType.instance, outputType, converter, useLegacyName);
+ }
+
+ @Override
+ public JavaCounterFunctionWrapper<O> withLegacyName()
+ {
+ return new JavaCounterFunctionWrapper<>(outputType(), converter, true);
}
protected Long compose(ByteBuffer bb)
@@ -305,19 +330,26 @@
AbstractType<O> outputType,
NativeScalarFunction delegate)
{
- return new CassandraFunctionWrapper<I, O>(inputType, outputType, delegate);
+ return new CassandraFunctionWrapper<>(inputType, outputType, delegate, false);
}
private CassandraFunctionWrapper(AbstractType<I> inputType,
AbstractType<O> outputType,
- NativeScalarFunction delegate)
+ NativeScalarFunction delegate,
+ boolean useLegacyName)
{
- super(inputType, outputType);
+ super(inputType, outputType, useLegacyName);
assert delegate.argTypes().size() == 1 && inputType.equals(delegate.argTypes().get(0));
assert outputType.equals(delegate.returnType());
this.delegate = delegate;
}
+ @Override
+ public CassandraFunctionWrapper<I, O> withLegacyName()
+ {
+ return new CassandraFunctionWrapper<>(inputType(), outputType(), delegate, true);
+ }
+
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
return delegate.execute(protocolVersion, parameters);
@@ -335,13 +367,18 @@
public static <I> CastAsTextFunction<I> create(AbstractType<I> inputType,
AbstractType<String> outputType)
{
- return new CastAsTextFunction<I>(inputType, outputType);
+ return new CastAsTextFunction<>(inputType, outputType, false);
}
- private CastAsTextFunction(AbstractType<I> inputType,
- AbstractType<String> outputType)
+ private CastAsTextFunction(AbstractType<I> inputType, AbstractType<String> outputType, boolean useLegacyName)
{
- super(inputType, outputType);
+ super(inputType, outputType, useLegacyName);
+ }
+
+ @Override
+ public CastAsTextFunction<I> withLegacyName()
+ {
+ return new CastAsTextFunction<>(inputType(), outputType(), true);
}
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
diff --git a/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java b/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java
new file mode 100644
index 0000000..ebc96b7
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/CollectionFcts.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * Native CQL functions for collections (sets, list and maps).
+ * <p>
+ * All the functions provided here are {@link NativeScalarFunction}, and they are meant to be applied to single
+ * collection values to perform some kind of aggregation with the elements of the collection argument. When possible,
+ * the implementation of these aggregation functions is based on the accross-rows aggregation functions available on
+ * {@link AggregateFcts}, so both across-rows and within-collection aggregations have the same behaviour.
+ */
+public class CollectionFcts
+{
+ public static void addFunctionsTo(NativeFunctions functions)
+ {
+ functions.add(new FunctionFactory("map_keys", FunctionParameter.anyMap())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeMapKeysFunction(name.name, (MapType<?, ?>) argTypes.get(0));
+ }
+ });
+
+ functions.add(new FunctionFactory("map_values", FunctionParameter.anyMap())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeMapValuesFunction(name.name, (MapType<?, ?>) argTypes.get(0));
+ }
+ });
+
+ functions.add(new FunctionFactory("collection_count", FunctionParameter.anyCollection())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeCollectionCountFunction(name.name, (CollectionType<?>) argTypes.get(0));
+ }
+ });
+
+ functions.add(new FunctionFactory("collection_min", FunctionParameter.setOrList())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeCollectionMinFunction(name.name, (CollectionType<?>) argTypes.get(0));
+ }
+ });
+
+ functions.add(new FunctionFactory("collection_max", FunctionParameter.setOrList())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeCollectionMaxFunction(name.name, (CollectionType<?>) argTypes.get(0));
+ }
+ });
+
+ functions.add(new FunctionFactory("collection_sum", FunctionParameter.numericSetOrList())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeCollectionSumFunction(name.name, (CollectionType<?>) argTypes.get(0));
+ }
+ });
+
+ functions.add(new FunctionFactory("collection_avg", FunctionParameter.numericSetOrList())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return makeCollectionAvgFunction(name.name, (CollectionType<?>) argTypes.get(0));
+ }
+ });
+ }
+
+ /**
+ * Returns a native scalar function for getting the keys of a map column, as a set.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the map argument by the returned function
+ * @param <K> the class of the map argument keys
+ * @param <V> the class of the map argument values
+ * @return a function returning a serialized set containing the keys of the map passed as argument
+ */
+ private static <K, V> NativeScalarFunction makeMapKeysFunction(String name, MapType<K, V> inputType)
+ {
+ SetType<K> outputType = SetType.getInstance(inputType.getKeysType(), false);
+
+ return new NativeScalarFunction(name, outputType, inputType)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer value = parameters.get(0);
+ if (value == null)
+ return null;
+
+ Map<K, V> map = inputType.compose(value);
+ Set<K> keys = map.keySet();
+ return outputType.decompose(keys);
+ }
+ };
+ }
+
+ /**
+ * Returns a native scalar function for getting the values of a map column, as a list.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the map argument accepted by the returned function
+ * @param <K> the class of the map argument keys
+ * @param <V> the class of the map argument values
+ * @return a function returning a serialized list containing the values of the map passed as argument
+ */
+ private static <K, V> NativeScalarFunction makeMapValuesFunction(String name, MapType<K, V> inputType)
+ {
+ ListType<V> outputType = ListType.getInstance(inputType.getValuesType(), false);
+
+ return new NativeScalarFunction(name, outputType, inputType)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer value = parameters.get(0);
+ if (value == null)
+ return null;
+
+ Map<K, V> map = inputType.compose(value);
+ List<V> values = ImmutableList.copyOf(map.values());
+ return outputType.decompose(values);
+ }
+ };
+ }
+
+ /**
+ * Returns a native scalar function for getting the number of elements in a collection.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the collection argument accepted by the returned function
+ * @param <T> the type of the elements of the collection argument
+ * @return a function returning the number of elements in the collection passed as argument, as a 32-bit integer
+ */
+ private static <T> NativeScalarFunction makeCollectionCountFunction(String name, CollectionType<T> inputType)
+ {
+ return new NativeScalarFunction(name, Int32Type.instance, inputType)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer value = parameters.get(0);
+ if (value == null)
+ return null;
+
+ int size = inputType.size(value);
+ return Int32Type.instance.decompose(size);
+ }
+ };
+ }
+
+ /**
+ * Returns a native scalar function for getting the min element in a collection.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the collection argument accepted by the returned function
+ * @param <T> the type of the elements of the collection argument
+ * @return a function returning the min element in the collection passed as argument
+ */
+ private static <T> NativeScalarFunction makeCollectionMinFunction(String name, CollectionType<T> inputType)
+ {
+ AbstractType<?> elementsType = elementsType(inputType);
+ NativeAggregateFunction function = elementsType.isCounter()
+ ? AggregateFcts.minFunctionForCounter
+ : AggregateFcts.makeMinFunction(elementsType);
+ return new CollectionAggregationFunction(name, inputType, function);
+ }
+
+ /**
+ * Returns a native scalar function for getting the max element in a collection.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the collection argument accepted by the returned function
+ * @param <T> the type of the elements of the collection argument
+ * @return a function returning the max element in the collection passed as argument
+ */
+ private static <T> NativeScalarFunction makeCollectionMaxFunction(String name, CollectionType<T> inputType)
+ {
+ AbstractType<?> elementsType = elementsType(inputType);
+ NativeAggregateFunction function = elementsType.isCounter()
+ ? AggregateFcts.maxFunctionForCounter
+ : AggregateFcts.makeMaxFunction(elementsType);
+ return new CollectionAggregationFunction(name, inputType, function);
+ }
+
+ /**
+ * Returns a native scalar function for getting the sum of the elements in a numeric collection.
+ * </p>
+ * The value returned by the function is of the same type as elements of its input collection, so there is a risk
+ * of overflow if the sum of the values exceeds the maximum value that the type can represent.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the collection argument accepted by the returned function
+ * @param <T> the type of the elements of the collection argument
+ * @return a function returning the sum of the elements in the collection passed as argument
+ */
+ private static <T> NativeScalarFunction makeCollectionSumFunction(String name, CollectionType<T> inputType)
+ {
+ CQL3Type elementsType = elementsType(inputType).asCQL3Type();
+ NativeAggregateFunction function = getSumFunction((CQL3Type.Native) elementsType);
+ return new CollectionAggregationFunction(name, inputType, function);
+ }
+
+ private static NativeAggregateFunction getSumFunction(CQL3Type.Native type)
+ {
+ switch (type)
+ {
+ case TINYINT:
+ return AggregateFcts.sumFunctionForByte;
+ case SMALLINT:
+ return AggregateFcts.sumFunctionForShort;
+ case INT:
+ return AggregateFcts.sumFunctionForInt32;
+ case BIGINT:
+ return AggregateFcts.sumFunctionForLong;
+ case FLOAT:
+ return AggregateFcts.sumFunctionForFloat;
+ case DOUBLE:
+ return AggregateFcts.sumFunctionForDouble;
+ case VARINT:
+ return AggregateFcts.sumFunctionForVarint;
+ case DECIMAL:
+ return AggregateFcts.sumFunctionForDecimal;
+ default:
+ throw new AssertionError("Expected numeric collection but found " + type);
+ }
+ }
+
+ /**
+ * Returns a native scalar function for getting the average of the elements in a numeric collection.
+ * </p>
+ * The average of an empty collection returns zero. The value returned by the function is of the same type as the
+ * elements of its input collection, so if those don't have a decimal part then the returned average won't have a
+ * decimal part either.
+ *
+ * @param name the name of the function
+ * @param inputType the type of the collection argument accepted by the returned function
+ * @param <T> the type of the elements of the collection argument
+ * @return a function returning the average value of the elements in the collection passed as argument
+ */
+ private static <T> NativeScalarFunction makeCollectionAvgFunction(String name, CollectionType<T> inputType)
+ {
+ CQL3Type elementsType = elementsType(inputType).asCQL3Type();
+ NativeAggregateFunction function = getAvgFunction((CQL3Type.Native) elementsType);
+ return new CollectionAggregationFunction(name, inputType, function);
+ }
+
+ private static NativeAggregateFunction getAvgFunction(CQL3Type.Native type)
+ {
+ switch (type)
+ {
+ case TINYINT:
+ return AggregateFcts.avgFunctionForByte;
+ case SMALLINT:
+ return AggregateFcts.avgFunctionForShort;
+ case INT:
+ return AggregateFcts.avgFunctionForInt32;
+ case BIGINT:
+ return AggregateFcts.avgFunctionForLong;
+ case FLOAT:
+ return AggregateFcts.avgFunctionForFloat;
+ case DOUBLE:
+ return AggregateFcts.avgFunctionForDouble;
+ case VARINT:
+ return AggregateFcts.avgFunctionForVarint;
+ case DECIMAL:
+ return AggregateFcts.avgFunctionForDecimal;
+ default:
+ throw new AssertionError("Expected numeric collection but found " + type);
+ }
+ }
+
+ /**
+ * @return the type of the elements of the specified collection type.
+ */
+ private static AbstractType<?> elementsType(CollectionType<?> type)
+ {
+ if (type.kind == CollectionType.Kind.LIST)
+ {
+ return ((ListType<?>) type).getElementsType();
+ }
+
+ if (type.kind == CollectionType.Kind.SET)
+ {
+ return ((SetType<?>) type).getElementsType();
+ }
+
+ throw new AssertionError("Cannot get the element type of: " + type);
+ }
+
+ /**
+ * A {@link NativeScalarFunction} for aggregating the elements of a collection according to the aggregator of
+ * a certain {@link NativeAggregateFunction}.
+ * <p>
+ * {@link NativeAggregateFunction} is meant to be used for aggregating values accross rows, but here we use that
+ * function to aggregate the elements of a single collection value. That way, functions such as {@code avg} and
+ * {@code collection_avg} should have the same behaviour when applied to row columns or collection elements.
+ */
+ private static class CollectionAggregationFunction extends NativeScalarFunction
+ {
+ private final CollectionType<?> inputType;
+ private final NativeAggregateFunction aggregateFunction;
+
+ public CollectionAggregationFunction(String name,
+ CollectionType<?> inputType,
+ NativeAggregateFunction aggregateFunction)
+ {
+ super(name, aggregateFunction.returnType, inputType);
+ this.inputType = inputType;
+ this.aggregateFunction = aggregateFunction;
+ }
+
+ @Override
+ public ByteBuffer execute(ProtocolVersion version, List<ByteBuffer> parameters)
+ {
+ ByteBuffer value = parameters.get(0);
+ if (value == null)
+ return null;
+
+ AggregateFunction.Aggregate aggregate = aggregateFunction.newAggregate();
+ inputType.forEach(value, element -> aggregate.addInput(version, Collections.singletonList(element)));
+ return aggregate.compute(version);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java b/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java
index 01a6e20..34a7799 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FromJsonFct.java
@@ -22,39 +22,41 @@
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.cassandra.cql3.Json;
+import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.FunctionExecutionException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.JsonUtils;
+
+import static java.lang.String.format;
public class FromJsonFct extends NativeScalarFunction
{
- public static final FunctionName NAME = FunctionName.nativeFunction("fromjson");
-
private static final Map<AbstractType<?>, FromJsonFct> instances = new ConcurrentHashMap<>();
- public static FromJsonFct getInstance(AbstractType<?> returnType)
+ public static FromJsonFct getInstance(FunctionName name, AbstractType<?> returnType)
{
FromJsonFct func = instances.get(returnType);
if (func == null)
{
- func = new FromJsonFct(returnType);
+ func = new FromJsonFct(name, returnType);
instances.put(returnType, func);
}
return func;
}
- private FromJsonFct(AbstractType<?> returnType)
+ private FromJsonFct(FunctionName name, AbstractType<?> returnType)
{
- super("fromjson", returnType, UTF8Type.instance);
+ super(name.name, returnType, UTF8Type.instance);
}
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
- assert parameters.size() == 1 : "Unexpectedly got " + parameters.size() + " arguments for fromJson()";
+ assert parameters.size() == 1 : format("Unexpectedly got %d arguments for %s()", parameters.size(), name.name);
ByteBuffer argument = parameters.get(0);
if (argument == null)
return null;
@@ -62,18 +64,43 @@
String jsonArg = UTF8Type.instance.getSerializer().deserialize(argument);
try
{
- Object object = Json.JSON_OBJECT_MAPPER.readValue(jsonArg, Object.class);
+ Object object = JsonUtils.JSON_OBJECT_MAPPER.readValue(jsonArg, Object.class);
if (object == null)
return null;
return returnType.fromJSONObject(object).bindAndGet(QueryOptions.forProtocolVersion(protocolVersion));
}
catch (IOException exc)
{
- throw FunctionExecutionException.create(NAME, Collections.singletonList("text"), String.format("Could not decode JSON string '%s': %s", jsonArg, exc.toString()));
+ throw FunctionExecutionException.create(name(),
+ Collections.singletonList("text"),
+ format("Could not decode JSON string '%s': %s", jsonArg, exc));
}
catch (MarshalException exc)
{
throw FunctionExecutionException.create(this, exc);
}
}
+
+ public static void addFunctionsTo(NativeFunctions functions)
+ {
+ functions.add(new Factory("from_json"));
+ functions.add(new Factory("fromjson"));
+ }
+
+ private static class Factory extends FunctionFactory
+ {
+ private Factory(String name)
+ {
+ super(name, FunctionParameter.fixed(CQL3Type.Native.TEXT));
+ }
+
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ if (receiverType == null)
+ throw new InvalidRequestException(format("%s() cannot be used in the selection clause of a SELECT statement", name.name));
+
+ return FromJsonFct.getInstance(name, receiverType);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
index 4fb1ba3..3369657 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
@@ -107,14 +107,14 @@
return null;
if (fun.returnType().isCollection())
{
- switch (((CollectionType) fun.returnType()).kind)
+ switch (((CollectionType<?>) fun.returnType()).kind)
{
case LIST:
- return Lists.Value.fromSerialized(result, (ListType) fun.returnType(), version);
+ return Lists.Value.fromSerialized(result, (ListType<?>) fun.returnType());
case SET:
- return Sets.Value.fromSerialized(result, (SetType) fun.returnType(), version);
+ return Sets.Value.fromSerialized(result, (SetType<?>) fun.returnType());
case MAP:
- return Maps.Value.fromSerialized(result, (MapType) fun.returnType(), version);
+ return Maps.Value.fromSerialized(result, (MapType<?, ?>) fun.returnType());
}
}
else if (fun.returnType().isUDT())
@@ -127,7 +127,7 @@
public static class Raw extends Term.Raw
{
- private FunctionName name;
+ private final FunctionName name;
private final List<Term.Raw> terms;
public Raw(FunctionName name, List<Term.Raw> terms)
@@ -202,10 +202,11 @@
{
Function fun = FunctionResolver.get(keyspace, name, terms, receiver.ksName, receiver.cfName, receiver.type);
- // Because fromJson() can return whatever type the receiver is, we'll always get EXACT_MATCH. To
- // handle potentially ambiguous function calls with fromJson() as an argument, always return
- // WEAKLY_ASSIGNABLE to force the user to typecast if necessary
- if (fun != null && fun.name().equals(FromJsonFct.NAME))
+ // Because the return type of functions built by factories is not fixed but depending on the types of
+ // their arguments, we'll always get EXACT_MATCH. To handle potentially ambiguous function calls with
+ // dynamically built functions as an argument, always return WEAKLY_ASSIGNABLE to force the user to
+ // typecast if necessary
+ if (fun != null && NativeFunctions.instance.hasFactory(fun.name()))
return TestResult.WEAKLY_ASSIGNABLE;
if (fun != null && receiver.type.udfType().equals(fun.returnType()))
@@ -221,14 +222,18 @@
}
}
+ @Override
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
- // We could implement this, but the method is only used in selection clause, where FunctionCall is not used
- // we use a Selectable.WithFunction instead). And if that method is later used in other places, better to
- // let that future patch make sure this can be implemented properly (note in particular we don't have access
- // to the receiver type, which FunctionResolver.get() takes) rather than provide an implementation that may
- // not work in all cases.
- throw new UnsupportedOperationException();
+ try
+ {
+ Function fun = FunctionResolver.get(keyspace, name, terms, null, null, null);
+ return fun == null ? null : fun.returnType();
+ }
+ catch (InvalidRequestException e)
+ {
+ return null;
+ }
}
public String getText()
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java b/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java
new file mode 100644
index 0000000..32db9fe
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionFactory.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.cql3.AssignmentTestable;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.SchemaConstants;
+
+/**
+ * Class for dynamically building different overloads of a CQL {@link Function} according to specific function calls.
+ * <p>
+ * For example, the factory for the {@code max} function will return a {@code (text) -> text} function if it's called
+ * with an {@code text} argument, like in {@code max('abc')}. It however will return a {@code (list<int>) -> list<int>}
+ * function if it's called with {@code max([1,2,3])}, etc.
+ * <p>
+ * This is meant to be used to create functions that require too many overloads to have them pre-created in memory. Note
+ * that in the case of functions accepting collections, tuples or UDTs the number of overloads is potentially infinite.
+ */
+public abstract class FunctionFactory
+{
+ /** The name of the built functions. */
+ protected final FunctionName name;
+
+ /** The accepted parameters. */
+ protected final List<FunctionParameter> parameters;
+
+ private final int numParameters;
+ private final int numMandatoryParameters;
+
+ /**
+ * @param name the name of the built functions
+ * @param parameters the accepted parameters
+ */
+ public FunctionFactory(String name, FunctionParameter... parameters)
+ {
+ this.name = FunctionName.nativeFunction(name);
+ this.parameters = Arrays.asList(parameters);
+ this.numParameters = parameters.length;
+ this.numMandatoryParameters = (int) this.parameters.stream().filter(p -> !p.isOptional()).count();
+ }
+
+ public FunctionName name()
+ {
+ return name;
+ }
+
+ /**
+ * Returns a function with a signature compatible with the specified function call.
+ *
+ * @param args the arguments in the function call for which the function is going to be built
+ * @param receiverType the expected return type of the function call for which the function is going to be built
+ * @param receiverKeyspace the name of the recevier keyspace
+ * @param receiverTable the name of the recevier table
+ * @return a function with a signature compatible with the specified function call, or {@code null} if the factory
+ * cannot create a function for the supplied arguments but there might be another factory with the same
+ * {@link #name()} able to do it.
+ */
+ @Nullable
+ public NativeFunction getOrCreateFunction(List<? extends AssignmentTestable> args,
+ AbstractType<?> receiverType,
+ String receiverKeyspace,
+ String receiverTable)
+ {
+ // validate the number of arguments
+ int numArgs = args.size();
+ if (numArgs < numMandatoryParameters || numArgs > numParameters)
+ throw invalidNumberOfArgumentsException();
+
+ // try to infer the types of the arguments
+ List<AbstractType<?>> types = new ArrayList<>(args.size());
+ for (int i = 0; i < args.size(); i++)
+ {
+ AssignmentTestable arg = args.get(i);
+ FunctionParameter parameter = parameters.get(i);
+ AbstractType<?> type = parameter.inferType(SchemaConstants.SYSTEM_KEYSPACE_NAME, arg, receiverType, types);
+ if (type == null)
+ throw new InvalidRequestException(String.format("Cannot infer type of argument %s in call to " +
+ "function %s: use type casts to disambiguate",
+ arg, this));
+ parameter.validateType(name, arg, type);
+ type = type.udfType();
+ types.add(type);
+ }
+
+ return doGetOrCreateFunction(types, receiverType);
+ }
+
+ public InvalidRequestException invalidNumberOfArgumentsException()
+ {
+ return new InvalidRequestException("Invalid number of arguments for function " + this);
+ }
+
+ /**
+ * Returns a function compatible with the specified signature.
+ *
+ * @param argTypes the types of the function arguments
+ * @param receiverType the expected return type of the function
+ * @return a function compatible with the specified signature, or {@code null} if this cannot create a function for
+ * the supplied arguments but there might be another factory with the same {@link #name()} able to do it.
+ */
+ @Nullable
+ protected abstract NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType);
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s(%s)", name, parameters.stream().map(Object::toString).collect(Collectors.joining(", ")));
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java b/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java
new file mode 100644
index 0000000..083b0f6
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionParameter.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.cql3.AssignmentTestable;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+
+/**
+ * Generic, loose definition of a function parameter, able to infer the specific data type of the parameter in the
+ * function specifically built by a {@link FunctionFactory} for a particular function call.
+ */
+public interface FunctionParameter
+{
+ /**
+ * Tries to infer the data type of the parameter for an argument in a call to the function.
+ *
+ * @param keyspace the current keyspace
+ * @param arg a parameter value in a specific function call
+ * @param receiverType the type of the object that will receive the result of the function call
+ * @return the inferred data type of the parameter, or {@link null} it isn't possible to infer it
+ */
+ @Nullable
+ default AbstractType<?> inferType(String keyspace,
+ AssignmentTestable arg,
+ @Nullable AbstractType<?> receiverType,
+ List<AbstractType<?>> previousTypes)
+ {
+ return arg.getCompatibleTypeIfKnown(keyspace);
+ }
+
+ void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType);
+
+ /**
+ * @return whether this parameter is optional
+ */
+ default boolean isOptional()
+ {
+ return false;
+ }
+
+ /**
+ * @param wrapped the wrapped parameter
+ * @return a function parameter definition that accepts the specified wrapped parameter, considering it optional as
+ * defined by {@link #isOptional()}.
+ */
+ static FunctionParameter optional(FunctionParameter wrapped)
+ {
+ return new FunctionParameter()
+ {
+ @Nullable
+ @Override
+ public AbstractType<?> inferType(String keyspace,
+ AssignmentTestable arg,
+ @Nullable AbstractType<?> receiverType,
+ List<AbstractType<?>> previousTypes)
+ {
+ return wrapped.inferType(keyspace, arg, receiverType, previousTypes);
+ }
+
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ wrapped.validateType(name, arg, argType);
+ }
+
+ @Override
+ public boolean isOptional()
+ {
+ return true;
+ }
+
+ @Override
+ public String toString()
+ {
+ return '[' + wrapped.toString() + ']';
+ }
+ };
+ }
+
+ /**
+ * @return a function parameter definition that accepts values of string-based data types (text, varchar and ascii)
+ */
+ static FunctionParameter string()
+ {
+ return fixed(CQL3Type.Native.TEXT, CQL3Type.Native.VARCHAR, CQL3Type.Native.ASCII);
+ }
+
+ /**
+ * @param types the accepted data types
+ * @return a function parameter definition that accepts values of a specific data type
+ */
+ static FunctionParameter fixed(CQL3Type... types)
+ {
+ assert types.length > 0;
+
+ return new FunctionParameter()
+ {
+ @Override
+ public AbstractType<?> inferType(String keyspace,
+ AssignmentTestable arg,
+ @Nullable AbstractType<?> receiverType,
+ List<AbstractType<?>> previousTypes)
+ {
+ AbstractType<?> inferred = arg.getCompatibleTypeIfKnown(keyspace);
+ return inferred != null ? inferred : types[0].getType();
+ }
+
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ if (Arrays.stream(types).allMatch(t -> argType.testAssignment(t.getType()) == NOT_ASSIGNABLE))
+ throw new InvalidRequestException(format("Function %s requires an argument of type %s, " +
+ "but found argument %s of type %s",
+ name, this, arg, argType.asCQL3Type()));
+ }
+
+ @Override
+ public String toString()
+ {
+ if (types.length == 1)
+ return types[0].toString();
+
+ return '[' + Arrays.stream(types).map(Object::toString).collect(Collectors.joining("|")) + ']';
+ }
+ };
+ }
+
+ /**
+ * @param inferFromReceiver whether the parameter should try to use the function receiver to infer its data type
+ * @return a function parameter definition that accepts columns of any data type
+ */
+ static FunctionParameter anyType(boolean inferFromReceiver)
+ {
+ return new FunctionParameter()
+ {
+ @Override
+ public AbstractType<?> inferType(String keyspace,
+ AssignmentTestable arg,
+ @Nullable AbstractType<?> receiverType,
+ List<AbstractType<?>> previousTypes)
+ {
+ AbstractType<?> type = arg.getCompatibleTypeIfKnown(keyspace);
+ return type == null && inferFromReceiver ? receiverType : type;
+ }
+
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ // nothing to do here, all types are accepted
+ }
+
+ @Override
+ public String toString()
+ {
+ return "any";
+ }
+ };
+ }
+
+ /**
+ * @return a function parameter definition that accepts values with the same type as the first parameter
+ */
+ static FunctionParameter sameAsFirst()
+ {
+ return new FunctionParameter()
+ {
+ @Override
+ public AbstractType<?> inferType(String keyspace,
+ AssignmentTestable arg,
+ @Nullable AbstractType<?> receiverType,
+ List<AbstractType<?>> previousTypes)
+ {
+ return previousTypes.get(0);
+ }
+
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ // nothing to do here, all types are accepted
+ }
+
+ @Override
+ public String toString()
+ {
+ return "same";
+ }
+ };
+ }
+
+ /**
+ * @return a function parameter definition that accepts values of type {@link CollectionType}, independently of the
+ * types of its elements.
+ */
+ static FunctionParameter anyCollection()
+ {
+ return new FunctionParameter()
+ {
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ if (!argType.isCollection())
+ throw new InvalidRequestException(format("Function %s requires a collection argument, " +
+ "but found argument %s of type %s",
+ name, arg, argType.asCQL3Type()));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "collection";
+ }
+ };
+ }
+
+ /**
+ * @return a function parameter definition that accepts values of type {@link SetType} or {@link ListType}.
+ */
+ static FunctionParameter setOrList()
+ {
+ return new FunctionParameter()
+ {
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ if (argType.isCollection())
+ {
+ CollectionType.Kind kind = ((CollectionType<?>) argType).kind;
+ if (kind == CollectionType.Kind.SET || kind == CollectionType.Kind.LIST)
+ return;
+ }
+
+ throw new InvalidRequestException(format("Function %s requires a set or list argument, " +
+ "but found argument %s of type %s",
+ name, arg, argType.asCQL3Type()));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "numeric_set_or_list";
+ }
+ };
+ }
+
+ /**
+ * @return a function parameter definition that accepts values of type {@link SetType} or {@link ListType},
+ * provided that its elements are numeric.
+ */
+ static FunctionParameter numericSetOrList()
+ {
+ return new FunctionParameter()
+ {
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ AbstractType<?> elementType = null;
+ if (argType.isCollection())
+ {
+ CollectionType<?> collectionType = (CollectionType<?>) argType;
+ if (collectionType.kind == CollectionType.Kind.SET)
+ {
+ elementType = ((SetType<?>) argType).getElementsType();
+ }
+ else if (collectionType.kind == CollectionType.Kind.LIST)
+ {
+ elementType = ((ListType<?>) argType).getElementsType();
+ }
+ }
+
+ if (!(elementType instanceof NumberType))
+ throw new InvalidRequestException(format("Function %s requires a numeric set/list argument, " +
+ "but found argument %s of type %s",
+ name, arg, argType.asCQL3Type()));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "numeric_set_or_list";
+ }
+ };
+ }
+
+ /**
+ * @return a function parameter definition that accepts values of type {@link MapType}, independently of the types
+ * of the map keys and values.
+ */
+ static FunctionParameter anyMap()
+ {
+ return new FunctionParameter()
+ {
+ @Override
+ public void validateType(FunctionName name, AssignmentTestable arg, AbstractType<?> argType)
+ {
+ if (!argType.isUDT() && !(argType instanceof MapType))
+ throw new InvalidRequestException(format("Function %s requires a map argument, " +
+ "but found argument %s of type %s",
+ name, arg, argType.asCQL3Type()));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "map";
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java b/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java
index 7717bdb..81e97a7 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionResolver.java
@@ -20,6 +20,10 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
import org.apache.cassandra.cql3.AbstractMarker;
import org.apache.cassandra.cql3.AssignmentTestable;
@@ -28,6 +32,7 @@
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.UserFunctions;
import static java.util.stream.Collectors.joining;
import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
@@ -38,15 +43,10 @@
{
}
- // We special case the token function because that's the only function whose argument types actually
- // depend on the table on which the function is called. Because it's the sole exception, it's easier
- // to handle it as a special case.
- private static final FunctionName TOKEN_FUNCTION_NAME = FunctionName.nativeFunction("token");
-
- public static ColumnSpecification makeArgSpec(String receiverKs, String receiverCf, Function fun, int i)
+ public static ColumnSpecification makeArgSpec(String receiverKeyspace, String receiverTable, Function fun, int i)
{
- return new ColumnSpecification(receiverKs,
- receiverCf,
+ return new ColumnSpecification(receiverKeyspace,
+ receiverTable,
new ColumnIdentifier("arg" + i + '(' + fun.name().toString().toLowerCase() + ')', true),
fun.argTypes().get(i));
}
@@ -55,21 +55,45 @@
* @param keyspace the current keyspace
* @param name the name of the function
* @param providedArgs the arguments provided for the function call
- * @param receiverKs the receiver's keyspace
- * @param receiverCf the receiver's table
+ * @param receiverKeyspace the receiver's keyspace
+ * @param receiverTable the receiver's table
* @param receiverType if the receiver type is known (during inserts, for example), this should be the type of
* the receiver
- * @throws InvalidRequestException
*/
+ @Nullable
public static Function get(String keyspace,
FunctionName name,
List<? extends AssignmentTestable> providedArgs,
- String receiverKs,
- String receiverCf,
+ String receiverKeyspace,
+ String receiverTable,
AbstractType<?> receiverType)
throws InvalidRequestException
{
- Collection<Function> candidates = collectCandidates(keyspace, name, receiverKs, receiverCf, receiverType);
+ return get(keyspace, name, providedArgs, receiverKeyspace, receiverTable, receiverType, UserFunctions.none());
+ }
+
+ /**
+ * @param keyspace the current keyspace
+ * @param name the name of the function
+ * @param providedArgs the arguments provided for the function call
+ * @param receiverKeyspace the receiver's keyspace
+ * @param receiverTable the receiver's table
+ * @param receiverType if the receiver type is known (during inserts, for example), this should be the type of
+ * the receiver
+ * @param functions a set of user functions that is not yet available in the schema, used during startup when those
+ * functions might not be yet available
+ */
+ @Nullable
+ public static Function get(String keyspace,
+ FunctionName name,
+ List<? extends AssignmentTestable> providedArgs,
+ String receiverKeyspace,
+ String receiverTable,
+ AbstractType<?> receiverType,
+ UserFunctions functions)
+ throws InvalidRequestException
+ {
+ Collection<Function> candidates = collectCandidates(keyspace, name, receiverKeyspace, receiverTable, providedArgs, receiverType, functions);
if (candidates.isEmpty())
return null;
@@ -78,50 +102,48 @@
if (candidates.size() == 1)
{
Function fun = candidates.iterator().next();
- validateTypes(keyspace, fun, providedArgs, receiverKs, receiverCf);
+ validateTypes(keyspace, fun, providedArgs, receiverKeyspace, receiverTable);
return fun;
}
- return pickBestMatch(keyspace, name, providedArgs, receiverKs, receiverCf, receiverType, candidates);
+ return pickBestMatch(keyspace, name, providedArgs, receiverKeyspace, receiverTable, receiverType, candidates);
}
private static Collection<Function> collectCandidates(String keyspace,
FunctionName name,
- String receiverKs,
- String receiverCf,
- AbstractType<?> receiverType)
+ String receiverKeyspace,
+ String receiverTable,
+ List<? extends AssignmentTestable> providedArgs,
+ AbstractType<?> receiverType,
+ UserFunctions functions)
{
Collection<Function> candidates = new ArrayList<>();
- if (name.equalsNativeFunction(TOKEN_FUNCTION_NAME))
- candidates.add(new TokenFct(Schema.instance.getTableMetadata(receiverKs, receiverCf)));
-
- // The toJson() function can accept any type of argument, so instances of it are not pre-declared. Instead,
- // we create new instances as needed while handling selectors (which is the only place that toJson() is supported,
- // due to needing to know the argument types in advance).
- if (name.equalsNativeFunction(ToJsonFct.NAME))
- throw new InvalidRequestException("toJson() may only be used within the selection clause of SELECT statements");
-
- // Similarly, we can only use fromJson when we know the receiver type (such as inserts)
- if (name.equalsNativeFunction(FromJsonFct.NAME))
+ if (name.hasKeyspace())
{
- if (receiverType == null)
- throw new InvalidRequestException("fromJson() cannot be used in the selection clause of a SELECT statement");
- candidates.add(FromJsonFct.getInstance(receiverType));
- }
-
- if (!name.hasKeyspace())
- {
- // function name not fully qualified
- // add 'SYSTEM' (native) candidates
- candidates.addAll(Schema.instance.getFunctions(name.asNativeFunction()));
- // add 'current keyspace' candidates
- candidates.addAll(Schema.instance.getFunctions(new FunctionName(keyspace, name.name)));
+ // function name is fully qualified (keyspace + name)
+ candidates.addAll(functions.get(name));
+ candidates.addAll(Schema.instance.getUserFunctions(name));
+ candidates.addAll(NativeFunctions.instance.getFunctions(name));
+ candidates.addAll(NativeFunctions.instance.getFactories(name).stream()
+ .map(f -> f.getOrCreateFunction(providedArgs, receiverType, receiverKeyspace, receiverTable))
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList()));
}
else
{
- // function name is fully qualified (keyspace + name)
- candidates.addAll(Schema.instance.getFunctions(name));
+ // function name is not fully qualified
+ // add 'current keyspace' candidates
+ FunctionName userName = new FunctionName(keyspace, name.name);
+ candidates.addAll(functions.get(userName));
+ candidates.addAll(Schema.instance.getUserFunctions(userName));
+ // add 'SYSTEM' (native) candidates
+ FunctionName nativeName = name.asNativeFunction();
+ candidates.addAll(NativeFunctions.instance.getFunctions(nativeName));
+ candidates.addAll(NativeFunctions.instance.getFactories(nativeName).stream()
+ .map(f -> f.getOrCreateFunction(providedArgs, receiverType, receiverKeyspace, receiverTable))
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList()));
}
return candidates;
@@ -130,8 +152,8 @@
private static Function pickBestMatch(String keyspace,
FunctionName name,
List<? extends AssignmentTestable> providedArgs,
- String receiverKs,
- String receiverCf,
+ String receiverKeyspace,
+ String receiverTable,
AbstractType<?> receiverType,
Collection<Function> candidates)
{
@@ -140,7 +162,7 @@
{
if (matchReturnType(toTest, receiverType))
{
- AssignmentTestable.TestResult r = matchAguments(keyspace, toTest, providedArgs, receiverKs, receiverCf);
+ AssignmentTestable.TestResult r = matchAguments(keyspace, toTest, providedArgs, receiverKeyspace, receiverTable);
switch (r)
{
case EXACT_MATCH:
@@ -221,8 +243,8 @@
private static void validateTypes(String keyspace,
Function fun,
List<? extends AssignmentTestable> providedArgs,
- String receiverKs,
- String receiverCf)
+ String receiverKeyspace,
+ String receiverTable)
{
if (providedArgs.size() != fun.argTypes().size())
throw invalidRequest("Invalid number of arguments in call to function %s: %d required but %d provided",
@@ -237,7 +259,7 @@
if (provided == null)
continue;
- ColumnSpecification expected = makeArgSpec(receiverKs, receiverCf, fun, i);
+ ColumnSpecification expected = makeArgSpec(receiverKeyspace, receiverTable, fun, i);
if (!provided.testAssignment(keyspace, expected).isAssignable())
throw invalidRequest("Type error: %s cannot be passed as argument %d of function %s of type %s",
provided, i, fun.name(), expected.type.asCQL3Type());
@@ -247,13 +269,13 @@
private static AssignmentTestable.TestResult matchAguments(String keyspace,
Function fun,
List<? extends AssignmentTestable> providedArgs,
- String receiverKs,
- String receiverCf)
+ String receiverKeyspace,
+ String receiverTable)
{
if (providedArgs.size() != fun.argTypes().size())
return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
- // It's an exact match if all are exact match, but is not assignable as soon as any is non assignable.
+ // It's an exact match if all are exact match, but is not assignable as soon as any is not assignable.
AssignmentTestable.TestResult res = AssignmentTestable.TestResult.EXACT_MATCH;
for (int i = 0; i < providedArgs.size(); i++)
{
@@ -264,7 +286,7 @@
continue;
}
- ColumnSpecification expected = makeArgSpec(receiverKs, receiverCf, fun, i);
+ ColumnSpecification expected = makeArgSpec(receiverKeyspace, receiverTable, fun, i);
AssignmentTestable.TestResult argRes = provided.testAssignment(keyspace, expected);
if (argRes == AssignmentTestable.TestResult.NOT_ASSIGNABLE)
return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
diff --git a/src/java/org/apache/cassandra/cql3/functions/MathFcts.java b/src/java/org/apache/cassandra/cql3/functions/MathFcts.java
new file mode 100644
index 0000000..553ac40
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/MathFcts.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+public final class MathFcts
+{
+ public static void addFunctionsTo(NativeFunctions functions)
+ {
+ final List<NumberType<?>> numericTypes = ImmutableList.of(ByteType.instance,
+ ShortType.instance,
+ Int32Type.instance,
+ FloatType.instance,
+ LongType.instance,
+ DoubleType.instance,
+ IntegerType.instance,
+ DecimalType.instance,
+ CounterColumnType.instance);
+
+ numericTypes.stream()
+ .map(t -> ImmutableList.of(absFct(t),
+ expFct(t),
+ logFct(t),
+ log10Fct(t),
+ roundFct(t)))
+ .flatMap(Collection::stream)
+ .forEach(f -> functions.add(f));
+ }
+
+ public static NativeFunction absFct(final NumberType<?> type)
+ {
+ return new NativeScalarFunction("abs", type, type)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null)
+ return null;
+ return type.abs(bb);
+ }
+ };
+ }
+
+ public static NativeFunction expFct(final NumberType<?> type)
+ {
+ return new NativeScalarFunction("exp", type, type)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null)
+ return null;
+ return type.exp(bb);
+ }
+ };
+ }
+
+ public static NativeFunction logFct(final NumberType<?> type)
+ {
+ return new NativeScalarFunction("log", type, type)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null)
+ return null;
+ return type.log(bb);
+
+ }
+ };
+ }
+
+ public static NativeFunction log10Fct(final NumberType<?> type)
+ {
+ return new NativeScalarFunction("log10", type, type)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null)
+ return null;
+ return type.log10(bb);
+
+ }
+ };
+ }
+
+ public static NativeFunction roundFct(final NumberType<?> type)
+ {
+ return new NativeScalarFunction("round", type, type)
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null)
+ return null;
+ return type.round(bb);
+
+ }
+ };
+ }
+
+ private MathFcts()
+ {
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java
index cafeca1..285cf63 100644
--- a/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java
@@ -19,6 +19,8 @@
import java.util.Arrays;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.db.marshal.AbstractType;
/**
@@ -32,7 +34,7 @@
}
@Override
- public boolean isNative()
+ public final boolean isNative()
{
return true;
}
@@ -43,4 +45,18 @@
// Most of our functions are pure, the other ones should override this
return true;
}
+
+ /**
+ * Returns a copy of this function using its old pre-5.0 name before the adoption of snake-cased function names.
+ * Those naming conventions were adopted in 5.0, but we still need to support the old names for
+ * compatibility. See CASSANDRA-18037 for further details.
+ *
+ * @return a copy of this function using its old pre-5.0 deprecated name, or {@code null} if the pre-5.0 function
+ * name already satisfied the naming conventions.
+ */
+ @Nullable
+ public NativeFunction withLegacyName()
+ {
+ return null;
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java
new file mode 100644
index 0000000..02939e9
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.Collection;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.cql3.functions.masking.MaskingFcts;
+
+/**
+ * A container of native functions. It stores both pre-built function overloads ({@link NativeFunction}) and
+ * dynamic generators of functions ({@link FunctionFactory}).
+ */
+public class NativeFunctions
+{
+ public static NativeFunctions instance = new NativeFunctions()
+ {
+ {
+ TokenFct.addFunctionsTo(this);
+ CastFcts.addFunctionsTo(this);
+ UuidFcts.addFunctionsTo(this);
+ TimeFcts.addFunctionsTo(this);
+ ToJsonFct.addFunctionsTo(this);
+ FromJsonFct.addFunctionsTo(this);
+ OperationFcts.addFunctionsTo(this);
+ AggregateFcts.addFunctionsTo(this);
+ CollectionFcts.addFunctionsTo(this);
+ BytesConversionFcts.addFunctionsTo(this);
+ MathFcts.addFunctionsTo(this);
+ MaskingFcts.addFunctionsTo(this);
+ }
+ };
+
+ /** Pre-built function overloads. */
+ private final Multimap<FunctionName, NativeFunction> functions = HashMultimap.create();
+
+ /** Dynamic function factories. */
+ private final Multimap<FunctionName, FunctionFactory> factories = HashMultimap.create();
+
+ public void add(NativeFunction function)
+ {
+ functions.put(function.name(), function);
+ NativeFunction legacyFunction = function.withLegacyName();
+ if (legacyFunction != null)
+ functions.put(legacyFunction.name(), legacyFunction);
+ }
+
+ public void addAll(NativeFunction... functions)
+ {
+ for (NativeFunction function : functions)
+ add(function);
+ }
+
+ public void add(FunctionFactory factory)
+ {
+ factories.put(factory.name(), factory);
+ }
+
+ /**
+ * Returns all the registered pre-built functions overloads with the specified name.
+ *
+ * @param name a function name
+ * @return the pre-built functions with the specified name
+ */
+ public Collection<NativeFunction> getFunctions(FunctionName name)
+ {
+ return functions.get(name);
+ }
+
+ /**
+ * @return all the registered pre-built functions.
+ */
+ public Collection<NativeFunction> getFunctions()
+ {
+ return functions.values();
+ }
+
+ /**
+ * Returns all the registered functions factories with the specified name.
+ *
+ * @param name a function name
+ * @return the function factories with the specified name
+ */
+ public Collection<FunctionFactory> getFactories(FunctionName name)
+ {
+ return factories.get(name);
+ }
+
+ /**
+ * @return all the registered functions factories.
+ */
+ public Collection<FunctionFactory> getFactories()
+ {
+ return factories.values();
+ }
+
+ /**
+ * Returns whether there is a function factory with the specified name.
+ *
+ * @param name a function name
+ * @return {@code true} if there is a factory with the specified name, {@code false} otherwise
+ */
+ public boolean hasFactory(FunctionName name)
+ {
+ return factories.containsKey(name);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java b/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java
index b00ced7..9e0fef1 100644
--- a/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/OperationFcts.java
@@ -18,8 +18,6 @@
package org.apache.cassandra.cql3.functions;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import org.apache.cassandra.schema.SchemaConstants;
@@ -219,10 +217,8 @@
*/
public static final String NEGATION_FUNCTION_NAME = "_negate";
- public static Collection<Function> all()
+ public static void addFunctionsTo(NativeFunctions functions)
{
- List<Function> functions = new ArrayList<>();
-
final NumberType<?>[] numericTypes = new NumberType[] { ByteType.instance,
ShortType.instance,
Int32Type.instance,
@@ -251,11 +247,9 @@
}
addStringConcatenations(functions);
-
- return functions;
}
- private static void addStringConcatenations(List<Function> functions)
+ private static void addStringConcatenations(NativeFunctions functions)
{
functions.add(new StringOperationFunction(UTF8Type.instance, UTF8Type.instance, OPERATION.ADDITION, UTF8Type.instance));
functions.add(new StringOperationFunction(AsciiType.instance, AsciiType.instance, OPERATION.ADDITION, AsciiType.instance));
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
deleted file mode 100644
index e42fbe9..0000000
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.cql3.functions;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.net.*;
-import java.nio.ByteBuffer;
-import java.security.*;
-import java.security.cert.Certificate;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import javax.script.*;
-
-import jdk.nashorn.api.scripting.AbstractJSObject;
-import jdk.nashorn.api.scripting.ClassFilter;
-import jdk.nashorn.api.scripting.NashornScriptEngine;
-import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.security.SecurityThreadGroup;
-import org.apache.cassandra.security.ThreadAwareSecurityManager;
-import org.apache.cassandra.transport.ProtocolVersion;
-
-final class ScriptBasedUDFunction extends UDFunction
-{
- private static final ProtectionDomain protectionDomain;
- private static final AccessControlContext accessControlContext;
-
- //
- // For scripted UDFs we have to rely on the security mechanisms of the scripting engine and
- // SecurityManager - especially SecurityManager.checkPackageAccess(). Unlike Java-UDFs, strict checking
- // of class access via the UDF class loader is not possible, since e.g. Nashorn builds its own class loader
- // (jdk.nashorn.internal.runtime.ScriptLoader / jdk.nashorn.internal.runtime.NashornLoader) configured with
- // a system class loader.
- //
- private static final String[] allowedPackagesArray =
- {
- // following required by jdk.nashorn.internal.objects.Global.initJavaAccess()
- "",
- "com",
- "edu",
- "java",
- "javax",
- "javafx",
- "org",
- // following required by Nashorn runtime
- "java.lang",
- "java.lang.invoke",
- "java.lang.reflect",
- "java.nio.charset",
- "java.util",
- "java.util.concurrent",
- "javax.script",
- "sun.reflect",
- "jdk.internal.org.objectweb.asm.commons",
- "jdk.nashorn.internal.runtime",
- "jdk.nashorn.internal.runtime.linker",
- // Nashorn / Java 11
- "java.lang.ref",
- "java.io",
- "java.util.function",
- "jdk.dynalink.linker",
- "jdk.internal.org.objectweb.asm",
- "jdk.internal.reflect",
- "jdk.nashorn.internal.scripts",
- // following required by Java Driver
- "java.math",
- "java.nio",
- "java.text",
- "com.google.common.base",
- "com.google.common.collect",
- "com.google.common.reflect",
- // following required by UDF
- "org.apache.cassandra.cql3.functions.types",
- "org.apache.cassandra.cql3.functions.types.exceptions",
- "org.apache.cassandra.cql3.functions.types.utils"
- };
-
- // use a JVM standard ExecutorService as ExecutorPlus references internal
- // classes, which triggers AccessControlException from the UDF sandbox
- private static final UDFExecutorService executor =
- new UDFExecutorService(new NamedThreadFactory("UserDefinedScriptFunctions",
- Thread.MIN_PRIORITY,
- udfClassLoader,
- new SecurityThreadGroup("UserDefinedScriptFunctions",
- Collections.unmodifiableSet(new HashSet<>(Arrays.asList(allowedPackagesArray))),
- UDFunction::initializeThread)),
- "userscripts");
-
- private static final ClassFilter classFilter = clsName -> secureResource(clsName.replace('.', '/') + ".class");
-
- private static final NashornScriptEngine scriptEngine;
-
-
- static
- {
- ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
- ScriptEngine engine = scriptEngineManager.getEngineByName("nashorn");
- NashornScriptEngineFactory factory = engine != null ? (NashornScriptEngineFactory) engine.getFactory() : null;
- scriptEngine = factory != null ? (NashornScriptEngine) factory.getScriptEngine(new String[]{}, udfClassLoader, classFilter) : null;
-
- try
- {
- protectionDomain = new ProtectionDomain(new CodeSource(new URL("udf", "localhost", 0, "/script", new URLStreamHandler()
- {
- protected URLConnection openConnection(URL u)
- {
- return null;
- }
- }), (Certificate[]) null), ThreadAwareSecurityManager.noPermissions);
- }
- catch (MalformedURLException e)
- {
- throw new RuntimeException(e);
- }
- accessControlContext = new AccessControlContext(new ProtectionDomain[]{ protectionDomain });
- }
-
- private final CompiledScript script;
- private final Object udfContextBinding;
-
- ScriptBasedUDFunction(FunctionName name,
- List<ColumnIdentifier> argNames,
- List<AbstractType<?>> argTypes,
- AbstractType<?> returnType,
- boolean calledOnNullInput,
- String language,
- String body)
- {
- super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
-
- if (!"JavaScript".equalsIgnoreCase(language) || scriptEngine == null)
- throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
-
- // execute compilation with no-permissions to prevent evil code e.g. via "static code blocks" / "class initialization"
- try
- {
- this.script = AccessController.doPrivileged((PrivilegedExceptionAction<CompiledScript>) () -> scriptEngine.compile(body),
- accessControlContext);
- }
- catch (PrivilegedActionException x)
- {
- Throwable e = x.getCause();
- logger.info("Failed to compile function '{}' for language {}: ", name, language, e);
- throw new InvalidRequestException(
- String.format("Failed to compile function '%s' for language %s: %s", name, language, e));
- }
-
- // It's not always possible to simply pass a plain Java object as a binding to Nashorn and
- // let the script execute methods on it.
- udfContextBinding = new UDFContextWrapper();
- }
-
- protected ExecutorService executor()
- {
- return executor;
- }
-
- public ByteBuffer executeUserDefined(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- Object[] params = new Object[argTypes.size()];
- for (int i = 0; i < params.length; i++)
- params[i] = compose(protocolVersion, i, parameters.get(i));
-
- Object result = executeScriptInternal(params);
-
- return decompose(protocolVersion, result);
- }
-
- /**
- * Like {@link UDFunction#executeUserDefined(ProtocolVersion, List)} but the first parameter is already in non-serialized form.
- * Remaining parameters (2nd paramters and all others) are in {@code parameters}.
- * This is used to prevent superfluous (de)serialization of the state of aggregates.
- * Means: scalar functions of aggregates are called using this variant.
- */
- protected Object executeAggregateUserDefined(ProtocolVersion protocolVersion, Object firstParam, List<ByteBuffer> parameters)
- {
- Object[] params = new Object[argTypes.size()];
- params[0] = firstParam;
- for (int i = 1; i < params.length; i++)
- params[i] = compose(protocolVersion, i, parameters.get(i - 1));
-
- return executeScriptInternal(params);
- }
-
- private Object executeScriptInternal(Object[] params)
- {
- ScriptContext scriptContext = new SimpleScriptContext();
- scriptContext.setAttribute("javax.script.filename", this.name.toString(), ScriptContext.ENGINE_SCOPE);
- Bindings bindings = scriptContext.getBindings(ScriptContext.ENGINE_SCOPE);
- for (int i = 0; i < params.length; i++)
- bindings.put(argNames.get(i).toString(), params[i]);
- bindings.put("udfContext", udfContextBinding);
-
- Object result;
- try
- {
- // How to prevent Class.forName() _without_ "help" from the script engine ?
- // NOTE: Nashorn enforces a special permission to allow class-loading, which is not granted - so it's fine.
-
- result = script.eval(scriptContext);
- }
- catch (ScriptException e)
- {
- throw new RuntimeException(e);
- }
- if (result == null)
- return null;
-
- Class<?> javaReturnType = UDHelper.asJavaClass(returnCodec);
- Class<?> resultType = result.getClass();
- if (!javaReturnType.isAssignableFrom(resultType))
- {
- if (result instanceof Number)
- {
- Number rNumber = (Number) result;
- if (javaReturnType == Integer.class)
- result = rNumber.intValue();
- else if (javaReturnType == Long.class)
- result = rNumber.longValue();
- else if (javaReturnType == Short.class)
- result = rNumber.shortValue();
- else if (javaReturnType == Byte.class)
- result = rNumber.byteValue();
- else if (javaReturnType == Float.class)
- result = rNumber.floatValue();
- else if (javaReturnType == Double.class)
- result = rNumber.doubleValue();
- else if (javaReturnType == BigInteger.class)
- {
- if (javaReturnType == Integer.class)
- result = rNumber.intValue();
- else if (javaReturnType == Short.class)
- result = rNumber.shortValue();
- else if (javaReturnType == Byte.class)
- result = rNumber.byteValue();
- else if (javaReturnType == Long.class)
- result = rNumber.longValue();
- else if (javaReturnType == Float.class)
- result = rNumber.floatValue();
- else if (javaReturnType == Double.class)
- result = rNumber.doubleValue();
- else if (javaReturnType == BigInteger.class)
- {
- if (rNumber instanceof BigDecimal)
- result = ((BigDecimal) rNumber).toBigInteger();
- else if (rNumber instanceof Double || rNumber instanceof Float)
- result = new BigDecimal(rNumber.toString()).toBigInteger();
- else
- result = BigInteger.valueOf(rNumber.longValue());
- }
- else if (javaReturnType == BigDecimal.class)
- // String c'tor of BigDecimal is more accurate than valueOf(double)
- result = new BigDecimal(rNumber.toString());
- }
- else if (javaReturnType == BigDecimal.class)
- // String c'tor of BigDecimal is more accurate than valueOf(double)
- result = new BigDecimal(rNumber.toString());
- }
- }
-
- return result;
- }
-
- private final class UDFContextWrapper extends AbstractJSObject
- {
- private final AbstractJSObject fRetUDT;
- private final AbstractJSObject fArgUDT;
- private final AbstractJSObject fRetTup;
- private final AbstractJSObject fArgTup;
-
- UDFContextWrapper()
- {
- fRetUDT = new AbstractJSObject()
- {
- public Object call(Object thiz, Object... args)
- {
- return udfContext.newReturnUDTValue();
- }
- };
- fArgUDT = new AbstractJSObject()
- {
- public Object call(Object thiz, Object... args)
- {
- if (args[0] instanceof String)
- return udfContext.newArgUDTValue((String) args[0]);
- if (args[0] instanceof Number)
- return udfContext.newArgUDTValue(((Number) args[0]).intValue());
- return super.call(thiz, args);
- }
- };
- fRetTup = new AbstractJSObject()
- {
- public Object call(Object thiz, Object... args)
- {
- return udfContext.newReturnTupleValue();
- }
- };
- fArgTup = new AbstractJSObject()
- {
- public Object call(Object thiz, Object... args)
- {
- if (args[0] instanceof String)
- return udfContext.newArgTupleValue((String) args[0]);
- if (args[0] instanceof Number)
- return udfContext.newArgTupleValue(((Number) args[0]).intValue());
- return super.call(thiz, args);
- }
- };
- }
-
- public Object getMember(String name)
- {
- switch(name)
- {
- case "newReturnUDTValue":
- return fRetUDT;
- case "newArgUDTValue":
- return fArgUDT;
- case "newReturnTupleValue":
- return fRetTup;
- case "newArgTupleValue":
- return fArgTup;
- }
- return super.getMember(name);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
index 2759210..e1557e6 100644
--- a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java
@@ -18,10 +18,9 @@
package org.apache.cassandra.cql3.functions;
import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.List;
-import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,61 +31,77 @@
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.UUIDGen;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
public abstract class TimeFcts
{
public static Logger logger = LoggerFactory.getLogger(TimeFcts.class);
- public static Collection<Function> all()
+ public static void addFunctionsTo(NativeFunctions functions)
{
- return ImmutableList.of(now("now", TimeUUIDType.instance),
- now("currenttimeuuid", TimeUUIDType.instance),
- now("currenttimestamp", TimestampType.instance),
- now("currentdate", SimpleDateType.instance),
- now("currenttime", TimeType.instance),
- minTimeuuidFct,
- maxTimeuuidFct,
- dateOfFct,
- unixTimestampOfFct,
- toDate(TimeUUIDType.instance),
- toTimestamp(TimeUUIDType.instance),
- toUnixTimestamp(TimeUUIDType.instance),
- toUnixTimestamp(TimestampType.instance),
- toDate(TimestampType.instance),
- toUnixTimestamp(SimpleDateType.instance),
- toTimestamp(SimpleDateType.instance),
- FloorTimestampFunction.newInstance(),
- FloorTimestampFunction.newInstanceWithStartTimeArgument(),
- FloorTimeUuidFunction.newInstance(),
- FloorTimeUuidFunction.newInstanceWithStartTimeArgument(),
- FloorDateFunction.newInstance(),
- FloorDateFunction.newInstanceWithStartTimeArgument(),
- floorTime);
+ functions.addAll(new NowFunction("now", TimeUUIDType.instance),
+ new NowFunction("current_timeuuid", TimeUUIDType.instance),
+ new NowFunction("current_timestamp", TimestampType.instance),
+ new NowFunction("current_date", SimpleDateType.instance),
+ new NowFunction("current_time", TimeType.instance),
+ minTimeuuidFct,
+ maxTimeuuidFct,
+ toDate(TimeUUIDType.instance),
+ toTimestamp(TimeUUIDType.instance),
+ toUnixTimestamp(TimeUUIDType.instance),
+ toUnixTimestamp(TimestampType.instance),
+ toDate(TimestampType.instance),
+ toUnixTimestamp(SimpleDateType.instance),
+ toTimestamp(SimpleDateType.instance),
+ FloorTimestampFunction.newInstance(),
+ FloorTimestampFunction.newInstanceWithStartTimeArgument(),
+ FloorTimeUuidFunction.newInstance(),
+ FloorTimeUuidFunction.newInstanceWithStartTimeArgument(),
+ FloorDateFunction.newInstance(),
+ FloorDateFunction.newInstanceWithStartTimeArgument(),
+ floorTime);
}
- public static final Function now(final String name, final TemporalType<?> type)
+ private static class NowFunction extends NativeScalarFunction
{
- return new NativeScalarFunction(name, type)
+ private final TemporalType<?> type;
+
+ public NowFunction(String name, TemporalType<?> type)
{
- @Override
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- return type.now();
- }
+ super(name, type);
+ this.type = type;
+ }
- @Override
- public boolean isPure()
- {
- return false; // as it returns non-identical results for identical arguments
- }
- };
- };
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ return type.now();
+ }
- public static final Function minTimeuuidFct = new NativeScalarFunction("mintimeuuid", TimeUUIDType.instance, TimestampType.instance)
+ @Override
+ public boolean isPure()
+ {
+ return false; // as it returns non-identical results for identical arguments
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ String name = name().name;
+ return name.contains("current") ? new NowFunction(StringUtils.remove(name, '_'), type) : null;
+ }
+ }
+
+ public static final NativeFunction minTimeuuidFct = new MinTimeuuidFunction(false);
+
+ private static final class MinTimeuuidFunction extends NativeScalarFunction
{
+ public MinTimeuuidFunction(boolean legacy)
+ {
+ super(legacy ? "mintimeuuid" : "min_timeuuid", TimeUUIDType.instance, TimestampType.instance);
+ }
+
+ @Override
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
ByteBuffer bb = parameters.get(0);
@@ -95,10 +110,24 @@
return TimeUUID.minAtUnixMillis(TimestampType.instance.compose(bb).getTime()).toBytes();
}
- };
- public static final Function maxTimeuuidFct = new NativeScalarFunction("maxtimeuuid", TimeUUIDType.instance, TimestampType.instance)
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new MinTimeuuidFunction(true);
+ }
+ }
+
+ public static final NativeFunction maxTimeuuidFct = new MaxTimeuuidFunction(false);
+
+ private static final class MaxTimeuuidFunction extends NativeScalarFunction
{
+ public MaxTimeuuidFunction(boolean legacy)
+ {
+ super(legacy ? "maxtimeuuid" : "max_timeuuid", TimeUUIDType.instance, TimestampType.instance);
+ }
+
+ @Override
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
ByteBuffer bb = parameters.get(0);
@@ -107,137 +136,146 @@
return TimeUUID.maxAtUnixMillis(TimestampType.instance.compose(bb).getTime()).toBytes();
}
- };
- /**
- * Function that convert a value of <code>TIMEUUID</code> into a value of type <code>TIMESTAMP</code>.
- * @deprecated Replaced by the {@link #toTimestamp} function
- */
- public static final NativeScalarFunction dateOfFct = new NativeScalarFunction("dateof", TimestampType.instance, TimeUUIDType.instance)
- {
- private volatile boolean hasLoggedDeprecationWarning;
-
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ @Override
+ public NativeFunction withLegacyName()
{
- if (!hasLoggedDeprecationWarning)
- {
- hasLoggedDeprecationWarning = true;
- logger.warn("The function 'dateof' is deprecated." +
- " Use the function 'toTimestamp' instead.");
- }
-
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
-
- long timeInMillis = TimeUUID.deserialize(bb).unix(MILLISECONDS);
- return ByteBufferUtil.bytes(timeInMillis);
+ return new MaxTimeuuidFunction(true);
}
- };
+ }
/**
- * Function that convert a value of type <code>TIMEUUID</code> into an UNIX timestamp.
- * @deprecated Replaced by the {@link #toUnixTimestamp} function
- */
- public static final NativeScalarFunction unixTimestampOfFct = new NativeScalarFunction("unixtimestampof", LongType.instance, TimeUUIDType.instance)
- {
- private volatile boolean hasLoggedDeprecationWarning;
-
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- if (!hasLoggedDeprecationWarning)
- {
- hasLoggedDeprecationWarning = true;
- logger.warn("The function 'unixtimestampof' is deprecated." +
- " Use the function 'toUnixTimestamp' instead.");
- }
-
- ByteBuffer bb = parameters.get(0);
- if (bb == null)
- return null;
-
- return ByteBufferUtil.bytes(TimeUUID.deserialize(bb).unix(MILLISECONDS));
- }
- };
-
- /**
- * Creates a function that convert a value of the specified type into a <code>DATE</code>.
- * @param type the temporal type
- * @return a function that convert a value of the specified type into a <code>DATE</code>.
- */
- public static final NativeScalarFunction toDate(final TemporalType<?> type)
- {
- return new NativeScalarFunction("todate", SimpleDateType.instance, type)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null || !bb.hasRemaining())
- return null;
-
- long millis = type.toTimeInMillis(bb);
- return SimpleDateType.instance.fromTimeInMillis(millis);
- }
-
- @Override
- public boolean isMonotonic()
- {
- return true;
- }
- };
- }
-
- /**
- * Creates a function that convert a value of the specified type into a <code>TIMESTAMP</code>.
- * @param type the temporal type
- * @return a function that convert a value of the specified type into a <code>TIMESTAMP</code>.
- */
- public static final NativeScalarFunction toTimestamp(final TemporalType<?> type)
- {
- return new NativeScalarFunction("totimestamp", TimestampType.instance, type)
- {
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null || !bb.hasRemaining())
- return null;
-
- long millis = type.toTimeInMillis(bb);
- return TimestampType.instance.fromTimeInMillis(millis);
- }
-
- @Override
- public boolean isMonotonic()
- {
- return true;
- }
- };
- }
-
- /**
- * Creates a function that convert a value of the specified type into an UNIX timestamp.
+ * Creates a function that converts a value of the specified type into a {@code DATE}.
+ *
* @param type the temporal type
- * @return a function that convert a value of the specified type into an UNIX timestamp.
+ * @return a function that convert a value of the specified type into a <code>DATE</code>.
*/
- public static final NativeScalarFunction toUnixTimestamp(final TemporalType<?> type)
+ public static NativeScalarFunction toDate(TemporalType<?> type)
{
- return new NativeScalarFunction("tounixtimestamp", LongType.instance, type)
+ return new ToDateFunction(type, false);
+ }
+
+ private static class ToDateFunction extends NativeScalarFunction
+ {
+ private final TemporalType<?> type;
+
+ public ToDateFunction(TemporalType<?> type, boolean useLegacyName)
{
- public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
- {
- ByteBuffer bb = parameters.get(0);
- if (bb == null || !bb.hasRemaining())
- return null;
+ super(useLegacyName ? "todate" : "to_date", SimpleDateType.instance, type);
+ this.type = type;
+ }
- return ByteBufferUtil.bytes(type.toTimeInMillis(bb));
- }
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null || !bb.hasRemaining())
+ return null;
- @Override
- public boolean isMonotonic()
- {
- return true;
- }
- };
+ long millis = type.toTimeInMillis(bb);
+ return SimpleDateType.instance.fromTimeInMillis(millis);
+ }
+
+ @Override
+ public boolean isMonotonic()
+ {
+ return true;
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new ToDateFunction(type, true);
+ }
+ }
+
+ /**
+ * Creates a function that converts a value of the specified type into a {@code TIMESTAMP}.
+ *
+ * @param type the temporal type
+ * @return a function that convert a value of the specified type into a {@code TIMESTAMP}.
+ */
+ public static NativeScalarFunction toTimestamp(TemporalType<?> type)
+ {
+ return new ToTimestampFunction(type, false);
+ }
+
+ private static class ToTimestampFunction extends NativeScalarFunction
+ {
+ private final TemporalType<?> type;
+
+ public ToTimestampFunction(TemporalType<?> type, boolean useLegacyName)
+ {
+ super(useLegacyName ? "totimestamp" : "to_timestamp", TimestampType.instance, type);
+ this.type = type;
+ }
+
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null || !bb.hasRemaining())
+ return null;
+
+ long millis = type.toTimeInMillis(bb);
+ return TimestampType.instance.fromTimeInMillis(millis);
+ }
+
+ @Override
+ public boolean isMonotonic()
+ {
+ return true;
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new ToTimestampFunction(type, true);
+ }
+ }
+
+ /**
+ * Creates a function that converts a value of the specified type into a UNIX timestamp.
+ *
+ * @param type the temporal type
+ * @return a function that convert a value of the specified type into a UNIX timestamp.
+ */
+ public static NativeScalarFunction toUnixTimestamp(TemporalType<?> type)
+ {
+ return new ToUnixTimestampFunction(type, false);
+ }
+
+ private static class ToUnixTimestampFunction extends NativeScalarFunction
+ {
+ private final TemporalType<?> type;
+
+ private ToUnixTimestampFunction(TemporalType<?> type, boolean useLegacyName)
+ {
+ super(useLegacyName ? "tounixtimestamp" : "to_unix_timestamp", LongType.instance, type);
+ this.type = type;
+ }
+
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer bb = parameters.get(0);
+ if (bb == null || !bb.hasRemaining())
+ return null;
+
+ return ByteBufferUtil.bytes(type.toTimeInMillis(bb));
+ }
+
+ @Override
+ public boolean isMonotonic()
+ {
+ return true;
+ }
+
+ @Override
+ public NativeFunction withLegacyName()
+ {
+ return new ToUnixTimestampFunction(type, true);
+ }
}
/**
@@ -245,14 +283,14 @@
*/
private static abstract class FloorFunction extends NativeScalarFunction
{
- private static final Long ZERO = Long.valueOf(0);
+ private static final Long ZERO = 0L;
protected FloorFunction(AbstractType<?> returnType,
AbstractType<?>... argsType)
{
super("floor", returnType, argsType);
// The function can accept either 2 parameters (time and duration) or 3 parameters (time, duration and startTime)r
- assert argsType.length == 2 || argsType.length == 3;
+ assert argsType.length == 2 || argsType.length == 3;
}
@Override
@@ -509,4 +547,3 @@
}
};
}
-
diff --git a/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java b/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java
index d0f2b0b..8dd7fd7 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ToJsonFct.java
@@ -28,39 +28,59 @@
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import static java.lang.String.format;
+
public class ToJsonFct extends NativeScalarFunction
{
- public static final FunctionName NAME = FunctionName.nativeFunction("tojson");
-
private static final Map<AbstractType<?>, ToJsonFct> instances = new ConcurrentHashMap<>();
- public static ToJsonFct getInstance(List<AbstractType<?>> argTypes) throws InvalidRequestException
+ public static ToJsonFct getInstance(String name, List<AbstractType<?>> argTypes) throws InvalidRequestException
{
if (argTypes.size() != 1)
- throw new InvalidRequestException(String.format("toJson() only accepts one argument (got %d)", argTypes.size()));
+ throw new InvalidRequestException(format("%s() only accepts one argument (got %d)", name, argTypes.size()));
AbstractType<?> fromType = argTypes.get(0);
ToJsonFct func = instances.get(fromType);
if (func == null)
{
- func = new ToJsonFct(fromType);
+ func = new ToJsonFct(name, fromType);
instances.put(fromType, func);
}
return func;
}
- private ToJsonFct(AbstractType<?> argType)
+ private ToJsonFct(String name, AbstractType<?> argType)
{
- super("tojson", UTF8Type.instance, argType);
+ super(name, UTF8Type.instance, argType);
}
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters) throws InvalidRequestException
{
- assert parameters.size() == 1 : "Expected 1 argument for toJson(), but got " + parameters.size();
+ assert parameters.size() == 1 : format("Expected 1 argument for %s(), but got %d", name.name, parameters.size());
ByteBuffer parameter = parameters.get(0);
if (parameter == null)
return ByteBufferUtil.bytes("null");
return ByteBufferUtil.bytes(argTypes.get(0).toJSONString(parameter, protocolVersion));
}
+
+ public static void addFunctionsTo(NativeFunctions functions)
+ {
+ functions.add(new Factory("to_json"));
+ functions.add(new Factory("tojson")); // deprecated pre-5.0 name
+ }
+
+ private static class Factory extends FunctionFactory
+ {
+ public Factory(String name)
+ {
+ super(name, FunctionParameter.anyType(false));
+ }
+
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return ToJsonFct.getInstance(name.name, argTypes);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
index e93084f..fb6185b 100644
--- a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
+++ b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java
@@ -20,7 +20,9 @@
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.cassandra.cql3.AssignmentTestable;
import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.CBuilder;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -37,9 +39,9 @@
this.metadata = metadata;
}
- private static AbstractType[] getKeyTypes(TableMetadata metadata)
+ private static AbstractType<?>[] getKeyTypes(TableMetadata metadata)
{
- AbstractType[] types = new AbstractType[metadata.partitionKeyColumns().size()];
+ AbstractType<?>[] types = new AbstractType[metadata.partitionKeyColumns().size()];
int i = 0;
for (ColumnMetadata def : metadata.partitionKeyColumns())
types[i++] = def.type;
@@ -58,4 +60,37 @@
}
return metadata.partitioner.getTokenFactory().toByteArray(metadata.partitioner.getToken(builder.build().serializeAsPartitionKey()));
}
+
+ public static void addFunctionsTo(NativeFunctions functions)
+ {
+ functions.add(new FunctionFactory("token")
+ {
+ @Override
+ public NativeFunction getOrCreateFunction(List<? extends AssignmentTestable> args,
+ AbstractType<?> receiverType,
+ String receiverKeyspace,
+ String receiverTable)
+ {
+ if (receiverKeyspace == null)
+ throw new InvalidRequestException("No receiver keyspace has been specified for function " + name);
+
+ if (receiverTable == null)
+ throw new InvalidRequestException("No receiver table has been specified for function " + name);
+
+ TableMetadata metadata = Schema.instance.getTableMetadata(receiverKeyspace, receiverTable);
+ if (metadata == null)
+ throw new InvalidRequestException(String.format("The receiver table %s.%s specified by call to " +
+ "function %s hasn't been found",
+ receiverKeyspace, receiverTable, name));
+
+ return new TokenFct(metadata);
+ }
+
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ throw new AssertionError("Should be unreachable");
+ }
+ });
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
index b686328..67e96cd 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java
@@ -26,14 +26,13 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.CqlBuilder;
-import org.apache.cassandra.cql3.SchemaElement;
import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.schema.Difference;
-import org.apache.cassandra.schema.Functions;
+import org.apache.cassandra.schema.UserFunctions;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.ProtocolVersion;
@@ -44,7 +43,7 @@
/**
* Base class for user-defined-aggregates.
*/
-public class UDAggregate extends AbstractFunction implements AggregateFunction, SchemaElement
+public class UDAggregate extends UserFunction implements AggregateFunction
{
protected static final Logger logger = LoggerFactory.getLogger(UDAggregate.class);
@@ -95,7 +94,7 @@
private static UDFunction findFunction(FunctionName udaName, Collection<UDFunction> functions, FunctionName name, List<AbstractType<?>> arguments)
{
return functions.stream()
- .filter(f -> f.name().equals(name) && Functions.typesMatch(f.argTypes(), arguments))
+ .filter(f -> f.name().equals(name) && f.typesMatch(arguments))
.findFirst()
.orElseThrow(() -> new ConfigurationException(String.format("Unable to find function %s referenced by UDA %s", name, udaName)));
}
@@ -150,11 +149,6 @@
return true;
}
- public boolean isNative()
- {
- return false;
- }
-
public ScalarFunction stateFunction()
{
return stateFunction;
@@ -329,7 +323,7 @@
@Override
public int hashCode()
{
- return Objects.hashCode(name, Functions.typeHashCode(argTypes), Functions.typeHashCode(returnType), stateFunction, finalFunction, stateType, initcond);
+ return Objects.hashCode(name, UserFunctions.typeHashCode(argTypes), UserFunctions.typeHashCode(returnType), stateFunction, finalFunction, stateType, initcond);
}
@Override
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java b/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java
index ab913b4..e0b1806 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFByteCodeVerifier.java
@@ -35,6 +35,8 @@
import org.objectweb.asm.MethodVisitor;
import org.objectweb.asm.Opcodes;
+import static org.apache.cassandra.utils.FBUtilities.ASM_BYTECODE_VERSION;
+
/**
* Verifies Java UDF byte code.
* Checks for disallowed method calls (e.g. {@code Object.finalize()}),
@@ -84,7 +86,7 @@
{
String clsNameSl = clsName.replace('.', '/');
Set<String> errors = new TreeSet<>(); // it's a TreeSet for unit tests
- ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM7)
+ ClassVisitor classVisitor = new ClassVisitor(ASM_BYTECODE_VERSION)
{
public FieldVisitor visitField(int access, String name, String desc, String signature, Object value)
{
@@ -160,7 +162,7 @@
ExecuteImplVisitor(Set<String> errors)
{
- super(Opcodes.ASM7);
+ super(ASM_BYTECODE_VERSION);
this.errors = errors;
}
@@ -210,7 +212,7 @@
ConstructorVisitor(Set<String> errors)
{
- super(Opcodes.ASM7);
+ super(ASM_BYTECODE_VERSION);
this.errors = errors;
}
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFExecutorService.java b/src/java/org/apache/cassandra/cql3/functions/UDFExecutorService.java
index 3b7631f..1970545 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFExecutorService.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFExecutorService.java
@@ -22,6 +22,7 @@
import org.apache.cassandra.concurrent.ThreadPoolExecutorBase;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.UDF_EXECUTOR_THREAD_KEEPALIVE_MS;
import static org.apache.cassandra.utils.FBUtilities.getAvailableProcessors;
import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQueue;
@@ -33,7 +34,7 @@
*/
final class UDFExecutorService extends ThreadPoolExecutorBase
{
- private static final int KEEPALIVE = Integer.getInteger("cassandra.udf_executor_thread_keepalive_ms", 30000);
+ private static final int KEEPALIVE = UDF_EXECUTOR_THREAD_KEEPALIVE_MS.getInt();
public UDFExecutorService(NamedThreadFactory threadFactory, String jmxPath)
{
diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
index 5ef065b..58454a4 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java
@@ -46,7 +46,6 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.CqlBuilder;
-import org.apache.cassandra.cql3.SchemaElement;
import org.apache.cassandra.cql3.functions.types.DataType;
import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -67,7 +66,7 @@
/**
* Base class for User Defined Functions.
*/
-public abstract class UDFunction extends AbstractFunction implements ScalarFunction, SchemaElement
+public abstract class UDFunction extends UserFunction implements ScalarFunction
{
protected static final Logger logger = LoggerFactory.getLogger(UDFunction.class);
@@ -267,13 +266,7 @@
{
assertUdfsEnabled(language);
- switch (language)
- {
- case "java":
- return new JavaBasedUDFunction(name, argNames, argTypes, returnType, calledOnNullInput, body);
- default:
- return new ScriptBasedUDFunction(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
- }
+ return new JavaBasedUDFunction(name, argNames, argTypes, returnType, calledOnNullInput, body);
}
/**
@@ -449,8 +442,8 @@
{
if (!DatabaseDescriptor.enableUserDefinedFunctions())
throw new InvalidRequestException("User-defined functions are disabled in cassandra.yaml - set user_defined_functions_enabled=true to enable");
- if (!"java".equalsIgnoreCase(language) && !DatabaseDescriptor.enableScriptedUserDefinedFunctions())
- throw new InvalidRequestException("Scripted user-defined functions are disabled in cassandra.yaml - set scripted_user_defined_functions_enabled=true to enable if you are aware of the security risks");
+ if (!"java".equalsIgnoreCase(language))
+ throw new InvalidRequestException("Currently only Java UDFs are available in Cassandra. For more information - CASSANDRA-18252 and CASSANDRA-17281");
}
static void initializeThread()
@@ -617,11 +610,6 @@
return false;
}
- public boolean isNative()
- {
- return false;
- }
-
public boolean isCalledOnNullInput()
{
return calledOnNullInput;
@@ -643,8 +631,7 @@
}
/**
- * Used by UDF implementations (both Java code generated by {@link JavaBasedUDFunction}
- * and script executor {@link ScriptBasedUDFunction}) to convert the C*
+ * Used by UDF implementations (both Java code generated by {@link JavaBasedUDFunction}) to convert the C*
* serialized representation to the Java object representation.
*
* @param protocolVersion the native protocol version used for serialization
@@ -661,8 +648,7 @@
}
/**
- * Used by UDF implementations (both Java code generated by {@link JavaBasedUDFunction}
- * and script executor {@link ScriptBasedUDFunction}) to convert the Java
+ * Used by UDF implementations (both Java code generated by {@link JavaBasedUDFunction}) to convert the Java
* object representation for the return value to the C* serialized representation.
*
* @param protocolVersion the native protocol version used for serialization
@@ -760,7 +746,7 @@
@Override
public int hashCode()
{
- return Objects.hashCode(name, Functions.typeHashCode(argTypes), Functions.typeHashCode(returnType), returnType, language, body);
+ return Objects.hashCode(name, UserFunctions.typeHashCode(argTypes), UserFunctions.typeHashCode(returnType), returnType, language, body);
}
private static class UDFClassLoader extends ClassLoader
diff --git a/src/java/org/apache/cassandra/cql3/functions/UserFunction.java b/src/java/org/apache/cassandra/cql3/functions/UserFunction.java
new file mode 100644
index 0000000..7fa10a6
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/UserFunction.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.List;
+
+import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * A non-native, user-defined function, like UDFs and UDAs.
+ */
+public abstract class UserFunction extends AbstractFunction implements SchemaElement
+{
+ public UserFunction(FunctionName name, List<AbstractType<?>> argTypes, AbstractType<?> returnType)
+ {
+ super(name, argTypes, returnType);
+ }
+
+ @Override
+ public final boolean isNative()
+ {
+ return false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java b/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java
index 3d82ece..0083883 100644
--- a/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java
+++ b/src/java/org/apache/cassandra/cql3/functions/UuidFcts.java
@@ -26,12 +26,12 @@
public abstract class UuidFcts
{
- public static Collection<Function> all()
+ public static void addFunctionsTo(NativeFunctions functions)
{
- return Collections.singleton(uuidFct);
+ functions.add(uuidFct);
}
- public static final Function uuidFct = new NativeScalarFunction("uuid", UUIDType.instance)
+ public static final NativeFunction uuidFct = new NativeScalarFunction("uuid", UUIDType.instance)
{
public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
{
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java b/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java
new file mode 100644
index 0000000..add2228
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.AssignmentTestable;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.CqlBuilder;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.Terms;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionResolver;
+import org.apache.cassandra.cql3.functions.ScalarFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
+/**
+ * Dynamic data mask that can be applied to a schema column.
+ * <p>
+ * It consists on a partial application of a certain {@link MaskingFunction} to the values of a column, with the
+ * precondition that the type of any masked column is compatible with the type of the first argument of the function.
+ * <p>
+ * This partial application is meant to be associated to specific columns in the schema, acting as a mask for the values
+ * of those columns. It's associated to queries such as:
+ * <pre>
+ * CREATE TABLE t (k int PRIMARY KEY, v int MASKED WITH mask_inner(1, 1));
+ * ALTER TABLE t ALTER v MASKED WITH mask_inner(2, 1);
+ * ALTER TABLE t ALTER v DROP MASKED;
+ * </pre>
+ * Note that in the example above we are referencing the {@code mask_inner} function with two arguments. However, that
+ * CQL function actually has three arguments. The first argument is always ommitted when attaching the function to a
+ * schema column. The value of that first argument is always the value of the masked column, in this case an int.
+ */
+public abstract class ColumnMask
+{
+ public static final String DISABLED_ERROR_MESSAGE = "Cannot mask columns because dynamic data masking is not " +
+ "enabled. You can enable it with the " +
+ "dynamic_data_masking_enabled property on cassandra.yaml";
+
+ /** The CQL function used for masking. */
+ public final ScalarFunction function;
+
+ /** The values of the arguments of the partially applied masking function. */
+ protected final ByteBuffer[] partialArgumentValues;
+
+ private ColumnMask(ScalarFunction function, ByteBuffer... partialArgumentValues)
+ {
+ assert function.argTypes().size() == partialArgumentValues.length + 1;
+ this.function = function;
+ this.partialArgumentValues = partialArgumentValues;
+ }
+
+ public static ColumnMask build(ScalarFunction function, ByteBuffer... partialArgumentValues)
+ {
+ return function.isNative()
+ ? new Native((MaskingFunction) function, partialArgumentValues)
+ : new Custom(function, partialArgumentValues);
+ }
+
+ /**
+ * @return The types of the arguments of the partially applied masking function, as an unmodifiable list.
+ */
+ public List<AbstractType<?>> partialArgumentTypes()
+ {
+ List<AbstractType<?>> argTypes = function.argTypes();
+ return argTypes.size() == 1
+ ? Collections.emptyList()
+ : Collections.unmodifiableList(argTypes.subList(1, argTypes.size()));
+ }
+
+ /**
+ * @return The values of the arguments of the partially applied masking function, as an unmodifiable list that can
+ * contain nulls.
+ */
+ public List<ByteBuffer> partialArgumentValues()
+ {
+ return Collections.unmodifiableList(Arrays.asList(partialArgumentValues));
+ }
+
+ /**
+ * @return A copy of this mask for a version of its masked column that has its type reversed.
+ */
+ public ColumnMask withReversedType()
+ {
+ AbstractType<?> reversed = ReversedType.getInstance(function.argTypes().get(0));
+ List<AbstractType<?>> args = ImmutableList.<AbstractType<?>>builder()
+ .add(reversed)
+ .addAll(partialArgumentTypes())
+ .build();
+ Function newFunction = FunctionResolver.get(function.name().keyspace, function.name(), args, null, null, null);
+ assert newFunction != null;
+ return build((ScalarFunction) newFunction, partialArgumentValues);
+ }
+
+ /**
+ * @param protocolVersion the used version of the transport protocol
+ * @param value a column value to be masked
+ * @return the specified value after having been masked by the masked function
+ */
+ public ByteBuffer mask(ProtocolVersion protocolVersion, ByteBuffer value)
+ {
+ if (!DatabaseDescriptor.getDynamicDataMaskingEnabled())
+ return value;
+
+ return maskInternal(protocolVersion, value);
+ }
+
+ protected abstract ByteBuffer maskInternal(ProtocolVersion protocolVersion, ByteBuffer value);
+
+ public static void ensureEnabled()
+ {
+ if (!DatabaseDescriptor.getDynamicDataMaskingEnabled())
+ throw new InvalidRequestException(DISABLED_ERROR_MESSAGE);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ ColumnMask mask = (ColumnMask) o;
+ return function.name().equals(mask.function.name())
+ && Arrays.equals(partialArgumentValues, mask.partialArgumentValues);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(function.name(), Arrays.hashCode(partialArgumentValues));
+ }
+
+ @Override
+ public String toString()
+ {
+ List<AbstractType<?>> types = partialArgumentTypes();
+ List<String> arguments = new ArrayList<>(types.size());
+ for (int i = 0; i < types.size(); i++)
+ {
+ CQL3Type type = types.get(i).asCQL3Type();
+ ByteBuffer value = partialArgumentValues[i];
+ arguments.add(type.toCQLLiteral(value, ProtocolVersion.CURRENT));
+ }
+ return format("%s(%s)", function.name(), StringUtils.join(arguments, ", "));
+ }
+
+ public void appendCqlTo(CqlBuilder builder)
+ {
+ builder.append(" MASKED WITH ").append(toString());
+ }
+
+ /**
+ * {@link ColumnMask} for native masking functions.
+ */
+ private static class Native extends ColumnMask
+ {
+ private final MaskingFunction.Masker masker;
+
+ public Native(MaskingFunction function, ByteBuffer... partialArgumentValues)
+ {
+ super(function, partialArgumentValues);
+ masker = function.masker(partialArgumentValues);
+ }
+
+ @Override
+ protected ByteBuffer maskInternal(ProtocolVersion protocolVersion, ByteBuffer value)
+ {
+ return masker.mask(value);
+ }
+ }
+
+ /**
+ * {@link ColumnMask} for user-defined masking functions.
+ */
+ private static class Custom extends ColumnMask
+ {
+ public Custom(ScalarFunction function, ByteBuffer... partialArgumentValues)
+ {
+ super(function, partialArgumentValues);
+ }
+
+ @Override
+ protected ByteBuffer maskInternal(ProtocolVersion protocolVersion, ByteBuffer value)
+ {
+ List<ByteBuffer> argumentValues;
+ int numPartialArgs = partialArgumentValues.length;
+ if (numPartialArgs == 0)
+ {
+ argumentValues = Collections.singletonList(value);
+ }
+ else
+ {
+ ByteBuffer[] args = new ByteBuffer[numPartialArgs + 1];
+ args[0] = value;
+ System.arraycopy(partialArgumentValues, 0, args, 1, numPartialArgs);
+ argumentValues = Arrays.asList(args);
+ }
+
+ return function.execute(protocolVersion, argumentValues);
+ }
+ }
+
+ /**
+ * A parsed but not prepared column mask.
+ */
+ public final static class Raw
+ {
+ public final FunctionName name;
+ public final List<Term.Raw> rawPartialArguments;
+
+ public Raw(FunctionName name, List<Term.Raw> rawPartialArguments)
+ {
+ this.name = name;
+ this.rawPartialArguments = rawPartialArguments;
+ }
+
+ public ColumnMask prepare(String keyspace, String table, ColumnIdentifier column, AbstractType<?> type)
+ {
+ ScalarFunction function = findMaskingFunction(keyspace, table, column, type);
+ ByteBuffer[] partialArguments = preparePartialArguments(keyspace, function);
+ return ColumnMask.build(function, partialArguments);
+ }
+
+ private ScalarFunction findMaskingFunction(String keyspace, String table, ColumnIdentifier column, AbstractType<?> type)
+ {
+ List<AssignmentTestable> args = new ArrayList<>(rawPartialArguments.size() + 1);
+ args.add(type);
+ args.addAll(rawPartialArguments);
+
+ Function function = FunctionResolver.get(keyspace, name, args, keyspace, table, type);
+
+ if (function == null)
+ throw invalidRequest("Unable to find masking function for %s, " +
+ "no declared function matches the signature %s",
+ column, this);
+
+ if (function.isAggregate())
+ throw invalidRequest("Aggregate function %s cannot be used for masking table columns", this);
+
+ if (function.isNative() && !(function instanceof MaskingFunction))
+ throw invalidRequest("Not-masking function %s cannot be used for masking table columns", this);
+
+ if (!function.isNative() && !function.name().keyspace.equals(keyspace))
+ throw invalidRequest("Masking function %s doesn't belong to the same keyspace as the table %s.%s",
+ this, keyspace, table);
+
+ CQL3Type returnType = function.returnType().asCQL3Type();
+ CQL3Type expectedType = type.asCQL3Type();
+ if (!returnType.equals(expectedType))
+ throw invalidRequest("Masking function %s return type is %s. " +
+ "This is different to the type of the masked column %s of type %s. " +
+ "Masking functions can only be attached to table columns " +
+ "if they return the same data type as the masked column.",
+ this, returnType, column, expectedType);
+
+ return (ScalarFunction) function;
+ }
+
+ private ByteBuffer[] preparePartialArguments(String keyspace, ScalarFunction function)
+ {
+ // Note that there could be null arguments
+ ByteBuffer[] arguments = new ByteBuffer[rawPartialArguments.size()];
+
+ for (int i = 0; i < rawPartialArguments.size(); i++)
+ {
+ String term = rawPartialArguments.get(i).toString();
+ AbstractType<?> type = function.argTypes().get(i + 1);
+ arguments[i] = Terms.asBytes(keyspace, term, type);
+ }
+
+ return arguments;
+ }
+
+ @Override
+ public String toString()
+ {
+ return format("%s(%s)", name, StringUtils.join(rawPartialArguments, ", "));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java
new file mode 100644
index 0000000..b111671
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunction.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * A {@link MaskingFunction} that returns a fixed replacement value for the data type of its single argument.
+ * <p>
+ * The default values are defined by {@link AbstractType#getMaskedValue()}, being {@code ****} for text fields,
+ * {@code false} for booleans, zero for numeric types, {@code 1970-01-01} for dates, etc.
+ * <p>
+ * For example, given a text column named "username", {@code mask_default(username)} will always return {@code ****},
+ * independently of the actual value of that column.
+ */
+public class DefaultMaskingFunction extends MaskingFunction
+{
+ public static final String NAME = "default";
+
+ private final Masker masker;
+
+ private <T> DefaultMaskingFunction(FunctionName name, AbstractType<T> inputType)
+ {
+ super(name, inputType, inputType);
+ masker = new Masker(inputType);
+ }
+
+ @Override
+ public Masker masker(ByteBuffer... parameters)
+ {
+ return masker;
+ }
+
+ private static class Masker implements MaskingFunction.Masker
+ {
+ private final ByteBuffer defaultValue;
+
+ private Masker(AbstractType<?> inputType)
+ {
+ defaultValue = inputType.getMaskedValue();
+ }
+
+ @Override
+ public ByteBuffer mask(ByteBuffer value)
+ {
+ return defaultValue;
+ }
+ }
+
+ /** @return a {@link FunctionFactory} to build new {@link DefaultMaskingFunction}s. */
+ public static FunctionFactory factory()
+ {
+ return new MaskingFunction.Factory(NAME, FunctionParameter.anyType(false))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return new DefaultMaskingFunction(name, argTypes.get(0));
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java
new file mode 100644
index 0000000..aa515dd
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/HashMaskingFunction.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.function.Supplier;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.StringType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * A {@link MaskingFunction} that replaces the specified column value by its hash according to the specified
+ * algorithm. The available algorithms are those defined by the registered security {@link java.security.Provider}s.
+ * If no algorithm is passed to the function, the {@link #DEFAULT_ALGORITHM} will be used.
+ */
+public class HashMaskingFunction extends MaskingFunction
+{
+ public static final String NAME = "hash";
+
+ /** The default hashing algorithm to be used if no other algorithm is specified in the call to the function. */
+ public static final String DEFAULT_ALGORITHM = "SHA-256";
+
+ // The default message digest is lazily built to prevent a failure during server startup if the algorithm is not
+ // available. That way, if the algorithm is not found only the calls to the function will fail.
+ private static final Supplier<MessageDigest> DEFAULT_DIGEST = Suppliers.memoize(() -> messageDigest(DEFAULT_ALGORITHM));
+ private static final AbstractType<?>[] DEFAULT_ARGUMENTS = {};
+
+ /** The type of the supplied algorithm argument, {@code null} if that argument isn't supplied. */
+ @Nullable
+ private final StringType algorithmArgumentType;
+
+ private HashMaskingFunction(FunctionName name, AbstractType<?> inputType, @Nullable StringType algorithmArgumentType)
+ {
+ super(name, BytesType.instance, inputType, argumentsType(algorithmArgumentType));
+ this.algorithmArgumentType = algorithmArgumentType;
+ }
+
+ private static AbstractType<?>[] argumentsType(@Nullable StringType algorithmArgumentType)
+ {
+ // the algorithm argument is optional, so we will have different signatures depending on whether that argument
+ // is supplied or not
+ return algorithmArgumentType == null
+ ? DEFAULT_ARGUMENTS
+ : new AbstractType<?>[]{ algorithmArgumentType };
+ }
+
+ @Override
+ public Masker masker(ByteBuffer... parameters)
+ {
+ return new Masker(algorithmArgumentType, parameters);
+ }
+
+ private static class Masker implements MaskingFunction.Masker
+ {
+ private final MessageDigest digest;
+
+ private Masker(StringType algorithmArgumentType, ByteBuffer... parameters)
+ {
+ if (algorithmArgumentType == null || parameters[0] == null)
+ {
+ digest = DEFAULT_DIGEST.get();
+ }
+ else
+ {
+ String algorithm = algorithmArgumentType.compose(parameters[0]);
+ digest = messageDigest(algorithm);
+ }
+ }
+
+ @Override
+ public ByteBuffer mask(ByteBuffer value)
+ {
+ return HashMaskingFunction.hash(digest, value);
+ }
+
+ }
+
+ @VisibleForTesting
+ @Nullable
+ static ByteBuffer hash(MessageDigest digest, ByteBuffer value)
+ {
+ if (value == null)
+ return null;
+
+ byte[] hash = digest.digest(ByteBufferUtil.getArray(value));
+ return BytesType.instance.compose(ByteBuffer.wrap(hash));
+ }
+
+ @VisibleForTesting
+ static MessageDigest messageDigest(String algorithm)
+ {
+ try
+ {
+ return MessageDigest.getInstance(algorithm);
+ }
+ catch (NoSuchAlgorithmException e)
+ {
+ throw new InvalidRequestException("Hash algorithm not found: " + algorithm);
+ }
+ }
+
+ /** @return a {@link FunctionFactory} to build new {@link HashMaskingFunction}s. */
+ public static FunctionFactory factory()
+ {
+ return new MaskingFunction.Factory(NAME,
+ FunctionParameter.anyType(false),
+ FunctionParameter.optional(FunctionParameter.fixed(CQL3Type.Native.TEXT)))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ switch (argTypes.size())
+ {
+ case 1:
+ return new HashMaskingFunction(name, argTypes.get(0), null);
+ case 2:
+ return new HashMaskingFunction(name, argTypes.get(0), UTF8Type.instance);
+ default:
+ throw invalidNumberOfArgumentsException();
+ }
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/MaskingFcts.java b/src/java/org/apache/cassandra/cql3/functions/masking/MaskingFcts.java
new file mode 100644
index 0000000..a1a3baf
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/MaskingFcts.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.apache.cassandra.cql3.functions.NativeFunctions;
+
+/**
+ * A collection of {@link MaskingFunction}s for dynamic data masking, meant to obscure the real value of a column.
+ */
+public class MaskingFcts
+{
+ /** Adds all the available native data masking functions to the specified native functions. */
+ public static void addFunctionsTo(NativeFunctions functions)
+ {
+ functions.add(NullMaskingFunction.factory());
+ functions.add(ReplaceMaskingFunction.factory());
+ functions.add(DefaultMaskingFunction.factory());
+ functions.add(HashMaskingFunction.factory());
+ PartialMaskingFunction.factories().forEach(functions::add);
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/MaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/MaskingFunction.java
new file mode 100644
index 0000000..0ed037d
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/MaskingFunction.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.google.common.collect.ObjectArrays;
+
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeScalarFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+/**
+ * A {@link NativeScalarFunction} that totally or partially replaces the original value of a column value,
+ * meant to obscure the real value of the column.
+ * <p>
+ * The names of all masking functions share a common prefix, {@link MaskingFunction#NAME_PREFIX}, to easily identify
+ * their purpose.
+ */
+public abstract class MaskingFunction extends NativeScalarFunction
+{
+ /** The common prefix for the names of all the native data masking functions. */
+ public static final String NAME_PREFIX = "mask_";
+
+ /**
+ * @param name the name of the function
+ * @param outputType the type of the values returned by the function
+ * @param inputType the type of the values accepted by the function, always be the first argument of the function
+ * @param argsType the type of the additional arguments of the function
+ */
+ protected MaskingFunction(FunctionName name,
+ AbstractType<?> outputType,
+ AbstractType<?> inputType,
+ AbstractType<?>... argsType)
+ {
+ super(name.name, outputType, ObjectArrays.concat(inputType, argsType));
+ }
+
+ @Override
+ public final ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters)
+ {
+ ByteBuffer[] partialParameters = new ByteBuffer[parameters.size() - 1];
+ for (int i = 0; i < partialParameters.length; i++)
+ partialParameters[i] = parameters.get(i + 1);
+
+ return masker(partialParameters).mask(parameters.get(0));
+ }
+
+ /**
+ * Returns a new {@link Masker} for the specified masking parameters.
+ * This is meant to be used by {@link ColumnMask}, so it doesn't need to evaluate the arguments on every call.
+ *
+ * @param parameters the masking parameters in the function call.
+ * @return a new {@link Masker} using the specified masking arguments
+ */
+ public abstract Masker masker(ByteBuffer... parameters);
+
+ /**
+ * Class that actually makes the masking of the first function parameter according to the masking arguments.
+ */
+ public interface Masker
+ {
+ public ByteBuffer mask(ByteBuffer value);
+ }
+
+ protected static abstract class Factory extends FunctionFactory
+ {
+ public Factory(String name, FunctionParameter... parameters)
+ {
+ super(NAME_PREFIX + name.toLowerCase(), parameters);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java
new file mode 100644
index 0000000..0d82a76
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/NullMaskingFunction.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * A {@link MaskingFunction} that always returns a {@code null} column. The returned value is always an absent column,
+ * as it didn't exist, and not a not-null column representing a {@code null} value.
+ * <p>
+ * For example, given a text column named "username", {@code mask_null(username)} will always return {@code null},
+ * independently of the actual value of that column.
+ */
+public class NullMaskingFunction extends MaskingFunction
+{
+ public static final String NAME = "null";
+ private static final Masker MASKER = new Masker();
+
+ private NullMaskingFunction(FunctionName name, AbstractType<?> inputType)
+ {
+ super(name, inputType, inputType);
+ }
+
+ @Override
+ public Masker masker(ByteBuffer... parameters)
+ {
+ return MASKER;
+ }
+
+ private static class Masker implements MaskingFunction.Masker
+ {
+ @Override
+ public ByteBuffer mask(ByteBuffer value)
+ {
+ return null;
+ }
+ }
+
+ /** @return a {@link FunctionFactory} to build new {@link NullMaskingFunction}s. */
+ public static FunctionFactory factory()
+ {
+ return new MaskingFunction.Factory(NAME, FunctionParameter.anyType(false))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return new NullMaskingFunction(name, argTypes.get(0));
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java
new file mode 100644
index 0000000..80826a4
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunction.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+/**
+ * A {@link MaskingFunction} applied to a {@link org.apache.cassandra.db.marshal.StringType} value that,
+ * depending on {@link Type}:
+ * <ul>
+ * <li>Replaces each character between the supplied positions by the supplied padding character. In other words,
+ * it will mask all the characters except the first m and last n.</li>
+ * <li>Replaces each character before and after the supplied positions by the supplied padding character. In other
+ * words, it will only mask all the first m and last n characters.</li>
+ * </ul>
+ * The returned value will allways be of the same type as the first string-based argument.
+ */
+public class PartialMaskingFunction extends MaskingFunction
+{
+ /** The character to be used as padding if no other character is supplied when calling the function. */
+ public static final char DEFAULT_PADDING_CHAR = '*';
+
+ /** The type of partial masking to perform, inner or outer. */
+ private final Type type;
+
+ /** The original type of the masked value. */
+ private final AbstractType<String> inputType;
+
+ /** Whether a padding argument hab been supplied. */
+ @Nullable
+ private final boolean hasPaddingArgument;
+
+ private PartialMaskingFunction(FunctionName name,
+ Type type,
+ AbstractType<String> inputType,
+ boolean hasPaddingArgument)
+ {
+ super(name, inputType, inputType, argumentsType(hasPaddingArgument));
+
+ this.type = type;
+ this.inputType = inputType;
+ this.hasPaddingArgument = hasPaddingArgument;
+ }
+
+ private static AbstractType<?>[] argumentsType(boolean hasPaddingArgument)
+ {
+ // The padding argument is optional, so we provide different signatures depending on whether it's present or not.
+ // Also, the padding argument should be a single character, but we don't have a data type for that, so we use
+ // a string-based argument. We will later validate on execution that the string argument is single-character.
+ return hasPaddingArgument
+ ? new AbstractType<?>[]{ Int32Type.instance, Int32Type.instance, UTF8Type.instance }
+ : new AbstractType<?>[]{ Int32Type.instance, Int32Type.instance };
+ }
+
+ @Override
+ public Masker masker(ByteBuffer... parameters)
+ {
+ return new Masker(parameters);
+ }
+
+ private class Masker implements MaskingFunction.Masker
+ {
+ private final int begin, end;
+ private final char padding;
+
+ private Masker(ByteBuffer... parameters)
+ {
+ // Parse the beginning and end positions. No validation is needed since the masker accepts negatives,
+ // but we should consider that the arguments migh be null.
+ begin = parameters[0] == null ? 0 : Int32Type.instance.compose(parameters[0]);
+ end = parameters[1] == null ? 0 : Int32Type.instance.compose(parameters[1]);
+
+ // Parse the padding character. The type of the argument is a string of any length because we don't have a
+ // character type in CQL, so we should verify that the passed string argument is single-character.
+ if (hasPaddingArgument && parameters[2] != null)
+ {
+ String parameter = UTF8Type.instance.compose(parameters[2]);
+ if (parameter.length() != 1)
+ {
+ throw new InvalidRequestException(String.format("The padding argument for function %s should " +
+ "be single-character, but '%s' has %d characters.",
+ name(), parameter, parameter.length()));
+ }
+ padding = parameter.charAt(0);
+ }
+ else
+ {
+ padding = DEFAULT_PADDING_CHAR;
+ }
+ }
+
+ @Override
+ public ByteBuffer mask(ByteBuffer value)
+ {
+ // Null column values aren't masked
+ if (value == null)
+ return null;
+
+ String stringValue = inputType.compose(value);
+ String maskedValue = type.mask(stringValue, begin, end, padding);
+ return inputType.decompose(maskedValue);
+ }
+ }
+
+ public enum Type
+ {
+ /** Masks everything except the first {@code begin} and last {@code end} characters. */
+ INNER
+ {
+ @Override
+ protected boolean shouldMask(int pos, int begin, int end)
+ {
+ return pos >= begin && pos <= end;
+ }
+ },
+ /** Masks only the first {@code begin} and last {@code end} characters. */
+ OUTER
+ {
+ @Override
+ protected boolean shouldMask(int pos, int begin, int end)
+ {
+ return pos < begin || pos > end;
+ }
+ };
+
+ protected abstract boolean shouldMask(int pos, int begin, int end);
+
+ @VisibleForTesting
+ public String mask(String value, int begin, int end, char padding)
+ {
+ if (StringUtils.isEmpty(value))
+ return value;
+
+ int size = value.length();
+ int endIndex = size - 1 - end;
+ char[] chars = new char[size];
+
+ for (int i = 0; i < size; i++)
+ {
+ chars[i] = shouldMask(i, begin, endIndex) ? padding : value.charAt(i);
+ }
+
+ return new String(chars);
+ }
+ }
+
+ /** @return a collection of function factories to build new {@code PartialMaskingFunction} functions. */
+ public static Collection<FunctionFactory> factories()
+ {
+ return Stream.of(Type.values())
+ .map(PartialMaskingFunction::factory)
+ .collect(Collectors.toSet());
+ }
+
+ private static FunctionFactory factory(Type type)
+ {
+ return new MaskingFunction.Factory(type.name(),
+ FunctionParameter.string(),
+ FunctionParameter.fixed(CQL3Type.Native.INT),
+ FunctionParameter.fixed(CQL3Type.Native.INT),
+ FunctionParameter.optional(FunctionParameter.fixed(CQL3Type.Native.TEXT)))
+ {
+ @Override
+ @SuppressWarnings("unchecked")
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ AbstractType<String> inputType = (AbstractType<String>) argTypes.get(0);
+ return new PartialMaskingFunction(name, type, inputType, argTypes.size() == 4);
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java b/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java
new file mode 100644
index 0000000..1510521
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunction.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeFunction;
+import org.apache.cassandra.db.marshal.AbstractType;
+
+/**
+ * A {@link MaskingFunction} that replaces the specified column value by a certain replacement value.
+ * <p>
+ * The returned replacement value needs to have the same type as the replaced value.
+ * <p>
+ * For example, given a text column named "username", {@code mask_replace(username, '****')} will return {@code ****}.
+ */
+public class ReplaceMaskingFunction extends MaskingFunction
+{
+ public static final String NAME = "replace";
+
+ private ReplaceMaskingFunction(FunctionName name, AbstractType<?> replacedType, AbstractType<?> replacementType)
+ {
+ super(name, replacementType, replacedType, replacementType);
+ }
+
+ @Override
+ public Masker masker(ByteBuffer... parameters)
+ {
+ return new Masker(parameters[0]);
+ }
+
+ private static class Masker implements MaskingFunction.Masker
+ {
+ private final ByteBuffer replacement;
+
+ private Masker(ByteBuffer replacement)
+ {
+ this.replacement = replacement;
+ }
+
+ @Override
+ public ByteBuffer mask(ByteBuffer value)
+ {
+ return replacement;
+ }
+ }
+
+ /** @return a {@link FunctionFactory} to build new {@link ReplaceMaskingFunction}s. */
+ public static FunctionFactory factory()
+ {
+ return new MaskingFunction.Factory(NAME, FunctionParameter.anyType(true), FunctionParameter.sameAsFirst())
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ AbstractType<?> replacedType = argTypes.get(0);
+ AbstractType<?> replacementType = argTypes.get(1);
+
+ assert replacedType == replacementType
+ : String.format("Both arguments should have the same type, but found %s(%s, %s)",
+ name, replacedType.asCQL3Type(), replacementType.asCQL3Type());
+
+ return new ReplaceMaskingFunction(name, replacedType, replacementType);
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java b/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java
index dc34bca..54ff540 100644
--- a/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java
+++ b/src/java/org/apache/cassandra/cql3/functions/types/TypeCodec.java
@@ -34,9 +34,9 @@
import com.google.common.io.ByteStreams;
import com.google.common.reflect.TypeToken;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.cql3.functions.types.exceptions.InvalidTypeException;
import org.apache.cassandra.cql3.functions.types.utils.Bytes;
+import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.vint.VIntCoding;
import static com.google.common.base.Preconditions.checkArgument;
@@ -3043,17 +3043,9 @@
+ VIntCoding.computeVIntSize(days)
+ VIntCoding.computeVIntSize(nanoseconds);
ByteBuffer bb = ByteBuffer.allocate(size);
- try
- {
- VIntCoding.writeVInt(months, bb);
- VIntCoding.writeVInt(days, bb);
- VIntCoding.writeVInt(nanoseconds, bb);
- }
- catch (IOException e)
- {
- // cannot happen
- throw new AssertionError();
- }
+ VIntCoding.writeVInt(months, bb);
+ VIntCoding.writeVInt(days, bb);
+ VIntCoding.writeVInt(nanoseconds, bb);
bb.flip();
return bb;
}
@@ -3071,8 +3063,8 @@
DataInput in = ByteStreams.newDataInput(Bytes.getArray(bytes));
try
{
- int months = (int) VIntCoding.readVInt(in);
- int days = (int) VIntCoding.readVInt(in);
+ int months = VIntCoding.readVInt32(in);
+ int days = VIntCoding.readVInt32(in);
long nanoseconds = VIntCoding.readVInt(in);
return Duration.newInstance(months, days, nanoseconds);
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
index c1d0c52..146ec20 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
@@ -228,5 +228,4 @@
{
return restriction.isContains() || restriction.isLIKE() || index != restriction.getFirstColumn().position();
}
-
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
index acbb48e..22c000e 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
@@ -18,12 +18,22 @@
package org.apache.cassandra.cql3.restrictions;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.serializers.ListSerializer;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.cql3.*;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+import org.apache.cassandra.cql3.AbstractMarker;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.Terms;
+import org.apache.cassandra.cql3.Tuples;
import org.apache.cassandra.cql3.Term.Terminal;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.statements.Bound;
@@ -31,8 +41,8 @@
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.IndexRegistry;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.ListSerializer;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
@@ -258,7 +268,7 @@
for (List<ByteBuffer> splitValue : splitValues)
values.add(splitValue.get(0));
- ByteBuffer buffer = ListSerializer.pack(values, values.size(), ProtocolVersion.V3);
+ ByteBuffer buffer = ListSerializer.pack(values, values.size());
filter.add(getFirstColumn(), Operator.IN, buffer);
}
else
@@ -376,7 +386,7 @@
{
boolean reversed = getFirstColumn().isReversedType();
- EnumMap<Bound, List<ByteBuffer>> componentBounds = new EnumMap<Bound, List<ByteBuffer>>(Bound.class);
+ EnumMap<Bound, List<ByteBuffer>> componentBounds = new EnumMap<>(Bound.class);
componentBounds.put(Bound.START, componentBounds(Bound.START, options));
componentBounds.put(Bound.END, componentBounds(Bound.END, options));
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
index 7a5d5b9..344498c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
@@ -105,6 +105,15 @@
return new ArrayList<>(restrictions.keySet());
}
+ /**
+ * @return a direct reference to the key set from {@link #restrictions} with no defenseive copying
+ */
+ @Override
+ public Collection<ColumnMetadata> getColumnDefinitions()
+ {
+ return restrictions.keySet();
+ }
+
@Override
public void addFunctionsTo(List<Function> functions)
{
@@ -332,7 +341,6 @@
/**
* {@code Iterator} decorator that removes duplicates in an ordered one.
*
- * @param iterator the decorated iterator
* @param <E> the iterator element type.
*/
private static final class DistinctIterator<E> extends AbstractIterator<E>
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
index 9803adc..8c89bca 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cql3.restrictions;
+import java.util.Collection;
import java.util.List;
import java.util.Set;
@@ -57,6 +58,17 @@
return restrictions.getColumnDefs();
}
+ @Override
+ public Collection<ColumnMetadata> getColumnDefinitions()
+ {
+ return restrictions.getColumnDefinitions();
+ }
+
+ public RestrictionSet getRestrictionSet()
+ {
+ return restrictions;
+ }
+
public void addFunctionsTo(List<Function> functions)
{
restrictions.addFunctionsTo(functions);
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
index 77e0dd9..0ad7530 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cql3.restrictions;
+import java.util.Collection;
import java.util.Set;
import org.apache.cassandra.schema.ColumnMetadata;
@@ -35,6 +36,15 @@
Set<Restriction> getRestrictions(ColumnMetadata columnDef);
/**
+ * This method exists in addition to {@link #getColumnDefs()} in case implementations want to
+ * provide columns definitions that are not strictly in position order.
+ */
+ default Collection<ColumnMetadata> getColumnDefinitions()
+ {
+ return getColumnDefs();
+ }
+
+ /**
* Checks if this <code>Restrictions</code> is empty or not.
*
* @return <code>true</code> if this <code>Restrictions</code> is empty, <code>false</code> otherwise.
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
index e5b2465..5ce78ae 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
@@ -24,7 +24,6 @@
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.ListSerializer;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.Term.Terminal;
import org.apache.cassandra.cql3.functions.Function;
@@ -221,7 +220,7 @@
QueryOptions options)
{
List<ByteBuffer> values = getValues(options);
- ByteBuffer buffer = ListSerializer.pack(values, values.size(), ProtocolVersion.V3);
+ ByteBuffer buffer = ListSerializer.pack(values, values.size());
filter.add(columnDef, Operator.IN, buffer);
}
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java
index 42b0b4e..989d5da 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleRestriction.java
@@ -51,6 +51,14 @@
return false;
}
+ /**
+ * @return <code>true</code> if this restriction is based on equality comparison rather than a range or negation
+ */
+ default boolean isEqualityBased()
+ {
+ return isEQ() || isIN() || isContains();
+ }
+
public default boolean isNotNull()
{
return false;
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index 8f8be94..d522f46 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -28,7 +28,10 @@
import org.apache.cassandra.cql3.statements.StatementType;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.Index;
@@ -50,10 +53,15 @@
*/
public final class StatementRestrictions
{
- public static final String REQUIRES_ALLOW_FILTERING_MESSAGE =
- "Cannot execute this query as it might involve data filtering and " +
- "thus may have unpredictable performance. If you want to execute " +
- "this query despite the performance unpredictability, use ALLOW FILTERING";
+ private static final String ALLOW_FILTERING_MESSAGE =
+ "Cannot execute this query as it might involve data filtering and thus may have unpredictable performance. ";
+
+ public static final String REQUIRES_ALLOW_FILTERING_MESSAGE = ALLOW_FILTERING_MESSAGE +
+ "If you want to execute this query despite the performance unpredictability, use ALLOW FILTERING";
+
+ public static final String CANNOT_USE_ALLOW_FILTERING_MESSAGE = ALLOW_FILTERING_MESSAGE +
+ "Executing this query despite the performance unpredictability with ALLOW FILTERING has been disabled " +
+ "by the allow_filtering_enabled property in cassandra.yaml";
/**
* The type of statement
@@ -125,7 +133,8 @@
this.notNullColumns = new HashSet<>();
}
- public StatementRestrictions(StatementType type,
+ public StatementRestrictions(ClientState state,
+ StatementType type,
TableMetadata table,
WhereClause whereClause,
VariableSpecifications boundNames,
@@ -133,14 +142,15 @@
boolean allowFiltering,
boolean forView)
{
- this(type, table, whereClause, boundNames, selectsOnlyStaticColumns, type.allowUseOfSecondaryIndices(), allowFiltering, forView);
+ this(state, type, table, whereClause, boundNames, selectsOnlyStaticColumns, type.allowUseOfSecondaryIndices(), allowFiltering, forView);
}
/*
* We want to override allowUseOfSecondaryIndices flag from the StatementType for MV statements
* to avoid initing the Keyspace and SecondaryIndexManager.
*/
- public StatementRestrictions(StatementType type,
+ public StatementRestrictions(ClientState state,
+ StatementType type,
TableMetadata table,
WhereClause whereClause,
VariableSpecifications boundNames,
@@ -214,7 +224,7 @@
}
// At this point, the select statement if fully constructed, but we still have a few things to validate
- processPartitionKeyRestrictions(hasQueriableIndex, allowFiltering, forView);
+ processPartitionKeyRestrictions(state, hasQueriableIndex, allowFiltering, forView);
// Some but not all of the partition key columns have been specified;
// hence we need turn these restrictions into a row filter.
@@ -266,8 +276,8 @@
}
if (hasQueriableIndex)
usesSecondaryIndexing = true;
- else if (!allowFiltering)
- throw invalidRequest(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+ else if (!allowFiltering && requiresAllowFilteringIfNotSpecified())
+ throw invalidRequest(allowFilteringMessage(state));
filterRestrictions.add(nonPrimaryKeyRestrictions);
}
@@ -276,6 +286,16 @@
validateSecondaryIndexSelections();
}
+ public boolean requiresAllowFilteringIfNotSpecified()
+ {
+ if (!table.isVirtual())
+ return true;
+
+ VirtualTable tableNullable = VirtualKeyspaceRegistry.instance.getTableNullable(table.id);
+ assert tableNullable != null;
+ return !tableNullable.allowFilteringImplicitly();
+ }
+
private void addRestriction(Restriction restriction)
{
ColumnMetadata def = restriction.getFirstColumn();
@@ -383,6 +403,54 @@
}
/**
+ * This method determines whether a specified column is restricted on equality or something equivalent, like IN.
+ * It can be used in conjunction with the columns selected by a query to determine which of those columns is
+ * already bound by the client (and from its perspective, not retrieved by the database).
+ *
+ * @param column a column from the same table these restrictions are against
+ *
+ * @return <code>true</code> if the given column is restricted on equality
+ */
+ public boolean isEqualityRestricted(ColumnMetadata column)
+ {
+ if (column.kind == ColumnMetadata.Kind.PARTITION_KEY)
+ {
+ if (partitionKeyRestrictions.hasOnlyEqualityRestrictions())
+ for (ColumnMetadata restricted : partitionKeyRestrictions.getColumnDefinitions())
+ if (restricted.name.equals(column.name))
+ return true;
+ }
+ else if (column.kind == ColumnMetadata.Kind.CLUSTERING)
+ {
+ if (hasClusteringColumnsRestrictions())
+ {
+ for (SingleRestriction restriction : clusteringColumnsRestrictions.getRestrictionSet())
+ {
+ if (restriction.isEqualityBased())
+ {
+ if (restriction.isMultiColumn())
+ {
+ for (ColumnMetadata restricted : restriction.getColumnDefs())
+ if (restricted.name.equals(column.name))
+ return true;
+ }
+ else if (restriction.getFirstColumn().name.equals(column.name))
+ return true;
+ }
+ }
+ }
+ }
+ else if (hasNonPrimaryKeyRestrictions())
+ {
+ for (SingleRestriction restriction : nonPrimaryKeyRestrictions)
+ if (restriction.getFirstColumn().name.equals(column.name) && restriction.isEqualityBased())
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
* Returns the <code>Restrictions</code> for the specified type of columns.
*
* @param kind the column type
@@ -408,7 +476,7 @@
return this.usesSecondaryIndexing;
}
- private void processPartitionKeyRestrictions(boolean hasQueriableIndex, boolean allowFiltering, boolean forView)
+ private void processPartitionKeyRestrictions(ClientState state, boolean hasQueriableIndex, boolean allowFiltering, boolean forView)
{
if (!type.allowPartitionKeyRanges())
{
@@ -444,8 +512,8 @@
// components must have a EQ. Only the last partition key component can be in IN relation.
if (partitionKeyRestrictions.needFiltering(table))
{
- if (!allowFiltering && !forView && !hasQueriableIndex)
- throw new InvalidRequestException(REQUIRES_ALLOW_FILTERING_MESSAGE);
+ if (!allowFiltering && !forView && !hasQueriableIndex && requiresAllowFilteringIfNotSpecified())
+ throw new InvalidRequestException(allowFilteringMessage(state));
isKeyRange = true;
usesSecondaryIndexing = hasQueriableIndex;
@@ -874,4 +942,11 @@
{
return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
}
+
+ private static String allowFilteringMessage(ClientState state)
+ {
+ return Guardrails.allowFilteringEnabled.isEnabled(state)
+ ? REQUIRES_ALLOW_FILTERING_MESSAGE
+ : CANNOT_USE_ALLOW_FILTERING_MESSAGE;
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
index bf2564e..b21b5f8 100644
--- a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java
@@ -22,14 +22,14 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Optional;
import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.text.StrBuilder;
+
+import org.apache.cassandra.cql3.functions.FunctionResolver;
import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -57,28 +57,26 @@
{
FunctionName name = new FunctionName(in.readUTF(), in.readUTF());
- int numberOfArguments = (int) in.readUnsignedVInt();
+ int numberOfArguments = in.readUnsignedVInt32();
List<AbstractType<?>> argTypes = new ArrayList<>(numberOfArguments);
for (int i = 0; i < numberOfArguments; i++)
{
argTypes.add(readType(metadata, in));
}
- Optional<Function> optional = Schema.instance.findFunction(name, argTypes);
+ Function function = FunctionResolver.get(metadata.keyspace, name, argTypes, metadata.keyspace, metadata.name, null);
- if (!optional.isPresent())
+ if (function == null)
throw new IOException(String.format("Unknown serialized function %s(%s)",
name,
argTypes.stream()
.map(p -> p.asCQL3Type().toString())
.collect(joining(", "))));
- Function function = optional.get();
-
boolean isPartial = in.readBoolean();
if (isPartial)
{
- int bitset = (int) in.readUnsignedVInt();
+ int bitset = in.readUnsignedVInt32();
List<ByteBuffer> partialParameters = new ArrayList<>(numberOfArguments);
for (int i = 0; i < numberOfArguments; i++)
{
@@ -91,7 +89,7 @@
function = ((ScalarFunction) function).partialApplication(ProtocolVersion.CURRENT, partialParameters);
}
- int numberOfRemainingArguments = (int) in.readUnsignedVInt();
+ int numberOfRemainingArguments = in.readUnsignedVInt32();
List<Selector> argSelectors = new ArrayList<>(numberOfRemainingArguments);
for (int i = 0; i < numberOfRemainingArguments; i++)
{
@@ -102,8 +100,8 @@
}
protected abstract Selector newFunctionSelector(Function function, List<Selector> argSelectors);
- };
-
+ }
+
protected final T fun;
/**
@@ -346,7 +344,7 @@
List<AbstractType<?>> argTypes = function.argTypes();
int numberOfArguments = argTypes.size();
- out.writeUnsignedVInt(numberOfArguments);
+ out.writeUnsignedVInt32(numberOfArguments);
for (int i = 0; i < numberOfArguments; i++)
writeType(out, argTypes.get(i));
@@ -358,7 +356,7 @@
List<ByteBuffer> partialParameters = ((PartialScalarFunction) fun).getPartialParameters();
// We use a bitset to track the position of the unresolved arguments
- out.writeUnsignedVInt(computeBitSet(partialParameters));
+ out.writeUnsignedVInt32(computeBitSet(partialParameters));
for (int i = 0, m = partialParameters.size(); i < m; i++)
{
@@ -369,7 +367,7 @@
}
int numberOfRemainingArguments = argSelectors.size();
- out.writeUnsignedVInt(numberOfRemainingArguments);
+ out.writeUnsignedVInt32(numberOfRemainingArguments);
for (int i = 0; i < numberOfRemainingArguments; i++)
serializer.serialize(argSelectors.get(i), out, version);
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java
index 8c4f745..8d21c1e 100644
--- a/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java
@@ -43,13 +43,15 @@
return true;
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
+ ProtocolVersion protocolVersion = input.getProtocolVersion();
+
// Aggregation of aggregation is not supported
for (int i = 0, m = argSelectors.size(); i < m; i++)
{
Selector s = argSelectors.get(i);
- s.addInput(protocolVersion, input);
+ s.addInput(input);
setArg(i, s.getOutput(protocolVersion));
s.reset();
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ColumnTimestamps.java b/src/java/org/apache/cassandra/cql3/selection/ColumnTimestamps.java
new file mode 100644
index 0000000..2cac925
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/ColumnTimestamps.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.selection;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.serializers.CollectionSerializer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Represents a list of timestamps associated to a CQL column. Those timestamps can either be writetimes or TTLs,
+ * according to {@link TimestampsType}.
+ */
+abstract class ColumnTimestamps
+{
+ /**
+ * The timestamps type.
+ */
+ protected final TimestampsType type;
+
+ protected ColumnTimestamps(TimestampsType type)
+ {
+ this.type = type;
+ }
+
+ /**
+ * @return the timestamps type
+ */
+ public TimestampsType type()
+ {
+ return type;
+ }
+
+ /**
+ * Retrieves the timestamps at the specified position.
+ *
+ * @param index the timestamps position
+ * @return the timestamps at the specified position or a {@link #NO_TIMESTAMP}
+ */
+ public abstract ColumnTimestamps get(int index);
+
+ public abstract ColumnTimestamps max();
+
+ /**
+ * Returns a view of the portion of the timestamps within the specified range.
+ *
+ * @param range the indexes range
+ * @return a view of the specified range within this {@link ColumnTimestamps}
+ */
+ public abstract ColumnTimestamps slice(Range<Integer> range);
+
+ /**
+ * Converts the timestamps into their serialized form.
+ *
+ * @param protocolVersion the protocol version to use for the serialization
+ * @return the serialized timestamps
+ */
+ public abstract ByteBuffer toByteBuffer(ProtocolVersion protocolVersion);
+
+ /**
+ * Appends an empty timestamp at the end of this list.
+ */
+ public abstract void addNoTimestamp();
+
+ /**
+ * Appends the timestamp of the specified cell at the end of this list.
+ */
+ public abstract void addTimestampFrom(Cell<?> cell, int nowInSecond);
+
+ /**
+ * Creates a new {@link ColumnTimestamps} instance for the specified column type.
+ *
+ * @param timestampType the timestamps type
+ * @param columnType the column type
+ * @return a {@link ColumnTimestamps} instance for the specified column type
+ */
+ static ColumnTimestamps newTimestamps(TimestampsType timestampType, AbstractType<?> columnType)
+ {
+ if (!columnType.isMultiCell())
+ return new SingleTimestamps(timestampType);
+
+ // For UserType we know that the size will not change, so we can initialize the array with the proper capacity.
+ if (columnType instanceof UserType)
+ return new MultipleTimestamps(timestampType, ((UserType) columnType).size());
+
+ return new MultipleTimestamps(timestampType, 0);
+ }
+
+ /**
+ * The type of represented timestamps.
+ */
+ public enum TimestampsType
+ {
+ WRITETIMES
+ {
+ @Override
+ long getTimestamp(Cell<?> cell, int nowInSecond)
+ {
+ return cell.timestamp();
+ }
+
+ @Override
+ long defaultValue()
+ {
+ return Long.MIN_VALUE;
+ }
+
+ @Override
+ ByteBuffer toByteBuffer(long timestamp)
+ {
+ return timestamp == defaultValue() ? null : ByteBufferUtil.bytes(timestamp);
+ }
+ },
+ TTLS
+ {
+ @Override
+ long getTimestamp(Cell<?> cell, int nowInSecond)
+ {
+ if (!cell.isExpiring())
+ return defaultValue();
+
+ int remaining = cell.localDeletionTime() - nowInSecond;
+ return remaining >= 0 ? remaining : defaultValue();
+ }
+
+ @Override
+ long defaultValue()
+ {
+ return -1;
+ }
+
+ @Override
+ ByteBuffer toByteBuffer(long timestamp)
+ {
+ return timestamp == defaultValue() ? null : ByteBufferUtil.bytes((int) timestamp);
+ }
+ };
+
+ /**
+ * Extracts the timestamp from the specified cell.
+ *
+ * @param cell the cell
+ * @param nowInSecond the query timestamp insecond
+ * @return the timestamp corresponding to this type
+ */
+ abstract long getTimestamp(Cell<?> cell, int nowInSecond);
+
+ /**
+ * Returns the value to use when there is no timestamp.
+ *
+ * @return the value to use when there is no timestamp
+ */
+ abstract long defaultValue();
+
+ /**
+ * Serializes the specified timestamp.
+ *
+ * @param timestamp the timestamp to serialize
+ * @return the bytes corresponding to the specified timestamp
+ */
+ abstract ByteBuffer toByteBuffer(long timestamp);
+ }
+
+ /**
+ * A {@link ColumnTimestamps} that doesn't contain any timestamps.
+ */
+ static final ColumnTimestamps NO_TIMESTAMP = new ColumnTimestamps(null)
+ {
+ @Override
+ public ColumnTimestamps get(int index)
+ {
+ return this;
+ }
+
+ @Override
+ public ColumnTimestamps max()
+ {
+ return this;
+ }
+
+ @Override
+ public ColumnTimestamps slice(Range<Integer> range)
+ {
+ return this;
+ }
+
+ @Override
+ public ByteBuffer toByteBuffer(ProtocolVersion protocolVersion)
+ {
+ return null;
+ }
+
+ @Override
+ public void addNoTimestamp()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addTimestampFrom(Cell<?> cell, int nowInSecond)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "no timestamp";
+ }
+ };
+
+ /**
+ * A {@link ColumnTimestamps} that can contains a single timestamp (for columns that aren't multicell).
+ */
+ private static class SingleTimestamps extends ColumnTimestamps
+ {
+ protected long timestamp;
+
+ public SingleTimestamps(TimestampsType type)
+ {
+ this(type, type.defaultValue());
+ }
+
+ public SingleTimestamps(TimestampsType type, long timestamp)
+ {
+ super(type);
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public void addNoTimestamp()
+ {
+ timestamp = type.defaultValue();
+ }
+
+ @Override
+ public void addTimestampFrom(Cell<?> cell, int nowInSecond)
+ {
+ timestamp = type.getTimestamp(cell, nowInSecond);
+ }
+
+ @Override
+ public ColumnTimestamps get(int index)
+ {
+ // If this method is called it means that it is an element selection on a frozen collection/UDT,
+ // so we can safely return this Timestamps as all the elements also share that timestamp
+ return this;
+ }
+
+ @Override
+ public ColumnTimestamps max()
+ {
+ return this;
+ }
+
+ @Override
+ public ColumnTimestamps slice(Range<Integer> range)
+ {
+ return range.isEmpty() ? NO_TIMESTAMP : this;
+ }
+
+ @Override
+ public ByteBuffer toByteBuffer(ProtocolVersion protocolVersion)
+ {
+ return timestamp == type.defaultValue() ? null : type.toByteBuffer(timestamp);
+ }
+
+ @Override
+ public String toString()
+ {
+ return type + ": " + timestamp;
+ }
+ }
+
+ /**
+ * A {@link ColumnTimestamps} that can contain multiple timestamps (for unfrozen collections or UDTs).
+ */
+ private static final class MultipleTimestamps extends ColumnTimestamps
+ {
+ private final List<Long> timestamps;
+
+ public MultipleTimestamps(TimestampsType type, int initialCapacity)
+ {
+ this(type, new ArrayList<>(initialCapacity));
+ }
+
+ public MultipleTimestamps(TimestampsType type, List<Long> timestamps)
+ {
+ super(type);
+ this.timestamps = timestamps;
+ }
+
+ @Override
+ public void addNoTimestamp()
+ {
+ timestamps.add(type.defaultValue());
+ }
+
+ @Override
+ public void addTimestampFrom(Cell<?> cell, int nowInSecond)
+ {
+ timestamps.add(type.getTimestamp(cell, nowInSecond));
+ }
+
+ @Override
+ public ColumnTimestamps get(int index)
+ {
+ return timestamps.isEmpty()
+ ? NO_TIMESTAMP
+ : new SingleTimestamps(type, timestamps.get(index));
+ }
+
+ @Override
+ public ColumnTimestamps max()
+ {
+ return timestamps.isEmpty()
+ ? NO_TIMESTAMP
+ : new SingleTimestamps(type, Collections.max(timestamps));
+ }
+
+ @Override
+ public ColumnTimestamps slice(Range<Integer> range)
+ {
+ if (range.isEmpty())
+ return NO_TIMESTAMP;
+
+ // Prepare the "from" argument for the call to List#sublist below. That argument is always specified and
+ // inclusive, whereas the range lower bound can be open, closed or not specified.
+ int from = 0;
+ if (range.hasLowerBound())
+ {
+ from = range.lowerBoundType() == BoundType.CLOSED
+ ? range.lowerEndpoint() // inclusive range lower bound, inclusive "from" is the same list position
+ : range.lowerEndpoint() + 1; // exclusive range lower bound, inclusive "from" is the next list position
+ }
+
+ // Prepare the "to" argument for the call to List#sublist below. That argument is always specified and
+ // exclusive, whereas the range upper bound can be open, closed or not specified.
+ int to = timestamps.size();
+ if (range.hasUpperBound())
+ {
+ to = range.upperBoundType() == BoundType.CLOSED
+ ? range.upperEndpoint() + 1 // inclusive range upper bound, exclusive "to" is the next list position
+ : range.upperEndpoint(); // exclusive range upper bound, exclusive "to" is the same list position
+ }
+
+ return new MultipleTimestamps(type, timestamps.subList(from, to));
+ }
+
+ @Override
+ public ByteBuffer toByteBuffer(ProtocolVersion protocolVersion)
+ {
+ if (timestamps.isEmpty())
+ return null;
+
+ List<ByteBuffer> buffers = new ArrayList<>(timestamps.size());
+ timestamps.forEach(timestamp -> buffers.add(type.toByteBuffer(timestamp)));
+
+ return CollectionSerializer.pack(buffers, timestamps.size());
+ }
+
+ @Override
+ public String toString()
+ {
+ return type + ": " + timestamps;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
index e520d0f..4644ba2 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java
@@ -21,6 +21,7 @@
import java.nio.ByteBuffer;
import com.google.common.base.Objects;
+import com.google.common.collect.Range;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
@@ -43,12 +44,19 @@
*/
abstract class ElementsSelector extends Selector
{
+ /**
+ * An empty collection is composed of an int size of zero.
+ */
+ private static final ByteBuffer EMPTY_FROZEN_COLLECTION = ByteBufferUtil.bytes(0);
+
protected final Selector selected;
+ protected final CollectionType<?> type;
protected ElementsSelector(Kind kind,Selector selected)
{
super(kind);
this.selected = selected;
+ this.type = getCollectionType(selected);
}
private static boolean isUnset(ByteBuffer bb)
@@ -237,9 +245,14 @@
protected abstract ByteBuffer extractSelection(ByteBuffer collection);
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
- selected.addInput(protocolVersion, input);
+ selected.addInput(input);
+ }
+
+ protected Range<Integer> getIndexRange(ByteBuffer output, ByteBuffer fromKey, ByteBuffer toKey)
+ {
+ return type.getSerializer().getIndexesRangeFromSerialized(output, fromKey, toKey, keyType(type));
}
public void reset()
@@ -266,13 +279,11 @@
}
};
- private final CollectionType<?> type;
private final ByteBuffer key;
private ElementSelector(Selector selected, ByteBuffer key)
{
super(Kind.ELEMENT_SELECTOR, selected);
- this.type = getCollectionType(selected);
this.key = key;
}
@@ -294,6 +305,31 @@
return type.getSerializer().getSerializedValue(collection, key, keyType(type));
}
+ protected int getElementIndex(ProtocolVersion protocolVersion, ByteBuffer key)
+ {
+ ByteBuffer output = selected.getOutput(protocolVersion);
+ return output == null ? -1 : type.getSerializer().getIndexFromSerialized(output, key, keyType(type));
+ }
+
+ @Override
+ protected ColumnTimestamps getWritetimes(ProtocolVersion protocolVersion)
+ {
+ return getElementTimestamps(protocolVersion, selected.getWritetimes(protocolVersion));
+ }
+
+ @Override
+ protected ColumnTimestamps getTTLs(ProtocolVersion protocolVersion)
+ {
+ return getElementTimestamps(protocolVersion, selected.getTTLs(protocolVersion));
+ }
+
+ private ColumnTimestamps getElementTimestamps(ProtocolVersion protocolVersion,
+ ColumnTimestamps timestamps)
+ {
+ int index = getElementIndex(protocolVersion, key);
+ return index == -1 ? ColumnTimestamps.NO_TIMESTAMP : timestamps.get(index);
+ }
+
public AbstractType<?> getType()
{
return valueType(type);
@@ -358,8 +394,6 @@
}
};
- private final CollectionType<?> type;
-
// Note that neither from nor to can be null, but they can both be ByteBufferUtil.UNSET_BYTE_BUFFER to represent no particular bound
private final ByteBuffer from;
private final ByteBuffer to;
@@ -368,7 +402,6 @@
{
super(Kind.SLICE_SELECTOR, selected);
assert from != null && to != null : "We can have unset buffers, but not nulls";
- this.type = getCollectionType(selected);
this.from = from;
this.to = to;
}
@@ -391,6 +424,36 @@
return type.getSerializer().getSliceFromSerialized(collection, from, to, type.nameComparator(), type.isFrozenCollection());
}
+ @Override
+ protected ColumnTimestamps getWritetimes(ProtocolVersion protocolVersion)
+ {
+ return getTimestampsSlice(protocolVersion, selected.getWritetimes(protocolVersion));
+ }
+
+ @Override
+ protected ColumnTimestamps getTTLs(ProtocolVersion protocolVersion)
+ {
+ return getTimestampsSlice(protocolVersion, selected.getTTLs(protocolVersion));
+ }
+
+ protected ColumnTimestamps getTimestampsSlice(ProtocolVersion protocolVersion, ColumnTimestamps timestamps)
+ {
+ ByteBuffer output = selected.getOutput(protocolVersion);
+ return (output == null || isCollectionEmpty(output))
+ ? ColumnTimestamps.NO_TIMESTAMP
+ : timestamps.slice(getIndexRange(output, from, to) );
+ }
+
+ /**
+ * Checks if the collection is empty. Only frozen collection can be empty.
+ * @param output the serialized collection
+ * @return {@code true} if the collection is empty {@code false} otherwise.
+ */
+ private boolean isCollectionEmpty(ByteBuffer output)
+ {
+ return EMPTY_FROZEN_COLLECTION.equals(output);
+ }
+
public AbstractType<?> getType()
{
return type;
diff --git a/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java b/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java
index 0c62397..20c1029 100644
--- a/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java
@@ -27,6 +27,7 @@
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -41,7 +42,7 @@
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
UserType type = (UserType) readType(metadata, in);
- int field = (int) in.readUnsignedVInt();
+ int field = in.readUnsignedVInt32();
Selector selected = Selector.serializer.deserialize(in, version, metadata);
return new FieldSelector(type, field, selected);
@@ -98,9 +99,9 @@
selected.addFetchedColumns(builder);
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
- selected.addInput(protocolVersion, input);
+ selected.addInput(input);
}
public ByteBuffer getOutput(ProtocolVersion protocolVersion)
@@ -108,10 +109,26 @@
ByteBuffer value = selected.getOutput(protocolVersion);
if (value == null)
return null;
- ByteBuffer[] buffers = type.split(value);
+ ByteBuffer[] buffers = type.split(ByteBufferAccessor.instance, value);
return field < buffers.length ? buffers[field] : null;
}
+ @Override
+ protected ColumnTimestamps getWritetimes(ProtocolVersion protocolVersion)
+ {
+ return getOutput(protocolVersion) == null
+ ? ColumnTimestamps.NO_TIMESTAMP
+ : selected.getWritetimes(protocolVersion).get(field);
+ }
+
+ @Override
+ protected ColumnTimestamps getTTLs(ProtocolVersion protocolVersion)
+ {
+ return getOutput(protocolVersion) == null
+ ? ColumnTimestamps.NO_TIMESTAMP
+ : selected.getTTLs(protocolVersion).get(field);
+ }
+
public AbstractType<?> getType()
{
return type.fieldType(field);
@@ -174,7 +191,7 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
writeType(out, type);
- out.writeUnsignedVInt(field);
+ out.writeUnsignedVInt32(field);
serializer.serialize(selected, out, version);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
index 9136ab2..5d095d5 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java
@@ -47,7 +47,7 @@
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
ListType<?> type = (ListType<?>) readType(metadata, in);
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
List<Selector> elements = new ArrayList<>(size);
for (int i = 0; i < size; i++)
elements.add(serializer.deserialize(in, version, metadata));
@@ -89,10 +89,10 @@
elements.get(i).addFetchedColumns(builder);
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
for (int i = 0, m = elements.size(); i < m; i++)
- elements.get(i).addInput(protocolVersion, input);
+ elements.get(i).addInput(input);
}
public ByteBuffer getOutput(ProtocolVersion protocolVersion)
@@ -102,7 +102,7 @@
{
buffers.add(elements.get(i).getOutput(protocolVersion));
}
- return CollectionSerializer.pack(buffers, buffers.size(), protocolVersion);
+ return CollectionSerializer.pack(buffers, buffers.size());
}
public void reset()
@@ -175,7 +175,7 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
writeType(out, type);
- out.writeUnsignedVInt(elements.size());
+ out.writeUnsignedVInt32(elements.size());
for (int i = 0, m = elements.size(); i < m; i++)
serializer.serialize(elements.get(i), out, version);
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/MapSelector.java b/src/java/org/apache/cassandra/cql3/selection/MapSelector.java
index dc811c0..26580e7 100644
--- a/src/java/org/apache/cassandra/cql3/selection/MapSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/MapSelector.java
@@ -54,7 +54,7 @@
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
MapType<?, ?> type = (MapType<?, ?>) readType(metadata, in);
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
List<Pair<Selector, Selector>> entries = new ArrayList<>(size);
for (int i = 0; i < size; i++)
{
@@ -193,13 +193,13 @@
}
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
for (int i = 0, m = elements.size(); i < m; i++)
{
Pair<Selector, Selector> pair = elements.get(i);
- pair.left.addInput(protocolVersion, input);
- pair.right.addInput(protocolVersion, input);
+ pair.left.addInput(input);
+ pair.right.addInput(input);
}
}
@@ -218,7 +218,7 @@
buffers.add(entry.getKey());
buffers.add(entry.getValue());
}
- return CollectionSerializer.pack(buffers, elements.size(), protocolVersion);
+ return CollectionSerializer.pack(buffers, elements.size());
}
public void reset()
@@ -301,7 +301,7 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
writeType(out, type);
- out.writeUnsignedVInt(elements.size());
+ out.writeUnsignedVInt32(elements.size());
for (int i = 0, m = elements.size(); i < m; i++)
{
diff --git a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
index 22566b2..3adcf0d 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java
@@ -28,6 +28,9 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.aggregation.GroupMaker;
import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.transport.ProtocolVersion;
public final class ResultSetBuilder
{
@@ -44,6 +47,11 @@
*/
private final GroupMaker groupMaker;
+ /**
+ * Whether masked columns should be unmasked.
+ */
+ private final boolean unmask;
+
/*
* We'll build CQL3 row one by one.
*/
@@ -52,16 +60,17 @@
private long size = 0;
private boolean sizeWarningEmitted = false;
- public ResultSetBuilder(ResultMetadata metadata, Selectors selectors)
+ public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, boolean unmask)
{
- this(metadata, selectors, null);
+ this(metadata, selectors, unmask, null);
}
- public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, GroupMaker groupMaker)
+ public ResultSetBuilder(ResultMetadata metadata, Selectors selectors, boolean unmask, GroupMaker groupMaker)
{
- this.resultSet = new ResultSet(metadata.copy(), new ArrayList<List<ByteBuffer>>());
+ this.resultSet = new ResultSet(metadata.copy(), new ArrayList<>());
this.selectors = selectors;
this.groupMaker = groupMaker;
+ this.unmask = unmask;
}
private void addSize(List<ByteBuffer> row)
@@ -103,13 +112,18 @@
inputRow.add(c, nowInSec);
}
+ public void add(ColumnData columnData, int nowInSec)
+ {
+ inputRow.add(columnData, nowInSec);
+ }
+
/**
* Notifies this <code>Builder</code> that a new row is being processed.
*
* @param partitionKey the partition key of the new row
* @param clustering the clustering of the new row
*/
- public void newRow(DecoratedKey partitionKey, Clustering<?> clustering)
+ public void newRow(ProtocolVersion protocolVersion, DecoratedKey partitionKey, Clustering<?> clustering, List<ColumnMetadata> columns)
{
// The groupMaker needs to be called for each row
boolean isNewAggregate = groupMaker == null || groupMaker.isNewGroup(partitionKey, clustering);
@@ -129,7 +143,11 @@
}
else
{
- inputRow = new Selector.InputRow(selectors.numberOfFetchedColumns(), selectors.collectTimestamps(), selectors.collectTTLs());
+ inputRow = new Selector.InputRow(protocolVersion,
+ columns,
+ unmask,
+ selectors.collectWritetimes(),
+ selectors.collectTTLs());
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/RowTimestamps.java b/src/java/org/apache/cassandra/cql3/selection/RowTimestamps.java
new file mode 100644
index 0000000..24d23ee
--- /dev/null
+++ b/src/java/org/apache/cassandra/cql3/selection/RowTimestamps.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.selection;
+
+import java.util.List;
+
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+/**
+ * The {@link ColumnTimestamps} associated to the given set of columns of a row.
+ */
+interface RowTimestamps
+{
+ /**
+ * Adds an empty timestamp for the specified column.
+ *
+ * @param index the column index
+ */
+ void addNoTimestamp(int index);
+
+ /**
+ * Adds the timestamp of the specified cell.
+ *
+ * @param index the column index
+ * @param cell the cell to get the timestamp from
+ * @param nowInSec the query timestamp in second
+ */
+ void addTimestamp(int index, Cell<?> cell, int nowInSec);
+
+ /**
+ * Returns the timestamp of the specified column.
+ *
+ * @param index the column index
+ * @return the timestamp of the specified column
+ */
+ ColumnTimestamps get(int index);
+
+ /**
+ * A {@code RowTimestamps} that does nothing.
+ */
+ RowTimestamps NOOP_ROW_TIMESTAMPS = new RowTimestamps()
+ {
+ @Override
+ public void addNoTimestamp(int index)
+ {
+ }
+
+ @Override
+ public void addTimestamp(int index, Cell<?> cell, int nowInSec)
+ {
+ }
+
+ @Override
+ public ColumnTimestamps get(int index)
+ {
+ return ColumnTimestamps.NO_TIMESTAMP;
+ }
+ };
+
+ static RowTimestamps newInstance(ColumnTimestamps.TimestampsType type, List<ColumnMetadata> columns)
+ {
+ final ColumnTimestamps[] array = new ColumnTimestamps[columns.size()];
+
+ for (int i = 0, m = columns.size(); i < m; i++)
+ array[i] = ColumnTimestamps.newTimestamps(type, columns.get(i).type);
+
+ return new RowTimestamps()
+ {
+ @Override
+ public void addNoTimestamp(int index)
+ {
+ array[index].addNoTimestamp();
+ }
+
+ @Override
+ public void addTimestamp(int index, Cell<?> cell, int nowInSec)
+ {
+ array[index].addTimestampFrom(cell, nowInSec);
+ }
+
+ @Override
+ public ColumnTimestamps get(int index)
+ {
+ return array[index];
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
index ed2a140..5e9711a 100644
--- a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java
@@ -37,12 +37,12 @@
}
};
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
for (int i = 0, m = argSelectors.size(); i < m; i++)
{
Selector s = argSelectors.get(i);
- s.addInput(protocolVersion, input);
+ s.addInput(input);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selectable.java b/src/java/org/apache/cassandra/cql3/selection/Selectable.java
index 6e653ba..9476dea 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selectable.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selectable.java
@@ -22,8 +22,6 @@
import java.util.function.Predicate;
import java.util.stream.Collectors;
-import org.apache.commons.lang3.text.StrBuilder;
-
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.*;
import org.apache.cassandra.cql3.selection.Selector.Factory;
@@ -81,7 +79,7 @@
*/
public default boolean processesSelection()
{
- // ColumnMetadata is the only case that returns false and override this
+ // ColumnMetadata is the only case that returns false (if the column is not masked) and overrides this
return true;
}
@@ -92,6 +90,12 @@
return type == null ? TestResult.NOT_ASSIGNABLE : type.testAssignment(keyspace, receiver);
}
+ @Override
+ public default AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return getExactTypeIfKnown(keyspace);
+ }
+
default int addAndGetIndex(ColumnMetadata def, List<ColumnMetadata> l)
{
int idx = l.indexOf(def);
@@ -152,7 +156,7 @@
/*
* expectedType will be null if we have no constraint on what the type should be. For instance, if this term is a bind marker:
* - it will be null if we do "SELECT ? FROM foo"
- * - it won't be null (and be LongType) if we do "SELECT bigintAsBlob(?) FROM foo" because the function constrain it.
+ * - it won't be null (and be LongType) if we do "SELECT bigint_as_blob(?) FROM foo" because the function constrain it.
*
* In the first case, we have to error out: we need to infer the type of the metadata of a SELECT at preparation time, which we can't
* here (users will have to do "SELECT (varint)? FROM foo" for instance).
@@ -193,6 +197,12 @@
}
@Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return rawTerm.getCompatibleTypeIfKnown(keyspace);
+ }
+
+ @Override
public boolean selectColumns(Predicate<ColumnMetadata> predicate)
{
return false;
@@ -222,19 +232,48 @@
public static class WritetimeOrTTL implements Selectable
{
- public final ColumnMetadata column;
- public final boolean isWritetime;
+ // The order of the variants in the Kind enum matters as they are used in ser/deser
+ public enum Kind
+ {
+ TTL("ttl", Int32Type.instance),
+ WRITE_TIME("writetime", LongType.instance),
+ MAX_WRITE_TIME("maxwritetime", LongType.instance); // maxwritetime is available after Cassandra 4.1 (exclusive)
- public WritetimeOrTTL(ColumnMetadata column, boolean isWritetime)
+ public final String name;
+ public final AbstractType<?> returnType;
+
+ public static Kind fromOrdinal(int ordinal)
+ {
+ return values()[ordinal];
+ }
+
+ Kind(String name, AbstractType<?> returnType)
+ {
+ this.name = name;
+ this.returnType = returnType;
+ }
+
+ public boolean aggregatesMultiCell()
+ {
+ return this == MAX_WRITE_TIME;
+ }
+ }
+
+ public final ColumnMetadata column;
+ public final Selectable selectable;
+ public final Kind kind;
+
+ public WritetimeOrTTL(ColumnMetadata column, Selectable selectable, Kind kind)
{
this.column = column;
- this.isWritetime = isWritetime;
+ this.selectable = selectable;
+ this.kind = kind;
}
@Override
public String toString()
{
- return (isWritetime ? "writetime" : "ttl") + "(" + column.name + ")";
+ return kind.name + "(" + selectable + ")";
}
public Selector.Factory newSelectorFactory(TableMetadata table,
@@ -245,43 +284,45 @@
if (column.isPrimaryKeyColumn())
throw new InvalidRequestException(
String.format("Cannot use selection function %s on PRIMARY KEY part %s",
- isWritetime ? "writeTime" : "ttl",
+ kind.name,
column.name));
- if (column.type.isMultiCell())
- throw new InvalidRequestException(String.format("Cannot use selection function %s on non-frozen %s %s",
- isWritetime ? "writeTime" : "ttl",
- column.type.isCollection() ? "collection" : "UDT",
- column.name));
- return WritetimeOrTTLSelector.newFactory(column, addAndGetIndex(column, defs), isWritetime);
+ Selector.Factory factory = selectable.newSelectorFactory(table, expectedType, defs, boundNames);
+ boolean isMultiCell = factory.getColumnSpecification(table).type.isMultiCell();
+
+ return WritetimeOrTTLSelector.newFactory(factory, addAndGetIndex(column, defs), kind, isMultiCell);
}
+ @Override
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
- return isWritetime ? LongType.instance : Int32Type.instance;
+ AbstractType<?> type = kind.returnType;
+ return column.type.isMultiCell() && !kind.aggregatesMultiCell() ? ListType.getInstance(type, false) : type;
}
@Override
public boolean selectColumns(Predicate<ColumnMetadata> predicate)
{
- return predicate.test(column);
+ return selectable.selectColumns(predicate);
}
public static class Raw implements Selectable.Raw
{
- private final Selectable.RawIdentifier id;
- private final boolean isWritetime;
+ private final Selectable.RawIdentifier column;
+ private final Selectable.Raw selected;
+ private final Kind kind;
- public Raw(Selectable.RawIdentifier id, boolean isWritetime)
+ public Raw(Selectable.RawIdentifier column, Selectable.Raw selected, Kind kind)
{
- this.id = id;
- this.isWritetime = isWritetime;
+ this.column = column;
+ this.selected = selected;
+ this.kind = kind;
}
@Override
public WritetimeOrTTL prepare(TableMetadata table)
{
- return new WritetimeOrTTL(id.prepare(table), isWritetime);
+ return new WritetimeOrTTL((ColumnMetadata) column.prepare(table), selected.prepare(table), kind);
}
}
}
@@ -357,17 +398,11 @@
preparedArgs.add(arg.prepare(table));
FunctionName name = functionName;
- // We need to circumvent the normal function lookup process for toJson() because instances of the function
- // are not pre-declared (because it can accept any type of argument). We also have to wait until we have the
- // selector factories of the argument so we can access their final type.
- if (functionName.equalsNativeFunction(ToJsonFct.NAME))
- {
- return new WithToJSonFunction(preparedArgs);
- }
- // Also, COUNT(x) is equivalent to COUNT(*) for any non-null term x (since count(x) don't care about it's argument outside of check for nullness) and
- // for backward compatibilty we want to support COUNT(1), but we actually have COUNT(x) method for every existing (simple) input types so currently COUNT(1)
- // will throw as ambiguous (since 1 works for any type). So we have have to special case COUNT.
- else if (functionName.equalsNativeFunction(FunctionName.nativeFunction("count"))
+ // COUNT(x) is equivalent to COUNT(*) for any non-null term x (since count(x) don't care about its
+ // argument outside of check for nullness) and for backward compatibilty we want to support COUNT(1),
+ // but we actually have COUNT(x) method for every existing (simple) input types so currently COUNT(1)
+ // will throw as ambiguous (since 1 works for any type). So we have to special case COUNT.
+ if (functionName.equalsNativeFunction(FunctionName.nativeFunction("count"))
&& preparedArgs.size() == 1
&& (preparedArgs.get(0) instanceof WithTerm)
&& (((WithTerm)preparedArgs.get(0)).rawTerm instanceof Constants.Literal))
@@ -390,44 +425,6 @@
}
}
- public static class WithToJSonFunction implements Selectable
- {
- public final List<Selectable> args;
-
- private WithToJSonFunction(List<Selectable> args)
- {
- this.args = args;
- }
-
- @Override
- public String toString()
- {
- return new StrBuilder().append(ToJsonFct.NAME)
- .append("(")
- .appendWithSeparators(args, ", ")
- .append(")")
- .toString();
- }
-
- public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames)
- {
- SelectorFactories factories = SelectorFactories.createFactoriesAndCollectColumnDefinitions(args, null, table, defs, boundNames);
- Function fun = ToJsonFct.getInstance(factories.getReturnTypes());
- return AbstractFunctionSelector.newFactory(fun, factories);
- }
-
- public AbstractType<?> getExactTypeIfKnown(String keyspace)
- {
- return UTF8Type.instance;
- }
-
- @Override
- public boolean selectColumns(Predicate<ColumnMetadata> predicate)
- {
- return Selectable.selectColumns(args, predicate);
- }
- }
-
public static class WithCast implements Selectable
{
private final CQL3Type type;
@@ -682,6 +679,17 @@
}
@Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ // If there is only one element we cannot know if it is an element between parentheses or a tuple
+ // with only one element. By consequence, we need to force the user to specify the type.
+ if (selectables.size() == 1)
+ return null;
+
+ return Tuples.getExactTupleTypeIfKnown(selectables, p -> p.getCompatibleTypeIfKnown(keyspace));
+ }
+
+ @Override
public boolean selectColumns(Predicate<ColumnMetadata> predicate)
{
return Selectable.selectColumns(selectables, predicate);
@@ -767,6 +775,12 @@
}
@Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return Lists.getPreferredCompatibleType(selectables, p -> p.getCompatibleTypeIfKnown(keyspace));
+ }
+
+ @Override
public boolean selectColumns(Predicate<ColumnMetadata> predicate)
{
return Selectable.selectColumns(selectables, predicate);
@@ -860,6 +874,12 @@
}
@Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return Sets.getPreferredCompatibleType(selectables, p -> p.getCompatibleTypeIfKnown(keyspace));
+ }
+
+ @Override
public boolean selectColumns(Predicate<ColumnMetadata> predicate)
{
return Selectable.selectColumns(selectables, predicate);
@@ -991,7 +1011,14 @@
@Override
public AbstractType<?> getExactTypeIfKnown(String keyspace)
{
- // Lets force the user to specify the type.
+ // Let's force the user to specify the type.
+ return null;
+ }
+
+ @Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ // Let's force the user to specify the type.
return null;
}
@@ -1213,12 +1240,17 @@
this.quoted = quoted;
}
- @Override
- public ColumnMetadata prepare(TableMetadata cfm)
+ public ColumnMetadata columnMetadata(TableMetadata cfm)
{
return cfm.getExistingColumn(ColumnIdentifier.getInterned(text, quoted));
}
+ @Override
+ public Selectable prepare(TableMetadata cfm)
+ {
+ return columnMetadata(cfm);
+ }
+
public FieldIdentifier toFieldIdentifier()
{
return quoted ? FieldIdentifier.forQuoted(text)
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selection.java b/src/java/org/apache/cassandra/cql3/selection/Selection.java
index f07184a..da87f26 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selection.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selection.java
@@ -35,6 +35,7 @@
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.JsonUtils;
public abstract class Selection
{
@@ -103,15 +104,20 @@
*/
public Integer getOrderingIndex(ColumnMetadata c)
{
- if (!isJson)
- return getResultSetIndex(c);
-
// If we order post-query in json, the first and only column that we ship to the client is the json column.
// In that case, we should keep ordering columns around to perform the ordering, then these columns will
// be placed after the json column. As a consequence of where the colums are placed, we should give the
// ordering index a value based on their position in the json encoding and discard the original index.
// (CASSANDRA-14286)
- return orderingColumns.indexOf(c) + 1;
+ if (isJson)
+ return orderingColumns.indexOf(c) + 1;
+
+ // If the column is masked it might appear twice, once masked in the selected column and once unmasked in
+ // the ordering columns. For ordering we are interested in that second unmasked value.
+ if (c.isMasked())
+ return columns.lastIndexOf(c);
+
+ return getResultSetIndex(c);
}
public ResultSet.ResultMetadata getResultMetadata()
@@ -133,15 +139,16 @@
return new SimpleSelection(table, all, Collections.emptySet(), true, isJson, returnStaticContentOnPartitionWithNoRows);
}
- public static Selection wildcardWithGroupBy(TableMetadata table,
- VariableSpecifications boundNames,
- boolean isJson,
- boolean returnStaticContentOnPartitionWithNoRows)
+ public static Selection wildcardWithGroupByOrMaskedColumns(TableMetadata table,
+ VariableSpecifications boundNames,
+ Set<ColumnMetadata> orderingColumns,
+ boolean isJson,
+ boolean returnStaticContentOnPartitionWithNoRows)
{
return fromSelectors(table,
Lists.newArrayList(table.allColumnsInSelectOrder()),
boundNames,
- Collections.emptySet(),
+ orderingColumns,
Collections.emptySet(),
true,
isJson,
@@ -225,7 +232,7 @@
for (ColumnMetadata orderingColumn : orderingColumns)
{
int index = selectedColumns.indexOf(orderingColumn);
- if (index >= 0 && factories.indexOfSimpleSelectorFactory(index) >= 0)
+ if (index >= 0 && factories.indexOfSimpleSelectorFactory(index) >= 0 && !orderingColumn.isMasked())
continue;
filteredOrderingColumns.add(orderingColumn);
@@ -324,7 +331,7 @@
columnName = "\"" + columnName + "\"";
sb.append('"');
- sb.append(Json.quoteAsJsonString(columnName));
+ sb.append(JsonUtils.quoteAsJsonString(columnName));
sb.append("\": ");
if (buffer == null)
sb.append("null");
@@ -371,10 +378,10 @@
public boolean collectTTLs();
/**
- * Checks if one of the selectors collect timestamps.
- * @return {@code true} if one of the selectors collect timestamps, {@code false} otherwise.
+ * Checks if one of the selectors collects write timestamps.
+ * @return {@code true} if one of the selectors collects write timestamps, {@code false} otherwise.
*/
- public boolean collectTimestamps();
+ public boolean collectWritetimes();
/**
* Adds the current row of the specified <code>ResultSetBuilder</code>.
@@ -501,7 +508,7 @@
}
@Override
- public boolean collectTimestamps()
+ public boolean collectWritetimes()
{
return false;
}
@@ -520,7 +527,8 @@
private static class SelectionWithProcessing extends Selection
{
private final SelectorFactories factories;
- private final boolean collectTimestamps;
+ private final boolean collectWritetimes;
+ private final boolean collectMaxWritetimes;
private final boolean collectTTLs;
public SelectionWithProcessing(TableMetadata table,
@@ -540,8 +548,9 @@
isJson);
this.factories = factories;
- this.collectTimestamps = factories.containsWritetimeSelectorFactory();
- this.collectTTLs = factories.containsTTLSelectorFactory();;
+ this.collectWritetimes = factories.containsWritetimeSelectorFactory();
+ this.collectMaxWritetimes = factories.containsMaxWritetimeSelectorFactory();
+ this.collectTTLs = factories.containsTTLSelectorFactory();
for (ColumnMetadata orderingColumn : orderingColumns)
{
@@ -601,7 +610,7 @@
public void addInputRow(InputRow input)
{
for (Selector selector : selectors)
- selector.addInput(options.getProtocolVersion(), input);
+ selector.addInput(input);
}
@Override
@@ -617,9 +626,9 @@
}
@Override
- public boolean collectTimestamps()
+ public boolean collectWritetimes()
{
- return collectTimestamps;
+ return collectWritetimes || collectMaxWritetimes;
}
@Override
diff --git a/src/java/org/apache/cassandra/cql3/selection/Selector.java b/src/java/org/apache/cassandra/cql3/selection/Selector.java
index 463382d..b5857d4 100644
--- a/src/java/org/apache/cassandra/cql3/selection/Selector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/Selector.java
@@ -22,22 +22,28 @@
import java.util.Arrays;
import java.util.List;
-import org.apache.cassandra.schema.CQLTypeParser;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.selection.ColumnTimestamps.TimestampsType;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.ComplexColumnData;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.CQLTypeParser;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -71,7 +77,7 @@
/**
* The <code>Selector</code> kinds.
*/
- public static enum Kind
+ public enum Kind
{
SIMPLE_SELECTOR(SimpleSelector.deserializer),
TERM_SELECTOR(TermSelector.deserializer),
@@ -152,6 +158,17 @@
}
/**
+ * Checks if this factory creates <code>maxwritetime</code> selector instances.
+ *
+ * @return <code>true</code> if this factory creates <code>maxwritetime</code> selectors instances,
+ * <code>false</code> otherwise
+ */
+ public boolean isMaxWritetimeSelectorFactory()
+ {
+ return false;
+ }
+
+ /**
* Checks if this factory creates <code>TTL</code> selectors instances.
*
* @return <code>true</code> if this factory creates <code>TTL</code> selectors instances,
@@ -165,7 +182,6 @@
/**
* Checks if this factory creates <code>Selector</code>s that simply return a column value.
*
- * @param index the column index
* @return <code>true</code> if this factory creates <code>Selector</code>s that simply return a column value,
* <code>false</code> otherwise.
*/
@@ -288,77 +304,143 @@
*/
public static final class InputRow
{
+ private final ProtocolVersion protocolVersion;
+ private final List<ColumnMetadata> columns;
+ private final boolean unmask;
+ private final boolean collectWritetimes;
+ private final boolean collectTTLs;
+
private ByteBuffer[] values;
- private final long[] timestamps;
- private final int[] ttls;
+ private RowTimestamps writetimes;
+ private RowTimestamps ttls;
private int index;
- public InputRow(int size, boolean collectTimestamps, boolean collectTTLs)
+ public InputRow(ProtocolVersion protocolVersion, List<ColumnMetadata> columns, boolean unmask)
{
- this.values = new ByteBuffer[size];
+ this(protocolVersion, columns, unmask, false, false);
+ }
- if (collectTimestamps)
- {
- this.timestamps = new long[size];
- // We use MIN_VALUE to indicate no timestamp
- Arrays.fill(timestamps, Long.MIN_VALUE);
- }
- else
- {
- timestamps = null;
- }
+ public InputRow(ProtocolVersion protocolVersion,
+ List<ColumnMetadata> columns,
+ boolean unmask,
+ boolean collectWritetimes,
+ boolean collectTTLs)
+ {
+ this.protocolVersion = protocolVersion;
+ this.columns = columns;
+ this.unmask = unmask;
+ this.collectWritetimes = collectWritetimes;
+ this.collectTTLs = collectTTLs;
- if (collectTTLs)
- {
- this.ttls = new int[size];
- // We use -1 to indicate no ttl
- Arrays.fill(ttls, -1);
- }
- else
- {
- ttls = null;
- }
+ values = new ByteBuffer[columns.size()];
+ writetimes = initTimestamps(TimestampsType.WRITETIMES, collectWritetimes, columns);
+ ttls = initTimestamps(TimestampsType.TTLS, collectTTLs, columns);
+ }
+
+ private RowTimestamps initTimestamps(TimestampsType type,
+ boolean collectWritetimes,
+ List<ColumnMetadata> columns)
+ {
+ return collectWritetimes ? RowTimestamps.newInstance(type, columns)
+ : RowTimestamps.NOOP_ROW_TIMESTAMPS;
+ }
+
+ public ProtocolVersion getProtocolVersion()
+ {
+ return protocolVersion;
+ }
+
+ public boolean unmask()
+ {
+ return unmask;
}
public void add(ByteBuffer v)
{
values[index] = v;
- if (timestamps != null)
- timestamps[index] = Long.MIN_VALUE;
-
- if (ttls != null)
- ttls[index] = -1;
-
+ if (v != null)
+ {
+ writetimes.addNoTimestamp(index);
+ ttls.addNoTimestamp(index);
+ }
index++;
}
- public void add(Cell<?> c, int nowInSec)
+ public void add(ColumnData columnData, int nowInSec)
{
- if (c == null)
+ ColumnMetadata column = columns.get(index);
+ if (columnData == null)
{
add(null);
- return;
}
+ else
+ {
+ if (column.isComplex())
+ {
+ add((ComplexColumnData) columnData, nowInSec);
+ }
+ else
+ {
+ add((Cell<?>) columnData, nowInSec);
+ }
+ }
+ }
+ private void add(Cell<?> c, int nowInSec)
+ {
values[index] = value(c);
-
- if (timestamps != null)
- timestamps[index] = c.timestamp();
-
- if (ttls != null)
- ttls[index] = remainingTTL(c, nowInSec);
-
+ writetimes.addTimestamp(index, c, nowInSec);
+ ttls.addTimestamp(index, c, nowInSec);
index++;
}
- private int remainingTTL(Cell<?> c, int nowInSec)
+ private void add(ComplexColumnData ccd, int nowInSec)
{
- if (!c.isExpiring())
- return -1;
+ AbstractType<?> type = columns.get(index).type;
+ if (type.isCollection())
+ {
+ values[index] = ((CollectionType<?>) type).serializeForNativeProtocol(ccd.iterator());
- int remaining = c.localDeletionTime() - nowInSec;
- return remaining >= 0 ? remaining : -1;
+ for (Cell<?> cell : ccd)
+ {
+ writetimes.addTimestamp(index, cell, nowInSec);
+ ttls.addTimestamp(index, cell, nowInSec);
+ }
+ }
+ else
+ {
+ UserType udt = (UserType) type;
+ int size = udt.size();
+
+ values[index] = udt.serializeForNativeProtocol(ccd.iterator(), protocolVersion);
+
+ short fieldPosition = 0;
+ for (Cell<?> cell : ccd)
+ {
+ // handle null fields that aren't at the end
+ short fieldPositionOfCell = ByteBufferUtil.toShort(cell.path().get(0));
+ while (fieldPosition < fieldPositionOfCell)
+ {
+ fieldPosition++;
+ writetimes.addNoTimestamp(index);
+ ttls.addNoTimestamp(index);
+ }
+
+ fieldPosition++;
+ writetimes.addTimestamp(index, cell, nowInSec);
+ ttls.addTimestamp(index, cell, nowInSec);
+ }
+
+ // append nulls for missing cells
+ while (fieldPosition < size)
+ {
+ fieldPosition++;
+ writetimes.addNoTimestamp(index);
+ ttls.addNoTimestamp(index);
+ }
+ }
+ index++;
}
private <V> ByteBuffer value(Cell<V> c)
@@ -390,35 +472,38 @@
public void reset(boolean deep)
{
index = 0;
+ this.writetimes = initTimestamps(TimestampsType.WRITETIMES, collectWritetimes, columns);
+ this.ttls = initTimestamps(TimestampsType.TTLS, collectTTLs, columns);
+
if (deep)
values = new ByteBuffer[values.length];
}
/**
- * Return the timestamp of the column with the specified index.
+ * Return the timestamp of the column component with the specified indexes.
*
- * @param index the column index
- * @return the timestamp of the column with the specified index
+ * @return the timestamp of the cell with the specified indexes
*/
- public long getTimestamp(int index)
+ ColumnTimestamps getWritetimes(int columnIndex)
{
- return timestamps[index];
+ return writetimes.get(columnIndex);
}
/**
- * Return the ttl of the column with the specified index.
+ * Return the ttl of the column component with the specified column and cell indexes.
*
- * @param index the column index
- * @return the ttl of the column with the specified index
+ * @param columnIndex the column index
+ * @return the ttl of the column with the specified indexes
*/
- public int getTtl(int index)
+ ColumnTimestamps getTtls(int columnIndex)
{
- return ttls[index];
+ return ttls.get(columnIndex);
}
/**
* Returns the column values as list.
* <p>This content of the list will be shared with the {@code InputRow} unless a deep reset has been done.</p>
+ *
* @return the column values as list.
*/
public List<ByteBuffer> getValues()
@@ -430,11 +515,10 @@
/**
* Add the current value from the specified <code>ResultSetBuilder</code>.
*
- * @param protocolVersion protocol version used for serialization
* @param input the input row
* @throws InvalidRequestException if a problem occurs while adding the input row
*/
- public abstract void addInput(ProtocolVersion protocolVersion, InputRow input);
+ public abstract void addInput(InputRow input);
/**
* Returns the selector output.
@@ -445,6 +529,16 @@
*/
public abstract ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException;
+ protected ColumnTimestamps getWritetimes(ProtocolVersion protocolVersion)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ protected ColumnTimestamps getTTLs(ProtocolVersion protocolVersion)
+ {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Returns the <code>Selector</code> output type.
*
diff --git a/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java b/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java
index 7f4bcb3..c5fbf44 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SelectorFactories.java
@@ -32,22 +32,27 @@
import org.apache.cassandra.exceptions.InvalidRequestException;
/**
- * A set of <code>Selector</code> factories.
+ * A set of {@code Selector} factories.
*/
final class SelectorFactories implements Iterable<Selector.Factory>
{
/**
- * The <code>Selector</code> factories.
+ * The {@code Selector} factories.
*/
private final List<Selector.Factory> factories;
/**
- * <code>true</code> if one of the factory creates writetime selectors.
+ * {@code true} if one of the factories creates writetime selectors.
*/
private boolean containsWritetimeFactory;
/**
- * <code>true</code> if one of the factory creates TTL selectors.
+ * {@code true} if one of the factories creates maxWritetime selectors.
+ */
+ private boolean containsMaxWritetimeFactory;
+
+ /**
+ * {@code true} if one of the factories creates TTL selectors.
*/
private boolean containsTTLFactory;
@@ -96,6 +101,7 @@
Factory factory = selectable.newSelectorFactory(table, expectedType, defs, boundNames);
containsWritetimeFactory |= factory.isWritetimeSelectorFactory();
containsTTLFactory |= factory.isTTLSelectorFactory();
+ containsMaxWritetimeFactory |= factory.isMaxWritetimeSelectorFactory();
if (factory.isAggregateSelectorFactory())
++numberOfAggregateFactories;
factories.add(factory);
@@ -141,7 +147,7 @@
*/
public void addSelectorForOrdering(ColumnMetadata def, int index)
{
- factories.add(SimpleSelector.newFactory(def, index));
+ factories.add(SimpleSelector.newFactory(def, index, true));
}
/**
@@ -166,6 +172,17 @@
}
/**
+ * Checks if this {@code SelectorFactories} contains at least one factory for maxWritetime selectors.
+ *
+ * @return {@link true} if this {@link SelectorFactories} contains at least one factory for maxWritetime
+ * selectors, {@link false} otherwise.
+ */
+ public boolean containsMaxWritetimeSelectorFactory()
+ {
+ return containsMaxWritetimeFactory;
+ }
+
+ /**
* Checks if this <code>SelectorFactories</code> contains at least one factory for TTL selectors.
*
* @return <code>true</code> if this <code>SelectorFactories</code> contains at least one factory for TTL
diff --git a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java
index b54b2d4..c6aa225 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java
@@ -49,7 +49,7 @@
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
SetType<?> type = (SetType<?>) readType(metadata, in);
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
List<Selector> elements = new ArrayList<>(size);
for (int i = 0; i < size; i++)
elements.add(serializer.deserialize(in, version, metadata));
@@ -91,10 +91,10 @@
elements.get(i).addFetchedColumns(builder);
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
for (int i = 0, m = elements.size(); i < m; i++)
- elements.get(i).addInput(protocolVersion, input);
+ elements.get(i).addInput(input);
}
public ByteBuffer getOutput(ProtocolVersion protocolVersion)
@@ -104,7 +104,7 @@
{
buffers.add(elements.get(i).getOutput(protocolVersion));
}
- return CollectionSerializer.pack(buffers, buffers.size(), protocolVersion);
+ return CollectionSerializer.pack(buffers, buffers.size());
}
public void reset()
@@ -178,7 +178,7 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
writeType(out, type);
- out.writeUnsignedVInt(elements.size());
+ out.writeUnsignedVInt32(elements.size());
for (int i = 0, m = elements.size(); i < m; i++)
serializer.serialize(elements.get(i), out, version);
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java b/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java
index a6ae446..22045eb 100644
--- a/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java
@@ -22,6 +22,7 @@
import com.google.common.base.Objects;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnSpecification;
@@ -45,7 +46,7 @@
ByteBuffer columnName = ByteBufferUtil.readWithVIntLength(in);
ColumnMetadata column = metadata.getColumn(columnName);
int idx = in.readInt();
- return new SimpleSelector(column, idx);
+ return new SimpleSelector(column, idx, false);
}
};
@@ -55,13 +56,14 @@
public static final class SimpleSelectorFactory extends Factory
{
private final int idx;
-
private final ColumnMetadata column;
+ private final boolean useForPostOrdering;
- private SimpleSelectorFactory(int idx, ColumnMetadata def)
+ private SimpleSelectorFactory(int idx, ColumnMetadata def, boolean useForPostOrdering)
{
this.idx = idx;
this.column = def;
+ this.useForPostOrdering = useForPostOrdering;
}
@Override
@@ -84,7 +86,7 @@
@Override
public Selector newInstance(QueryOptions options)
{
- return new SimpleSelector(column, idx);
+ return new SimpleSelector(column, idx, useForPostOrdering);
}
@Override
@@ -117,12 +119,15 @@
public final ColumnMetadata column;
private final int idx;
+ private final boolean useForPostOrdering;
private ByteBuffer current;
+ private ColumnTimestamps writetimes;
+ private ColumnTimestamps ttls;
private boolean isSet;
- public static Factory newFactory(final ColumnMetadata def, final int idx)
+ public static Factory newFactory(final ColumnMetadata def, final int idx, boolean useForPostOrdering)
{
- return new SimpleSelectorFactory(idx, def);
+ return new SimpleSelectorFactory(idx, def, useForPostOrdering);
}
@Override
@@ -132,12 +137,23 @@
}
@Override
- public void addInput(ProtocolVersion protocolVersion, InputRow input) throws InvalidRequestException
+ public void addInput(InputRow input) throws InvalidRequestException
{
if (!isSet)
{
isSet = true;
- current = input.getValue(idx);
+ writetimes = input.getWritetimes(idx);
+ ttls = input.getTtls(idx);
+
+ /*
+ We apply the column mask of the column unless:
+ - The column doesn't have a mask
+ - This selector is for a query with ORDER BY post-ordering, indicated by this.useForPostOrdering
+ - The input row is for a user with UNMASK permission, indicated by input.unmask()
+ */
+ ColumnMask mask = useForPostOrdering || input.unmask() ? null : column.getMask();
+ ByteBuffer value = input.getValue(idx);
+ current = mask == null ? value : mask.mask(input.getProtocolVersion(), value);
}
}
@@ -148,10 +164,24 @@
}
@Override
+ protected ColumnTimestamps getWritetimes(ProtocolVersion protocolVersion)
+ {
+ return writetimes;
+ }
+
+ @Override
+ protected ColumnTimestamps getTTLs(ProtocolVersion protocolVersion)
+ {
+ return ttls;
+ }
+
+ @Override
public void reset()
{
isSet = false;
current = null;
+ writetimes = null;
+ ttls = null;
}
@Override
@@ -166,11 +196,12 @@
return column.name.toString();
}
- private SimpleSelector(ColumnMetadata column, int idx)
+ private SimpleSelector(ColumnMetadata column, int idx, boolean useForPostOrdering)
{
super(Kind.SIMPLE_SELECTOR);
this.column = column;
this.idx = idx;
+ this.useForPostOrdering = useForPostOrdering;
}
@Override
@@ -210,6 +241,6 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
- out.writeInt(idx);;
+ out.writeInt(idx);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/TermSelector.java b/src/java/org/apache/cassandra/cql3/selection/TermSelector.java
index 6f0c844..19a60ac 100644
--- a/src/java/org/apache/cassandra/cql3/selection/TermSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/TermSelector.java
@@ -101,7 +101,7 @@
{
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
}
diff --git a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java
index 0c06bc2..111d63c 100644
--- a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java
@@ -47,7 +47,7 @@
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
AbstractType<?> type = readType(metadata, in);
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
List<Selector> elements = new ArrayList<>(size);
for (int i = 0; i < size; i++)
elements.add(serializer.deserialize(in, version, metadata));
@@ -89,10 +89,10 @@
elements.get(i).addFetchedColumns(builder);
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
for (int i = 0, m = elements.size(); i < m; i++)
- elements.get(i).addInput(protocolVersion, input);
+ elements.get(i).addInput(input);
}
public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException
@@ -176,7 +176,7 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
writeType(out, type);
- out.writeUnsignedVInt(elements.size());
+ out.writeUnsignedVInt32(elements.size());
for (int i = 0, m = elements.size(); i < m; i++)
serializer.serialize(elements.get(i), out, version);
diff --git a/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java b/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java
index 8007467..bf71c73 100644
--- a/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java
@@ -55,7 +55,7 @@
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
UserType type = (UserType) readType(metadata, in);
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
Map<FieldIdentifier, Selector> fields = new HashMap<>(size);
for (int i = 0; i < size; i++)
{
@@ -182,10 +182,10 @@
field.addFetchedColumns(builder);
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
for (Selector field : fields.values())
- field.addInput(protocolVersion, input);
+ field.addInput(input);
}
public ByteBuffer getOutput(ProtocolVersion protocolVersion)
@@ -272,7 +272,7 @@
protected void serialize(DataOutputPlus out, int version) throws IOException
{
writeType(out, type);
- out.writeUnsignedVInt(fields.size());
+ out.writeUnsignedVInt32(fields.size());
for (Map.Entry<FieldIdentifier, Selector> field : fields.entrySet())
{
diff --git a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
index 2c56f5c..9be0b45 100644
--- a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
+++ b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java
@@ -22,107 +22,127 @@
import com.google.common.base.Objects;
-import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
final class WritetimeOrTTLSelector extends Selector
{
- protected static final SelectorDeserializer deserializer = new SelectorDeserializer()
+ static final SelectorDeserializer deserializer = new SelectorDeserializer()
{
+ @Override
protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
- ByteBuffer columnName = ByteBufferUtil.readWithVIntLength(in);
- ColumnMetadata column = metadata.getColumn(columnName);
+ Selector selected = serializer.deserialize(in, version, metadata);
int idx = in.readInt();
- boolean isWritetime = in.readBoolean();
- return new WritetimeOrTTLSelector(column, idx, isWritetime);
+ int ordinal = in.readByte();
+ Selectable.WritetimeOrTTL.Kind kind = Selectable.WritetimeOrTTL.Kind.fromOrdinal(ordinal);
+ boolean isMultiCell = in.readBoolean();
+ return new WritetimeOrTTLSelector(selected, idx, kind, isMultiCell);
}
};
- private final ColumnMetadata column;
- private final int idx;
- private final boolean isWritetime;
+ private final Selector selected;
+ private final int columnIndex;
+ private final Selectable.WritetimeOrTTL.Kind kind;
private ByteBuffer current;
+ private final boolean isMultiCell;
private boolean isSet;
- public static Factory newFactory(final ColumnMetadata def, final int idx, final boolean isWritetime)
+ public static Factory newFactory(final Selector.Factory factory, final int columnIndex, final Selectable.WritetimeOrTTL.Kind kind, boolean isMultiCell)
{
return new Factory()
{
+ @Override
protected String getColumnName()
{
- return String.format("%s(%s)", isWritetime ? "writetime" : "ttl", def.name.toString());
+ return String.format("%s(%s)", kind.name, factory.getColumnName());
}
+ @Override
protected AbstractType<?> getReturnType()
{
- return isWritetime ? LongType.instance : Int32Type.instance;
+ AbstractType<?> type = kind.returnType;
+ return isMultiCell && !kind.aggregatesMultiCell() ? ListType.getInstance(type, false) : type;
}
+ @Override
protected void addColumnMapping(SelectionColumnMapping mapping, ColumnSpecification resultsColumn)
{
- mapping.addMapping(resultsColumn, def);
+ factory.addColumnMapping(mapping, resultsColumn);
}
+ @Override
public Selector newInstance(QueryOptions options)
{
- return new WritetimeOrTTLSelector(def, idx, isWritetime);
+ return new WritetimeOrTTLSelector(factory.newInstance(options), columnIndex, kind, isMultiCell);
}
+ @Override
public boolean isWritetimeSelectorFactory()
{
- return isWritetime;
+ return kind != Selectable.WritetimeOrTTL.Kind.TTL;
}
+ @Override
public boolean isTTLSelectorFactory()
{
- return !isWritetime;
+ return kind == Selectable.WritetimeOrTTL.Kind.TTL;
}
+ @Override
+ public boolean isMaxWritetimeSelectorFactory()
+ {
+ return kind == Selectable.WritetimeOrTTL.Kind.MAX_WRITE_TIME;
+ }
+
+ @Override
public boolean areAllFetchedColumnsKnown()
{
return true;
}
+ @Override
public void addFetchedColumns(ColumnFilter.Builder builder)
{
- builder.add(def);
+ factory.addFetchedColumns(builder);
}
};
}
public void addFetchedColumns(ColumnFilter.Builder builder)
{
- builder.add(column);
+ selected.addFetchedColumns(builder);
}
- public void addInput(ProtocolVersion protocolVersion, InputRow input)
+ public void addInput(InputRow input)
{
if (isSet)
return;
isSet = true;
- if (isWritetime)
+ selected.addInput(input);
+ ProtocolVersion protocolVersion = input.getProtocolVersion();
+
+ switch (kind)
{
- long ts = input.getTimestamp(idx);
- current = ts != Long.MIN_VALUE ? ByteBufferUtil.bytes(ts) : null;
- }
- else
- {
- int ttl = input.getTtl(idx);
- current = ttl > 0 ? ByteBufferUtil.bytes(ttl) : null;
+ case WRITE_TIME:
+ current = selected.getWritetimes(protocolVersion).toByteBuffer(protocolVersion);
+ break;
+ case MAX_WRITE_TIME:
+ current = selected.getWritetimes(protocolVersion).max().toByteBuffer(protocolVersion);
+ break;
+ case TTL:
+ current = selected.getTTLs(protocolVersion).toByteBuffer(protocolVersion);
+ break;
}
}
@@ -133,27 +153,30 @@
public void reset()
{
+ selected.reset();
isSet = false;
current = null;
}
public AbstractType<?> getType()
{
- return isWritetime ? LongType.instance : Int32Type.instance;
+ AbstractType<?> type = kind.returnType;
+ return isMultiCell ? ListType.getInstance(type, false) : type;
}
@Override
public String toString()
{
- return column.name.toString();
+ return selected.toString();
}
- private WritetimeOrTTLSelector(ColumnMetadata column, int idx, boolean isWritetime)
+ private WritetimeOrTTLSelector(Selector selected, int idx, Selectable.WritetimeOrTTL.Kind kind, boolean isMultiCell)
{
super(Kind.WRITETIME_OR_TTL_SELECTOR);
- this.column = column;
- this.idx = idx;
- this.isWritetime = isWritetime;
+ this.selected = selected;
+ this.columnIndex = idx;
+ this.kind = kind;
+ this.isMultiCell = isMultiCell;
}
@Override
@@ -167,30 +190,30 @@
WritetimeOrTTLSelector s = (WritetimeOrTTLSelector) o;
- return Objects.equal(column, s.column)
- && Objects.equal(idx, s.idx)
- && Objects.equal(isWritetime, s.isWritetime);
+ return Objects.equal(selected, s.selected) && kind == s.kind;
}
@Override
public int hashCode()
{
- return Objects.hashCode(column, idx, isWritetime);
+ return Objects.hashCode(selected, kind);
}
@Override
protected int serializedSize(int version)
{
- return ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes)
- + TypeSizes.sizeof(idx)
- + TypeSizes.sizeof(isWritetime);
+ return serializer.serializedSize(selected, version)
+ + TypeSizes.sizeof(columnIndex)
+ + TypeSizes.sizeofUnsignedVInt(kind.ordinal())
+ + TypeSizes.sizeof(isMultiCell);
}
@Override
protected void serialize(DataOutputPlus out, int version) throws IOException
{
- ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
- out.writeInt(idx);
- out.writeBoolean(isWritetime);
+ serializer.serialize(selected, out, version);
+ out.writeInt(columnIndex);
+ out.writeByte(kind.ordinal());
+ out.writeBoolean(isMultiCell);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
index 61e4934..b034db4 100644
--- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
@@ -42,6 +42,7 @@
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.metrics.BatchMetrics;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
import org.apache.cassandra.service.*;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.messages.ResultMessage;
@@ -442,6 +443,7 @@
boolean mutateAtomic = (isLogged() && mutations.size() > 1);
StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, queryStartNanoTime);
+ ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(mutations);
}
private void updatePartitionsPerBatchMetrics(int updatedPartitions)
@@ -647,7 +649,7 @@
public BatchStatement prepare(ClientState state)
{
List<ModificationStatement> statements = new ArrayList<>(parsedStatements.size());
- parsedStatements.forEach(s -> statements.add(s.prepare(bindVariables)));
+ parsedStatements.forEach(s -> statements.add(s.prepare(state, bindVariables)));
Attributes prepAttrs = attrs.prepare("[batch]", "[batch]");
prepAttrs.collectMarkerSpecification(bindVariables);
diff --git a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
index be01481..fff6dd3 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java
@@ -31,6 +31,7 @@
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.utils.Pair;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
@@ -140,7 +141,8 @@
@Override
- protected ModificationStatement prepareInternal(TableMetadata metadata,
+ protected ModificationStatement prepareInternal(ClientState state,
+ TableMetadata metadata,
VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
@@ -160,7 +162,8 @@
operations.add(op);
}
- StatementRestrictions restrictions = newRestrictions(metadata,
+ StatementRestrictions restrictions = newRestrictions(state,
+ metadata,
bindVariables,
operations,
whereClause,
diff --git a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
index b1f576e..96a0247 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java
@@ -67,17 +67,17 @@
private static final String CF = "describe";
/**
- * The columns returned by the describe queries that only list elements names (e.g. DESCRIBE KEYSPACES, DESCRIBE TABLES...)
+ * The columns returned by the describe queries that only list elements names (e.g. DESCRIBE KEYSPACES, DESCRIBE TABLES...)
*/
- private static final List<ColumnSpecification> LIST_METADATA =
+ private static final List<ColumnSpecification> LIST_METADATA =
ImmutableList.of(new ColumnSpecification(KS, CF, new ColumnIdentifier("keyspace_name", true), UTF8Type.instance),
new ColumnSpecification(KS, CF, new ColumnIdentifier("type", true), UTF8Type.instance),
new ColumnSpecification(KS, CF, new ColumnIdentifier("name", true), UTF8Type.instance));
/**
- * The columns returned by the describe queries that returns the CREATE STATEMENT for the different elements (e.g. DESCRIBE KEYSPACE, DESCRIBE TABLE ...)
+ * The columns returned by the describe queries that returns the CREATE STATEMENT for the different elements (e.g. DESCRIBE KEYSPACE, DESCRIBE TABLE ...)
*/
- private static final List<ColumnSpecification> ELEMENT_METADATA =
+ private static final List<ColumnSpecification> ELEMENT_METADATA =
ImmutableList.<ColumnSpecification>builder().addAll(LIST_METADATA)
.add(new ColumnSpecification(KS, CF, new ColumnIdentifier("create_statement", true), UTF8Type.instance))
.build();
@@ -308,7 +308,7 @@
*/
public static DescribeStatement<SchemaElement> functions()
{
- return new Listing(ks -> ks.functions.udfs());
+ return new Listing(ks -> ks.userFunctions.udfs());
}
/**
@@ -316,7 +316,7 @@
*/
public static DescribeStatement<SchemaElement> aggregates()
{
- return new Listing(ks -> ks.functions.udas());
+ return new Listing(ks -> ks.userFunctions.udas());
}
/**
@@ -387,7 +387,7 @@
public static class Element extends DescribeStatement<SchemaElement>
{
/**
- * The keyspace name
+ * The keyspace name
*/
private final String keyspace;
@@ -445,8 +445,8 @@
if (!onlyKeyspace)
{
s = Stream.concat(s, ks.types.sortedStream());
- s = Stream.concat(s, ks.functions.udfs().sorted(SchemaElement.NAME_COMPARATOR));
- s = Stream.concat(s, ks.functions.udas().sorted(SchemaElement.NAME_COMPARATOR));
+ s = Stream.concat(s, ks.userFunctions.udfs().sorted(SchemaElement.NAME_COMPARATOR));
+ s = Stream.concat(s, ks.userFunctions.udas().sorted(SchemaElement.NAME_COMPARATOR));
s = Stream.concat(s, ks.tables.stream().sorted(SchemaElement.NAME_COMPARATOR)
.flatMap(tm -> getTableElements(ks, tm)));
}
@@ -534,7 +534,7 @@
{
return new Element(keyspace, name, (ks, n) -> {
- return checkNotEmpty(ks.functions.getUdfs(new FunctionName(ks.name, n)),
+ return checkNotEmpty(ks.userFunctions.getUdfs(new FunctionName(ks.name, n)),
"User defined function '%s' not found in '%s'", n, ks.name).stream()
.sorted(SchemaElement.NAME_COMPARATOR);
});
@@ -547,7 +547,7 @@
{
return new Element(keyspace, name, (ks, n) -> {
- return checkNotEmpty(ks.functions.getUdas(new FunctionName(ks.name, n)),
+ return checkNotEmpty(ks.userFunctions.getUdas(new FunctionName(ks.name, n)),
"User defined aggregate '%s' not found in '%s'", n, ks.name).stream()
.sorted(SchemaElement.NAME_COMPARATOR);
});
@@ -681,7 +681,7 @@
list.add(trimIfPresent(DatabaseDescriptor.getPartitionerName(), "org.apache.cassandra.dht."));
list.add(trimIfPresent(DatabaseDescriptor.getEndpointSnitch().getClass().getName(),
"org.apache.cassandra.locator."));
-
+
String useKs = state.getRawKeyspace();
if (mustReturnsRangeOwnerships(useKs))
{
@@ -721,7 +721,7 @@
@Override
protected List<ByteBuffer> toRow(List<Object> elements, boolean withInternals)
{
- ImmutableList.Builder<ByteBuffer> builder = ImmutableList.builder();
+ ImmutableList.Builder<ByteBuffer> builder = ImmutableList.builder();
builder.add(UTF8Type.instance.decompose((String) elements.get(CLUSTER_NAME_INDEX)),
UTF8Type.instance.decompose((String) elements.get(PARTITIONER_NAME_INDEX)),
diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
index ab36ec9..0cf6771 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.ViewMetadata;
import org.apache.cassandra.cql3.*;
@@ -51,6 +52,7 @@
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.view.View;
import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageProxy;
@@ -300,6 +302,16 @@
}
}
+ public void validateTimestamp(QueryState queryState, QueryOptions options)
+ {
+ if (!isTimestampSet())
+ return;
+
+ long ts = attrs.getTimestamp(options.getTimestamp(queryState), options);
+ Guardrails.maximumAllowableTimestamp.guard(ts, table(), false, queryState.getClientState());
+ Guardrails.minimumAllowableTimestamp.guard(ts, table(), false, queryState.getClientState());
+ }
+
public RegularAndStaticColumns updatedColumns()
{
return updatedColumns;
@@ -504,6 +516,7 @@
cl.validateForWrite();
validateDiskUsage(options, queryState.getClientState());
+ validateTimestamp(queryState, options);
List<? extends IMutation> mutations =
getMutations(queryState.getClientState(),
@@ -513,8 +526,13 @@
options.getNowInSeconds(queryState),
queryStartNanoTime);
if (!mutations.isEmpty())
+ {
StorageProxy.mutateWithTriggers(mutations, cl, false, queryStartNanoTime);
+ if (!SchemaConstants.isSystemKeyspace(metadata.keyspace))
+ ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(mutations);
+ }
+
return null;
}
@@ -652,7 +670,7 @@
}
Selectors selectors = selection.newSelectors(options);
- ResultSetBuilder builder = new ResultSetBuilder(selection.getResultMetadata(), selectors);
+ ResultSetBuilder builder = new ResultSetBuilder(selection.getResultMetadata(), selectors, false);
SelectStatement.forSelection(metadata, selection)
.processPartition(partition, options, builder, nowInSeconds);
@@ -940,10 +958,10 @@
public ModificationStatement prepare(ClientState state)
{
- return prepare(bindVariables);
+ return prepare(state, bindVariables);
}
- public ModificationStatement prepare(VariableSpecifications bindVariables)
+ public ModificationStatement prepare(ClientState state, VariableSpecifications bindVariables)
{
TableMetadata metadata = Schema.instance.validateTable(keyspace(), name());
@@ -952,7 +970,7 @@
Conditions preparedConditions = prepareConditions(metadata, bindVariables);
- return prepareInternal(metadata, bindVariables, preparedConditions, preparedAttributes);
+ return prepareInternal(state, metadata, bindVariables, preparedConditions, preparedAttributes);
}
/**
@@ -1011,7 +1029,8 @@
return builder.build();
}
- protected abstract ModificationStatement prepareInternal(TableMetadata metadata,
+ protected abstract ModificationStatement prepareInternal(ClientState state,
+ TableMetadata metadata,
VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs);
@@ -1026,7 +1045,8 @@
* @param conditions the conditions
* @return the restrictions
*/
- protected StatementRestrictions newRestrictions(TableMetadata metadata,
+ protected StatementRestrictions newRestrictions(ClientState state,
+ TableMetadata metadata,
VariableSpecifications boundNames,
Operations operations,
WhereClause where,
@@ -1036,7 +1056,7 @@
throw new InvalidRequestException(CUSTOM_EXPRESSIONS_NOT_ALLOWED);
boolean applyOnlyToStaticColumns = appliesOnlyToStaticColumns(operations, conditions);
- return new StatementRestrictions(type, metadata, where, boundNames, applyOnlyToStaticColumns, false, false);
+ return new StatementRestrictions(state, type, metadata, where, boundNames, applyOnlyToStaticColumns, false, false);
}
public List<Pair<ColumnIdentifier, ColumnCondition.Raw>> getConditions()
diff --git a/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java b/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
index aa7e85b..e809a27 100644
--- a/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/PermissionsManagementStatement.java
@@ -21,7 +21,6 @@
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.RoleName;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.RequestValidationException;
@@ -55,13 +54,6 @@
// called both here and in authorize(), as in some cases we do not call the latter.
resource = maybeCorrectResource(resource, state);
- // altering permissions on builtin functions is not supported
- if (resource instanceof FunctionResource
- && SchemaConstants.SYSTEM_KEYSPACE_NAME.equals(((FunctionResource)resource).getKeyspace()))
- {
- throw new InvalidRequestException("Altering permissions on builtin functions is not supported");
- }
-
if (!resource.exists())
throw new InvalidRequestException(String.format("Resource %s doesn't exist", resource));
}
@@ -78,7 +70,7 @@
for (Permission p : permissions)
state.ensurePermission(p, resource);
}
-
+
@Override
public String toString()
{
diff --git a/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java b/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java
index b6112fa..65ec8fc 100644
--- a/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java
+++ b/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java
@@ -17,7 +17,9 @@
*/
package org.apache.cassandra.cql3.statements;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
import java.util.regex.Pattern;
import org.slf4j.Logger;
@@ -25,6 +27,8 @@
import org.apache.cassandra.exceptions.SyntaxException;
+import static java.lang.String.format;
+
public class PropertyDefinitions
{
private static final Pattern POSITIVE_PATTERN = Pattern.compile("(1|true|yes)");
@@ -32,18 +36,18 @@
protected static final Logger logger = LoggerFactory.getLogger(PropertyDefinitions.class);
- protected final Map<String, Object> properties = new HashMap<String, Object>();
+ protected final Map<String, Object> properties = new HashMap<>();
public void addProperty(String name, String value) throws SyntaxException
{
if (properties.put(name, value) != null)
- throw new SyntaxException(String.format("Multiple definition for property '%s'", name));
+ throw new SyntaxException(format("Multiple definitions for property '%s'", name));
}
public void addProperty(String name, Map<String, String> value) throws SyntaxException
{
if (properties.put(name, value) != null)
- throw new SyntaxException(String.format("Multiple definition for property '%s'", name));
+ throw new SyntaxException(format("Multiple definitions for property '%s'", name));
}
public void validate(Set<String> keywords, Set<String> obsolete) throws SyntaxException
@@ -56,7 +60,7 @@
if (obsolete.contains(name))
logger.warn("Ignoring obsolete property {}", name);
else
- throw new SyntaxException(String.format("Unknown property '%s'", name));
+ throw new SyntaxException(format("Unknown property '%s'", name));
}
}
@@ -73,13 +77,18 @@
properties.remove(name);
}
- protected String getSimple(String name) throws SyntaxException
+ public boolean hasProperty(String name)
+ {
+ return properties.containsKey(name);
+ }
+
+ protected String getString(String name) throws SyntaxException
{
Object val = properties.get(name);
if (val == null)
return null;
if (!(val instanceof String))
- throw new SyntaxException(String.format("Invalid value for property '%s'. It should be a string", name));
+ throw new SyntaxException(format("Invalid value for property '%s'. It should be a string", name));
return (String)val;
}
@@ -89,13 +98,14 @@
if (val == null)
return null;
if (!(val instanceof Map))
- throw new SyntaxException(String.format("Invalid value for property '%s'. It should be a map.", name));
+ throw new SyntaxException(format("Invalid value for property '%s'. It should be a map.", name));
return (Map<String, String>)val;
}
- public Boolean hasProperty(String name)
+ public boolean getBoolean(String key, boolean defaultValue) throws SyntaxException
{
- return properties.containsKey(name);
+ String value = getString(key);
+ return value != null ? parseBoolean(key, value) : defaultValue;
}
public static boolean parseBoolean(String key, String value) throws SyntaxException
@@ -110,63 +120,51 @@
else if (NEGATIVE_PATTERN.matcher(lowerCasedValue).matches())
return false;
- throw new SyntaxException(String.format("Invalid boolean value %s for '%s'. " +
- "Positive values can be '1', 'true' or 'yes'. " +
- "Negative values can be '0', 'false' or 'no'.",
- value, key));
+ throw new SyntaxException(format("Invalid boolean value %s for '%s'. " +
+ "Positive values can be '1', 'true' or 'yes'. " +
+ "Negative values can be '0', 'false' or 'no'.",
+ value, key));
}
- // Return a property value, typed as a Boolean
- public Boolean getBoolean(String key, Boolean defaultValue) throws SyntaxException
+ public int getInt(String key, int defaultValue) throws SyntaxException
{
- String value = getSimple(key);
- return (value == null) ? defaultValue : parseBoolean(key, value);
+ String value = getString(key);
+ return value != null ? parseInt(key, value) : defaultValue;
}
- // Return a property value, typed as a double
+ public static int parseInt(String key, String value) throws SyntaxException
+ {
+ if (null == value)
+ throw new IllegalArgumentException("value argument can't be null");
+
+ try
+ {
+ return Integer.parseInt(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new SyntaxException(format("Invalid integer value %s for '%s'", value, key));
+ }
+ }
+
public double getDouble(String key, double defaultValue) throws SyntaxException
{
- String value = getSimple(key);
- if (value == null)
- {
- return defaultValue;
- }
- else
- {
- try
- {
- return Double.parseDouble(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(String.format("Invalid double value %s for '%s'", value, key));
- }
- }
+ String value = getString(key);
+ return value != null ? parseDouble(key, value) : defaultValue;
}
- // Return a property value, typed as an Integer
- public Integer getInt(String key, Integer defaultValue) throws SyntaxException
+ public static double parseDouble(String key, String value) throws SyntaxException
{
- String value = getSimple(key);
- return toInt(key, value, defaultValue);
- }
+ if (null == value)
+ throw new IllegalArgumentException("value argument can't be null");
- public static Integer toInt(String key, String value, Integer defaultValue) throws SyntaxException
- {
- if (value == null)
+ try
{
- return defaultValue;
+ return Double.parseDouble(value);
}
- else
+ catch (NumberFormatException e)
{
- try
- {
- return Integer.valueOf(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(String.format("Invalid integer value %s for '%s'", value, key));
- }
+ throw new SyntaxException(format("Invalid double value %s for '%s'", value, key));
}
}
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 8a48935..4c43cf8 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -22,6 +22,8 @@
import java.util.stream.Collectors;
import java.util.concurrent.TimeUnit;
+import javax.annotation.concurrent.ThreadSafe;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
@@ -53,17 +55,15 @@
import org.apache.cassandra.db.aggregation.AggregationSpecification;
import org.apache.cassandra.db.aggregation.GroupMaker;
import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.rows.ComplexColumnData;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.view.View;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.ClientState;
@@ -82,6 +82,7 @@
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
+import static java.lang.String.format;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
import static org.apache.cassandra.cql3.statements.RequestValidations.checkNull;
@@ -98,7 +99,10 @@
* many of these are made accessible for the benefit of custom
* QueryHandler implementations, so before reducing their accessibility
* due consideration should be given.
+ *
+ * Note that select statements can be accessed by multiple threads, so we cannot rely on mutable attributes.
*/
+@ThreadSafe
public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
{
private static final Logger logger = LoggerFactory.getLogger(SelectStatement.class);
@@ -238,6 +242,21 @@
for (Function function : getFunctions())
state.ensurePermission(Permission.EXECUTE, function);
+
+ if (!state.hasTablePermission(table, Permission.UNMASK) &&
+ !state.hasTablePermission(table, Permission.SELECT_MASKED))
+ {
+ List<ColumnMetadata> queriedMaskedColumns = table.columns()
+ .stream()
+ .filter(ColumnMetadata::isMasked)
+ .filter(restrictions::isRestricted)
+ .collect(Collectors.toList());
+
+ if (!queriedMaskedColumns.isEmpty())
+ throw new UnauthorizedException(format("User %s has no UNMASK nor SELECT_MASKED permission on table %s.%s, " +
+ "cannot query masked columns %s",
+ state.getUser().getName(), keyspace(), table(), queriedMaskedColumns));
+ }
}
public void validate(ClientState state) throws InvalidRequestException
@@ -258,6 +277,7 @@
int userLimit = getLimit(options);
int userPerPartitionLimit = getPerPartitionLimit(options);
int pageSize = options.getPageSize();
+ boolean unmask = !table.hasMaskedColumns() || state.getClientState().hasTablePermission(table, Permission.UNMASK);
Selectors selectors = selection.newSelectors(options);
AggregationSpecification aggregationSpec = getAggregationSpec(options);
@@ -266,21 +286,31 @@
if (options.isReadThresholdsEnabled())
query.trackWarnings();
+ ResultMessage.Rows rows;
if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize)))
- return execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, null, queryStartNanoTime);
+ {
+ rows = execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, null, queryStartNanoTime, unmask);
+ }
+ else
+ {
+ QueryPager pager = getPager(query, options);
- QueryPager pager = getPager(query, options);
+ rows = execute(state,
+ Pager.forDistributedQuery(pager, cl, state.getClientState()),
+ options,
+ selectors,
+ pageSize,
+ nowInSec,
+ userLimit,
+ aggregationSpec,
+ queryStartNanoTime,
+ unmask);
+ }
+ if (!SchemaConstants.isSystemKeyspace(table.keyspace))
+ ClientRequestSizeMetrics.recordReadResponseMetrics(rows, restrictions, selection);
- return execute(state,
- Pager.forDistributedQuery(pager, cl, state.getClientState()),
- options,
- selectors,
- pageSize,
- nowInSec,
- userLimit,
- aggregationSpec,
- queryStartNanoTime);
+ return rows;
}
@@ -328,11 +358,12 @@
int nowInSec,
int userLimit,
AggregationSpecification aggregationSpec,
- long queryStartNanoTime)
+ long queryStartNanoTime,
+ boolean unmask)
{
try (PartitionIterator data = query.execute(options.getConsistency(), state, queryStartNanoTime))
{
- return processResults(data, options, selectors, nowInSec, userLimit, aggregationSpec);
+ return processResults(data, options, selectors, nowInSec, userLimit, aggregationSpec, unmask);
}
}
@@ -417,7 +448,8 @@
int nowInSec,
int userLimit,
AggregationSpecification aggregationSpec,
- long queryStartNanoTime)
+ long queryStartNanoTime,
+ boolean unmask)
{
Guardrails.pageSize.guard(pageSize, table(), false, state.getClientState());
@@ -446,7 +478,7 @@
ResultMessage.Rows msg;
try (PartitionIterator page = pager.fetchPage(pageSize, queryStartNanoTime))
{
- msg = processResults(page, options, selectors, nowInSec, userLimit, aggregationSpec);
+ msg = processResults(page, options, selectors, nowInSec, userLimit, aggregationSpec, unmask);
}
// Please note that the isExhausted state of the pager only gets updated when we've closed the page, so this
@@ -468,9 +500,10 @@
Selectors selectors,
int nowInSec,
int userLimit,
- AggregationSpecification aggregationSpec) throws RequestValidationException
+ AggregationSpecification aggregationSpec,
+ boolean unmask) throws RequestValidationException
{
- ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit, aggregationSpec);
+ ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit, aggregationSpec, unmask);
return new ResultMessage.Rows(rset);
}
@@ -487,6 +520,7 @@
int userLimit = getLimit(options);
int userPerPartitionLimit = getPerPartitionLimit(options);
int pageSize = options.getPageSize();
+ boolean unmask = state.getClientState().hasTablePermission(table, Permission.UNMASK);
Selectors selectors = selection.newSelectors(options);
AggregationSpecification aggregationSpec = getAggregationSpec(options);
@@ -505,7 +539,7 @@
{
try (PartitionIterator data = query.executeInternal(executionController))
{
- return processResults(data, options, selectors, nowInSec, userLimit, null);
+ return processResults(data, options, selectors, nowInSec, userLimit, null, unmask);
}
}
@@ -519,7 +553,8 @@
nowInSec,
userLimit,
aggregationSpec,
- queryStartNanoTime);
+ queryStartNanoTime,
+ unmask);
}
}
@@ -577,11 +612,11 @@
}
}
- public ResultSet process(PartitionIterator partitions, int nowInSec) throws InvalidRequestException
+ public ResultSet process(PartitionIterator partitions, int nowInSec, boolean unmask) throws InvalidRequestException
{
QueryOptions options = QueryOptions.DEFAULT;
Selectors selectors = selection.newSelectors(options);
- return process(partitions, options, selectors, nowInSec, getLimit(options), getAggregationSpec(options));
+ return process(partitions, options, selectors, nowInSec, getLimit(options), getAggregationSpec(options), unmask);
}
@Override
@@ -887,10 +922,11 @@
Selectors selectors,
int nowInSec,
int userLimit,
- AggregationSpecification aggregationSpec) throws InvalidRequestException
+ AggregationSpecification aggregationSpec,
+ boolean unmask) throws InvalidRequestException
{
GroupMaker groupMaker = aggregationSpec == null ? null : aggregationSpec.newGroupMaker();
- ResultSetBuilder result = new ResultSetBuilder(getResultMetadata(), selectors, groupMaker);
+ ResultSetBuilder result = new ResultSetBuilder(getResultMetadata(), selectors, unmask, groupMaker);
while (partitions.hasNext())
{
@@ -987,7 +1023,7 @@
{
if (!staticRow.isEmpty() && restrictions.returnStaticContentOnPartitionWithNoRows())
{
- result.newRow(partition.partitionKey(), staticRow.clustering());
+ result.newRow(protocolVersion, partition.partitionKey(), staticRow.clustering(), selection.getColumns());
maybeFail(result, options);
for (ColumnMetadata def : selection.getColumns())
{
@@ -997,7 +1033,7 @@
result.add(keyComponents[def.position()]);
break;
case STATIC:
- addValue(result, def, staticRow, nowInSec, protocolVersion);
+ result.add(partition.staticRow().getColumnData(def), nowInSec);
break;
default:
result.add((ByteBuffer)null);
@@ -1010,7 +1046,7 @@
while (partition.hasNext())
{
Row row = partition.next();
- result.newRow( partition.partitionKey(), row.clustering());
+ result.newRow(protocolVersion, partition.partitionKey(), row.clustering(), selection.getColumns());
// reads aren't failed as soon the size exceeds the failure threshold, they're failed once the failure
// threshold has been exceeded and we start adding more data. We're slightly more permissive to avoid
@@ -1030,35 +1066,16 @@
result.add(row.clustering().bufferAt(def.position()));
break;
case REGULAR:
- addValue(result, def, row, nowInSec, protocolVersion);
+ result.add(row.getColumnData(def), nowInSec);
break;
case STATIC:
- addValue(result, def, staticRow, nowInSec, protocolVersion);
+ result.add(staticRow.getColumnData(def), nowInSec);
break;
}
}
}
}
- private static void addValue(ResultSetBuilder result, ColumnMetadata def, Row row, int nowInSec, ProtocolVersion protocolVersion)
- {
- if (def.isComplex())
- {
- assert def.type.isMultiCell();
- ComplexColumnData complexData = row.getComplexColumnData(def);
- if (complexData == null)
- result.add(null);
- else if (def.type.isCollection())
- result.add(((CollectionType) def.type).serializeForNativeProtocol(complexData.iterator(), protocolVersion));
- else
- result.add(((UserType) def.type).serializeForNativeProtocol(complexData.iterator(), protocolVersion));
- }
- else
- {
- result.add(row.getCell(def), nowInSec);
- }
- }
-
private boolean needsPostQueryOrdering()
{
// We need post-query ordering only for queries with IN on the partition key and an ORDER BY.
@@ -1104,17 +1121,17 @@
{
// Cache locally for use by Guardrails
this.state = state;
- return prepare(false);
+ return prepare(state, false);
}
- public SelectStatement prepare(boolean forView) throws InvalidRequestException
+ public SelectStatement prepare(ClientState state, boolean forView) throws InvalidRequestException
{
TableMetadata table = Schema.instance.validateTable(keyspace(), name());
List<Selectable> selectables = RawSelector.toSelectables(selectClause, table);
boolean containsOnlyStaticColumns = selectOnlyStaticColumns(table, selectables);
- StatementRestrictions restrictions = prepareRestrictions(table, bindVariables, containsOnlyStaticColumns, forView);
+ StatementRestrictions restrictions = prepareRestrictions(state, table, bindVariables, containsOnlyStaticColumns, forView);
// If we order post-query, the sorted column needs to be in the ResultSet for sorting,
// even if we don't ultimately ship them to the client (CASSANDRA-4911).
@@ -1182,10 +1199,14 @@
if (hasGroupBy)
Guardrails.groupByEnabled.ensureEnabled(state);
+ boolean isJson = parameters.isJson;
+ boolean returnStaticContentOnPartitionWithNoRows = restrictions.returnStaticContentOnPartitionWithNoRows();
+
if (selectables.isEmpty()) // wildcard query
{
- return hasGroupBy ? Selection.wildcardWithGroupBy(table, boundNames, parameters.isJson, restrictions.returnStaticContentOnPartitionWithNoRows())
- : Selection.wildcard(table, parameters.isJson, restrictions.returnStaticContentOnPartitionWithNoRows());
+ return hasGroupBy || table.hasMaskedColumns()
+ ? Selection.wildcardWithGroupByOrMaskedColumns(table, boundNames, resultSetOrderingColumns, isJson, returnStaticContentOnPartitionWithNoRows)
+ : Selection.wildcard(table, isJson, returnStaticContentOnPartitionWithNoRows);
}
return Selection.fromSelectors(table,
@@ -1194,8 +1215,8 @@
resultSetOrderingColumns,
restrictions.nonPKRestrictedColumns(false),
hasGroupBy,
- parameters.isJson,
- restrictions.returnStaticContentOnPartitionWithNoRows());
+ isJson,
+ returnStaticContentOnPartitionWithNoRows);
}
/**
@@ -1244,12 +1265,14 @@
* @return the restrictions
* @throws InvalidRequestException if a problem occurs while building the restrictions
*/
- private StatementRestrictions prepareRestrictions(TableMetadata metadata,
+ private StatementRestrictions prepareRestrictions(ClientState state,
+ TableMetadata metadata,
VariableSpecifications boundNames,
boolean selectsOnlyStaticColumns,
boolean forView) throws InvalidRequestException
{
- return new StatementRestrictions(StatementType.SELECT,
+ return new StatementRestrictions(state,
+ StatementType.SELECT,
metadata,
whereClause,
boundNames,
@@ -1323,6 +1346,7 @@
int clusteringPrefixSize = 0;
Iterator<ColumnMetadata> pkColumns = metadata.primaryKeyColumns().iterator();
+ List<ColumnMetadata> columns = null;
Selector.Factory selectorFactory = null;
for (Selectable.Raw raw : parameters.groups)
{
@@ -1334,7 +1358,7 @@
{
WithFunction withFunction = (WithFunction) selectable;
validateGroupByFunction(withFunction);
- List<ColumnMetadata> columns = new ArrayList<ColumnMetadata>();
+ columns = new ArrayList<ColumnMetadata>();
selectorFactory = selectable.newSelectorFactory(metadata, null, columns, boundNames);
checkFalse(columns.isEmpty(), "GROUP BY functions must have one clustering column name as parameter");
if (columns.size() > 1)
@@ -1382,7 +1406,8 @@
return selectorFactory == null ? AggregationSpecification.aggregatePkPrefixFactory(metadata.comparator, clusteringPrefixSize)
: AggregationSpecification.aggregatePkPrefixFactoryWithSelector(metadata.comparator,
clusteringPrefixSize,
- selectorFactory);
+ selectorFactory,
+ columns);
}
/**
@@ -1405,8 +1430,8 @@
if (!restrictions.keyIsInRelation())
return null;
- List<Integer> idToSort = new ArrayList<Integer>(orderingColumns.size());
- List<Comparator<ByteBuffer>> sorters = new ArrayList<Comparator<ByteBuffer>>(orderingColumns.size());
+ List<Integer> idToSort = new ArrayList<>(orderingColumns.size());
+ List<Comparator<ByteBuffer>> sorters = new ArrayList<>(orderingColumns.size());
for (ColumnMetadata orderingColumn : orderingColumns.keySet())
{
@@ -1466,7 +1491,8 @@
// We will potentially filter data if either:
// - Have more than one IndexExpression
// - Have no index expression and the row filter is not the identity
- checkFalse(restrictions.needFiltering(), StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+ if (restrictions.requiresAllowFilteringIfNotSpecified())
+ checkFalse(restrictions.needFiltering(), StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
}
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
index 20df151..258fbf3 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
@@ -32,6 +32,7 @@
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -135,7 +136,8 @@
}
@Override
- protected ModificationStatement prepareInternal(TableMetadata metadata,
+ protected ModificationStatement prepareInternal(ClientState state,
+ TableMetadata metadata,
VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
@@ -176,7 +178,8 @@
boolean applyOnlyToStaticColumns = !hasClusteringColumnsSet && appliesOnlyToStaticColumns(operations, conditions);
- StatementRestrictions restrictions = new StatementRestrictions(type,
+ StatementRestrictions restrictions = new StatementRestrictions(state,
+ type,
metadata,
whereClause.build(),
bindVariables,
@@ -210,7 +213,8 @@
}
@Override
- protected ModificationStatement prepareInternal(TableMetadata metadata,
+ protected ModificationStatement prepareInternal(ClientState state,
+ TableMetadata metadata,
VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
@@ -244,7 +248,8 @@
boolean applyOnlyToStaticColumns = !hasClusteringColumnsSet && appliesOnlyToStaticColumns(operations, conditions);
- StatementRestrictions restrictions = new StatementRestrictions(type,
+ StatementRestrictions restrictions = new StatementRestrictions(state,
+ type,
metadata,
whereClause.build(),
bindVariables,
@@ -291,7 +296,8 @@
}
@Override
- protected ModificationStatement prepareInternal(TableMetadata metadata,
+ protected ModificationStatement prepareInternal(ClientState state,
+ TableMetadata metadata,
VariableSpecifications bindVariables,
Conditions conditions,
Attributes attrs)
@@ -309,7 +315,8 @@
operations.add(operation);
}
- StatementRestrictions restrictions = newRestrictions(metadata,
+ StatementRestrictions restrictions = newRestrictions(state,
+ metadata,
bindVariables,
operations,
whereClause,
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
index 87377d7..b8a27af 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
@@ -26,17 +26,18 @@
import org.apache.cassandra.audit.AuditLogContext;
import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.Permission;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.ReplicationFactor;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
import org.apache.cassandra.schema.Keyspaces;
@@ -46,10 +47,13 @@
import org.apache.cassandra.transport.Event.SchemaChange.Change;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_TRANSIENT_CHANGES;
+
public final class AlterKeyspaceStatement extends AlterSchemaStatement
{
- private static final boolean allow_alter_rf_during_range_movement = Boolean.getBoolean(Config.PROPERTY_PREFIX + "allow_alter_rf_during_range_movement");
- private static final boolean allow_unsafe_transient_changes = Boolean.getBoolean(Config.PROPERTY_PREFIX + "allow_unsafe_transient_changes");
+ private static final boolean allow_alter_rf_during_range_movement = ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT.getBoolean();
+ private static final boolean allow_unsafe_transient_changes = ALLOW_UNSAFE_TRANSIENT_CHANGES.getBoolean();
private final HashSet<String> clientWarnings = new HashSet<>();
private final KeyspaceAttributes attrs;
@@ -76,6 +80,9 @@
KeyspaceMetadata newKeyspace = keyspace.withSwapped(attrs.asAlteredKeyspaceParams(keyspace.params));
+ if (attrs.getReplicationStrategyClass() != null && attrs.getReplicationStrategyClass().equals(SimpleStrategy.class.getSimpleName()))
+ Guardrails.simpleStrategyEnabled.ensureEnabled(state);
+
if (newKeyspace.params.replication.klass.equals(LocalStrategy.class))
throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
index fdc4921..a539ea7 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java
@@ -26,6 +26,8 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
+import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -140,6 +142,14 @@
}
}
+ protected void validateDefaultTimeToLive(TableParams params)
+ {
+ if (params.defaultTimeToLive == 0
+ && !SchemaConstants.isSystemKeyspace(keyspaceName)
+ && TimeWindowCompactionStrategy.class.isAssignableFrom(params.compaction.klass()))
+ Guardrails.zeroTTLOnTWCSEnabled.ensureEnabled(state);
+ }
+
private void grantPermissionsOnResource(IResource resource, AuthenticatedUser user)
{
try
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
index 08c5f04..81f096a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
@@ -23,9 +23,12 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
@@ -40,6 +43,7 @@
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -78,6 +82,7 @@
{
protected final String tableName;
private final boolean ifExists;
+ protected ClientState state;
public AlterTableStatement(String keyspaceName, String tableName, boolean ifExists)
{
@@ -86,6 +91,15 @@
this.ifExists = ifExists;
}
+ @Override
+ public void validate(ClientState state)
+ {
+ super.validate(state);
+
+ // save the query state to use it for guardrails validation in #apply
+ this.state = state;
+ }
+
public Keyspaces apply(Keyspaces schema)
{
KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
@@ -149,6 +163,78 @@
}
/**
+ * ALTER TABLE [IF EXISTS] <table> ALTER [IF EXISTS] <column> ( MASKED WITH <newMask> | DROP MASKED )
+ */
+ public static class MaskColumn extends AlterTableStatement
+ {
+ private final ColumnIdentifier columnName;
+ @Nullable
+ private final ColumnMask.Raw rawMask;
+ private final boolean ifColumnExists;
+
+ MaskColumn(String keyspaceName,
+ String tableName,
+ ColumnIdentifier columnName,
+ @Nullable ColumnMask.Raw rawMask,
+ boolean ifTableExists,
+ boolean ifColumnExists)
+ {
+ super(keyspaceName, tableName, ifTableExists);
+ this.columnName = columnName;
+ this.rawMask = rawMask;
+ this.ifColumnExists = ifColumnExists;
+ }
+
+ @Override
+ public void validate(ClientState state)
+ {
+ super.validate(state);
+
+ // we don't allow creating masks if they are disabled, but we still allow dropping them
+ if (rawMask != null)
+ ColumnMask.ensureEnabled();
+ }
+
+ @Override
+ public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+ {
+ ColumnMetadata column = table.getColumn(columnName);
+
+ if (column == null)
+ {
+ if (!ifColumnExists)
+ throw ire("Column with name '%s' doesn't exist on table '%s'", columnName, tableName);
+
+ return keyspace;
+ }
+
+ ColumnMask oldMask = table.getColumn(columnName).getMask();
+ ColumnMask newMask = rawMask == null ? null : rawMask.prepare(keyspace.name, table.name, columnName, column.type);
+
+ if (Objects.equals(oldMask, newMask))
+ return keyspace;
+
+ TableMetadata.Builder tableBuilder = table.unbuild();
+ tableBuilder.alterColumnMask(columnName, newMask);
+ TableMetadata newTable = tableBuilder.build();
+ newTable.validate();
+
+ // Update any reference on materialized views, so the mask is consistent among the base table and its views.
+ Views.Builder viewsBuilder = keyspace.views.unbuild();
+ for (ViewMetadata view : keyspace.views.forTable(table.id))
+ {
+ if (view.includes(columnName))
+ {
+ viewsBuilder.put(viewsBuilder.get(view.name()).withNewColumnMask(columnName, newMask));
+ }
+ }
+
+ return keyspace.withSwapped(keyspace.tables.withSwapped(newTable))
+ .withSwapped(viewsBuilder.build());
+ }
+ }
+
+ /**
* ALTER TABLE [IF EXISTS] <table> ADD [IF NOT EXISTS] <column> <newtype>
* ALTER TABLE [IF EXISTS] <table> ADD [IF NOT EXISTS] (<column> <newtype>, <column1> <newtype1>, ... <columnn> <newtypen>)
*/
@@ -159,12 +245,15 @@
private final ColumnIdentifier name;
private final CQL3Type.Raw type;
private final boolean isStatic;
+ @Nullable
+ private final ColumnMask.Raw mask;
- Column(ColumnIdentifier name, CQL3Type.Raw type, boolean isStatic)
+ Column(ColumnIdentifier name, CQL3Type.Raw type, boolean isStatic, @Nullable ColumnMask.Raw mask)
{
this.name = name;
this.type = type;
this.isStatic = isStatic;
+ this.mask = mask;
}
}
@@ -178,14 +267,9 @@
this.ifColumnNotExists = ifColumnNotExists;
}
- @Override
- public void validate(ClientState state)
- {
- super.validate(state);
- }
-
public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
{
+ Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);
TableMetadata.Builder tableBuilder = table.unbuild();
Views.Builder viewsBuilder = keyspace.views.unbuild();
newColumns.forEach(c -> addColumn(keyspace, table, c, ifColumnNotExists, tableBuilder, viewsBuilder));
@@ -209,6 +293,7 @@
ColumnIdentifier name = column.name;
AbstractType<?> type = column.type.prepare(keyspaceName, keyspace.types).getType();
boolean isStatic = column.isStatic;
+ ColumnMask mask = column.mask == null ? null : column.mask.prepare(keyspaceName, tableName, name, type);
if (null != tableBuilder.getColumn(name)) {
if (!ifColumnNotExists)
@@ -249,9 +334,9 @@
}
if (isStatic)
- tableBuilder.addStaticColumn(name, type);
+ tableBuilder.addStaticColumn(name, type, mask);
else
- tableBuilder.addRegularColumn(name, type);
+ tableBuilder.addRegularColumn(name, type, mask);
if (!isStatic)
{
@@ -259,7 +344,8 @@
{
if (view.includeAllColumns)
{
- ColumnMetadata viewColumn = ColumnMetadata.regularColumn(view.metadata, name.bytes, type);
+ ColumnMetadata viewColumn = ColumnMetadata.regularColumn(view.metadata, name.bytes, type)
+ .withNewMask(mask);
viewsBuilder.put(viewsBuilder.get(view.name()).withAddedRegularColumn(viewColumn));
}
}
@@ -288,6 +374,7 @@
public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
{
+ Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);
TableMetadata.Builder builder = table.unbuild();
removedColumns.forEach(c -> dropColumn(keyspace, table, c, ifColumnExists, builder));
return keyspace.withSwapped(keyspace.tables.withSwapped(builder.build()));
@@ -355,6 +442,7 @@
public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
{
+ Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);
TableMetadata.Builder tableBuilder = table.unbuild();
Views.Builder viewsBuilder = keyspace.views.unbuild();
renamedColumns.forEach((o, n) -> renameColumn(keyspace, table, o, n, ifColumnsExists, tableBuilder, viewsBuilder));
@@ -430,6 +518,8 @@
super.validate(state);
Guardrails.tableProperties.guard(attrs.updatedProperties(), attrs::removeProperty, state);
+
+ validateDefaultTimeToLive(attrs.asNewTableParams());
}
public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
@@ -567,7 +657,13 @@
{
private enum Kind
{
- ALTER_COLUMN, ADD_COLUMNS, DROP_COLUMNS, RENAME_COLUMNS, ALTER_OPTIONS, DROP_COMPACT_STORAGE
+ ALTER_COLUMN,
+ MASK_COLUMN,
+ ADD_COLUMNS,
+ DROP_COLUMNS,
+ RENAME_COLUMNS,
+ ALTER_OPTIONS,
+ DROP_COMPACT_STORAGE
}
private final QualifiedName name;
@@ -580,6 +676,10 @@
// ADD
private final List<AddColumns.Column> addedColumns = new ArrayList<>();
+ // ALTER MASK
+ private ColumnIdentifier maskedColumn = null;
+ private ColumnMask.Raw rawMask = null;
+
// DROP
private final Set<ColumnIdentifier> droppedColumns = new HashSet<>();
private Long timestamp = null; // will use execution timestamp if not provided by query
@@ -604,6 +704,7 @@
switch (kind)
{
case ALTER_COLUMN: return new AlterColumn(keyspaceName, tableName, ifTableExists);
+ case MASK_COLUMN: return new MaskColumn(keyspaceName, tableName, maskedColumn, rawMask, ifTableExists, ifColumnExists);
case ADD_COLUMNS: return new AddColumns(keyspaceName, tableName, addedColumns, ifTableExists, ifColumnNotExists);
case DROP_COLUMNS: return new DropColumns(keyspaceName, tableName, droppedColumns, ifTableExists, ifColumnExists, timestamp);
case RENAME_COLUMNS: return new RenameColumns(keyspaceName, tableName, renamedColumns, ifTableExists, ifColumnExists);
@@ -619,10 +720,17 @@
kind = Kind.ALTER_COLUMN;
}
- public void add(ColumnIdentifier name, CQL3Type.Raw type, boolean isStatic)
+ public void mask(ColumnIdentifier name, ColumnMask.Raw mask)
+ {
+ kind = Kind.MASK_COLUMN;
+ maskedColumn = name;
+ rawMask = mask;
+ }
+
+ public void add(ColumnIdentifier name, CQL3Type.Raw type, boolean isStatic, @Nullable ColumnMask.Raw mask)
{
kind = Kind.ADD_COLUMNS;
- addedColumns.add(new AddColumns.Column(name, type, isStatic));
+ addedColumns.add(new AddColumns.Column(name, type, isStatic, mask));
}
public void drop(ColumnIdentifier name)
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
index 9c3be11..89cb990 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
@@ -179,7 +179,7 @@
UserType apply(KeyspaceMetadata keyspace, UserType userType)
{
List<String> dependentAggregates =
- keyspace.functions
+ keyspace.userFunctions
.udas()
.filter(uda -> null != uda.initialCondition() && uda.stateType().referencesUserType(userType.name))
.map(uda -> uda.name().toString())
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
index 0550515..010f464 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java
@@ -31,9 +31,14 @@
import org.apache.cassandra.auth.IResource;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.ScalarFunction;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.functions.UDHelper;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.schema.Functions.FunctionsDiff;
+import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -117,8 +122,8 @@
AbstractType<?> stateType = rawStateType.prepare(keyspaceName, keyspace.types).getType().udfType();
List<AbstractType<?>> stateFunctionArguments = Lists.newArrayList(concat(singleton(stateType), argumentTypes));
- Function stateFunction =
- keyspace.functions
+ UserFunction stateFunction =
+ keyspace.userFunctions
.find(stateFunctionName, stateFunctionArguments)
.orElseThrow(() -> ire("State function %s doesn't exist", stateFunctionString()));
@@ -135,12 +140,12 @@
* Resolve the final function and return type
*/
- Function finalFunction = null;
+ UserFunction finalFunction = null;
AbstractType<?> returnType = stateFunction.returnType();
if (null != finalFunctionName)
{
- finalFunction = keyspace.functions.find(finalFunctionName, singletonList(stateType)).orElse(null);
+ finalFunction = keyspace.userFunctions.find(finalFunctionName, singletonList(stateType)).orElse(null);
if (null == finalFunction)
throw ire("Final function %s doesn't exist", finalFunctionString());
@@ -199,7 +204,7 @@
(ScalarFunction) finalFunction,
initialValue);
- Function existingAggregate = keyspace.functions.find(aggregate.name(), argumentTypes).orElse(null);
+ UserFunction existingAggregate = keyspace.userFunctions.find(aggregate.name(), argumentTypes).orElse(null);
if (null != existingAggregate)
{
if (!existingAggregate.isAggregate())
@@ -220,7 +225,7 @@
}
}
- return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.withAddedOrUpdated(aggregate)));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.userFunctions.withAddedOrUpdated(aggregate)));
}
SchemaChange schemaChangeEvent(KeyspacesDiff diff)
@@ -242,7 +247,7 @@
{
FunctionName name = new FunctionName(keyspaceName, aggregateName);
- if (Schema.instance.findFunction(name, Lists.transform(rawArgumentTypes, t -> t.prepare(keyspaceName).getType())).isPresent() && orReplace)
+ if (Schema.instance.findUserFunction(name, Lists.transform(rawArgumentTypes, t -> t.prepare(keyspaceName).getType())).isPresent() && orReplace)
client.ensurePermission(Permission.ALTER, FunctionResource.functionFromCql(keyspaceName, aggregateName, rawArgumentTypes));
else
client.ensurePermission(Permission.CREATE, FunctionResource.keyspace(keyspaceName));
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
index adb8f40..8e3c79c 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java
@@ -30,11 +30,11 @@
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.schema.Functions.FunctionsDiff;
+import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -122,7 +122,7 @@
language,
body);
- Function existingFunction = keyspace.functions.find(function.name(), argumentTypes).orElse(null);
+ UserFunction existingFunction = keyspace.userFunctions.find(function.name(), argumentTypes).orElse(null);
if (null != existingFunction)
{
if (existingFunction.isAggregate())
@@ -152,7 +152,7 @@
// TODO: update dependent aggregates
}
- return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.withAddedOrUpdated(function)));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.userFunctions.withAddedOrUpdated(function)));
}
SchemaChange schemaChangeEvent(KeyspacesDiff diff)
@@ -174,7 +174,7 @@
{
FunctionName name = new FunctionName(keyspaceName, functionName);
- if (Schema.instance.findFunction(name, Lists.transform(rawArgumentTypes, t -> t.prepare(keyspaceName).getType().udfType())).isPresent() && orReplace)
+ if (Schema.instance.findUserFunction(name, Lists.transform(rawArgumentTypes, t -> t.prepare(keyspaceName).getType().udfType())).isPresent() && orReplace)
client.ensurePermission(Permission.ALTER, FunctionResource.functionFromCql(keyspaceName, functionName, rawArgumentTypes));
else
client.ensurePermission(Permission.CREATE, FunctionResource.keyspace(keyspaceName));
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
index dc82f93..ad6bcc4 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
@@ -36,6 +36,7 @@
import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.exceptions.AlreadyExistsException;
import org.apache.cassandra.locator.LocalStrategy;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams.Option;
import org.apache.cassandra.schema.Keyspaces;
@@ -67,6 +68,9 @@
if (!attrs.hasOption(Option.REPLICATION))
throw ire("Missing mandatory option '%s'", Option.REPLICATION);
+ if (attrs.getReplicationStrategyClass() != null && attrs.getReplicationStrategyClass().equals(SimpleStrategy.class.getSimpleName()))
+ Guardrails.simpleStrategyEnabled.ensureEnabled("SimpleStrategy", state);
+
if (schema.containsKeyspace(keyspaceName))
{
if (ifNotExists)
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
index 1a62538..45f23e9 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
@@ -19,6 +19,8 @@
import java.util.*;
+import javax.annotation.Nullable;
+
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
@@ -33,6 +35,7 @@
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.db.marshal.*;
@@ -54,7 +57,7 @@
private static final Logger logger = LoggerFactory.getLogger(CreateTableStatement.class);
private final String tableName;
- private final Map<ColumnIdentifier, CQL3Type.Raw> rawColumns;
+ private final Map<ColumnIdentifier, ColumnProperties.Raw> rawColumns;
private final Set<ColumnIdentifier> staticColumns;
private final List<ColumnIdentifier> partitionKeyColumns;
private final List<ColumnIdentifier> clusteringColumns;
@@ -67,15 +70,12 @@
public CreateTableStatement(String keyspaceName,
String tableName,
-
- Map<ColumnIdentifier, CQL3Type.Raw> rawColumns,
+ Map<ColumnIdentifier, ColumnProperties.Raw> rawColumns,
Set<ColumnIdentifier> staticColumns,
List<ColumnIdentifier> partitionKeyColumns,
List<ColumnIdentifier> clusteringColumns,
-
LinkedHashMap<ColumnIdentifier, Boolean> clusteringOrder,
TableAttributes attrs,
-
boolean ifNotExists,
boolean useCompactStorage)
{
@@ -149,6 +149,15 @@
// Guardrail to check whether creation of new COMPACT STORAGE tables is allowed
if (useCompactStorage)
Guardrails.compactTablesEnabled.ensureEnabled(state);
+
+ validateDefaultTimeToLive(attrs.asNewTableParams());
+
+ // Verify that dynamic data masking is enabled if there are masked columns
+ for (ColumnProperties.Raw raw : rawColumns.values())
+ {
+ if (raw.rawMask != null)
+ ColumnMask.ensureEnabled();
+ }
}
SchemaChange schemaChangeEvent(KeyspacesDiff diff)
@@ -184,15 +193,16 @@
TableParams params = attrs.asNewTableParams();
// use a TreeMap to preserve ordering across JDK versions (see CASSANDRA-9492) - important for stable unit tests
- Map<ColumnIdentifier, CQL3Type> columns = new TreeMap<>(comparing(o -> o.bytes));
- rawColumns.forEach((column, type) -> columns.put(column, type.prepare(keyspaceName, types)));
+ Map<ColumnIdentifier, ColumnProperties> columns = new TreeMap<>(comparing(o -> o.bytes));
+ rawColumns.forEach((column, properties) -> columns.put(column, properties.prepare(keyspaceName, tableName, column, types)));
// check for nested non-frozen UDTs or collections in a non-frozen UDT
- columns.forEach((column, type) ->
+ columns.forEach((column, properties) ->
{
- if (type.isUDT() && type.getType().isMultiCell())
+ AbstractType<?> type = properties.type;
+ if (type.isUDT() && type.isMultiCell())
{
- ((UserType) type.getType()).fieldTypes().forEach(field ->
+ ((UserType) type).fieldTypes().forEach(field ->
{
if (field.isMultiCell())
throw ire("Non-frozen UDTs with nested non-frozen collections are not supported");
@@ -207,45 +217,47 @@
HashSet<ColumnIdentifier> primaryKeyColumns = new HashSet<>();
concat(partitionKeyColumns, clusteringColumns).forEach(column ->
{
- CQL3Type type = columns.get(column);
- if (null == type)
+ ColumnProperties properties = columns.get(column);
+ if (null == properties)
throw ire("Unknown column '%s' referenced in PRIMARY KEY for table '%s'", column, tableName);
if (!primaryKeyColumns.add(column))
throw ire("Duplicate column '%s' in PRIMARY KEY clause for table '%s'", column, tableName);
- if (type.getType().isMultiCell())
+ AbstractType<?> type = properties.type;
+ if (type.isMultiCell())
{
+ CQL3Type cqlType = properties.cqlType;
if (type.isCollection())
- throw ire("Invalid non-frozen collection type %s for PRIMARY KEY column '%s'", type, column);
+ throw ire("Invalid non-frozen collection type %s for PRIMARY KEY column '%s'", cqlType, column);
else
- throw ire("Invalid non-frozen user-defined type %s for PRIMARY KEY column '%s'", type, column);
+ throw ire("Invalid non-frozen user-defined type %s for PRIMARY KEY column '%s'", cqlType, column);
}
- if (type.getType().isCounter())
+ if (type.isCounter())
throw ire("counter type is not supported for PRIMARY KEY column '%s'", column);
- if (type.getType().referencesDuration())
+ if (type.referencesDuration())
throw ire("duration type is not supported for PRIMARY KEY column '%s'", column);
if (staticColumns.contains(column))
throw ire("Static column '%s' cannot be part of the PRIMARY KEY", column);
});
- List<AbstractType<?>> partitionKeyTypes = new ArrayList<>();
- List<AbstractType<?>> clusteringTypes = new ArrayList<>();
+ List<ColumnProperties> partitionKeyColumnProperties = new ArrayList<>();
+ List<ColumnProperties> clusteringColumnProperties = new ArrayList<>();
partitionKeyColumns.forEach(column ->
{
- CQL3Type type = columns.remove(column);
- partitionKeyTypes.add(type.getType());
+ ColumnProperties columnProperties = columns.remove(column);
+ partitionKeyColumnProperties.add(columnProperties);
});
clusteringColumns.forEach(column ->
{
- CQL3Type type = columns.remove(column);
+ ColumnProperties columnProperties = columns.remove(column);
boolean reverse = !clusteringOrder.getOrDefault(column, true);
- clusteringTypes.add(reverse ? ReversedType.getInstance(type.getType()) : type.getType());
+ clusteringColumnProperties.add(reverse ? columnProperties.withReversedType() : columnProperties);
});
if (clusteringOrder.size() > clusteringColumns.size())
@@ -268,7 +280,7 @@
// For COMPACT STORAGE, we reject any "feature" that we wouldn't be able to translate back to thrift.
if (useCompactStorage)
{
- validateCompactTable(clusteringTypes, columns);
+ validateCompactTable(clusteringColumnProperties, columns);
}
else
{
@@ -281,12 +293,12 @@
* Counter table validation
*/
- boolean hasCounters = rawColumns.values().stream().anyMatch(CQL3Type.Raw::isCounter);
+ boolean hasCounters = rawColumns.values().stream().anyMatch(c -> c.rawType.isCounter());
if (hasCounters)
{
// We've handled anything that is not a PRIMARY KEY so columns only contains NON-PK columns. So
// if it's a counter table, make sure we don't have non-counter types
- if (columns.values().stream().anyMatch(t -> !t.getType().isCounter()))
+ if (columns.values().stream().anyMatch(t -> !t.type.isCounter()))
throw ire("Cannot mix counter and non counter columns in the same table");
if (params.defaultTimeToLive > 0)
@@ -306,38 +318,44 @@
.params(params);
for (int i = 0; i < partitionKeyColumns.size(); i++)
- builder.addPartitionKeyColumn(partitionKeyColumns.get(i), partitionKeyTypes.get(i));
+ {
+ ColumnProperties properties = partitionKeyColumnProperties.get(i);
+ builder.addPartitionKeyColumn(partitionKeyColumns.get(i), properties.type, properties.mask);
+ }
for (int i = 0; i < clusteringColumns.size(); i++)
- builder.addClusteringColumn(clusteringColumns.get(i), clusteringTypes.get(i));
+ {
+ ColumnProperties properties = clusteringColumnProperties.get(i);
+ builder.addClusteringColumn(clusteringColumns.get(i), properties.type, properties.mask);
+ }
if (useCompactStorage)
{
- fixupCompactTable(clusteringTypes, columns, hasCounters, builder);
+ fixupCompactTable(clusteringColumnProperties, columns, hasCounters, builder);
}
else
{
- columns.forEach((column, type) -> {
+ columns.forEach((column, properties) -> {
if (staticColumns.contains(column))
- builder.addStaticColumn(column, type.getType());
+ builder.addStaticColumn(column, properties.type, properties.mask);
else
- builder.addRegularColumn(column, type.getType());
+ builder.addRegularColumn(column, properties.type, properties.mask);
});
}
return builder;
}
- private void validateCompactTable(List<AbstractType<?>> clusteringTypes,
- Map<ColumnIdentifier, CQL3Type> columns)
+ private void validateCompactTable(List<ColumnProperties> clusteringColumnProperties,
+ Map<ColumnIdentifier, ColumnProperties> columns)
{
- boolean isDense = !clusteringTypes.isEmpty();
+ boolean isDense = !clusteringColumnProperties.isEmpty();
- if (columns.values().stream().anyMatch(c -> c.getType().isMultiCell()))
+ if (columns.values().stream().anyMatch(c -> c.type.isMultiCell()))
throw ire("Non-frozen collections and UDTs are not supported with COMPACT STORAGE");
if (!staticColumns.isEmpty())
throw ire("Static columns are not supported in COMPACT STORAGE tables");
- if (clusteringTypes.isEmpty())
+ if (clusteringColumnProperties.isEmpty())
{
// It's a thrift "static CF" so there should be some columns definition
if (columns.isEmpty())
@@ -359,8 +377,8 @@
}
}
- private void fixupCompactTable(List<AbstractType<?>> clusteringTypes,
- Map<ColumnIdentifier, CQL3Type> columns,
+ private void fixupCompactTable(List<ColumnProperties> clusteringTypes,
+ Map<ColumnIdentifier, ColumnProperties> columns,
boolean hasCounters,
TableMetadata.Builder builder)
{
@@ -379,12 +397,12 @@
builder.flags(flags);
- columns.forEach((name, type) -> {
+ columns.forEach((name, properties) -> {
// Note that for "static" no-clustering compact storage we use static for the defined columns
if (staticColumns.contains(name) || isStaticCompact)
- builder.addStaticColumn(name, type.getType());
+ builder.addStaticColumn(name, properties.type, properties.mask);
else
- builder.addRegularColumn(name, type.getType());
+ builder.addRegularColumn(name, properties.type, properties.mask);
});
DefaultNames names = new DefaultNames(builder.columnNames());
@@ -469,7 +487,7 @@
private final boolean ifNotExists;
private boolean useCompactStorage = false;
- private final Map<ColumnIdentifier, CQL3Type.Raw> rawColumns = new HashMap<>();
+ private final Map<ColumnIdentifier, ColumnProperties.Raw> rawColumns = new HashMap<>();
private final Set<ColumnIdentifier> staticColumns = new HashSet<>();
private final List<ColumnIdentifier> clusteringColumns = new ArrayList<>();
@@ -493,15 +511,12 @@
return new CreateTableStatement(keyspaceName,
name.getName(),
-
rawColumns,
staticColumns,
partitionKeyColumns,
clusteringColumns,
-
clusteringOrder,
attrs,
-
ifNotExists,
useCompactStorage);
}
@@ -522,9 +537,10 @@
return name.getName();
}
- public void addColumn(ColumnIdentifier column, CQL3Type.Raw type, boolean isStatic)
+ public void addColumn(ColumnIdentifier column, CQL3Type.Raw type, boolean isStatic, ColumnMask.Raw mask)
{
- if (null != rawColumns.put(column, type))
+
+ if (null != rawColumns.put(column, new ColumnProperties.Raw(type, mask)))
throw ire("Duplicate column '%s' declaration for table '%s'", column, name);
if (isStatic)
@@ -560,4 +576,52 @@
throw ire("Duplicate column '%s' in CLUSTERING ORDER BY clause for table '%s'", column, name);
}
}
+
+ /**
+ * Class encapsulating the properties of a column, which are its type and mask.
+ */
+ private final static class ColumnProperties
+ {
+ public final AbstractType<?> type;
+ public final CQL3Type cqlType; // we keep the original CQL type for printing fully qualified user type names
+
+ @Nullable
+ public final ColumnMask mask;
+
+ public ColumnProperties(AbstractType<?> type, CQL3Type cqlType, @Nullable ColumnMask mask)
+ {
+ this.type = type;
+ this.cqlType = cqlType;
+ this.mask = mask;
+ }
+
+ public ColumnProperties withReversedType()
+ {
+ return new ColumnProperties(ReversedType.getInstance(type),
+ cqlType,
+ mask == null ? null : mask.withReversedType());
+ }
+
+ public final static class Raw
+ {
+ public final CQL3Type.Raw rawType;
+
+ @Nullable
+ public final ColumnMask.Raw rawMask;
+
+ public Raw(CQL3Type.Raw rawType, @Nullable ColumnMask.Raw rawMask)
+ {
+ this.rawType = rawType;
+ this.rawMask = rawMask;
+ }
+
+ public ColumnProperties prepare(String keyspace, String table, ColumnIdentifier column, Types udts)
+ {
+ CQL3Type cqlType = rawType.prepare(keyspace, udts);
+ AbstractType<?> type = cqlType.getType();
+ ColumnMask mask = rawMask == null ? null : rawMask.prepare(keyspace, table, column, type);
+ return new ColumnProperties(type, cqlType, mask);
+ }
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
index 145c8fc..ed7a9a1 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java
@@ -50,6 +50,7 @@
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE;
public final class CreateViewStatement extends AlterSchemaStatement
{
@@ -188,7 +189,8 @@
throw ire("Can only select columns by name when defining a materialized view (got %s)", selector.selectable);
// will throw IRE if the column doesn't exist in the base table
- ColumnMetadata column = (ColumnMetadata) selector.selectable.prepare(table);
+ Selectable.RawIdentifier rawIdentifier = (Selectable.RawIdentifier) selector.selectable;
+ ColumnMetadata column = rawIdentifier.columnMetadata(table);
selectedColumns.add(column.name);
});
@@ -272,7 +274,8 @@
throw ire("WHERE clause for materialized view '%s' cannot contain custom index expressions", viewName);
StatementRestrictions restrictions =
- new StatementRestrictions(StatementType.SELECT,
+ new StatementRestrictions(state,
+ StatementType.SELECT,
table,
whereClause,
VariableSpecifications.empty(),
@@ -292,7 +295,7 @@
// See CASSANDRA-13798
Set<ColumnMetadata> restrictedNonPrimaryKeyColumns = restrictions.nonPKRestrictedColumns(false);
- if (!restrictedNonPrimaryKeyColumns.isEmpty() && !Boolean.getBoolean("cassandra.mv.allow_filtering_nonkey_columns_unsafe"))
+ if (!restrictedNonPrimaryKeyColumns.isEmpty() && !MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE.getBoolean())
{
throw ire("Non-primary key columns can only be restricted with 'IS NOT NULL' (got: %s restricted illegally)",
join(",", transform(restrictedNonPrimaryKeyColumns, ColumnMetadata::toString)));
@@ -324,12 +327,18 @@
builder.params(attrs.asNewTableParams())
.kind(TableMetadata.Kind.VIEW);
- partitionKeyColumns.forEach(name -> builder.addPartitionKeyColumn(name, getType(table, name)));
- clusteringColumns.forEach(name -> builder.addClusteringColumn(name, getType(table, name)));
+ partitionKeyColumns.stream()
+ .map(table::getColumn)
+ .forEach(column -> builder.addPartitionKeyColumn(column.name, getType(column), column.getMask()));
+
+ clusteringColumns.stream()
+ .map(table::getColumn)
+ .forEach(column -> builder.addClusteringColumn(column.name, getType(column), column.getMask()));
selectedColumns.stream()
.filter(name -> !primaryKeyColumns.contains(name))
- .forEach(name -> builder.addRegularColumn(name, getType(table, name)));
+ .map(table::getColumn)
+ .forEach(column -> builder.addRegularColumn(column.name, getType(column), column.getMask()));
ViewMetadata view = new ViewMetadata(table.id, table.name, rawColumns.isEmpty(), whereClause, builder.build());
view.metadata.validate();
@@ -347,15 +356,15 @@
client.ensureTablePermission(keyspaceName, tableName, Permission.ALTER);
}
- private AbstractType<?> getType(TableMetadata table, ColumnIdentifier name)
+ private AbstractType<?> getType(ColumnMetadata column)
{
- AbstractType<?> type = table.getColumn(name).type;
- if (clusteringOrder.containsKey(name))
+ AbstractType<?> type = column.type;
+ if (clusteringOrder.containsKey(column.name))
{
- boolean reverse = !clusteringOrder.get(name);
+ boolean reverse = !clusteringOrder.get(column.name);
if (type.isReversed() && !reverse)
- return ((ReversedType) type).baseType;
+ return ((ReversedType<?>) type).baseType;
if (!type.isReversed() && reverse)
return ReversedType.getInstance(type);
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
index 0cb1cbe..186891e 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java
@@ -28,9 +28,9 @@
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.CQLStatement;
-import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -80,7 +80,7 @@
throw ire("Aggregate '%s' doesn't exist", name);
}
- Collection<Function> aggregates = keyspace.functions.get(new FunctionName(keyspaceName, aggregateName));
+ Collection<UserFunction> aggregates = keyspace.userFunctions.get(new FunctionName(keyspaceName, aggregateName));
if (aggregates.size() > 1 && !argumentsSpeficied)
{
throw ire("'DROP AGGREGATE %s' matches multiple function definitions; " +
@@ -97,11 +97,11 @@
List<AbstractType<?>> argumentTypes = prepareArgumentTypes(keyspace.types);
- Predicate<Function> filter = Functions.Filter.UDA;
+ Predicate<UserFunction> filter = UserFunctions.Filter.UDA;
if (argumentsSpeficied)
- filter = filter.and(f -> Functions.typesMatch(f.argTypes(), argumentTypes));
+ filter = filter.and(f -> f.typesMatch(argumentTypes));
- Function aggregate = aggregates.stream().filter(filter).findAny().orElse(null);
+ UserFunction aggregate = aggregates.stream().filter(filter).findAny().orElse(null);
if (null == aggregate)
{
if (ifExists)
@@ -110,12 +110,12 @@
throw ire("Aggregate '%s' doesn't exist", name);
}
- return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.without(aggregate)));
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.userFunctions.without(aggregate)));
}
SchemaChange schemaChangeEvent(KeyspacesDiff diff)
{
- Functions dropped = diff.altered.get(0).udas.dropped;
+ UserFunctions dropped = diff.altered.get(0).udas.dropped;
assert dropped.size() == 1;
return SchemaChange.forAggregate(Change.DROPPED, (UDAggregate) dropped.iterator().next());
}
@@ -126,9 +126,9 @@
if (null == keyspace)
return;
- Stream<Function> functions = keyspace.functions.get(new FunctionName(keyspaceName, aggregateName)).stream();
+ Stream<UserFunction> functions = keyspace.userFunctions.get(new FunctionName(keyspaceName, aggregateName)).stream();
if (argumentsSpeficied)
- functions = functions.filter(f -> Functions.typesMatch(f.argTypes(), prepareArgumentTypes(keyspace.types)));
+ functions = functions.filter(f -> f.typesMatch(prepareArgumentTypes(keyspace.types)));
functions.forEach(f -> client.ensurePermission(Permission.DROP, FunctionResource.function(f)));
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
index d9d637d..e743a1a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java
@@ -28,7 +28,9 @@
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.CQLStatement;
-import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -79,7 +81,7 @@
throw ire("Function '%s' doesn't exist", name);
}
- Collection<Function> functions = keyspace.functions.get(new FunctionName(keyspaceName, functionName));
+ Collection<UserFunction> functions = keyspace.userFunctions.get(new FunctionName(keyspaceName, functionName));
if (functions.size() > 1 && !argumentsSpeficied)
{
throw ire("'DROP FUNCTION %s' matches multiple function definitions; " +
@@ -96,11 +98,11 @@
List<AbstractType<?>> argumentTypes = prepareArgumentTypes(keyspace.types);
- Predicate<Function> filter = Functions.Filter.UDF;
+ Predicate<UserFunction> filter = UserFunctions.Filter.UDF;
if (argumentsSpeficied)
- filter = filter.and(f -> Functions.typesMatch(f.argTypes(), argumentTypes));
+ filter = filter.and(f -> f.typesMatch(argumentTypes));
- Function function = functions.stream().filter(filter).findAny().orElse(null);
+ UserFunction function = functions.stream().filter(filter).findAny().orElse(null);
if (null == function)
{
if (ifExists)
@@ -110,7 +112,7 @@
}
String dependentAggregates =
- keyspace.functions
+ keyspace.userFunctions
.aggregatesUsingFunction(function)
.map(a -> a.name().toString())
.collect(joining(", "));
@@ -118,12 +120,19 @@
if (!dependentAggregates.isEmpty())
throw ire("Function '%s' is still referenced by aggregates %s", name, dependentAggregates);
- return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.functions.without(function)));
+ String dependentTables = keyspace.tablesUsingFunction(function)
+ .map(table -> table.name)
+ .collect(joining(", "));
+
+ if (!dependentTables.isEmpty())
+ throw ire("Function '%s' is still referenced by column masks in tables %s", name, dependentTables);
+
+ return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.userFunctions.without(function)));
}
SchemaChange schemaChangeEvent(KeyspacesDiff diff)
{
- Functions dropped = diff.altered.get(0).udfs.dropped;
+ UserFunctions dropped = diff.altered.get(0).udfs.dropped;
assert dropped.size() == 1;
return SchemaChange.forFunction(Change.DROPPED, (UDFunction) dropped.iterator().next());
}
@@ -134,9 +143,9 @@
if (null == keyspace)
return;
- Stream<Function> functions = keyspace.functions.get(new FunctionName(keyspaceName, functionName)).stream();
+ Stream<UserFunction> functions = keyspace.userFunctions.get(new FunctionName(keyspaceName, functionName)).stream();
if (argumentsSpeficied)
- functions = functions.filter(f -> Functions.typesMatch(f.argTypes(), prepareArgumentTypes(keyspace.types)));
+ functions = functions.filter(f -> f.typesMatch(prepareArgumentTypes(keyspace.types)));
functions.forEach(f -> client.ensurePermission(Permission.DROP, FunctionResource.function(f)));
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
index f2bd30b..47e514a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java
@@ -21,6 +21,7 @@
import org.apache.cassandra.audit.AuditLogEntryType;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
import org.apache.cassandra.service.ClientState;
@@ -39,6 +40,8 @@
public Keyspaces apply(Keyspaces schema)
{
+ Guardrails.dropKeyspaceEnabled.ensureEnabled(state);
+
if (schema.containsKeyspace(keyspaceName))
return schema.without(keyspaceName);
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
index d188bdb..97830c8 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java
@@ -24,7 +24,7 @@
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.UTName;
-import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -82,7 +82,7 @@
* 2) other user type that can nest the one we drop and
* 3) existing tables referencing the type (maybe in a nested way).
*/
- Iterable<Function> functions = keyspace.functions.referencingUserType(name);
+ Iterable<UserFunction> functions = keyspace.userFunctions.referencingUserType(name);
if (!isEmpty(functions))
{
throw ire("Cannot drop user type '%s.%s' as it is still used by functions %s",
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
index 42fcaf4..d4d5b98 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java
@@ -50,7 +50,7 @@
throw new ConfigurationException("Missing replication strategy class");
}
- private String getReplicationStrategyClass()
+ public String getReplicationStrategyClass()
{
return getAllReplicationOptions().get(ReplicationParams.CLASS);
}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
index 829a53a..c55d79e 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
@@ -25,7 +25,6 @@
import org.apache.cassandra.cql3.statements.PropertyDefinitions;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.CompressionParams;
@@ -37,6 +36,7 @@
import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
import static java.lang.String.format;
+import static org.apache.cassandra.schema.TableParams.Option.*;
public final class TableAttributes extends PropertyDefinitions
{
@@ -74,7 +74,7 @@
public TableId getId() throws ConfigurationException
{
- String id = getSimple(ID);
+ String id = getString(ID);
try
{
return id != null ? TableId.fromString(id) : null;
@@ -97,111 +97,81 @@
private TableParams build(TableParams.Builder builder)
{
- if (hasOption(Option.BLOOM_FILTER_FP_CHANCE))
- builder.bloomFilterFpChance(getDouble(Option.BLOOM_FILTER_FP_CHANCE));
+ if (hasOption(Option.ALLOW_AUTO_SNAPSHOT))
+ builder.allowAutoSnapshot(getBoolean(Option.ALLOW_AUTO_SNAPSHOT.toString(), true));
- if (hasOption(Option.CACHING))
- builder.caching(CachingParams.fromMap(getMap(Option.CACHING)));
+ if (hasOption(BLOOM_FILTER_FP_CHANCE))
+ builder.bloomFilterFpChance(getDouble(BLOOM_FILTER_FP_CHANCE));
- if (hasOption(Option.COMMENT))
- builder.comment(getString(Option.COMMENT));
+ if (hasOption(CACHING))
+ builder.caching(CachingParams.fromMap(getMap(CACHING)));
- if (hasOption(Option.COMPACTION))
- builder.compaction(CompactionParams.fromMap(getMap(Option.COMPACTION)));
+ if (hasOption(COMMENT))
+ builder.comment(getString(COMMENT));
- if (hasOption(Option.COMPRESSION))
+ if (hasOption(COMPACTION))
+ builder.compaction(CompactionParams.fromMap(getMap(COMPACTION)));
+
+ if (hasOption(COMPRESSION))
{
- //crc_check_chance was "promoted" from a compression property to a top-level-property after #9839
+ //crc_check_chance was "promoted" from a compression property to a top-level-property after #9839,
//so we temporarily accept it to be defined as a compression option, to maintain backwards compatibility
- Map<String, String> compressionOpts = getMap(Option.COMPRESSION);
- if (compressionOpts.containsKey(Option.CRC_CHECK_CHANCE.toString().toLowerCase()))
+ Map<String, String> compressionOpts = getMap(COMPRESSION);
+ if (compressionOpts.containsKey(CRC_CHECK_CHANCE.toString().toLowerCase()))
{
- Double crcCheckChance = getDeprecatedCrcCheckChance(compressionOpts);
+ double crcCheckChance = getDeprecatedCrcCheckChance(compressionOpts);
builder.crcCheckChance(crcCheckChance);
}
- builder.compression(CompressionParams.fromMap(getMap(Option.COMPRESSION)));
+ builder.compression(CompressionParams.fromMap(getMap(COMPRESSION)));
}
if (hasOption(Option.MEMTABLE))
builder.memtable(MemtableParams.get(getString(Option.MEMTABLE)));
- if (hasOption(Option.DEFAULT_TIME_TO_LIVE))
- builder.defaultTimeToLive(getInt(Option.DEFAULT_TIME_TO_LIVE));
+ if (hasOption(DEFAULT_TIME_TO_LIVE))
+ builder.defaultTimeToLive(getInt(DEFAULT_TIME_TO_LIVE));
- if (hasOption(Option.GC_GRACE_SECONDS))
- builder.gcGraceSeconds(getInt(Option.GC_GRACE_SECONDS));
+ if (hasOption(GC_GRACE_SECONDS))
+ builder.gcGraceSeconds(getInt(GC_GRACE_SECONDS));
+
+ if (hasOption(INCREMENTAL_BACKUPS))
+ builder.incrementalBackups(getBoolean(INCREMENTAL_BACKUPS.toString(), true));
- if (hasOption(Option.MAX_INDEX_INTERVAL))
- builder.maxIndexInterval(getInt(Option.MAX_INDEX_INTERVAL));
+ if (hasOption(MAX_INDEX_INTERVAL))
+ builder.maxIndexInterval(getInt(MAX_INDEX_INTERVAL));
- if (hasOption(Option.MEMTABLE_FLUSH_PERIOD_IN_MS))
- builder.memtableFlushPeriodInMs(getInt(Option.MEMTABLE_FLUSH_PERIOD_IN_MS));
+ if (hasOption(MEMTABLE_FLUSH_PERIOD_IN_MS))
+ builder.memtableFlushPeriodInMs(getInt(MEMTABLE_FLUSH_PERIOD_IN_MS));
- if (hasOption(Option.MIN_INDEX_INTERVAL))
- builder.minIndexInterval(getInt(Option.MIN_INDEX_INTERVAL));
+ if (hasOption(MIN_INDEX_INTERVAL))
+ builder.minIndexInterval(getInt(MIN_INDEX_INTERVAL));
- if (hasOption(Option.SPECULATIVE_RETRY))
- builder.speculativeRetry(SpeculativeRetryPolicy.fromString(getString(Option.SPECULATIVE_RETRY)));
+ if (hasOption(SPECULATIVE_RETRY))
+ builder.speculativeRetry(SpeculativeRetryPolicy.fromString(getString(SPECULATIVE_RETRY)));
- if (hasOption(Option.ADDITIONAL_WRITE_POLICY))
- builder.additionalWritePolicy(SpeculativeRetryPolicy.fromString(getString(Option.ADDITIONAL_WRITE_POLICY)));
+ if (hasOption(ADDITIONAL_WRITE_POLICY))
+ builder.additionalWritePolicy(SpeculativeRetryPolicy.fromString(getString(ADDITIONAL_WRITE_POLICY)));
- if (hasOption(Option.CRC_CHECK_CHANCE))
- builder.crcCheckChance(getDouble(Option.CRC_CHECK_CHANCE));
+ if (hasOption(CRC_CHECK_CHANCE))
+ builder.crcCheckChance(getDouble(CRC_CHECK_CHANCE));
- if (hasOption(Option.CDC))
- builder.cdc(getBoolean(Option.CDC.toString(), false));
+ if (hasOption(CDC))
+ builder.cdc(getBoolean(CDC));
- if (hasOption(Option.READ_REPAIR))
- builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR)));
+ if (hasOption(READ_REPAIR))
+ builder.readRepair(ReadRepairStrategy.fromString(getString(READ_REPAIR)));
return builder.build();
}
- private Double getDeprecatedCrcCheckChance(Map<String, String> compressionOpts)
+ public boolean hasOption(Option option)
{
- String value = compressionOpts.get(Option.CRC_CHECK_CHANCE.toString().toLowerCase());
- try
- {
- return Double.valueOf(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(String.format("Invalid double value %s for crc_check_chance.'", value));
- }
- }
-
- private double getDouble(Option option)
- {
- String value = getString(option);
-
- try
- {
- return Double.parseDouble(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(format("Invalid double value %s for '%s'", value, option));
- }
- }
-
- private int getInt(Option option)
- {
- String value = getString(option);
-
- try
- {
- return Integer.parseInt(value);
- }
- catch (NumberFormatException e)
- {
- throw new SyntaxException(String.format("Invalid integer value %s for '%s'", value, option));
- }
+ return hasProperty(option.toString());
}
private String getString(Option option)
{
- String value = getSimple(option.toString());
+ String value = getString(option.toString());
if (value == null)
throw new IllegalStateException(format("Option '%s' is absent", option));
return value;
@@ -215,8 +185,24 @@
return value;
}
- public boolean hasOption(Option option)
+ private boolean getBoolean(Option option)
{
- return hasProperty(option.toString());
+ return parseBoolean(option.toString(), getString(option));
+ }
+
+ private int getInt(Option option)
+ {
+ return parseInt(option.toString(), getString(option));
+ }
+
+ private double getDouble(Option option)
+ {
+ return parseDouble(option.toString(), getString(option));
+ }
+
+ private double getDeprecatedCrcCheckChance(Map<String, String> compressionOpts)
+ {
+ String value = compressionOpts.get(CRC_CHECK_CHANCE.toString().toLowerCase());
+ return parseDouble(CRC_CHECK_CHANCE.toString(), value);
}
}
diff --git a/src/java/org/apache/cassandra/db/AbstractArrayClusteringPrefix.java b/src/java/org/apache/cassandra/db/AbstractArrayClusteringPrefix.java
index 211eeb0..16fb080 100644
--- a/src/java/org/apache/cassandra/db/AbstractArrayClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/AbstractArrayClusteringPrefix.java
@@ -49,7 +49,8 @@
return out;
}
- public ClusteringPrefix<byte[]> minimize()
+ @Override
+ public ClusteringPrefix<byte[]> retainable()
{
return this;
}
diff --git a/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java b/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java
index 457d0c4..b47a3a3 100644
--- a/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/AbstractBufferClusteringPrefix.java
@@ -42,10 +42,19 @@
return getRawValues();
}
- public ClusteringPrefix<ByteBuffer> minimize()
+ @Override
+ public ClusteringPrefix<ByteBuffer> retainable()
{
if (!ByteBufferUtil.canMinimize(values))
return this;
- return new BufferClustering(ByteBufferUtil.minimizeBuffers(values));
+
+ ByteBuffer[] minimizedValues = ByteBufferUtil.minimizeBuffers(this.values);
+ if (kind.isBoundary())
+ return accessor().factory().boundary(kind, minimizedValues);
+ if (kind.isBound())
+ return accessor().factory().bound(kind, minimizedValues);
+
+ assert kind() != Kind.STATIC_CLUSTERING; // not minimizable
+ return accessor().factory().clustering(minimizedValues);
}
}
diff --git a/src/java/org/apache/cassandra/db/AbstractReadQuery.java b/src/java/org/apache/cassandra/db/AbstractReadQuery.java
index 374d2b2..c2e4258 100644
--- a/src/java/org/apache/cassandra/db/AbstractReadQuery.java
+++ b/src/java/org/apache/cassandra/db/AbstractReadQuery.java
@@ -118,4 +118,4 @@
}
protected abstract void appendCQLWhereClause(StringBuilder sb);
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ArrayClustering.java b/src/java/org/apache/cassandra/db/ArrayClustering.java
index 53d45e7..b04910c 100644
--- a/src/java/org/apache/cassandra/db/ArrayClustering.java
+++ b/src/java/org/apache/cassandra/db/ArrayClustering.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
import org.apache.cassandra.utils.ObjectSizes;
public class ArrayClustering extends AbstractArrayClusteringPrefix implements Clustering<byte[]>
@@ -31,6 +32,8 @@
public long unsharedHeapSize()
{
+ if (this == ByteArrayAccessor.factory.clustering() || this == ByteArrayAccessor.factory.staticClustering())
+ return 0;
long arrayRefSize = ObjectSizes.sizeOfArray(values);
long elementsSize = 0;
for (int i = 0; i < values.length; i++)
@@ -40,6 +43,8 @@
public long unsharedHeapSizeExcludingData()
{
+ if (this == ByteArrayAccessor.factory.clustering() || this == ByteArrayAccessor.factory.staticClustering())
+ return 0;
return EMPTY_SIZE + ObjectSizes.sizeOfArray(values);
}
diff --git a/src/java/org/apache/cassandra/db/BufferClustering.java b/src/java/org/apache/cassandra/db/BufferClustering.java
index e3592e1..ed6d61c 100644
--- a/src/java/org/apache/cassandra/db/BufferClustering.java
+++ b/src/java/org/apache/cassandra/db/BufferClustering.java
@@ -43,11 +43,15 @@
public long unsharedHeapSize()
{
+ if (this == Clustering.EMPTY || this == Clustering.STATIC_CLUSTERING)
+ return 0;
return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(values);
}
public long unsharedHeapSizeExcludingData()
{
+ if (this == Clustering.EMPTY || this == Clustering.STATIC_CLUSTERING)
+ return 0;
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values);
}
diff --git a/src/java/org/apache/cassandra/db/BufferDecoratedKey.java b/src/java/org/apache/cassandra/db/BufferDecoratedKey.java
index d375162..07f610f 100644
--- a/src/java/org/apache/cassandra/db/BufferDecoratedKey.java
+++ b/src/java/org/apache/cassandra/db/BufferDecoratedKey.java
@@ -19,7 +19,9 @@
import java.nio.ByteBuffer;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
public class BufferDecoratedKey extends DecoratedKey
{
@@ -36,4 +38,34 @@
{
return key;
}
+
+ @Override
+ public int getKeyLength()
+ {
+ return key.remaining();
+ }
+
+ /**
+ * A factory method that translates the given byte-comparable representation to a {@link BufferDecoratedKey}
+ * instance. If the given byte comparable doesn't represent the encoding of a buffer decorated key, anything from a
+ * wide variety of throwables may be thrown (e.g. {@link AssertionError}, {@link IndexOutOfBoundsException},
+ * {@link IllegalStateException}, etc.).
+ *
+ * @param byteComparable A byte-comparable representation (presumably of a {@link BufferDecoratedKey} instance).
+ * @param version The encoding version used for the given byte comparable.
+ * @param partitioner The partitioner of the encoded decorated key. Needed in order to correctly decode the token
+ * bytes of the key.
+ * @return A new {@link BufferDecoratedKey} instance, corresponding to the given byte-comparable representation. If
+ * we were to call {@link #asComparableBytes(Version)} on the returned object, we should get a {@link ByteSource}
+ * equal to the one of the input byte comparable.
+ */
+ public static BufferDecoratedKey fromByteComparable(ByteComparable byteComparable,
+ Version version,
+ IPartitioner partitioner)
+ {
+ return DecoratedKey.fromByteComparable(byteComparable,
+ version,
+ partitioner,
+ (token, keyBytes) -> new BufferDecoratedKey(token, ByteBuffer.wrap(keyBytes)));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/Clustering.java b/src/java/org/apache/cassandra/db/Clustering.java
index dd91fea..426d327 100644
--- a/src/java/org/apache/cassandra/db/Clustering.java
+++ b/src/java/org/apache/cassandra/db/Clustering.java
@@ -56,6 +56,18 @@
return new BufferClustering(newValues);
}
+ @Override
+ default ClusteringBound<V> asStartBound()
+ {
+ return ClusteringBound.inclusiveStartOf(this);
+ }
+
+ @Override
+ default ClusteringBound<V> asEndBound()
+ {
+ return ClusteringBound.inclusiveEndOf(this);
+ }
+
public default String toString(TableMetadata metadata)
{
StringBuilder sb = new StringBuilder();
@@ -178,4 +190,4 @@
}
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ClusteringBound.java b/src/java/org/apache/cassandra/db/ClusteringBound.java
index 60a1c6c..4afdfe6 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBound.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBound.java
@@ -21,22 +21,27 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
-import java.util.List;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.utils.memory.ByteBufferCloner;
+import static org.apache.cassandra.db.AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY;
+
/**
* The start or end of a range of clusterings, either inclusive or exclusive.
*/
public interface ClusteringBound<V> extends ClusteringBoundOrBoundary<V>
{
/** The smallest start bound, i.e. the one that starts before any row. */
- public static final ClusteringBound<?> BOTTOM = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND, BufferClusteringBound.EMPTY_VALUES_ARRAY);
+ ClusteringBound<?> BOTTOM = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND, EMPTY_VALUES_ARRAY);
/** The biggest end bound, i.e. the one that ends after any row. */
- public static final ClusteringBound<?> TOP = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, BufferClusteringBound.EMPTY_VALUES_ARRAY);
+ ClusteringBound<?> TOP = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, EMPTY_VALUES_ARRAY);
- public static ClusteringPrefix.Kind boundKind(boolean isStart, boolean isInclusive)
+ /** The biggest start bound, i.e. the one that starts after any row. */
+ ClusteringBound<?> MAX_START = new BufferClusteringBound(Kind.EXCL_START_BOUND, EMPTY_VALUES_ARRAY);
+ /** The smallest end bound, i.e. the one that end before any row. */
+ ClusteringBound<?> MIN_END = new BufferClusteringBound(Kind.EXCL_END_BOUND, EMPTY_VALUES_ARRAY);
+
+ static ClusteringPrefix.Kind boundKind(boolean isStart, boolean isInclusive)
{
return isStart
? (isInclusive ? ClusteringPrefix.Kind.INCL_START_BOUND : ClusteringPrefix.Kind.EXCL_START_BOUND)
@@ -69,32 +74,14 @@
return kind() == Kind.EXCL_START_BOUND || kind() == Kind.EXCL_END_BOUND;
}
- // For use by intersects, it's called with the sstable bound opposite to the slice bound
- // (so if the slice bound is a start, it's call with the max sstable bound)
- default int compareTo(ClusteringComparator comparator, List<ByteBuffer> sstableBound)
+ default boolean isArtificial()
{
- for (int i = 0; i < sstableBound.size(); i++)
- {
- // Say the slice bound is a start. It means we're in the case where the max
- // sstable bound is say (1:5) while the slice start is (1). So the start
- // does start before the sstable end bound (and intersect it). It's the exact
- // inverse with a end slice bound.
- if (i >= size())
- return isStart() ? -1 : 1;
+ return kind() == Kind.SSTABLE_LOWER_BOUND || kind() == Kind.SSTABLE_UPPER_BOUND;
+ }
- int cmp = comparator.compareComponent(i, get(i), accessor(), sstableBound.get(i), ByteBufferAccessor.instance);
- if (cmp != 0)
- return cmp;
- }
-
- // Say the slice bound is a start. I means we're in the case where the max
- // sstable bound is say (1), while the slice start is (1:5). This again means
- // that the slice start before the end bound.
- if (size() > sstableBound.size())
- return isStart() ? -1 : 1;
-
- // The slice bound is equal to the sstable bound. Results depends on whether the slice is inclusive or not
- return isInclusive() ? 0 : (isStart() ? 1 : -1);
+ default ClusteringBound<V> artificialLowerBound(boolean isReversed)
+ {
+ return create(!isReversed ? Kind.SSTABLE_LOWER_BOUND : Kind.SSTABLE_UPPER_BOUND, this);
}
static <V> ClusteringBound<V> create(ClusteringPrefix.Kind kind, ClusteringPrefix<V> from)
@@ -102,27 +89,27 @@
return from.accessor().factory().bound(kind, from.getRawValues());
}
- public static ClusteringBound<?> inclusiveStartOf(ClusteringPrefix<?> from)
+ static <V> ClusteringBound<V> inclusiveStartOf(ClusteringPrefix<V> from)
{
return create(ClusteringPrefix.Kind.INCL_START_BOUND, from);
}
- public static ClusteringBound<?> inclusiveEndOf(ClusteringPrefix<?> from)
+ static <V> ClusteringBound<V> inclusiveEndOf(ClusteringPrefix<V> from)
{
return create(ClusteringPrefix.Kind.INCL_END_BOUND, from);
}
- public static ClusteringBound<?> exclusiveStartOf(ClusteringPrefix<?> from)
+ static <V> ClusteringBound<V> exclusiveStartOf(ClusteringPrefix<V> from)
{
return create(ClusteringPrefix.Kind.EXCL_START_BOUND, from);
}
- public static ClusteringBound<?> exclusiveEndOf(ClusteringPrefix<?> from)
+ static <V> ClusteringBound<V> exclusiveEndOf(ClusteringPrefix<V> from)
{
return create(ClusteringPrefix.Kind.EXCL_END_BOUND, from);
}
- public static ClusteringBound<?> create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values)
+ static ClusteringBound<?> create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values)
{
CBuilder builder = CBuilder.create(comparator);
for (Object val : values)
@@ -134,4 +121,18 @@
}
return builder.buildBound(isStart, isInclusive);
}
-}
+
+ @Override
+ default ClusteringBound<V> asStartBound()
+ {
+ assert isStart();
+ return this;
+ }
+
+ @Override
+ default ClusteringBound<V> asEndBound()
+ {
+ assert isEnd();
+ return this;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ClusteringBoundary.java b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
index 3e50f52..9e0a87c 100644
--- a/src/java/org/apache/cassandra/db/ClusteringBoundary.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
@@ -37,4 +37,16 @@
{
return from.accessor().factory().boundary(kind, from.getRawValues());
}
-}
+
+ @Override
+ default ClusteringBound<V> asStartBound()
+ {
+ return openBound(false);
+ }
+
+ @Override
+ default ClusteringBound<V> asEndBound()
+ {
+ return closeBound(false);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ClusteringComparator.java b/src/java/org/apache/cassandra/db/ClusteringComparator.java
index fdc4508..d007c5d 100644
--- a/src/java/org/apache/cassandra/db/ClusteringComparator.java
+++ b/src/java/org/apache/cassandra/db/ClusteringComparator.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
@@ -31,6 +32,15 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.EXCLUDED;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_EMPTY;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_EMPTY_REVERSED;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_NULL;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.TERMINATOR;
/**
* A comparator of clustering prefixes (or more generally of {@link Clusterable}}.
@@ -233,6 +243,272 @@
}
/**
+ * Produce a prefix-free byte-comparable representation of the given value, i.e. such a sequence of bytes that any
+ * pair x, y of valid values of this type
+ * compare(x, y) == compareLexicographicallyUnsigned(asByteComparable(x), asByteComparable(y))
+ * and
+ * asByteComparable(x) is not a prefix of asByteComparable(y)
+ */
+ public <V> ByteComparable asByteComparable(ClusteringPrefix<V> clustering)
+ {
+ return new ByteComparableClustering<>(clustering);
+ }
+
+ /**
+ * A prefix-free byte-comparable representation for a clustering or prefix.
+ *
+ * Adds a NEXT_COMPONENT byte before each component (allowing inclusive/exclusive bounds over incomplete prefixes
+ * of that length) and finishes with a suitable byte for the clustering kind. Also deals with null entries.
+ *
+ * Since all types' encodings are weakly prefix-free, this is guaranteed to be prefix-free as long as the
+ * bound/ClusteringPrefix terminators are different from the separator byte. It is okay for the terminator for
+ * Clustering to be the same as the separator, as all Clusterings must be completely specified.
+ *
+ * See also {@link AbstractType#asComparableBytes}.
+ *
+ * Some examples:
+ * "A", 0005, Clustering -> 40 4100 40 0005 40
+ * "B", 0006, InclusiveEnd -> 40 4200 40 0006 60
+ * "A", ExclusiveStart -> 40 4100 60
+ * "", null, Clustering -> 40 00 3F 40
+ * "", 0000, Clustering -> 40 00 40 0000 40
+ * BOTTOM -> 20
+ */
+ private class ByteComparableClustering<V> implements ByteComparable
+ {
+ private final ClusteringPrefix<V> src;
+
+ ByteComparableClustering(ClusteringPrefix<V> src)
+ {
+ this.src = src;
+ }
+
+ @Override
+ public ByteSource asComparableBytes(Version version)
+ {
+ return new ByteSource()
+ {
+ private ByteSource current = null;
+ private int srcnum = -1;
+
+ @Override
+ public int next()
+ {
+ if (current != null)
+ {
+ int b = current.next();
+ if (b > END_OF_STREAM)
+ return b;
+ current = null;
+ }
+
+ int sz = src.size();
+ if (srcnum == sz)
+ return END_OF_STREAM;
+
+ ++srcnum;
+ if (srcnum == sz)
+ return src.kind().asByteComparableValue(version);
+
+ final V nextComponent = src.get(srcnum);
+ // We can have a null as the clustering component (this is a relic of COMPACT STORAGE, but also
+ // can appear in indexed partitions with no rows but static content),
+ if (nextComponent == null)
+ {
+ if (version != Version.LEGACY)
+ return NEXT_COMPONENT_NULL; // always sorts before non-nulls, including for reversed types
+ else
+ {
+ // legacy version did not permit nulls in clustering keys and treated these as null values
+ return subtype(srcnum).isReversed() ? NEXT_COMPONENT_EMPTY_REVERSED : NEXT_COMPONENT_EMPTY;
+ }
+ }
+
+ current = subtype(srcnum).asComparableBytes(src.accessor(), nextComponent, version);
+ // and also null values for some types (e.g. int, varint but not text) that are encoded as empty
+ // buffers.
+ if (current == null)
+ return subtype(srcnum).isReversed() ? NEXT_COMPONENT_EMPTY_REVERSED : NEXT_COMPONENT_EMPTY;
+
+ return NEXT_COMPONENT;
+ }
+ };
+ }
+
+ public String toString()
+ {
+ return src.clusteringString(subtypes());
+ }
+ }
+
+ /**
+ * Produces a clustering from the given byte-comparable value. The method will throw an exception if the value
+ * does not correctly encode a clustering of this type, including if it encodes a position before or after a
+ * clustering (i.e. a bound/boundary).
+ *
+ * @param accessor Accessor to use to construct components.
+ * @param comparable The clustering encoded as a byte-comparable sequence.
+ */
+ public <V> Clustering<V> clusteringFromByteComparable(ValueAccessor<V> accessor, ByteComparable comparable)
+ {
+ ByteComparable.Version version = ByteComparable.Version.OSS50;
+ ByteSource.Peekable orderedBytes = ByteSource.peekable(comparable.asComparableBytes(version));
+
+ // First check for special cases (partition key only, static clustering) that can do without buffers.
+ int sep = orderedBytes.next();
+ switch (sep)
+ {
+ case TERMINATOR:
+ assert size() == 0 : "Terminator should be after " + size() + " components, got 0";
+ return accessor.factory().clustering();
+ case EXCLUDED:
+ return accessor.factory().staticClustering();
+ default:
+ // continue with processing
+ }
+
+ int cc = 0;
+ V[] components = accessor.createArray(size());
+
+ while (true)
+ {
+ switch (sep)
+ {
+ case NEXT_COMPONENT_NULL:
+ components[cc] = null;
+ break;
+ case NEXT_COMPONENT_EMPTY:
+ case NEXT_COMPONENT_EMPTY_REVERSED:
+ components[cc] = subtype(cc).fromComparableBytes(accessor, null, version);
+ break;
+ case NEXT_COMPONENT:
+ // Decode the next component, consuming bytes from orderedBytes.
+ components[cc] = subtype(cc).fromComparableBytes(accessor, orderedBytes, version);
+ break;
+ case TERMINATOR:
+ assert cc == size() : "Terminator should be after " + size() + " components, got " + cc;
+ return accessor.factory().clustering(components);
+ case EXCLUDED:
+ throw new AssertionError("Unexpected static terminator after the first component");
+ default:
+ throw new AssertionError("Unexpected separator " + Integer.toHexString(sep) + " in Clustering encoding");
+ }
+ ++cc;
+ sep = orderedBytes.next();
+ }
+ }
+
+ /**
+ * Produces a clustering bound from the given byte-comparable value. The method will throw an exception if the value
+ * does not correctly encode a bound position of this type, including if it encodes an exact clustering.
+ *
+ * Note that the encoded clustering position cannot specify the type of bound (i.e. start/end/boundary) because to
+ * correctly compare clustering positions the encoding must be the same for the different types (e.g. the position
+ * for a exclusive end and an inclusive start is the same, before the exact clustering). The type must be supplied
+ * separately (in the bound... vs boundary... call and isEnd argument).
+ *
+ * @param accessor Accessor to use to construct components.
+ * @param comparable The clustering position encoded as a byte-comparable sequence.
+ * @param isEnd true if the bound marks the end of a range, false is it marks the start.
+ */
+ public <V> ClusteringBound<V> boundFromByteComparable(ValueAccessor<V> accessor,
+ ByteComparable comparable,
+ boolean isEnd)
+ {
+ ByteComparable.Version version = ByteComparable.Version.OSS50;
+ ByteSource.Peekable orderedBytes = ByteSource.peekable(comparable.asComparableBytes(version));
+
+ int sep = orderedBytes.next();
+ int cc = 0;
+ V[] components = accessor.createArray(size());
+
+ while (true)
+ {
+ switch (sep)
+ {
+ case NEXT_COMPONENT_NULL:
+ components[cc] = null;
+ break;
+ case NEXT_COMPONENT_EMPTY:
+ case NEXT_COMPONENT_EMPTY_REVERSED:
+ components[cc] = subtype(cc).fromComparableBytes(accessor, null, version);
+ break;
+ case NEXT_COMPONENT:
+ // Decode the next component, consuming bytes from orderedBytes.
+ components[cc] = subtype(cc).fromComparableBytes(accessor, orderedBytes, version);
+ break;
+ case ByteSource.LT_NEXT_COMPONENT:
+ return accessor.factory().bound(isEnd ? ClusteringPrefix.Kind.EXCL_END_BOUND
+ : ClusteringPrefix.Kind.INCL_START_BOUND,
+ Arrays.copyOf(components, cc));
+ case ByteSource.GT_NEXT_COMPONENT:
+ return accessor.factory().bound(isEnd ? ClusteringPrefix.Kind.INCL_END_BOUND
+ : ClusteringPrefix.Kind.EXCL_START_BOUND,
+ Arrays.copyOf(components, cc));
+
+ case ByteSource.LTLT_NEXT_COMPONENT:
+ case ByteSource.GTGT_NEXT_COMPONENT:
+ throw new AssertionError("Unexpected sstable lower/upper bound - byte comparable representation of artificial sstable bounds is not supported");
+
+ default:
+ throw new AssertionError("Unexpected separator " + Integer.toHexString(sep) + " in ClusteringBound encoding");
+ }
+ ++cc;
+ sep = orderedBytes.next();
+ }
+ }
+
+ /**
+ * Produces a clustering boundary from the given byte-comparable value. The method will throw an exception if the
+ * value does not correctly encode a bound position of this type, including if it encodes an exact clustering.
+ *
+ * Note that the encoded clustering position cannot specify the type of bound (i.e. start/end/boundary) because to
+ * correctly compare clustering positions the encoding must be the same for the different types (e.g. the position
+ * for a exclusive end and an inclusive start is the same, before the exact clustering). The type must be supplied
+ * separately (in the bound... vs boundary... call and isEnd argument).
+ *
+ * @param accessor Accessor to use to construct components.
+ * @param comparable The clustering position encoded as a byte-comparable sequence.
+ */
+ public <V> ClusteringBoundary<V> boundaryFromByteComparable(ValueAccessor<V> accessor, ByteComparable comparable)
+ {
+ ByteComparable.Version version = ByteComparable.Version.OSS50;
+ ByteSource.Peekable orderedBytes = ByteSource.peekable(comparable.asComparableBytes(version));
+
+ int sep = orderedBytes.next();
+ int cc = 0;
+ V[] components = accessor.createArray(size());
+
+ while (true)
+ {
+ switch (sep)
+ {
+ case NEXT_COMPONENT_NULL:
+ components[cc] = null;
+ break;
+ case NEXT_COMPONENT_EMPTY:
+ case NEXT_COMPONENT_EMPTY_REVERSED:
+ components[cc] = subtype(cc).fromComparableBytes(accessor, null, version);
+ break;
+ case NEXT_COMPONENT:
+ // Decode the next component, consuming bytes from orderedBytes.
+ components[cc] = subtype(cc).fromComparableBytes(accessor, orderedBytes, version);
+ break;
+ case ByteSource.LT_NEXT_COMPONENT:
+ return accessor.factory().boundary(ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY,
+ Arrays.copyOf(components, cc));
+ case ByteSource.GT_NEXT_COMPONENT:
+ return accessor.factory().boundary(ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY,
+ Arrays.copyOf(components, cc));
+ default:
+ throw new AssertionError("Unexpected separator " + Integer.toHexString(sep) + " in ClusteringBoundary encoding");
+ }
+ ++cc;
+ sep = orderedBytes.next();
+ }
+ }
+
+ /**
* A comparator for rows.
*
* A {@code Row} is a {@code Clusterable} so {@code ClusteringComparator} can be used
@@ -279,4 +555,4 @@
{
return Objects.hashCode(clusteringTypes);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ClusteringPrefix.java b/src/java/org/apache/cassandra/db/ClusteringPrefix.java
index a1291c8..be81e5d 100644
--- a/src/java/org/apache/cassandra/db/ClusteringPrefix.java
+++ b/src/java/org/apache/cassandra/db/ClusteringPrefix.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.function.ToIntFunction;
import org.apache.cassandra.cache.IMeasurableMemory;
import org.apache.cassandra.config.*;
@@ -33,7 +34,8 @@
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteArrayUtil;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
/**
* A clustering prefix is the unit of what a {@link ClusteringComparator} can compare.
@@ -62,14 +64,20 @@
{
// WARNING: the ordering of that enum matters because we use ordinal() in the serialization
- EXCL_END_BOUND (0, -1),
- INCL_START_BOUND (0, -1),
- EXCL_END_INCL_START_BOUNDARY(0, -1),
- STATIC_CLUSTERING (1, -1),
- CLUSTERING (2, 0),
- INCL_END_EXCL_START_BOUNDARY(3, 1),
- INCL_END_BOUND (3, 1),
- EXCL_START_BOUND (3, 1);
+ // @formatter:off
+ EXCL_END_BOUND ( 0, -1, v -> ByteSource.LT_NEXT_COMPONENT),
+ INCL_START_BOUND ( 0, -1, v -> ByteSource.LT_NEXT_COMPONENT),
+ EXCL_END_INCL_START_BOUNDARY ( 0, -1, v -> ByteSource.LT_NEXT_COMPONENT),
+ STATIC_CLUSTERING ( 1, -1, v -> v == Version.LEGACY ? ByteSource.LT_NEXT_COMPONENT + 1
+ : ByteSource.EXCLUDED),
+ CLUSTERING ( 2, 0, v -> v == Version.LEGACY ? ByteSource.NEXT_COMPONENT
+ : ByteSource.TERMINATOR),
+ INCL_END_EXCL_START_BOUNDARY ( 3, 1, v -> ByteSource.GT_NEXT_COMPONENT),
+ INCL_END_BOUND ( 3, 1, v -> ByteSource.GT_NEXT_COMPONENT),
+ EXCL_START_BOUND ( 3, 1, v -> ByteSource.GT_NEXT_COMPONENT),
+ SSTABLE_LOWER_BOUND (-1, -1, v -> ByteSource.LTLT_NEXT_COMPONENT),
+ SSTABLE_UPPER_BOUND ( 4, 1, v -> ByteSource.GTGT_NEXT_COMPONENT);
+ // @formatter:on
private final int comparison;
@@ -79,10 +87,13 @@
*/
public final int comparedToClustering;
- Kind(int comparison, int comparedToClustering)
+ public final ToIntFunction<Version> asByteComparable;
+
+ Kind(int comparison, int comparedToClustering, ToIntFunction<Version> asByteComparable)
{
this.comparison = comparison;
this.comparedToClustering = comparedToClustering;
+ this.asByteComparable = asByteComparable;
}
/**
@@ -197,6 +208,16 @@
? (this == INCL_END_EXCL_START_BOUNDARY ? INCL_END_BOUND : EXCL_END_BOUND)
: (this == INCL_END_EXCL_START_BOUNDARY ? EXCL_START_BOUND : INCL_START_BOUND);
}
+
+ /*
+ * Returns a terminator value for this clustering type that is suitable for byte comparison.
+ * Inclusive starts / exclusive ends need a lower value than ByteSource.NEXT_COMPONENT and the clustering byte,
+ * exclusive starts / inclusive ends -- a higher.
+ */
+ public int asByteComparableValue(Version version)
+ {
+ return asByteComparable.applyAsInt(version);
+ }
}
default boolean isBottom()
@@ -292,6 +313,22 @@
*/
public String toString(TableMetadata metadata);
+ /**
+ * Returns this prefix as a start bound.
+ * If this prefix is a bound, just returns it asserting that it is a start bound.
+ * If this prefix is a clustering, returns an included start bound.
+ * If this prefix is a boundary, returns an open bound of it
+ */
+ ClusteringBound<V> asStartBound();
+
+ /**
+ * Returns this prefix as an end bound.
+ * If this prefix is a bound, just returns it asserting that it is an end bound.
+ * If this prefix is a clustering, returns an included end bound.
+ * In this prefix is a boundary, returns a close bound of it.
+ */
+ ClusteringBound<V> asEndBound();
+
/*
* TODO: we should stop using Clustering for partition keys. Maybe we can add
* a few methods to DecoratedKey so we don't have to (note that while using a Clustering
@@ -308,6 +345,24 @@
values[i] = accessor().toBuffer(get(i));
return CompositeType.build(ByteBufferAccessor.instance, values);
}
+
+ /**
+ * Produce a human-readable representation of the clustering given the list of types.
+ * Easier to access than metadata for debugging.
+ */
+ public default String clusteringString(List<AbstractType<?>> types)
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append(kind()).append('(');
+ for (int i = 0; i < size(); i++)
+ {
+ if (i > 0)
+ sb.append(", ");
+ sb.append(types.get(i).getString(get(i), accessor()));
+ }
+ return sb.append(')').toString();
+ }
+
/**
* The values of this prefix as an array.
* <p>
@@ -322,10 +377,11 @@
public ByteBuffer[] getBufferArray();
/**
- * If the prefix contains byte buffers that can be minimized (see {@link ByteBufferUtil#minimalBufferFor(ByteBuffer)}),
- * this will return a copy of the prefix with minimized values, otherwise it returns itself.
+ * Return the key in a form that can be retained for longer-term use. This means extracting keys stored in shared
+ * memory (i.e. in memtables) to minimized on-heap versions.
+ * If the object is already in minimal form, no action will be taken.
*/
- public ClusteringPrefix<V> minimize();
+ public ClusteringPrefix<V> retainable();
public static class Serializer
{
@@ -676,4 +732,4 @@
return equals(prefix, (ClusteringPrefix<?>) o);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 3619520..1be2085 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -22,7 +22,6 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@@ -39,6 +38,7 @@
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -84,20 +84,20 @@
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
+import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.CompactionStrategyManager;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.DataLimits;
-import org.apache.cassandra.db.memtable.Flushing;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.memtable.ShardBoundaries;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.Tracker;
import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.memtable.Flushing;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.ShardBoundaries;
import org.apache.cassandra.db.partitions.CachedPartition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.repair.CassandraTableRepairManager;
@@ -119,17 +119,19 @@
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.IVerifier;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.Sampler;
import org.apache.cassandra.metrics.Sampler.Sample;
import org.apache.cassandra.metrics.Sampler.SamplerType;
@@ -155,6 +157,7 @@
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.TablePaxosRepairHistory;
+import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.streaming.TableStreamManager;
@@ -163,8 +166,10 @@
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.JsonUtils;
import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
@@ -173,7 +178,6 @@
import org.apache.cassandra.utils.concurrent.Refs;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
-import static com.google.common.base.Throwables.propagate;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.DatabaseDescriptor.getFlushWriters;
import static org.apache.cassandra.db.commitlog.CommitLogPosition.NONE;
@@ -184,7 +188,7 @@
import static org.apache.cassandra.utils.Throwables.merge;
import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
-public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
+public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner, SSTable.Owner
{
private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class);
@@ -320,6 +324,9 @@
private volatile boolean compactionSpaceCheck = true;
+ // Tombtone partitions that ignore the gc_grace_seconds during compaction
+ private final Set<DecoratedKey> partitionKeySetIgnoreGcGrace = ConcurrentHashMap.newKeySet();
+
@VisibleForTesting
final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager();
private volatile ShardBoundaries cachedShardBoundaries = null;
@@ -378,7 +385,7 @@
for (ColumnFamilyStore cfs : concatWithIndexes())
cfs.crcCheckChance = new DefaultValue(metadata().params.crcCheckChance);
- compactionStrategyManager.maybeReload(metadata());
+ compactionStrategyManager.maybeReloadParamsFromSchema(metadata().params.compaction);
indexManager.reload();
@@ -402,7 +409,7 @@
public String getCompactionParametersJson()
{
- return FBUtilities.json(getCompactionParameters());
+ return JsonUtils.writeAsJsonString(getCompactionParameters());
}
public void setCompactionParameters(Map<String, String> options)
@@ -411,7 +418,7 @@
{
CompactionParams compactionParams = CompactionParams.fromMap(options);
compactionParams.validate();
- compactionStrategyManager.setNewLocalCompactionStrategy(compactionParams);
+ compactionStrategyManager.overrideLocalParams(compactionParams);
}
catch (Throwable t)
{
@@ -423,7 +430,7 @@
public void setCompactionParametersJson(String options)
{
- setCompactionParameters(FBUtilities.fromJsonMap(options));
+ setCompactionParameters(JsonUtils.fromJsonMap(options));
}
public Map<String,String> getCompressionParameters()
@@ -433,7 +440,7 @@
public String getCompressionParametersJson()
{
- return FBUtilities.json(getCompressionParameters());
+ return JsonUtils.writeAsJsonString(getCompressionParameters());
}
public void setCompressionParameters(Map<String,String> opts)
@@ -452,7 +459,7 @@
public void setCompressionParametersJson(String options)
{
- setCompressionParameters(FBUtilities.fromJsonMap(options));
+ setCompressionParameters(JsonUtils.fromJsonMap(options));
}
@VisibleForTesting
@@ -502,7 +509,7 @@
if (data.loadsstables)
{
Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
- sstables = SSTableReader.openAll(sstableFiles.list().entrySet(), metadata);
+ sstables = SSTableReader.openAll(this, sstableFiles.list().entrySet(), metadata);
data.addInitialSSTablesWithoutUpdatingSize(sstables);
}
@@ -607,6 +614,7 @@
return directories;
}
+ @Override
public List<String> getDataPaths() throws IOException
{
List<String> dataPaths = new ArrayList<>();
@@ -792,19 +800,7 @@
}
}
- File dataFile = new File(desc.filenameFor(Component.DATA));
- if (components.contains(Component.DATA) && dataFile.length() > 0)
- // everything appears to be in order... moving on.
- continue;
-
- // missing the DATA file! all components are orphaned
- logger.warn("Removing orphans for {}: {}", desc, components);
- for (Component component : components)
- {
- File file = new File(desc.filenameFor(component));
- if (file.exists())
- FileUtils.deleteWithConfirm(desc.filenameFor(component));
- }
+ desc.getFormat().deleteOrphanedComponents(desc, components);
}
// cleanup incomplete saved caches
@@ -883,10 +879,9 @@
descriptor.cfname,
// Increment the generation until we find a filename that doesn't exist. This is needed because the new
// SSTables that are being loaded might already use these generation numbers.
- sstableIdGenerator.get(),
- descriptor.formatType);
+ sstableIdGenerator.get());
}
- while (newDescriptor.fileFor(Component.DATA).exists());
+ while (newDescriptor.fileFor(Components.DATA).exists());
return newDescriptor;
}
@@ -930,23 +925,22 @@
public Descriptor newSSTableDescriptor(File directory)
{
- return newSSTableDescriptor(directory, SSTableFormat.Type.current().info.getLatestVersion(), SSTableFormat.Type.current());
+ return newSSTableDescriptor(directory, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion());
}
- public Descriptor newSSTableDescriptor(File directory, SSTableFormat.Type format)
+ public Descriptor newSSTableDescriptor(File directory, SSTableFormat<?, ?> format)
{
- return newSSTableDescriptor(directory, format.info.getLatestVersion(), format);
+ return newSSTableDescriptor(directory, format.getLatestVersion());
}
- public Descriptor newSSTableDescriptor(File directory, Version version, SSTableFormat.Type format)
+ public Descriptor newSSTableDescriptor(File directory, Version version)
{
Descriptor newDescriptor = new Descriptor(version,
directory,
keyspace.getName(),
name,
- sstableIdGenerator.get(),
- format);
- assert !newDescriptor.fileFor(Component.DATA).exists();
+ sstableIdGenerator.get());
+ assert !newDescriptor.fileFor(Components.DATA).exists();
return newDescriptor;
}
@@ -1112,7 +1106,10 @@
metric.pendingFlushes.dec();
if (flushFailure != null)
- throw propagate(flushFailure);
+ {
+ Throwables.throwIfUnchecked(flushFailure);
+ throw new RuntimeException(flushFailure);
+ }
return commitLogUpperBound;
}
@@ -1273,7 +1270,8 @@
{
t = Flushing.abortRunnables(flushRunnables, t);
t = txn.abort(t);
- throw Throwables.propagate(t);
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
try
@@ -1299,7 +1297,8 @@
for (SSTableMultiWriter writer : flushResults)
t = writer.abort(t);
t = txn.abort(t);
- Throwables.propagate(t);
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
txn.prepareToCommit();
@@ -1448,41 +1447,63 @@
@Override
public ShardBoundaries localRangeSplits(int shardCount)
{
- if (shardCount == 1 || !getPartitioner().splitter().isPresent() || SchemaConstants.isLocalSystemKeyspace(keyspace.getName()))
+ if (shardCount == 1 || !getPartitioner().splitter().isPresent())
return ShardBoundaries.NONE;
ShardBoundaries shardBoundaries = cachedShardBoundaries;
+
if (shardBoundaries == null ||
shardBoundaries.shardCount() != shardCount ||
- shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion())
+ shardBoundaries.ringVersion != -1 && shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion())
{
- DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
- Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
List<Splitter.WeightedRange> weightedRanges;
- if (localRanges.isEmpty())
- weightedRanges = ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken())));
+ long ringVersion;
+ if (!SchemaConstants.isLocalSystemKeyspace(keyspace.getName())
+ && getPartitioner() == StorageService.instance.getTokenMetadata().partitioner)
+ {
+ DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
+ Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
+ ringVersion = versionedLocalRanges.ringVersion;
+
+ if (!localRanges.isEmpty())
+ {
+ weightedRanges = new ArrayList<>(localRanges.size());
+ for (Range<Token> r : localRanges)
+ {
+ // WeightedRange supports only unwrapped ranges as it relies
+ // on right - left == num tokens equality
+ for (Range<Token> u: r.unwrap())
+ weightedRanges.add(new Splitter.WeightedRange(1.0, u));
+ }
+ weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
+ }
+ else
+ {
+ weightedRanges = fullWeightedRange();
+ }
+ }
else
{
- weightedRanges = new ArrayList<>(localRanges.size());
- for (Range<Token> r : localRanges)
- {
- // WeightedRange supports only unwrapped ranges as it relies
- // on right - left == num tokens equality
- for (Range<Token> u: r.unwrap())
- weightedRanges.add(new Splitter.WeightedRange(1.0, u));
- }
- weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
+ // Local tables need to cover the full token range and don't care about ring changes.
+ // We also end up here if the table's partitioner is not the database's, which can happen in tests.
+ weightedRanges = fullWeightedRange();
+ ringVersion = -1;
}
List<Token> boundaries = getPartitioner().splitter().get().splitOwnedRanges(shardCount, weightedRanges, false);
shardBoundaries = new ShardBoundaries(boundaries.subList(0, boundaries.size() - 1),
- versionedLocalRanges.ringVersion);
+ ringVersion);
cachedShardBoundaries = shardBoundaries;
logger.debug("Memtable shard boundaries for {}.{}: {}", keyspace.getName(), getTableName(), boundaries);
}
return shardBoundaries;
}
+ private ImmutableList<Splitter.WeightedRange> fullWeightedRange()
+ {
+ return ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken())));
+ }
+
/**
* @param sstables
* @return sstables whose key range overlaps with that of the given sstables, not including itself.
@@ -1500,7 +1521,7 @@
View view = data.getView();
List<SSTableReader> sortedByFirst = Lists.newArrayList(sstables);
- Collections.sort(sortedByFirst, (o1, o2) -> o1.first.compareTo(o2.first));
+ sortedByFirst.sort(SSTableReader.sstableComparator);
List<AbstractBounds<PartitionPosition>> bounds = new ArrayList<>();
DecoratedKey first = null, last = null;
@@ -1517,21 +1538,21 @@
{
if (first == null)
{
- first = sstable.first;
- last = sstable.last;
+ first = sstable.getFirst();
+ last = sstable.getLast();
}
else
{
- if (sstable.first.compareTo(last) <= 0) // we do overlap
+ if (sstable.getFirst().compareTo(last) <= 0) // we do overlap
{
- if (sstable.last.compareTo(last) > 0)
- last = sstable.last;
+ if (sstable.getLast().compareTo(last) > 0)
+ last = sstable.getLast();
}
else
{
bounds.add(AbstractBounds.bounds(first, true, last, true));
- first = sstable.first;
- last = sstable.last;
+ first = sstable.getFirst();
+ last = sstable.getLast();
}
}
}
@@ -1639,13 +1660,13 @@
return CompactionManager.instance.performCleanup(ColumnFamilyStore.this, jobs);
}
- public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs) throws ExecutionException, InterruptedException
+ public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, IScrubber.Options options, int jobs) throws ExecutionException, InterruptedException
{
- return scrub(disableSnapshot, skipCorrupted, reinsertOverflowedTTL, false, checkData, jobs);
+ return scrub(disableSnapshot, false, options, jobs);
}
@VisibleForTesting
- public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean reinsertOverflowedTTL, boolean alwaysFail, boolean checkData, int jobs) throws ExecutionException, InterruptedException
+ public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean alwaysFail, IScrubber.Options options, int jobs) throws ExecutionException, InterruptedException
{
// skip snapshot creation during scrub, SEE JIRA 5891
if(!disableSnapshot)
@@ -1657,7 +1678,7 @@
try
{
- return CompactionManager.instance.performScrub(ColumnFamilyStore.this, skipCorrupted, checkData, reinsertOverflowedTTL, jobs);
+ return CompactionManager.instance.performScrub(ColumnFamilyStore.this, options, jobs);
}
catch(Throwable t)
{
@@ -1692,7 +1713,7 @@
return true;
}
- public CompactionManager.AllSSTableOpStatus verify(Verifier.Options options) throws ExecutionException, InterruptedException
+ public CompactionManager.AllSSTableOpStatus verify(IVerifier.Options options) throws ExecutionException, InterruptedException
{
return CompactionManager.instance.performVerify(ColumnFamilyStore.this, options);
}
@@ -1700,9 +1721,9 @@
/**
* Rewrites all SSTables according to specified parameters
*
- * @param skipIfCurrentVersion - if {@link true}, will rewrite only SSTables that have version older than the current one ({@link BigFormat#latestVersion})
+ * @param skipIfCurrentVersion - if {@link true}, will rewrite only SSTables that have version older than the current one ({@link SSTableFormat#getLatestVersion()})
* @param skipIfNewerThanTimestamp - max timestamp (local creation time) for SSTable; SSTables created _after_ this timestamp will be excluded from compaction
- * @param skipIfCompressionMatches - if {@link true}, will rewrite only SSTables whose compression parameters are different from {@link CFMetaData#compressionParams()}
+ * @param skipIfCompressionMatches - if {@link true}, will rewrite only SSTables whose compression parameters are different from {@link TableMetadata#params#getCompressionParameters()} ()}
* @param jobs number of jobs for parallel execution
*/
public CompactionManager.AllSSTableOpStatus sstablesRewrite(final boolean skipIfCurrentVersion,
@@ -1802,7 +1823,7 @@
return session != null && sessions.contains(session);
};
return runWithCompactionsDisabled(() -> compactionStrategyManager.releaseRepairData(sessions),
- predicate, false, true, true);
+ predicate, OperationType.STREAM, false, true, true);
}
else
{
@@ -1911,27 +1932,53 @@
public List<String> getSSTablesForKey(String key, boolean hexFormat)
{
+ return withSSTablesForKey(key, hexFormat, SSTableReader::getFilename);
+ }
+
+ public Map<Integer, Set<String>> getSSTablesForKeyWithLevel(String key, boolean hexFormat)
+ {
+ List<Pair<Integer, String>> ssts = withSSTablesForKey(key, hexFormat, sstr -> Pair.create(sstr.getSSTableLevel(), sstr.getFilename()));
+ HashMap<Integer, Set<String>> result = new HashMap<>();
+ for (Pair<Integer, String> sst : ssts)
+ {
+ Set<String> perLevel = result.get(sst.left);
+ if (perLevel == null)
+ {
+ perLevel = new HashSet<>();
+ result.put(sst.left, perLevel);
+ }
+
+ perLevel.add(sst.right);
+ }
+
+ return result;
+ }
+
+ public <T> List<T> withSSTablesForKey(String key, boolean hexFormat, Function<SSTableReader, T> mapper)
+ {
ByteBuffer keyBuffer = hexFormat ? ByteBufferUtil.hexToBytes(key) : metadata().partitionKeyType.fromString(key);
DecoratedKey dk = decorateKey(keyBuffer);
try (OpOrder.Group op = readOrdering.start())
{
- List<String> files = new ArrayList<>();
+ List<T> mapped = new ArrayList<>();
for (SSTableReader sstr : select(View.select(SSTableSet.LIVE, dk)).sstables)
{
// check if the key actually exists in this sstable, without updating cache and stats
- if (sstr.getPosition(dk, SSTableReader.Operator.EQ, false) != null)
- files.add(sstr.getFilename());
+ if (sstr.getPosition(dk, SSTableReader.Operator.EQ, false) >= 0)
+ mapped.add(mapper.apply(sstr));
}
- return files;
+ return mapped;
}
}
+ @Override
public void beginLocalSampling(String sampler, int capacity, int durationMillis)
{
metric.samplers.get(SamplerType.valueOf(sampler)).beginSampling(capacity, durationMillis);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
+ @Override
public List<CompositeData> finishLocalSampling(String sampler, int count) throws OpenDataException
{
Sampler samplerImpl = metric.samplers.get(SamplerType.valueOf(sampler));
@@ -1950,11 +1997,13 @@
return result;
}
+ @Override
public boolean isCompactionDiskSpaceCheckEnabled()
{
return compactionSpaceCheck;
}
+ @Override
public void compactionDiskSpaceCheck(boolean enable)
{
compactionSpaceCheck = enable;
@@ -2023,7 +2072,6 @@
{
File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName);
ssTable.createLinks(snapshotDirectory.path(), rateLimiter); // hard links
-
if (logger.isTraceEnabled())
logger.trace("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory);
snapshottedSSTables.add(ssTable);
@@ -2041,7 +2089,7 @@
.collect(Collectors.toCollection(HashSet::new));
// Create and write snapshot manifest
- SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime);
+ SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime, ephemeral);
File manifestFile = getDirectories().getSnapshotManifestFile(tag);
writeSnapshotManifest(manifest, manifestFile);
snapshotDirs.add(manifestFile.parent().toAbsolute()); // manifest may create empty snapshot dir
@@ -2054,16 +2102,9 @@
snapshotDirs.add(schemaFile.parent().toAbsolute()); // schema may create empty snapshot dir
}
- // Maybe create ephemeral marker
- if (ephemeral)
- {
- File ephemeralSnapshotMarker = getDirectories().getNewEphemeralSnapshotMarkerFile(tag);
- createEphemeralSnapshotMarkerFile(tag, ephemeralSnapshotMarker);
- snapshotDirs.add(ephemeralSnapshotMarker.parent().toAbsolute()); // marker may create empty snapshot dir
- }
-
- TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag,
- manifest.createdAt, manifest.expiresAt, snapshotDirs);
+ TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(),
+ tag, manifest.createdAt, manifest.expiresAt, snapshotDirs,
+ manifest.ephemeral);
StorageService.instance.addSnapshot(snapshot);
return snapshot;
@@ -2085,7 +2126,7 @@
private List<String> mapToDataFilenames(Collection<SSTableReader> sstables)
{
- return sstables.stream().map(s -> s.descriptor.relativeFilenameFor(Component.DATA)).collect(Collectors.toList());
+ return sstables.stream().map(s -> s.descriptor.relativeFilenameFor(Components.DATA)).collect(Collectors.toList());
}
private void writeSnapshotSchema(File schemaFile)
@@ -2108,34 +2149,19 @@
}
}
- private void createEphemeralSnapshotMarkerFile(final String snapshot, File ephemeralSnapshotMarker)
- {
- try
- {
- if (!ephemeralSnapshotMarker.parent().exists())
- ephemeralSnapshotMarker.parent().tryCreateDirectories();
-
- Files.createFile(ephemeralSnapshotMarker.toPath());
- if (logger.isTraceEnabled())
- logger.trace("Created ephemeral snapshot marker file on {}.", ephemeralSnapshotMarker.absolutePath());
- }
- catch (IOException e)
- {
- logger.warn(String.format("Could not create marker file %s for ephemeral snapshot %s. " +
- "In case there is a failure in the operation that created " +
- "this snapshot, you may need to clean it manually afterwards.",
- ephemeralSnapshotMarker.absolutePath(), snapshot), e);
- }
- }
-
protected static void clearEphemeralSnapshots(Directories directories)
{
RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
- for (String ephemeralSnapshot : directories.listEphemeralSnapshots())
+ List<TableSnapshot> ephemeralSnapshots = new SnapshotLoader(directories).loadSnapshots()
+ .stream()
+ .filter(TableSnapshot::isEphemeral)
+ .collect(Collectors.toList());
+
+ for (TableSnapshot ephemeralSnapshot : ephemeralSnapshots)
{
- logger.trace("Clearing ephemeral snapshot {} leftover from previous session.", ephemeralSnapshot);
- Directories.clearSnapshot(ephemeralSnapshot, directories.getCFDirectories(), clearSnapshotRateLimiter);
+ logger.trace("Clearing ephemeral snapshot {} leftover from previous session.", ephemeralSnapshot.getId());
+ Directories.clearSnapshot(ephemeralSnapshot.getTag(), directories.getCFDirectories(), clearSnapshotRateLimiter);
}
}
@@ -2158,7 +2184,7 @@
if (logger.isTraceEnabled())
logger.trace("using snapshot sstable {}", entries.getKey());
// open offline so we don't modify components or track hotness.
- sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, true, true);
+ sstable = SSTableReader.open(this, entries.getKey(), entries.getValue(), metadata, true, true);
refs.tryRef(sstable);
// release the self ref as we never add the snapshot sstable to DataTracker where it is otherwise released
sstable.selfRef().release();
@@ -2409,6 +2435,31 @@
CompactionManager.instance.forceCompactionForKey(this, key);
}
+ public void forceCompactionKeysIgnoringGcGrace(String... partitionKeysIgnoreGcGrace)
+ {
+ List<DecoratedKey> decoratedKeys = new ArrayList<>();
+ try
+ {
+ partitionKeySetIgnoreGcGrace.clear();
+
+ for (String key : partitionKeysIgnoreGcGrace) {
+ DecoratedKey dk = decorateKey(metadata().partitionKeyType.fromString(key));
+ partitionKeySetIgnoreGcGrace.add(dk);
+ decoratedKeys.add(dk);
+ }
+
+ CompactionManager.instance.forceCompactionForKeys(this, decoratedKeys);
+ } finally
+ {
+ partitionKeySetIgnoreGcGrace.clear();
+ }
+ }
+
+ public boolean shouldIgnoreGcGraceForKey(DecoratedKey dk)
+ {
+ return partitionKeySetIgnoreGcGrace.contains(dk);
+ }
+
public static Iterable<ColumnFamilyStore> all()
{
List<Iterable<ColumnFamilyStore>> stores = new ArrayList<>(Schema.instance.getKeyspaces().size());
@@ -2474,7 +2525,8 @@
catch (Throwable t)
{
memtableContent.close();
- Throwables.propagate(t);
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
}
}
@@ -2559,7 +2611,7 @@
cfs.runWithCompactionsDisabled((Callable<Void>) () -> {
cfs.data.reset(memtableFactory.create(new AtomicReference<>(CommitLogPosition.NONE), cfs.metadata, cfs));
return null;
- }, true, false);
+ }, OperationType.P0, true, false);
}
}
@@ -2600,7 +2652,7 @@
if (!noSnapshot &&
((keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable()) // need to clear dirty regions
- || DatabaseDescriptor.isAutoSnapshot())) // need sstable for snapshot
+ || isAutoSnapshotEnabled()))
{
replayAfter = forceBlockingFlush(FlushReason.TRUNCATE);
viewManager.forceBlockingFlush(FlushReason.TRUNCATE);
@@ -2633,7 +2685,7 @@
"Stopping parent sessions {} due to truncation of tableId="+metadata.id);
data.notifyTruncated(truncatedAt);
- if (!noSnapshot && DatabaseDescriptor.isAutoSnapshot())
+ if (!noSnapshot && isAutoSnapshotEnabled())
snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, SNAPSHOT_TRUNCATE_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
discardSSTables(truncatedAt);
@@ -2648,7 +2700,7 @@
}
};
- runWithCompactionsDisabled(FutureTask.callable(truncateRunnable), true, true);
+ runWithCompactionsDisabled(FutureTask.callable(truncateRunnable), OperationType.P0, true, true);
viewManager.build();
@@ -2679,9 +2731,9 @@
FBUtilities.waitOnFuture(dumpMemtable());
}
- public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
+ public <V> V runWithCompactionsDisabled(Callable<V> callable, OperationType operationType, boolean interruptValidation, boolean interruptViews)
{
- return runWithCompactionsDisabled(callable, (sstable) -> true, interruptValidation, interruptViews, true);
+ return runWithCompactionsDisabled(callable, (sstable) -> true, operationType, interruptValidation, interruptViews, true);
}
/**
@@ -2694,13 +2746,13 @@
* @param interruptIndexes if we should interrupt compactions on indexes. NOTE: if you set this to true your sstablePredicate
* must be able to handle LocalPartitioner sstables!
*/
- public <V> V runWithCompactionsDisabled(Callable<V> callable, Predicate<SSTableReader> sstablesPredicate, boolean interruptValidation, boolean interruptViews, boolean interruptIndexes)
+ public <V> V runWithCompactionsDisabled(Callable<V> callable, Predicate<SSTableReader> sstablesPredicate, OperationType operationType, boolean interruptValidation, boolean interruptViews, boolean interruptIndexes)
{
// synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
// and so we only run one major compaction at a time
synchronized (this)
{
- logger.trace("Cancelling in-progress compactions for {}", metadata.name);
+ logger.debug("Cancelling in-progress compactions for {}", metadata.name);
Iterable<ColumnFamilyStore> toInterruptFor = interruptIndexes
? concatWithIndexes()
: Collections.singleton(this);
@@ -2709,9 +2761,24 @@
? Iterables.concat(toInterruptFor, viewManager.allViewsCfs())
: toInterruptFor;
+ Iterable<TableMetadata> toInterruptForMetadata = Iterables.transform(toInterruptFor, ColumnFamilyStore::metadata);
+
try (CompactionManager.CompactionPauser pause = CompactionManager.instance.pauseGlobalCompaction();
CompactionManager.CompactionPauser pausedStrategies = pauseCompactionStrategies(toInterruptFor))
{
+ List<CompactionInfo.Holder> uninterruptibleTasks = CompactionManager.instance.getCompactionsMatching(toInterruptForMetadata,
+ (info) -> info.getTaskType().priority <= operationType.priority);
+ if (!uninterruptibleTasks.isEmpty())
+ {
+ logger.info("Unable to cancel in-progress compactions, since they're running with higher or same priority: {}. You can abort these operations using `nodetool stop`.",
+ uninterruptibleTasks.stream().map((compaction) -> String.format("%s@%s (%s)",
+ compaction.getCompactionInfo().getTaskType(),
+ compaction.getCompactionInfo().getTable(),
+ compaction.getCompactionInfo().getTaskId()))
+ .collect(Collectors.joining(",")));
+ return null;
+ }
+
// interrupt in-progress compactions
CompactionManager.instance.interruptCompactionForCFs(toInterruptFor, sstablesPredicate, interruptValidation);
CompactionManager.instance.waitForCessation(toInterruptFor, sstablesPredicate);
@@ -2721,7 +2788,9 @@
{
if (cfs.getTracker().getCompacting().stream().anyMatch(sstablesPredicate))
{
- logger.warn("Unable to cancel in-progress compactions for {}. Perhaps there is an unusually large row in progress somewhere, or the system is simply overloaded.", metadata.name);
+ logger.warn("Unable to cancel in-progress compactions for {}. " +
+ "Perhaps there is an unusually large row in progress somewhere, or the system is simply overloaded.",
+ metadata.name);
return null;
}
}
@@ -2776,7 +2845,7 @@
return accumulate;
}
- public LifecycleTransaction markAllCompacting(final OperationType operationType)
+ public <T> T withAllSSTables(final OperationType operationType, Function<LifecycleTransaction, T> op)
{
Callable<LifecycleTransaction> callable = () -> {
assert data.getCompacting().isEmpty() : data.getCompacting();
@@ -2787,10 +2856,12 @@
return modifier;
};
- return runWithCompactionsDisabled(callable, false, false);
+ try (LifecycleTransaction compacting = runWithCompactionsDisabled(callable, operationType, false, false))
+ {
+ return op.apply(compacting);
+ }
}
-
@Override
public String toString()
{
@@ -2863,6 +2934,7 @@
}
+ @Override
public Double getCrcCheckChance()
{
return crcCheckChance.value();
@@ -2987,21 +3059,37 @@
return indexManager.getBuiltIndexNames();
}
+ @Override
public int getUnleveledSSTables()
{
return compactionStrategyManager.getUnleveledSSTables();
}
+ @Override
public int[] getSSTableCountPerLevel()
{
return compactionStrategyManager.getSSTableCountPerLevel();
}
+ @Override
public long[] getPerLevelSizeBytes()
{
return compactionStrategyManager.getPerLevelSizeBytes();
}
+ @Override
+ public boolean isLeveledCompaction()
+ {
+ return compactionStrategyManager.isLeveledCompaction();
+ }
+
+ @Override
+ public int[] getSSTableCountPerTWCSBucket()
+ {
+ return compactionStrategyManager.getSSTableCountPerTWCSBucket();
+ }
+
+ @Override
public int getLevelFanoutSize()
{
return compactionStrategyManager.getLevelFanoutSize();
@@ -3063,6 +3151,16 @@
return metadata().params.caching.cacheKeys() && CacheService.instance.keyCache.getCapacity() > 0;
}
+ public boolean isAutoSnapshotEnabled()
+ {
+ return metadata().params.allowAutoSnapshot && DatabaseDescriptor.isAutoSnapshot();
+ }
+
+ public boolean isTableIncrementalBackupsEnabled()
+ {
+ return DatabaseDescriptor.isIncrementalBackupsEnabled() && metadata().params.incrementalBackups;
+ }
+
/**
* Discard all SSTables that were created before given timestamp.
*
@@ -3097,6 +3195,7 @@
}
}
+ @Override
public double getDroppableTombstoneRatio()
{
double allDroppable = 0;
@@ -3111,6 +3210,7 @@
return allColumns > 0 ? allDroppable / allColumns : 0;
}
+ @Override
public long trueSnapshotsSize()
{
return getDirectories().trueSnapshotsSize();
@@ -3160,6 +3260,36 @@
return Objects.requireNonNull(getIfExists(tableId)).metric;
}
+ /**
+ * Grabs the global first/last tokens among sstables and returns the range of data directories that start/end with those tokens.
+ *
+ * This is done to avoid grabbing the disk boundaries for every sstable in case of huge compactions.
+ */
+ public List<File> getDirectoriesForFiles(Set<SSTableReader> sstables)
+ {
+ Directories.DataDirectory[] writeableLocations = directories.getWriteableLocations();
+ if (writeableLocations.length == 1 || sstables.isEmpty())
+ {
+ List<File> ret = new ArrayList<>(writeableLocations.length);
+ for (Directories.DataDirectory ddir : writeableLocations)
+ ret.add(getDirectories().getLocationForDisk(ddir));
+ return ret;
+ }
+
+ DecoratedKey first = null;
+ DecoratedKey last = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (first == null || first.compareTo(sstable.getFirst()) > 0)
+ first = sstable.getFirst();
+ if (last == null || last.compareTo(sstable.getLast()) < 0)
+ last = sstable.getLast();
+ }
+
+ DiskBoundaries diskBoundaries = getDiskBoundaries();
+ return diskBoundaries.getDisksInBounds(first, last).stream().map(directories::getLocationForDisk).collect(Collectors.toList());
+ }
+
public DiskBoundaries getDiskBoundaries()
{
return diskBoundaryManager.getDiskBoundaries(this);
@@ -3195,7 +3325,7 @@
CompactionManager.instance.interruptCompactionForCFs(concatWithIndexes(), (sstable) -> true, true);
- if (DatabaseDescriptor.isAutoSnapshot())
+ if (isAutoSnapshotEnabled())
snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id));
@@ -3307,6 +3437,18 @@
}
@Override
+ public long getMaxSSTableSize()
+ {
+ return metric.maxSSTableSize.getValue();
+ }
+
+ @Override
+ public long getMaxSSTableDuration()
+ {
+ return metric.maxSSTableDuration.getValue();
+ }
+
+ @Override
public Map<String, Long> getTopSizePartitions()
{
if (topPartitions == null)
@@ -3337,4 +3479,16 @@
return null;
return topPartitions.topTombstones().lastUpdate;
}
+
+ @Override
+ public OpOrder.Barrier newReadOrderingBarrier()
+ {
+ return readOrdering.newBarrier();
+ }
+
+ @Override
+ public TableMetrics getMetrics()
+ {
+ return metric;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index de7c933..2ae77b4 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -162,6 +162,15 @@
public List<String> getSSTablesForKey(String key, boolean hexFormat);
/**
+ * Returns a list of filenames that contain the given key and which level they belong to.
+ * Requires table to be compacted with {@link org.apache.cassandra.db.compaction.LeveledCompactionStrategy}
+ * @param key
+ * @param hexFormat
+ * @return list of filenames and levels containing the key
+ */
+ public Map<Integer, Set<String>> getSSTablesForKeyWithLevel(String key, boolean hexFormat);
+
+ /**
* Load new sstables from the given directory
*
* @param srcPaths the path to the new sstables - if it is an empty set, the data directories will be scanned
@@ -226,6 +235,17 @@
public long[] getPerLevelSizeBytes();
/**
+ * @return true if the table is using LeveledCompactionStrategy. false otherwise.
+ */
+ public boolean isLeveledCompaction();
+
+ /**
+ * @return sstable count for each bucket in TWCS. null unless time window compaction is used.
+ * array index corresponds to bucket(int[0] is for most recent, ...).
+ */
+ public int[] getSSTableCountPerTWCSBucket();
+
+ /**
* @return sstable fanout size for level compaction strategy.
*/
public int getLevelFanoutSize();
@@ -279,4 +299,23 @@
public Long getTopSizePartitionsLastUpdate();
public Map<String, Long> getTopTombstonePartitions();
public Long getTopTombstonePartitionsLastUpdate();
+
+ /**
+ * Returns the size of the biggest SSTable of this table.
+ *
+ * @return (physical) size of the biggest SSTable of this table on disk or 0 if no SSTable is present
+ */
+ public long getMaxSSTableSize();
+
+ /**
+ * Returns the longest duration of an SSTable, in milliseconds, of this table,
+ * computed as {@code maxTimestamp - minTimestamp}.
+ *
+ * It returns 0 if there are no SSTables or if {@code maxTimestamp} or {@code minTimestamp} is
+ * equal to {@code Long.MAX_VALUE}. Effectively non-zero for tables on {@code TimeWindowCompactionStrategy}.
+ *
+ * @return the biggest {@code maxTimestamp - minTimestamp} among all SSTables of this table
+ * or 0 if no SSTable is present
+ */
+ public long getMaxSSTableDuration();
}
diff --git a/src/java/org/apache/cassandra/db/ColumnIndex.java b/src/java/org/apache/cassandra/db/ColumnIndex.java
deleted file mode 100644
index f7860df..0000000
--- a/src/java/org/apache/cassandra/db/ColumnIndex.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.primitives.Ints;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.IndexInfo;
-import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-/**
- * Column index builder used by {@link org.apache.cassandra.io.sstable.format.big.BigTableWriter}.
- * For index entries that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size},
- * this uses the serialization logic as in {@link RowIndexEntry}.
- */
-public class ColumnIndex
-{
- // used, if the row-index-entry reaches config column_index_cache_size
- private DataOutputBuffer buffer;
- // used to track the size of the serialized size of row-index-entry (unused for buffer)
- private int indexSamplesSerializedSize;
- // used, until the row-index-entry reaches config column_index_cache_size
- private final List<IndexInfo> indexSamples = new ArrayList<>();
-
- private DataOutputBuffer reusableBuffer;
-
- public int columnIndexCount;
- private int[] indexOffsets;
-
- private final SerializationHelper helper;
- private final SerializationHeader header;
- private final int version;
- private final SequentialWriter writer;
- private long initialPosition;
- private final ISerializer<IndexInfo> idxSerializer;
- public long headerLength;
- private long startPosition;
-
- private int written;
- private long previousRowStart;
-
- private ClusteringPrefix<?> firstClustering;
- private ClusteringPrefix<?> lastClustering;
-
- private DeletionTime openMarker;
-
- private int cacheSizeThreshold;
-
- private final Collection<SSTableFlushObserver> observers;
-
- public ColumnIndex(SerializationHeader header,
- SequentialWriter writer,
- Version version,
- Collection<SSTableFlushObserver> observers,
- ISerializer<IndexInfo> indexInfoSerializer)
- {
- this.helper = new SerializationHelper(header);
- this.header = header;
- this.writer = writer;
- this.version = version.correspondingMessagingVersion();
- this.observers = observers;
- this.idxSerializer = indexInfoSerializer;
- }
-
- public void reset()
- {
- this.initialPosition = writer.position();
- this.headerLength = -1;
- this.startPosition = -1;
- this.previousRowStart = 0;
- this.columnIndexCount = 0;
- this.written = 0;
- this.indexSamplesSerializedSize = 0;
- this.indexSamples.clear();
- this.firstClustering = null;
- this.lastClustering = null;
- this.openMarker = null;
-
- int newCacheSizeThreshold = DatabaseDescriptor.getColumnIndexCacheSize();
- if (this.buffer != null && this.cacheSizeThreshold == newCacheSizeThreshold)
- this.reusableBuffer = this.buffer;
- this.buffer = null;
- this.cacheSizeThreshold = newCacheSizeThreshold;
- }
-
- public void buildRowIndex(UnfilteredRowIterator iterator) throws IOException
- {
- writePartitionHeader(iterator);
- this.headerLength = writer.position() - initialPosition;
-
- while (iterator.hasNext())
- {
- Unfiltered unfiltered = iterator.next();
- SSTableWriter.guardCollectionSize(iterator.metadata(), iterator.partitionKey(), unfiltered);
- add(unfiltered);
- }
-
- finish();
- }
-
- private void writePartitionHeader(UnfilteredRowIterator iterator) throws IOException
- {
- ByteBufferUtil.writeWithShortLength(iterator.partitionKey().getKey(), writer);
- DeletionTime.serializer.serialize(iterator.partitionLevelDeletion(), writer);
- if (header.hasStatic())
- {
- Row staticRow = iterator.staticRow();
-
- UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version);
- if (!observers.isEmpty())
- observers.forEach((o) -> o.nextUnfilteredCluster(staticRow));
- }
- }
-
- private long currentPosition()
- {
- return writer.position() - initialPosition;
- }
-
- public ByteBuffer buffer()
- {
- return buffer != null ? buffer.buffer() : null;
- }
-
- public List<IndexInfo> indexSamples()
- {
- if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= cacheSizeThreshold)
- {
- return indexSamples;
- }
-
- return null;
- }
-
- public int[] offsets()
- {
- return indexOffsets != null
- ? Arrays.copyOf(indexOffsets, columnIndexCount)
- : null;
- }
-
- private void addIndexBlock() throws IOException
- {
- IndexInfo cIndexInfo = new IndexInfo(firstClustering,
- lastClustering,
- startPosition,
- currentPosition() - startPosition,
- openMarker);
-
- // indexOffsets is used for both shallow (ShallowIndexedEntry) and non-shallow IndexedEntry.
- // For shallow ones, we need it to serialize the offsts in finish().
- // For non-shallow ones, the offsts are passed into IndexedEntry, so we don't have to
- // calculate the offsets again.
-
- // indexOffsets contains the offsets of the serialized IndexInfo objects.
- // I.e. indexOffsets[0] is always 0 so we don't have to deal with a special handling
- // for index #0 and always subtracting 1 for the index (which could be error-prone).
- if (indexOffsets == null)
- indexOffsets = new int[10];
- else
- {
- if (columnIndexCount >= indexOffsets.length)
- indexOffsets = Arrays.copyOf(indexOffsets, indexOffsets.length + 10);
-
- //the 0th element is always 0
- if (columnIndexCount == 0)
- {
- indexOffsets[columnIndexCount] = 0;
- }
- else
- {
- indexOffsets[columnIndexCount] =
- buffer != null
- ? Ints.checkedCast(buffer.position())
- : indexSamplesSerializedSize;
- }
- }
- columnIndexCount++;
-
- // First, we collect the IndexInfo objects until we reach Config.column_index_cache_size in an ArrayList.
- // When column_index_cache_size is reached, we switch to byte-buffer mode.
- if (buffer == null)
- {
- indexSamplesSerializedSize += idxSerializer.serializedSize(cIndexInfo);
- if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) > cacheSizeThreshold)
- {
- buffer = reuseOrAllocateBuffer();
- for (IndexInfo indexSample : indexSamples)
- {
- idxSerializer.serialize(indexSample, buffer);
- }
- }
- else
- {
- indexSamples.add(cIndexInfo);
- }
- }
- // don't put an else here since buffer may be allocated in preceding if block
- if (buffer != null)
- {
- idxSerializer.serialize(cIndexInfo, buffer);
- }
-
- firstClustering = null;
- }
-
- private DataOutputBuffer reuseOrAllocateBuffer()
- {
- // Check whether a reusable DataOutputBuffer already exists for this
- // ColumnIndex instance and return it.
- if (reusableBuffer != null) {
- DataOutputBuffer buffer = reusableBuffer;
- buffer.clear();
- return buffer;
- }
- // don't use the standard RECYCLER as that only recycles up to 1MB and requires proper cleanup
- return new DataOutputBuffer(cacheSizeThreshold * 2);
- }
-
- private void add(Unfiltered unfiltered) throws IOException
- {
- long pos = currentPosition();
-
- if (firstClustering == null)
- {
- // Beginning of an index block. Remember the start and position
- firstClustering = unfiltered.clustering();
- startPosition = pos;
- }
-
- UnfilteredSerializer.serializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version);
-
- // notify observers about each new row
- if (!observers.isEmpty())
- observers.forEach((o) -> o.nextUnfilteredCluster(unfiltered));
-
- lastClustering = unfiltered.clustering();
- previousRowStart = pos;
- ++written;
-
- if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
- {
- RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered;
- openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null;
- }
-
- // if we hit the column index size that we have to index after, go ahead and index it.
- if (currentPosition() - startPosition >= DatabaseDescriptor.getColumnIndexSize())
- addIndexBlock();
- }
-
- private void finish() throws IOException
- {
- UnfilteredSerializer.serializer.writeEndOfPartition(writer);
-
- // It's possible we add no rows, just a top level deletion
- if (written == 0)
- return;
-
- // the last column may have fallen on an index boundary already. if not, index it explicitly.
- if (firstClustering != null)
- addIndexBlock();
-
- // If we serialize the IndexInfo objects directly in the code above into 'buffer',
- // we have to write the offsts to these here. The offsets have already been are collected
- // in indexOffsets[]. buffer is != null, if it exceeds Config.column_index_cache_size.
- // In the other case, when buffer==null, the offsets are serialized in RowIndexEntry.IndexedEntry.serialize().
- if (buffer != null)
- RowIndexEntry.Serializer.serializeOffsets(buffer, indexOffsets, columnIndexCount);
-
- // we should always have at least one computed index block, but we only write it out if there is more than that.
- assert columnIndexCount > 0 && headerLength >= 0;
- }
-
- public int indexInfoSerializedSize()
- {
- return buffer != null
- ? buffer.buffer().limit()
- : indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0);
- }
-}
diff --git a/src/java/org/apache/cassandra/db/Columns.java b/src/java/org/apache/cassandra/db/Columns.java
index 88185ab..275d000 100644
--- a/src/java/org/apache/cassandra/db/Columns.java
+++ b/src/java/org/apache/cassandra/db/Columns.java
@@ -18,17 +18,15 @@
package org.apache.cassandra.db;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
-import java.nio.ByteBuffer;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterators;
import net.nicoulaj.compilecommand.annotations.DontInline;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.UTF8Type;
@@ -36,12 +34,14 @@
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
-import org.apache.cassandra.utils.btree.BTreeSearchIterator;
import org.apache.cassandra.utils.btree.BTreeRemoval;
+import org.apache.cassandra.utils.btree.BTreeSearchIterator;
/**
* An immutable and sorted list of (non-PK) columns for a given table.
@@ -61,7 +61,8 @@
ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
SetType.getInstance(UTF8Type.instance, true),
ColumnMetadata.NO_POSITION,
- ColumnMetadata.Kind.STATIC);
+ ColumnMetadata.Kind.STATIC,
+ null);
public static final ColumnMetadata FIRST_COMPLEX_REGULAR =
new ColumnMetadata("",
@@ -69,7 +70,8 @@
ColumnIdentifier.getInterned(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance),
SetType.getInstance(UTF8Type.instance, true),
ColumnMetadata.NO_POSITION,
- ColumnMetadata.Kind.REGULAR);
+ ColumnMetadata.Kind.REGULAR,
+ null);
private final Object[] columns;
private final int complexIdx; // Index of the first complex column
@@ -436,7 +438,7 @@
if(this == NONE)
return 0;
- return EMPTY_SIZE;
+ return EMPTY_SIZE + BTree.sizeOfStructureOnHeap(columns);
}
@Override
@@ -456,7 +458,7 @@
{
public void serialize(Columns columns, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(columns.size());
+ out.writeUnsignedVInt32(columns.size());
for (ColumnMetadata column : columns)
ByteBufferUtil.writeWithVIntLength(column.name.bytes, out);
}
@@ -471,7 +473,7 @@
public Columns deserialize(DataInputPlus in, TableMetadata metadata) throws IOException
{
- int length = (int)in.readUnsignedVInt();
+ int length = in.readUnsignedVInt32();
try (BTree.FastBuilder<ColumnMetadata> builder = BTree.fastBuilder())
{
for (int i = 0; i < length; i++)
@@ -516,7 +518,7 @@
int supersetCount = superset.size();
if (columnCount == supersetCount)
{
- out.writeUnsignedVInt(0);
+ out.writeUnsignedVInt32(0);
}
else if (supersetCount < 64)
{
@@ -609,7 +611,7 @@
private void serializeLargeSubset(Collection<ColumnMetadata> columns, int columnCount, Columns superset, int supersetCount, DataOutputPlus out) throws IOException
{
// write flag indicating we're in lengthy mode
- out.writeUnsignedVInt(supersetCount - columnCount);
+ out.writeUnsignedVInt32(supersetCount - columnCount);
BTreeSearchIterator<ColumnMetadata, ColumnMetadata> iter = superset.iterator();
if (columnCount < supersetCount / 2)
{
@@ -618,7 +620,7 @@
{
if (iter.next(column) == null)
throw new IllegalStateException();
- out.writeUnsignedVInt(iter.indexOfCurrent());
+ out.writeUnsignedVInt32(iter.indexOfCurrent());
}
}
else
@@ -631,10 +633,10 @@
throw new IllegalStateException();
int cur = iter.indexOfCurrent();
while (++prev != cur)
- out.writeUnsignedVInt(prev);
+ out.writeUnsignedVInt32(prev);
}
while (++prev != supersetCount)
- out.writeUnsignedVInt(prev);
+ out.writeUnsignedVInt32(prev);
}
}
@@ -650,7 +652,7 @@
{
for (int i = 0 ; i < columnCount ; i++)
{
- int idx = (int) in.readUnsignedVInt();
+ int idx = in.readUnsignedVInt32();
builder.add(BTree.findByIndex(superset.columns, idx));
}
}
@@ -661,7 +663,7 @@
int skipped = 0;
while (true)
{
- int nextMissingIndex = skipped < delta ? (int)in.readUnsignedVInt() : supersetCount;
+ int nextMissingIndex = skipped < delta ? in.readUnsignedVInt32() : supersetCount;
while (idx < nextMissingIndex)
{
ColumnMetadata def = iter.next();
diff --git a/src/java/org/apache/cassandra/db/DataRange.java b/src/java/org/apache/cassandra/db/DataRange.java
index 52162be..9912ac5 100644
--- a/src/java/org/apache/cassandra/db/DataRange.java
+++ b/src/java/org/apache/cassandra/db/DataRange.java
@@ -27,6 +27,7 @@
import org.apache.cassandra.dht.*;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
/**
* Groups both the range of partitions to query, and the clustering index filter to
@@ -139,6 +140,34 @@
}
/**
+ * The start of the partition key range queried by this {@code DataRange}.
+ *
+ * @return the start of the partition key range expressed as a ByteComparable.
+ */
+ public ByteComparable startAsByteComparable()
+ {
+ PartitionPosition bound = keyRange.left;
+ if (bound.isMinimum())
+ return null;
+
+ return bound.asComparableBound(keyRange.inclusiveLeft());
+ }
+
+ /**
+ * The end of the partition key range queried by this {@code DataRange}.
+ *
+ * @return the end of the partition key range expressed as a ByteComparable.
+ */
+ public ByteComparable stopAsByteComparable()
+ {
+ PartitionPosition bound = keyRange.right;
+ if (bound.isMinimum())
+ return null;
+
+ return bound.asComparableBound(!keyRange.inclusiveRight());
+ }
+
+ /**
* Whether the underlying clustering index filter is a names filter or not.
*
* @return Whether the underlying clustering index filter is a names filter or not.
diff --git a/src/java/org/apache/cassandra/db/DecoratedKey.java b/src/java/org/apache/cassandra/db/DecoratedKey.java
index 4dd87d0..03d6374 100644
--- a/src/java/org/apache/cassandra/db/DecoratedKey.java
+++ b/src/java/org/apache/cassandra/db/DecoratedKey.java
@@ -21,6 +21,7 @@
import java.util.Comparator;
import java.util.List;
import java.util.StringJoiner;
+import java.util.function.BiFunction;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.dht.IPartitioner;
@@ -29,8 +30,12 @@
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.MurmurHash;
import org.apache.cassandra.utils.IFilter.FilterKey;
+import org.apache.cassandra.utils.MurmurHash;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.apache.cassandra.utils.memory.HeapCloner;
/**
* Represents a decorated key, handy for certain operations
@@ -43,13 +48,7 @@
*/
public abstract class DecoratedKey implements PartitionPosition, FilterKey
{
- public static final Comparator<DecoratedKey> comparator = new Comparator<DecoratedKey>()
- {
- public int compare(DecoratedKey o1, DecoratedKey o2)
- {
- return o1.compareTo(o2);
- }
- };
+ public static final Comparator<DecoratedKey> comparator = DecoratedKey::compareTo;
private final Token token;
@@ -102,6 +101,37 @@
return cmp == 0 ? ByteBufferUtil.compareUnsigned(key, otherKey.getKey()) : cmp;
}
+ @Override
+ public ByteSource asComparableBytes(Version version)
+ {
+ // Note: In the legacy version one encoding could be a prefix of another as the escaping is only weakly
+ // prefix-free (see ByteSourceTest.testDecoratedKeyPrefixes()).
+ // The OSS50 version avoids this by adding a terminator.
+ return ByteSource.withTerminatorMaybeLegacy(version,
+ ByteSource.END_OF_STREAM,
+ token.asComparableBytes(version),
+ keyComparableBytes(version));
+ }
+
+ @Override
+ public ByteComparable asComparableBound(boolean before)
+ {
+ return version ->
+ {
+ assert (version != Version.LEGACY) : "Decorated key bounds are not supported by the legacy encoding.";
+
+ return ByteSource.withTerminator(
+ before ? ByteSource.LT_NEXT_COMPONENT : ByteSource.GT_NEXT_COMPONENT,
+ token.asComparableBytes(version),
+ keyComparableBytes(version));
+ };
+ }
+
+ protected ByteSource keyComparableBytes(Version version)
+ {
+ return ByteSource.of(getKey(), version);
+ }
+
public IPartitioner getPartitioner()
{
return getToken().getPartitioner();
@@ -163,10 +193,57 @@
}
public abstract ByteBuffer getKey();
+ public abstract int getKeyLength();
+
+ /**
+ * If this key occupies only part of a larger buffer, allocate a new buffer that is only as large as necessary.
+ * Otherwise, it returns this key.
+ */
+ public DecoratedKey retainable()
+ {
+ return ByteBufferUtil.canMinimize(getKey())
+ ? new BufferDecoratedKey(getToken(), HeapCloner.instance.clone(getKey()))
+ : this;
+ }
public void filterHash(long[] dest)
{
ByteBuffer key = getKey();
MurmurHash.hash3_x64_128(key, key.position(), key.remaining(), 0, dest);
}
-}
+
+ /**
+ * A template factory method for creating decorated keys from their byte-comparable representation.
+ */
+ static <T extends DecoratedKey> T fromByteComparable(ByteComparable byteComparable,
+ Version version,
+ IPartitioner partitioner,
+ BiFunction<Token, byte[], T> decoratedKeyFactory)
+ {
+ ByteSource.Peekable peekable = ByteSource.peekable(byteComparable.asComparableBytes(version));
+ // Decode the token from the first component of the multi-component sequence representing the whole decorated key.
+ Token token = partitioner.getTokenFactory().fromComparableBytes(ByteSourceInverse.nextComponentSource(peekable), version);
+ // Decode the key bytes from the second component.
+ byte[] keyBytes = ByteSourceInverse.getUnescapedBytes(ByteSourceInverse.nextComponentSource(peekable));
+ // Consume the terminator byte.
+ int terminator = peekable.next();
+ assert terminator == ByteSource.TERMINATOR : "Decorated key encoding must end in terminator.";
+ // Instantiate a decorated key from the decoded token and key bytes, using the provided factory method.
+ return decoratedKeyFactory.apply(token, keyBytes);
+ }
+
+ public static byte[] keyFromByteSource(ByteSource.Peekable peekableByteSource,
+ Version version,
+ IPartitioner partitioner)
+ {
+ assert version != Version.LEGACY; // reverse translation is not supported for LEGACY version.
+ // Decode the token from the first component of the multi-component sequence representing the whole decorated key.
+ // We won't use it, but the decoding also positions the byte source after it.
+ partitioner.getTokenFactory().fromComparableBytes(ByteSourceInverse.nextComponentSource(peekableByteSource), version);
+ // Decode the key bytes from the second component.
+ byte[] keyBytes = ByteSourceInverse.getUnescapedBytes(ByteSourceInverse.nextComponentSource(peekableByteSource));
+ int terminator = peekableByteSource.next();
+ assert terminator == ByteSource.TERMINATOR : "Decorated key encoding must end in terminator.";
+ return keyBytes;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/DeletionTime.java b/src/java/org/apache/cassandra/db/DeletionTime.java
index 105e10d..e6080f6 100644
--- a/src/java/org/apache/cassandra/db/DeletionTime.java
+++ b/src/java/org/apache/cassandra/db/DeletionTime.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db;
import java.io.IOException;
+import java.nio.ByteBuffer;
import com.google.common.base.Objects;
@@ -180,6 +181,15 @@
: new DeletionTime(mfda, ldt);
}
+ public DeletionTime deserialize(ByteBuffer buf, int offset)
+ {
+ int ldt = buf.getInt(offset);
+ long mfda = buf.getLong(offset + 4);
+ return mfda == Long.MIN_VALUE && ldt == Integer.MAX_VALUE
+ ? LIVE
+ : new DeletionTime(mfda, ldt);
+ }
+
public void skip(DataInputPlus in) throws IOException
{
in.skipBytesFully(4 + 8);
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index b16dd97..4e7347a 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -17,43 +17,60 @@
*/
package org.apache.cassandra.db;
-import java.time.Instant;
-import java.util.*;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.function.BiPredicate;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-import com.google.common.annotations.VisibleForTesting;
-
import java.io.IOError;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.BiPredicate;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.RateLimiter;
-
-import org.apache.cassandra.io.util.File;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.FSDiskFullWriteError;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSNoDiskAvailableForWriteError;
+import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableId;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileStoreUtils;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
@@ -272,7 +289,7 @@
if (file.isDirectory())
return false;
- Descriptor desc = SSTable.tryDescriptorFromFilename(file);
+ Descriptor desc = SSTable.tryDescriptorFromFile(file);
return desc != null && desc.ksname.equals(metadata.keyspace) && desc.cfname.equals(metadata.name);
});
for (File indexFile : indexFiles)
@@ -319,7 +336,7 @@
{
File file = new File(dir, filename);
if (file.exists())
- return Descriptor.fromFilename(file);
+ return Descriptor.fromFileWithComponent(file, false).left;
}
return null;
}
@@ -488,48 +505,106 @@
Collections.sort(candidates);
}
- public boolean hasAvailableDiskSpace(long estimatedSSTables, long expectedTotalWriteSize)
+ /**
+ * Sums up the space required for ongoing streams + compactions + expected new write size per FileStore and checks
+ * if there is enough space available.
+ *
+ * @param expectedNewWriteSizes where we expect to write the new compactions
+ * @param totalCompactionWriteRemaining approximate amount of data current compactions are writing - keyed by
+ * the file store they are writing to (or, reading from actually, but since
+ * CASSANDRA-6696 we expect compactions to read and written from the same dir)
+ * @return true if we expect to be able to write expectedNewWriteSizes to the available file stores
+ */
+ public boolean hasDiskSpaceForCompactionsAndStreams(Map<File, Long> expectedNewWriteSizes,
+ Map<File, Long> totalCompactionWriteRemaining)
{
- long writeSize = expectedTotalWriteSize / estimatedSSTables;
- long totalAvailable = 0L;
+ return hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, Directories::getFileStore);
+ }
- for (DataDirectory dataDir : paths)
- {
- if (DisallowedDirectories.isUnwritable(getLocationForDisk(dataDir)))
- continue;
- DataDirectoryCandidate candidate = new DataDirectoryCandidate(dataDir);
- // exclude directory if its total writeSize does not fit to data directory
- logger.debug("DataDirectory {} has {} bytes available, checking if we can write {} bytes", dataDir.location, candidate.availableSpace, writeSize);
- if (candidate.availableSpace < writeSize)
- {
- logger.warn("DataDirectory {} can't be used for compaction. Only {} is available, but {} is the minimum write size.",
- candidate.dataDirectory.location,
- FileUtils.stringifyFileSize(candidate.availableSpace),
- FileUtils.stringifyFileSize(writeSize));
- continue;
- }
- totalAvailable += candidate.availableSpace;
- }
+ @VisibleForTesting
+ public static boolean hasDiskSpaceForCompactionsAndStreams(Map<File, Long> expectedNewWriteSizes,
+ Map<File, Long> totalCompactionWriteRemaining,
+ Function<File, FileStore> filestoreMapper)
+ {
+ Map<FileStore, Long> newWriteSizesPerFileStore = perFileStore(expectedNewWriteSizes, filestoreMapper);
+ Map<FileStore, Long> compactionsRemainingPerFileStore = perFileStore(totalCompactionWriteRemaining, filestoreMapper);
- if (totalAvailable <= expectedTotalWriteSize)
+ Map<FileStore, Long> totalPerFileStore = new HashMap<>();
+ for (Map.Entry<FileStore, Long> entry : newWriteSizesPerFileStore.entrySet())
{
- StringJoiner pathString = new StringJoiner(",", "[", "]");
- for (DataDirectory p: paths)
- {
- pathString.add(p.location.toJavaIOFile().getAbsolutePath());
- }
- logger.warn("Insufficient disk space for compaction. Across {} there's only {} available, but {} is needed.",
- pathString.toString(),
- FileUtils.stringifyFileSize(totalAvailable),
- FileUtils.stringifyFileSize(expectedTotalWriteSize));
- return false;
+ long addedForFilestore = entry.getValue() + compactionsRemainingPerFileStore.getOrDefault(entry.getKey(), 0L);
+ totalPerFileStore.merge(entry.getKey(), addedForFilestore, Long::sum);
}
- return true;
+ return hasDiskSpaceForCompactionsAndStreams(totalPerFileStore);
+ }
+
+ /**
+ * Checks if there is enough space on all file stores to write the given amount of data.
+ * The data to write should be the total amount, ongoing writes + new writes.
+ */
+ public static boolean hasDiskSpaceForCompactionsAndStreams(Map<FileStore, Long> totalToWrite)
+ {
+ boolean hasSpace = true;
+ for (Map.Entry<FileStore, Long> toWrite : totalToWrite.entrySet())
+ {
+ long availableForCompaction = getAvailableSpaceForCompactions(toWrite.getKey());
+ logger.debug("FileStore {} has {} bytes available, checking if we can write {} bytes", toWrite.getKey(), availableForCompaction, toWrite.getValue());
+ if (availableForCompaction < toWrite.getValue())
+ {
+ logger.warn("FileStore {} has only {} available, but {} is needed",
+ toWrite.getKey(),
+ FileUtils.stringifyFileSize(availableForCompaction),
+ FileUtils.stringifyFileSize((long) toWrite.getValue()));
+ hasSpace = false;
+ }
+ }
+ return hasSpace;
+ }
+
+ public static long getAvailableSpaceForCompactions(FileStore fileStore)
+ {
+ long availableSpace = 0;
+ availableSpace = FileStoreUtils.tryGetSpace(fileStore, FileStore::getUsableSpace, e -> { throw new FSReadError(e, fileStore.name()); })
+ - DatabaseDescriptor.getMinFreeSpacePerDriveInBytes();
+ return Math.max(0L, Math.round(availableSpace * DatabaseDescriptor.getMaxSpaceForCompactionsPerDrive()));
+ }
+
+ public static Map<FileStore, Long> perFileStore(Map<File, Long> perDirectory, Function<File, FileStore> filestoreMapper)
+ {
+ return perDirectory.entrySet()
+ .stream()
+ .collect(Collectors.toMap(entry -> filestoreMapper.apply(entry.getKey()),
+ Map.Entry::getValue,
+ Long::sum));
+ }
+
+ public Set<FileStore> allFileStores(Function<File, FileStore> filestoreMapper)
+ {
+ return Arrays.stream(getWriteableLocations())
+ .map(this::getLocationForDisk)
+ .map(filestoreMapper)
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Gets the filestore for the actual directory where the sstables are stored.
+ * Handles the fact that an operator can symlink a table directory to a different filestore.
+ */
+ public static FileStore getFileStore(File directory)
+ {
+ try
+ {
+ return Files.getFileStore(directory.toPath());
+ }
+ catch (IOException e)
+ {
+ throw new FSReadError(e, directory);
+ }
}
public DataDirectory[] getWriteableLocations()
{
- List<DataDirectory> allowedDirs = new ArrayList<>();
+ List<DataDirectory> allowedDirs = new ArrayList<>(paths.length);
for (DataDirectory dir : paths)
{
if (!DisallowedDirectories.isUnwritable(dir.location))
@@ -593,17 +668,6 @@
return new File(snapshotDir, "schema.cql");
}
- public File getNewEphemeralSnapshotMarkerFile(String snapshotName)
- {
- File snapshotDir = new File(getWriteableLocationAsFile(1L), join(SNAPSHOT_SUBDIR, snapshotName));
- return getEphemeralSnapshotMarkerFile(snapshotDir);
- }
-
- private static File getEphemeralSnapshotMarkerFile(File snapshotDirectory)
- {
- return new File(snapshotDirectory, "ephemeral.snapshot");
- }
-
public static File getBackupsDirectory(Descriptor desc)
{
return getBackupsDirectory(desc.directory);
@@ -810,6 +874,16 @@
// last resort
return System.identityHashCode(this) - System.identityHashCode(o);
}
+
+ @Override
+ public String toString()
+ {
+ return "DataDirectoryCandidate{" +
+ "dataDirectory=" + dataDirectory +
+ ", availableSpace=" + availableSpace +
+ ", perc=" + perc +
+ '}';
+ }
}
/** The type of files that can be listed by SSTableLister, we never return txn logs,
@@ -928,7 +1002,7 @@
{
for (Component c : entry.getValue())
{
- l.add(new File(entry.getKey().filenameFor(c)));
+ l.add(entry.getKey().fileFor(c));
}
}
return l;
@@ -1016,18 +1090,25 @@
return snapshots;
}
- protected TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set<File> snapshotDirs) {
+ private TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set<File> snapshotDirs)
+ {
+ boolean ephemeral = manifest != null ? manifest.isEphemeral() : isLegacyEphemeralSnapshot(snapshotDirs);
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
return new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag, createdAt, expiresAt,
- snapshotDirs);
+ snapshotDirs, ephemeral);
+ }
+
+ private static boolean isLegacyEphemeralSnapshot(Set<File> snapshotDirs)
+ {
+ return snapshotDirs.stream().map(d -> new File(d, "ephemeral.snapshot")).anyMatch(File::exists);
}
@VisibleForTesting
protected static SnapshotManifest maybeLoadManifest(String keyspace, String table, String tag, Set<File> snapshotDirs)
{
List<File> manifests = snapshotDirs.stream().map(d -> new File(d, "manifest.json"))
- .filter(d -> d.exists()).collect(Collectors.toList());
+ .filter(File::exists).collect(Collectors.toList());
if (manifests.isEmpty())
{
@@ -1051,42 +1132,6 @@
return null;
}
- public List<String> listEphemeralSnapshots()
- {
- final List<String> ephemeralSnapshots = new LinkedList<>();
- for (File snapshot : listAllSnapshots())
- {
- if (getEphemeralSnapshotMarkerFile(snapshot).exists())
- ephemeralSnapshots.add(snapshot.name());
- }
- return ephemeralSnapshots;
- }
-
- private List<File> listAllSnapshots()
- {
- final List<File> snapshots = new LinkedList<>();
- for (final File dir : dataPaths)
- {
- File snapshotDir = isSecondaryIndexFolder(dir)
- ? new File(dir.parentPath(), SNAPSHOT_SUBDIR)
- : new File(dir, SNAPSHOT_SUBDIR);
- if (snapshotDir.exists() && snapshotDir.isDirectory())
- {
- final File[] snapshotDirs = snapshotDir.tryList();
- if (snapshotDirs != null)
- {
- for (final File snapshot : snapshotDirs)
- {
- if (snapshot.isDirectory())
- snapshots.add(snapshot);
- }
- }
- }
- }
-
- return snapshots;
- }
-
@VisibleForTesting
protected Map<String, Set<File>> listSnapshotDirsByTag()
{
@@ -1298,7 +1343,7 @@
public boolean isAcceptable(Path path)
{
File file = new File(path);
- Descriptor desc = SSTable.tryDescriptorFromFilename(file);
+ Descriptor desc = SSTable.tryDescriptorFromFile(file);
return desc != null
&& desc.ksname.equals(metadata.keyspace)
&& desc.cfname.equals(metadata.name)
diff --git a/src/java/org/apache/cassandra/db/DiskBoundaries.java b/src/java/org/apache/cassandra/db/DiskBoundaries.java
index f33b43e..32edcac 100644
--- a/src/java/org/apache/cassandra/db/DiskBoundaries.java
+++ b/src/java/org/apache/cassandra/db/DiskBoundaries.java
@@ -20,6 +20,7 @@
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@@ -154,4 +155,20 @@
assert pos < 0;
return -pos - 1;
}
+
+ public List<Directories.DataDirectory> getDisksInBounds(DecoratedKey first, DecoratedKey last)
+ {
+ if (positions == null || first == null || last == null)
+ return directories;
+ int firstIndex = getDiskIndex(first);
+ int lastIndex = getDiskIndex(last);
+ return directories.subList(firstIndex, lastIndex + 1);
+ }
+
+ public boolean isEquivalentTo(DiskBoundaries oldBoundaries)
+ {
+ return oldBoundaries != null &&
+ Objects.equals(positions, oldBoundaries.positions) &&
+ Objects.equals(directories, oldBoundaries.directories);
+ }
}
diff --git a/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java b/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
index 7f81b5c..f9207ce 100644
--- a/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
+++ b/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
@@ -25,6 +25,7 @@
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -32,13 +33,14 @@
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.utils.NoSpamLogger;
+import static org.apache.cassandra.config.CassandraRelevantProperties.EXPIRATION_DATE_OVERFLOW_POLICY;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class ExpirationDateOverflowHandling
{
private static final Logger logger = LoggerFactory.getLogger(ExpirationDateOverflowHandling.class);
- private static final int EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES = Integer.getInteger("cassandra.expiration_overflow_warning_interval_minutes", 5);
+ private static final int EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES = CassandraRelevantProperties.EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES.getInt();
public enum ExpirationDateOverflowPolicy
{
@@ -49,14 +51,13 @@
public static ExpirationDateOverflowPolicy policy;
static {
- String policyAsString = System.getProperty("cassandra.expiration_date_overflow_policy", ExpirationDateOverflowPolicy.REJECT.name());
try
{
- policy = ExpirationDateOverflowPolicy.valueOf(policyAsString.toUpperCase());
+ policy = EXPIRATION_DATE_OVERFLOW_POLICY.getEnum(ExpirationDateOverflowPolicy.REJECT);
}
catch (RuntimeException e)
{
- logger.warn("Invalid expiration date overflow policy: {}. Using default: {}", policyAsString, ExpirationDateOverflowPolicy.REJECT.name());
+ logger.warn("Invalid expiration date overflow policy. Using default: {}", ExpirationDateOverflowPolicy.REJECT.name());
policy = ExpirationDateOverflowPolicy.REJECT;
}
}
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index d6db700..22721bd 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -40,6 +40,7 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.lifecycle.SSTableSet;
@@ -87,9 +88,9 @@
{
private static final Logger logger = LoggerFactory.getLogger(Keyspace.class);
- private static final String TEST_FAIL_WRITES_KS = System.getProperty("cassandra.test.fail_writes_ks", "");
+ private static final String TEST_FAIL_WRITES_KS = CassandraRelevantProperties.TEST_FAIL_WRITES_KS.getString();
private static final boolean TEST_FAIL_WRITES = !TEST_FAIL_WRITES_KS.isEmpty();
- private static int TEST_FAIL_MV_LOCKS_COUNT = Integer.getInteger("cassandra.test.fail_mv_locks_count", 0);
+ private static int TEST_FAIL_MV_LOCKS_COUNT = CassandraRelevantProperties.TEST_FAIL_MV_LOCKS_COUNT.getInt();
public final KeyspaceMetrics metric;
@@ -303,20 +304,6 @@
}
/**
- * Clear all the snapshots for a given keyspace.
- *
- * @param snapshotName the user supplied snapshot name. It empty or null,
- * all the snapshots will be cleaned
- */
- public static void clearSnapshot(String snapshotName, String keyspace)
- {
- RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
-
- List<File> tableDirectories = Directories.getKSChildDirectories(keyspace);
- Directories.clearSnapshot(snapshotName, tableDirectories, clearSnapshotRateLimiter);
- }
-
- /**
* @return A list of open SSTableReaders
*/
public List<SSTableReader> getAllSSTables(SSTableSet sstableSet)
diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java
index 7b6a686..ad43b16 100644
--- a/src/java/org/apache/cassandra/db/Mutation.java
+++ b/src/java/org/apache/cassandra/db/Mutation.java
@@ -23,26 +23,32 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.DeserializationHelper;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.TeeDataInputPlus;
+import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.AbstractWriteResponseHandler;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.concurrent.Future;
-import static org.apache.cassandra.net.MessagingService.VERSION_30;
-import static org.apache.cassandra.net.MessagingService.VERSION_3014;
-import static org.apache.cassandra.net.MessagingService.VERSION_40;
+import static org.apache.cassandra.net.MessagingService.*;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
public class Mutation implements IMutation, Supplier<Mutation>
@@ -64,6 +70,15 @@
private final boolean cdcEnabled;
+ private static final int SERIALIZATION_VERSION_COUNT = MessagingService.Version.values().length;
+ // Contains serialized representations of this mutation.
+ // Note: there is no functionality to clear/remove serialized instances, because a mutation must never
+ // be modified (e.g. calling add(PartitionUpdate)) when it's being serialized.
+ private final Serialization[] cachedSerializations = new Serialization[SERIALIZATION_VERSION_COUNT];
+
+ /** @see CassandraRelevantProperties#CACHEABLE_MUTATION_SIZE_LIMIT */
+ private static final long CACHEABLE_MUTATION_SIZE_LIMIT = CassandraRelevantProperties.CACHEABLE_MUTATION_SIZE_LIMIT.getLong();
+
public Mutation(PartitionUpdate update)
{
this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc);
@@ -298,6 +313,7 @@
}
return buff.append("])").toString();
}
+
private int serializedSize30;
private int serializedSize3014;
private int serializedSize40;
@@ -393,34 +409,115 @@
{
public void serialize(Mutation mutation, DataOutputPlus out, int version) throws IOException
{
+ serialization(mutation, version).serialize(PartitionUpdate.serializer, mutation, out, version);
+ }
+
+ /**
+ * Called early during request processing to prevent that {@link #serialization(Mutation, int)} is
+ * called concurrently.
+ * See {@link org.apache.cassandra.service.StorageProxy#sendToHintedReplicas(Mutation, ReplicaPlan.ForWrite, AbstractWriteResponseHandler, String, Stage)}
+ */
+ @SuppressWarnings("JavadocReference")
+ public void prepareSerializedBuffer(Mutation mutation, int version)
+ {
+ serialization(mutation, version);
+ }
+
+ /**
+ * Retrieve the cached serialization of this mutation, or compute and cache said serialization if it doesn't
+ * exist yet. Note that this method is _not_ synchronized even though it may (and will often) be called
+ * concurrently. Concurrent calls are still safe however, the only risk is that the value is not cached yet,
+ * multiple concurrent calls may compute it multiple times instead of just once. This is ok as in practice
+ * as we make sure this doesn't happen in the hot path by forcing the initial caching in
+ * {@link org.apache.cassandra.service.StorageProxy#sendToHintedReplicas(Mutation, ReplicaPlan.ForWrite, AbstractWriteResponseHandler, String, Stage)}
+ * via {@link #prepareSerializedBuffer(Mutation)}, which is the only caller that passes
+ * {@code isPrepare==true}.
+ */
+ @SuppressWarnings("JavadocReference")
+ private Serialization serialization(Mutation mutation, int version)
+ {
+ int versionOrdinal = MessagingService.getVersionOrdinal(version);
+ // Retrieves the cached version, or build+cache it if it's not cached already.
+ Serialization serialization = mutation.cachedSerializations[versionOrdinal];
+ if (serialization == null)
+ {
+ serialization = new SizeOnlyCacheableSerialization();
+ long serializedSize = serialization.serializedSize(PartitionUpdate.serializer, mutation, version);
+
+ // Excessively large mutation objects cause GC pressure and huge allocations when serialized.
+ // so we only cache serialized mutations when they are below the defined limit.
+ if (serializedSize < CACHEABLE_MUTATION_SIZE_LIMIT)
+ {
+ try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
+ {
+ serializeInternal(PartitionUpdate.serializer, mutation, dob, version);
+ serialization = new CachedSerialization(dob.toByteArray());
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ mutation.cachedSerializations[versionOrdinal] = serialization;
+ }
+
+ return serialization;
+ }
+
+ static void serializeInternal(PartitionUpdate.PartitionUpdateSerializer serializer,
+ Mutation mutation,
+ DataOutputPlus out,
+ int version) throws IOException
+ {
+ Map<TableId, PartitionUpdate> modifications = mutation.modifications;
+
/* serialize the modifications in the mutation */
- int size = mutation.modifications.size();
- out.writeUnsignedVInt(size);
+ int size = modifications.size();
+ out.writeUnsignedVInt32(size);
assert size > 0;
- for (Map.Entry<TableId, PartitionUpdate> entry : mutation.modifications.entrySet())
- PartitionUpdate.serializer.serialize(entry.getValue(), out, version);
+ for (PartitionUpdate partitionUpdate : modifications.values())
+ {
+ serializer.serialize(partitionUpdate, out, version);
+ }
}
public Mutation deserialize(DataInputPlus in, int version, DeserializationHelper.Flag flag) throws IOException
{
- int size = (int)in.readUnsignedVInt();
- assert size > 0;
-
- PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, flag);
- if (size == 1)
- return new Mutation(update);
-
- ImmutableMap.Builder<TableId, PartitionUpdate> modifications = new ImmutableMap.Builder<>();
- DecoratedKey dk = update.partitionKey();
-
- modifications.put(update.metadata().id, update);
- for (int i = 1; i < size; ++i)
+ Mutation m;
+ TeeDataInputPlus teeIn;
+ try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
{
- update = PartitionUpdate.serializer.deserialize(in, version, flag);
- modifications.put(update.metadata().id, update);
+ teeIn = new TeeDataInputPlus(in, dob, CACHEABLE_MUTATION_SIZE_LIMIT);
+
+ int size = teeIn.readUnsignedVInt32();
+ assert size > 0;
+
+ PartitionUpdate update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
+ if (size == 1)
+ {
+ m = new Mutation(update);
+ }
+ else
+ {
+ ImmutableMap.Builder<TableId, PartitionUpdate> modifications = new ImmutableMap.Builder<>();
+ DecoratedKey dk = update.partitionKey();
+
+ modifications.put(update.metadata().id, update);
+ for (int i = 1; i < size; ++i)
+ {
+ update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
+ modifications.put(update.metadata().id, update);
+ }
+ m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now());
+ }
+
+ //Only cache serializations that don't hit the limit
+ if (!teeIn.isLimitReached())
+ m.cachedSerializations[MessagingService.getVersionOrdinal(version)] = new CachedSerialization(dob.toByteArray());
+
+ return m;
}
- return new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now());
}
public Mutation deserialize(DataInputPlus in, int version) throws IOException
@@ -430,10 +527,71 @@
public long serializedSize(Mutation mutation, int version)
{
- int size = TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
- for (Map.Entry<TableId, PartitionUpdate> entry : mutation.modifications.entrySet())
- size += PartitionUpdate.serializer.serializedSize(entry.getValue(), version);
+ return serialization(mutation, version).serializedSize(PartitionUpdate.serializer, mutation, version);
+ }
+ }
+ /**
+ * There are two implementations of this class. One that keeps the serialized representation on-heap for later
+ * reuse and one that doesn't. Keeping all sized mutations around may lead to "bad" GC pressure (G1 GC) due to humongous objects.
+ * By default serialized mutations up to 2MB are kept on-heap - see {@link org.apache.cassandra.config.CassandraRelevantProperties#CACHEABLE_MUTATION_SIZE_LIMIT}.
+ */
+ private static abstract class Serialization
+ {
+ abstract void serialize(PartitionUpdate.PartitionUpdateSerializer serializer, Mutation mutation, DataOutputPlus out, int version) throws IOException;
+
+ abstract long serializedSize(PartitionUpdate.PartitionUpdateSerializer serializer, Mutation mutation, int version);
+ }
+
+ /**
+ * Represents the cached serialization of a {@link Mutation} as a {@code byte[]}.
+ */
+ private static final class CachedSerialization extends Serialization
+ {
+ private final byte[] serialized;
+
+ CachedSerialization(byte[] serialized)
+ {
+ this.serialized = Preconditions.checkNotNull(serialized);
+ }
+
+ @Override
+ void serialize(PartitionUpdate.PartitionUpdateSerializer serializer, Mutation mutation, DataOutputPlus out, int version) throws IOException
+ {
+ out.write(serialized);
+ }
+
+ @Override
+ long serializedSize(PartitionUpdate.PartitionUpdateSerializer serializer, Mutation mutation, int version)
+ {
+ return serialized.length;
+ }
+ }
+
+ /**
+ * Represents a non-cacheable serialization of a {@link Mutation}, only the size of the mutation is lazily cached.
+ */
+ private static final class SizeOnlyCacheableSerialization extends Serialization
+ {
+ private volatile long size;
+
+ @Override
+ void serialize(PartitionUpdate.PartitionUpdateSerializer serializer, Mutation mutation, DataOutputPlus out, int version) throws IOException
+ {
+ MutationSerializer.serializeInternal(serializer, mutation, out, version);
+ }
+
+ @Override
+ long serializedSize(PartitionUpdate.PartitionUpdateSerializer serializer, Mutation mutation, int version)
+ {
+ long size = this.size;
+ if (size == 0L)
+ {
+ size = TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
+ for (PartitionUpdate partitionUpdate : mutation.modifications.values())
+ size += serializer.serializedSize(partitionUpdate, version);
+ this.size = size;
+ }
return size;
}
}
diff --git a/src/java/org/apache/cassandra/db/NativeClustering.java b/src/java/org/apache/cassandra/db/NativeClustering.java
index 0e4c19d..1b6761d 100644
--- a/src/java/org/apache/cassandra/db/NativeClustering.java
+++ b/src/java/org/apache/cassandra/db/NativeClustering.java
@@ -25,6 +25,7 @@
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.HeapCloner;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
@@ -36,11 +37,6 @@
private NativeClustering() { peer = 0; }
- public ClusteringPrefix<ByteBuffer> minimize()
- {
- return this;
- }
-
public NativeClustering(NativeAllocator allocator, OpOrder.Group writeOp, Clustering<?> clustering)
{
int count = clustering.size();
@@ -157,4 +153,20 @@
{
return ClusteringPrefix.equals(this, o);
}
+
+ @Override
+ public ClusteringPrefix<ByteBuffer> retainable()
+ {
+ assert kind() == Kind.CLUSTERING; // tombstones are never stored natively
+
+ // always extract
+ ByteBuffer[] values = new ByteBuffer[size()];
+ for (int i = 0; i < values.length; ++i)
+ {
+ ByteBuffer value = get(i);
+ values[i] = value != null ? HeapCloner.instance.clone(value) : null;
+ }
+
+ return accessor().factory().clustering(values);
+ }
}
diff --git a/src/java/org/apache/cassandra/db/NativeDecoratedKey.java b/src/java/org/apache/cassandra/db/NativeDecoratedKey.java
index add5218..bc14908 100644
--- a/src/java/org/apache/cassandra/db/NativeDecoratedKey.java
+++ b/src/java/org/apache/cassandra/db/NativeDecoratedKey.java
@@ -20,7 +20,9 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import net.nicoulaj.compilecommand.annotations.Inline;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
@@ -41,8 +43,44 @@
MemoryUtil.setBytes(peer + 4, key);
}
+ public NativeDecoratedKey(Token token, NativeAllocator allocator, OpOrder.Group writeOp, byte[] keyBytes)
+ {
+ super(token);
+ assert keyBytes != null;
+
+ int size = keyBytes.length;
+ this.peer = allocator.allocate(4 + size, writeOp);
+ MemoryUtil.setInt(peer, size);
+ MemoryUtil.setBytes(peer + 4, keyBytes, 0, size);
+ }
+
+ @Inline
+ int length()
+ {
+ return MemoryUtil.getInt(peer);
+ }
+
+ @Inline
+ long address()
+ {
+ return this.peer + 4;
+ }
+
+ @Override
public ByteBuffer getKey()
{
- return MemoryUtil.getByteBuffer(peer + 4, MemoryUtil.getInt(peer), ByteOrder.BIG_ENDIAN);
+ return MemoryUtil.getByteBuffer(address(), length(), ByteOrder.BIG_ENDIAN);
+ }
+
+ @Override
+ public int getKeyLength()
+ {
+ return MemoryUtil.getInt(peer);
+ }
+
+ @Override
+ protected ByteSource keyComparableBytes(Version version)
+ {
+ return ByteSource.ofMemory(address(), length(), version);
}
}
diff --git a/src/java/org/apache/cassandra/db/PartitionPosition.java b/src/java/org/apache/cassandra/db/PartitionPosition.java
index 3b45c6c..5e1d618 100644
--- a/src/java/org/apache/cassandra/db/PartitionPosition.java
+++ b/src/java/org/apache/cassandra/db/PartitionPosition.java
@@ -24,8 +24,10 @@
import org.apache.cassandra.dht.*;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
-public interface PartitionPosition extends RingPosition<PartitionPosition>
+public interface PartitionPosition extends RingPosition<PartitionPosition>, ByteComparable
{
public static enum Kind
{
@@ -54,6 +56,27 @@
public Kind kind();
public boolean isMinimum();
+ /**
+ * Produce a prefix-free byte-comparable representation of the key, i.e. such a sequence of bytes that any pair x, y
+ * of valid positions (with the same key column types and partitioner),
+ * x.compareTo(y) == compareLexicographicallyUnsigned(x.asComparableBytes(), y.asComparableBytes())
+ * and
+ * x.asComparableBytes() is not a prefix of y.asComparableBytes()
+ *
+ * We use a two-component tuple for decorated keys, and a one-component tuple for key bounds, where the terminator
+ * byte is chosen to yield the correct comparison result. No decorated key can be a prefix of another (per the tuple
+ * encoding), and no key bound can be a prefix of one because it uses a terminator byte that is different from the
+ * tuple separator.
+ */
+ public abstract ByteSource asComparableBytes(Version version);
+
+ /**
+ * Produce a byte-comparable representation for the position before or after the key.
+ * This does nothing for token boundaries (which are already at a position between valid keys), and changes
+ * the terminator byte for keys.
+ */
+ public abstract ByteComparable asComparableBound(boolean before);
+
public static class RowPositionSerializer implements IPartitionerDependentSerializer<PartitionPosition>
{
/*
diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
index f000d63..164e03c 100644
--- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
@@ -18,12 +18,11 @@
package org.apache.cassandra.db;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
-import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
@@ -36,14 +35,17 @@
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
import org.apache.cassandra.db.rows.BaseRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.transform.RTBoundValidator;
import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.metrics.TableMetrics;
@@ -62,6 +64,7 @@
protected static final SelectionDeserializer selectionDeserializer = new Deserializer();
protected final DataRange dataRange;
+ protected final Slices requestedSlices;
private PartitionRangeReadCommand(boolean isDigest,
int digestVersion,
@@ -77,6 +80,8 @@
{
super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index, trackWarnings);
this.dataRange = dataRange;
+ this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata());
+
}
private static PartitionRangeReadCommand create(boolean isDigest,
@@ -311,6 +316,7 @@
}
@VisibleForTesting
+ @SuppressWarnings("resource")
public UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadExecutionController controller)
{
ColumnFamilyStore.ViewFragment view = cfs.select(View.selectLive(dataRange().keyRange()));
@@ -329,20 +335,43 @@
inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
}
+ int selectedSSTablesCnt = 0;
for (SSTableReader sstable : view.sstables)
{
+ boolean intersects = intersects(sstable);
+ boolean hasPartitionLevelDeletions = hasPartitionLevelDeletions(sstable);
+ boolean hasRequiredStatics = hasRequiredStatics(sstable);
+
+ if (!intersects && !hasPartitionLevelDeletions && !hasRequiredStatics)
+ continue;
+
@SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
UnfilteredPartitionIterator iter = sstable.partitionIterator(columnFilter(), dataRange(), readCountUpdater);
inputCollector.addSSTableIterator(sstable, RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false));
if (!sstable.isRepaired())
controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime());
+
+ selectedSSTablesCnt++;
}
+
+ final int finalSelectedSSTables = selectedSSTablesCnt;
+
// iterators can be empty for offline tools
if (inputCollector.isEmpty())
return EmptyIterators.unfilteredPartition(metadata());
- return checkCacheFilter(UnfilteredPartitionIterators.mergeLazily(inputCollector.finalizeIterators(cfs, nowInSec(), controller.oldestUnrepairedTombstone())), cfs);
+ List<UnfilteredPartitionIterator> finalizedIterators = inputCollector.finalizeIterators(cfs, nowInSec(), controller.oldestUnrepairedTombstone());
+ UnfilteredPartitionIterator merged = UnfilteredPartitionIterators.mergeLazily(finalizedIterators);
+ return checkCacheFilter(Transformation.apply(merged, new Transformation<UnfilteredRowIterator>()
+ {
+ @Override
+ protected void onClose()
+ {
+ super.onClose();
+ cfs.metric.updateSSTableIteratedInRangeRead(finalSelectedSSTables);
+ }
+ }), cfs);
}
catch (RuntimeException | Error e)
{
@@ -358,6 +387,12 @@
}
}
+ @Override
+ protected boolean intersects(SSTableReader sstable)
+ {
+ return requestedSlices.intersects(sstable.getSSTableMetadata().coveredClustering);
+ }
+
/**
* Creates a new {@code SSTableReadsListener} to update the SSTables read counts.
* @return a new {@code SSTableReadsListener} to update the SSTables read counts.
@@ -545,4 +580,4 @@
return executionController();
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index 358d408..9cfcb3a 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -36,6 +36,7 @@
import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.net.MessageFlag;
import org.apache.cassandra.net.ParamType;
import org.apache.cassandra.net.Verb;
@@ -84,7 +85,7 @@
*/
public abstract class ReadCommand extends AbstractReadQuery
{
- private static final int TEST_ITERATION_DELAY_MILLIS = Integer.parseInt(System.getProperty("cassandra.test.read_iteration_delay_ms", "0"));
+ private static final int TEST_ITERATION_DELAY_MILLIS = CassandraRelevantProperties.TEST_READ_ITERATION_DELAY_MS.getInt();
protected static final Logger logger = LoggerFactory.getLogger(ReadCommand.class);
public static final IVersionedSerializer<ReadCommand> serializer = new Serializer();
@@ -422,7 +423,8 @@
try
{
iterator = withQuerySizeTracking(iterator);
- iterator = withStateTracking(iterator);
+ iterator = maybeSlowDownForTesting(iterator);
+ iterator = withQueryCancellation(iterator);
iterator = RTBoundValidator.validate(withoutPurgeableTombstones(iterator, cfs, executionController), Stage.PURGED, false);
iterator = withMetricsRecording(iterator, cfs.metric, startTimeNanos);
@@ -499,7 +501,9 @@
private final boolean enforceStrictLiveness = metadata().enforceStrictLiveness();
private int liveRows = 0;
+ private int lastReportedLiveRows = 0;
private int tombstones = 0;
+ private int lastReportedTombstones = 0;
private DecoratedKey currentKey;
@@ -567,6 +571,22 @@
}
@Override
+ protected void onPartitionClose()
+ {
+ int lr = liveRows - lastReportedLiveRows;
+ int ts = tombstones - lastReportedTombstones;
+
+ if (lr > 0)
+ metric.topReadPartitionRowCount.addSample(currentKey.getKey(), lr);
+
+ if (ts > 0)
+ metric.topReadPartitionTombstoneCount.addSample(currentKey.getKey(), ts);
+
+ lastReportedLiveRows = liveRows;
+ lastReportedTombstones = tombstones;
+ }
+
+ @Override
public void onClose()
{
recordLatency(metric, nanoTime() - startTimeNanos);
@@ -601,58 +621,6 @@
return Transformation.apply(iter, new MetricRecording());
}
- protected class CheckForAbort extends StoppingTransformation<UnfilteredRowIterator>
- {
- long lastChecked = 0;
-
- protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
- {
- if (maybeAbort())
- {
- partition.close();
- return null;
- }
-
- return Transformation.apply(partition, this);
- }
-
- protected Row applyToRow(Row row)
- {
- if (TEST_ITERATION_DELAY_MILLIS > 0)
- maybeDelayForTesting();
-
- return maybeAbort() ? null : row;
- }
-
- private boolean maybeAbort()
- {
- /**
- * TODO: this is not a great way to abort early; why not expressly limit checks to 10ms intervals?
- * The value returned by approxTime.now() is updated only every
- * {@link org.apache.cassandra.utils.MonotonicClock.SampledClock.CHECK_INTERVAL_MS}, by default 2 millis. Since MonitorableImpl
- * relies on approxTime, we don't need to check unless the approximate time has elapsed.
- */
- if (lastChecked == approxTime.now())
- return false;
-
- lastChecked = approxTime.now();
-
- if (isAborted())
- {
- stop();
- return true;
- }
-
- return false;
- }
-
- private void maybeDelayForTesting()
- {
- if (!metadata().keyspace.startsWith("system"))
- FBUtilities.sleepQuietly(TEST_ITERATION_DELAY_MILLIS);
- }
- }
-
private boolean shouldTrackSize(DataStorageSpec.LongBytesBound warnThresholdBytes, DataStorageSpec.LongBytesBound abortThresholdBytes)
{
return trackWarnings
@@ -737,9 +705,74 @@
return iterator;
}
- protected UnfilteredPartitionIterator withStateTracking(UnfilteredPartitionIterator iter)
+ private class QueryCancellationChecker extends StoppingTransformation<UnfilteredRowIterator>
{
- return Transformation.apply(iter, new CheckForAbort());
+ long lastCheckedAt = 0;
+
+ @Override
+ protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ maybeCancel();
+ return Transformation.apply(partition, this);
+ }
+
+ @Override
+ protected Row applyToRow(Row row)
+ {
+ maybeCancel();
+ return row;
+ }
+
+ private void maybeCancel()
+ {
+ /*
+ * The value returned by approxTime.now() is updated only every
+ * {@link org.apache.cassandra.utils.MonotonicClock.SampledClock.CHECK_INTERVAL_MS}, by default 2 millis.
+ * Since MonitorableImpl relies on approxTime, we don't need to check unless the approximate time has elapsed.
+ */
+ if (lastCheckedAt == approxTime.now())
+ return;
+ lastCheckedAt = approxTime.now();
+
+ if (isAborted())
+ {
+ stop();
+ throw new QueryCancelledException(ReadCommand.this);
+ }
+ }
+ }
+
+ private UnfilteredPartitionIterator withQueryCancellation(UnfilteredPartitionIterator iter)
+ {
+ return Transformation.apply(iter, new QueryCancellationChecker());
+ }
+
+ /**
+ * A transformation used for simulating slow queries by tests.
+ */
+ private static class DelayInjector extends Transformation<UnfilteredRowIterator>
+ {
+ @Override
+ protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ FBUtilities.sleepQuietly(TEST_ITERATION_DELAY_MILLIS);
+ return Transformation.apply(partition, this);
+ }
+
+ @Override
+ protected Row applyToRow(Row row)
+ {
+ FBUtilities.sleepQuietly(TEST_ITERATION_DELAY_MILLIS);
+ return row;
+ }
+ }
+
+ private UnfilteredPartitionIterator maybeSlowDownForTesting(UnfilteredPartitionIterator iter)
+ {
+ if (TEST_ITERATION_DELAY_MILLIS > 0 && !SchemaConstants.isSystemKeyspace(metadata().keyspace))
+ return Transformation.apply(iter, new DelayInjector());
+ else
+ return iter;
}
/**
@@ -755,6 +788,19 @@
return msg;
}
+ protected abstract boolean intersects(SSTableReader sstable);
+
+ protected boolean hasRequiredStatics(SSTableReader sstable) {
+ // If some static columns are queried, we should always include the sstable: the clustering values stats of the sstable
+ // don't tell us if the sstable contains static values in particular.
+ return !columnFilter().fetchedColumns().statics.isEmpty() && sstable.header.hasStatic();
+ }
+
+ protected boolean hasPartitionLevelDeletions(SSTableReader sstable)
+ {
+ return sstable.getSSTableMetadata().hasPartitionLevelDeletions;
+ }
+
public abstract Verb verb();
protected abstract void appendCQLWhereClause(StringBuilder sb);
@@ -1022,7 +1068,7 @@
| acceptsTransientFlag(command.acceptsTransient())
);
if (command.isDigestQuery())
- out.writeUnsignedVInt(command.digestVersion());
+ out.writeUnsignedVInt32(command.digestVersion());
command.metadata().id.serialize(out);
out.writeInt(command.nowInSec());
ColumnFilter.serializer.serialize(command.columnFilter(), out, version);
@@ -1049,7 +1095,7 @@
+ "upgrading to 4.0");
boolean hasIndex = hasIndex(flags);
- int digestVersion = isDigest ? (int)in.readUnsignedVInt() : 0;
+ int digestVersion = isDigest ? in.readUnsignedVInt32() : 0;
TableMetadata metadata = schema.getExistingTableMetadata(TableId.deserialize(in));
int nowInSec = in.readInt();
ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata);
@@ -1090,4 +1136,4 @@
+ command.indexSerializedSize(version);
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
index 9226568..f693bbc 100644
--- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
@@ -24,6 +24,7 @@
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
@@ -77,18 +78,29 @@
MessagingService.instance().send(reply, message.from());
return;
}
+ catch (AssertionError t)
+ {
+ throw new AssertionError(String.format("Caught an error while trying to process the command: %s", command.toCQLString()), t);
+ }
+ catch (QueryCancelledException e)
+ {
+ logger.debug("Query cancelled (timeout)", e);
+ response = null;
+ assert !command.isCompleted() : "Read marked as completed despite being aborted by timeout to table " + command.metadata();
+ }
- if (!command.complete())
+ if (command.complete())
+ {
+ Tracing.trace("Enqueuing response to {}", message.from());
+ Message<ReadResponse> reply = message.responseWith(response);
+ reply = MessageParams.addToMessage(reply);
+ MessagingService.instance().send(reply, message.from());
+ }
+ else
{
Tracing.trace("Discarding partial response to {} (timed out)", message.from());
MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
- return;
}
-
- Tracing.trace("Enqueuing response to {}", message.from());
- Message<ReadResponse> reply = message.responseWith(response);
- reply = MessageParams.addToMessage(reply);
- MessagingService.instance().send(reply, message.from());
}
private void validateTransientStatus(Message<ReadCommand> message)
diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/db/RowIndexEntry.java
deleted file mode 100644
index 80f53a9..0000000
--- a/src/java/org/apache/cassandra/db/RowIndexEntry.java
+++ /dev/null
@@ -1,857 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import com.codahale.metrics.Histogram;
-import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.config.DataStorageSpec;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.filter.RowIndexEntryReadSizeTooLargeException;
-import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.IndexInfo;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.TrackedDataInputPlus;
-import org.apache.cassandra.metrics.DefaultNameFactory;
-import org.apache.cassandra.metrics.MetricNameFactory;
-import org.apache.cassandra.net.ParamType;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.vint.VIntCoding;
-import org.github.jamm.Unmetered;
-
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-
-/**
- * Binary format of {@code RowIndexEntry} is defined as follows:
- * {@code
- * (long) position (64 bit long, vint encoded)
- * (int) serialized size of data that follows (32 bit int, vint encoded)
- * -- following for indexed entries only (so serialized size > 0)
- * (int) DeletionTime.localDeletionTime
- * (long) DeletionTime.markedForDeletionAt
- * (int) number of IndexInfo objects (32 bit int, vint encoded)
- * (*) serialized IndexInfo objects, see below
- * (*) offsets of serialized IndexInfo objects, since version "ma" (3.0)
- * Each IndexInfo object's offset is relative to the first IndexInfo object.
- * }
- * <p>
- * See {@link IndexInfo} for a description of the serialized format.
- * </p>
- *
- * <p>
- * For each partition, the layout of the index file looks like this:
- * </p>
- * <ol>
- * <li>partition key - prefixed with {@code short} length</li>
- * <li>serialized {@code RowIndexEntry} objects</li>
- * </ol>
- *
- * <p>
- * Generally, we distinguish between index entries that have <i>index
- * samples</i> (list of {@link IndexInfo} objects) and those who don't.
- * For each <i>portion</i> of data for a single partition in the data file,
- * an index sample is created. The size of that <i>portion</i> is defined
- * by {@link org.apache.cassandra.config.Config#column_index_size}.
- * </p>
- * <p>
- * Index entries with less than 2 index samples, will just store the
- * position in the data file.
- * </p>
- * <p>
- * Note: legacy sstables for index entries are those sstable formats that
- * do <i>not</i> have an offsets table to index samples ({@link IndexInfo}
- * objects). These are those sstables created on Cassandra versions
- * earlier than 3.0.
- * </p>
- * <p>
- * For index entries with index samples we store the index samples
- * ({@link IndexInfo} objects). The bigger the partition, the more
- * index samples are created. Since a huge amount of index samples
- * will "pollute" the heap and cause huge GC pressure, Cassandra 3.6
- * (CASSANDRA-11206) distinguishes between index entries with an
- * "acceptable" amount of index samples per partition and those
- * with an "enormous" amount of index samples. The barrier
- * is controlled by the configuration parameter
- * {@link org.apache.cassandra.config.Config#column_index_cache_size}.
- * Index entries with a total serialized size of index samples up to
- * {@code column_index_cache_size} will be held in an array.
- * Index entries exceeding that value will always be accessed from
- * disk.
- * </p>
- * <p>
- * This results in these classes:
- * </p>
- * <ul>
- * <li>{@link RowIndexEntry} just stores the offset in the data file.</li>
- * <li>{@link IndexedEntry} is for index entries with index samples
- * and used for both current and legacy sstables, which do not exceed
- * {@link org.apache.cassandra.config.Config#column_index_cache_size}.</li>
- * <li>{@link ShallowIndexedEntry} is for index entries with index samples
- * that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size}
- * for sstables with an offset table to the index samples.</li>
- * </ul>
- * <p>
- * Since access to index samples on disk (obviously) requires some file
- * reader, that functionality is encapsulated in implementations of
- * {@link IndexInfoRetriever}. There is an implementation to access
- * index samples of legacy sstables (without the offsets table),
- * an implementation of access sstables with an offsets table.
- * </p>
- * <p>
- * Until now (Cassandra 3.x), we still support reading from <i>legacy</i> sstables -
- * i.e. sstables created by Cassandra < 3.0 (see {@link org.apache.cassandra.io.sstable.format.big.BigFormat}.
- * </p>
- *
- */
-public class RowIndexEntry<T> implements IMeasurableMemory
-{
- private static final long EMPTY_SIZE = ObjectSizes.measure(new RowIndexEntry(0));
-
- // constants for type of row-index-entry as serialized for saved-cache
- static final int CACHE_NOT_INDEXED = 0;
- static final int CACHE_INDEXED = 1;
- static final int CACHE_INDEXED_SHALLOW = 2;
-
- static final Histogram indexEntrySizeHistogram;
- static final Histogram indexInfoCountHistogram;
- static final Histogram indexInfoGetsHistogram;
- static final Histogram indexInfoReadsHistogram;
- static
- {
- MetricNameFactory factory = new DefaultNameFactory("Index", "RowIndexEntry");
- indexEntrySizeHistogram = Metrics.histogram(factory.createMetricName("IndexedEntrySize"), false);
- indexInfoCountHistogram = Metrics.histogram(factory.createMetricName("IndexInfoCount"), false);
- indexInfoGetsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoGets"), false);
- indexInfoReadsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoReads"), false);
- }
-
- public final long position;
-
- public RowIndexEntry(long position)
- {
- this.position = position;
- }
-
- /**
- * @return true if this index entry contains the row-level tombstone and column summary. Otherwise,
- * caller should fetch these from the row header.
- */
- public boolean isIndexed()
- {
- return columnsIndexCount() > 1;
- }
-
- public boolean indexOnHeap()
- {
- return false;
- }
-
- public DeletionTime deletionTime()
- {
- throw new UnsupportedOperationException();
- }
-
- public int columnsIndexCount()
- {
- return 0;
- }
-
- public long unsharedHeapSize()
- {
- return EMPTY_SIZE;
- }
-
- /**
- * @param dataFilePosition position of the partition in the {@link org.apache.cassandra.io.sstable.Component.Type#DATA} file
- * @param indexFilePosition position in the {@link org.apache.cassandra.io.sstable.Component.Type#PRIMARY_INDEX} of the {@link RowIndexEntry}
- * @param deletionTime deletion time of {@link RowIndexEntry}
- * @param headerLength deletion time of {@link RowIndexEntry}
- * @param columnIndexCount number of {@link IndexInfo} entries in the {@link RowIndexEntry}
- * @param indexedPartSize serialized size of all serialized {@link IndexInfo} objects and their offsets
- * @param indexSamples list with IndexInfo offsets (if total serialized size is less than {@link org.apache.cassandra.config.Config#column_index_cache_size}
- * @param offsets offsets of IndexInfo offsets
- * @param idxInfoSerializer the {@link IndexInfo} serializer
- */
- public static RowIndexEntry<IndexInfo> create(long dataFilePosition, long indexFilePosition,
- DeletionTime deletionTime, long headerLength, int columnIndexCount,
- int indexedPartSize,
- List<IndexInfo> indexSamples, int[] offsets,
- ISerializer<IndexInfo> idxInfoSerializer)
- {
- // If the "partition building code" in BigTableWriter.append() via ColumnIndex returns a list
- // of IndexInfo objects, which is the case if the serialized size is less than
- // Config.column_index_cache_size, AND we have more than one IndexInfo object, we
- // construct an IndexedEntry object. (note: indexSamples.size() and columnIndexCount have the same meaning)
- if (indexSamples != null && indexSamples.size() > 1)
- return new IndexedEntry(dataFilePosition, deletionTime, headerLength,
- indexSamples.toArray(new IndexInfo[indexSamples.size()]), offsets,
- indexedPartSize, idxInfoSerializer);
- // Here we have to decide whether we have serialized IndexInfo objects that exceeds
- // Config.column_index_cache_size (not exceeding case covered above).
- // Such a "big" indexed-entry is represented as a shallow one.
- if (columnIndexCount > 1)
- return new ShallowIndexedEntry(dataFilePosition, indexFilePosition,
- deletionTime, headerLength, columnIndexCount,
- indexedPartSize, idxInfoSerializer);
- // Last case is that there are no index samples.
- return new RowIndexEntry<>(dataFilePosition);
- }
-
- public IndexInfoRetriever openWithIndex(FileHandle indexFile)
- {
- return null;
- }
-
- public interface IndexSerializer<T>
- {
- void serialize(RowIndexEntry<T> rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException;
-
- RowIndexEntry<T> deserialize(DataInputPlus in, long indexFilePosition) throws IOException;
- default RowIndexEntry<T> deserialize(RandomAccessReader reader) throws IOException
- {
- return deserialize(reader, reader.getFilePointer());
-
- }
-
- default RowIndexEntry<T> deserialize(FileDataInput input) throws IOException
- {
- return deserialize(input, input.getFilePointer());
-
- }
-
- void serializeForCache(RowIndexEntry<T> rie, DataOutputPlus out) throws IOException;
- RowIndexEntry<T> deserializeForCache(DataInputPlus in) throws IOException;
-
- long deserializePositionAndSkip(DataInputPlus in) throws IOException;
-
- ISerializer<T> indexInfoSerializer();
- }
-
- public static final class Serializer implements IndexSerializer<IndexInfo>
- {
- private final IndexInfo.Serializer idxInfoSerializer;
- private final Version version;
-
- public Serializer(Version version, SerializationHeader header)
- {
- this.idxInfoSerializer = IndexInfo.serializer(version, header);
- this.version = version;
- }
-
- public IndexInfo.Serializer indexInfoSerializer()
- {
- return idxInfoSerializer;
- }
-
- public void serialize(RowIndexEntry<IndexInfo> rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException
- {
- rie.serialize(out, indexInfo);
- }
-
- public void serializeForCache(RowIndexEntry<IndexInfo> rie, DataOutputPlus out) throws IOException
- {
- rie.serializeForCache(out);
- }
-
- public RowIndexEntry<IndexInfo> deserializeForCache(DataInputPlus in) throws IOException
- {
- long position = in.readUnsignedVInt();
-
- switch (in.readByte())
- {
- case CACHE_NOT_INDEXED:
- return new RowIndexEntry<>(position);
- case CACHE_INDEXED:
- return new IndexedEntry(position, in, idxInfoSerializer);
- case CACHE_INDEXED_SHALLOW:
- return new ShallowIndexedEntry(position, in, idxInfoSerializer);
- default:
- throw new AssertionError();
- }
- }
-
- public static void skipForCache(DataInputPlus in) throws IOException
- {
- in.readUnsignedVInt();
- switch (in.readByte())
- {
- case CACHE_NOT_INDEXED:
- break;
- case CACHE_INDEXED:
- IndexedEntry.skipForCache(in);
- break;
- case CACHE_INDEXED_SHALLOW:
- ShallowIndexedEntry.skipForCache(in);
- break;
- default:
- assert false;
- }
- }
-
- public RowIndexEntry<IndexInfo> deserialize(DataInputPlus in, long indexFilePosition) throws IOException
- {
- long position = in.readUnsignedVInt();
-
- int size = (int)in.readUnsignedVInt();
- if (size == 0)
- {
- return new RowIndexEntry<>(position);
- }
- else
- {
- long headerLength = in.readUnsignedVInt();
- DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
- int columnsIndexCount = (int) in.readUnsignedVInt();
-
- checkSize(columnsIndexCount, size);
-
- int indexedPartSize = size - serializedSize(deletionTime, headerLength, columnsIndexCount);
-
- if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
- {
- return new IndexedEntry(position, in, deletionTime, headerLength, columnsIndexCount,
- idxInfoSerializer, indexedPartSize);
- }
- else
- {
- in.skipBytes(indexedPartSize);
-
- return new ShallowIndexedEntry(position,
- indexFilePosition,
- deletionTime, headerLength, columnsIndexCount,
- indexedPartSize, idxInfoSerializer);
- }
- }
- }
-
- private void checkSize(int entries, int bytes)
- {
- ReadCommand command = ReadCommand.getCommand();
- if (command == null || SchemaConstants.isSystemKeyspace(command.metadata().keyspace) || !DatabaseDescriptor.getReadThresholdsEnabled())
- return;
-
- DataStorageSpec.LongBytesBound warnThreshold = DatabaseDescriptor.getRowIndexReadSizeWarnThreshold();
- DataStorageSpec.LongBytesBound failThreshold = DatabaseDescriptor.getRowIndexReadSizeFailThreshold();
- if (warnThreshold == null && failThreshold == null)
- return;
-
- long estimatedMemory = estimateMaterializedIndexSize(entries, bytes);
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(command.metadata().id);
- if (cfs != null)
- cfs.metric.rowIndexSize.update(estimatedMemory);
-
- if (failThreshold != null && estimatedMemory > failThreshold.toBytes())
- {
- String msg = String.format("Query %s attempted to access a large RowIndexEntry estimated to be %d bytes " +
- "in-memory (total entries: %d, total bytes: %d) but the max allowed is %s;" +
- " query aborted (see row_index_read_size_fail_threshold)",
- command.toCQLString(), estimatedMemory, entries, bytes, failThreshold);
- MessageParams.remove(ParamType.ROW_INDEX_READ_SIZE_WARN);
- MessageParams.add(ParamType.ROW_INDEX_READ_SIZE_FAIL, estimatedMemory);
-
- throw new RowIndexEntryReadSizeTooLargeException(msg);
- }
- else if (warnThreshold != null && estimatedMemory > warnThreshold.toBytes())
- {
- // use addIfLarger rather than add as a previous partition may be larger than this one
- Long current = MessageParams.get(ParamType.ROW_INDEX_READ_SIZE_WARN);
- if (current == null || current.compareTo(estimatedMemory) < 0)
- MessageParams.add(ParamType.ROW_INDEX_READ_SIZE_WARN, estimatedMemory);
- }
- }
-
- private static long estimateMaterializedIndexSize(int entries, int bytes)
- {
- long overhead = IndexInfo.EMPTY_SIZE
- + ArrayClustering.EMPTY_SIZE
- + DeletionTime.EMPTY_SIZE;
-
- return (overhead * entries) + bytes;
- }
-
- public long deserializePositionAndSkip(DataInputPlus in) throws IOException
- {
- long position = in.readUnsignedVInt();
-
- int size = (int) in.readUnsignedVInt();
- if (size > 0)
- in.skipBytesFully(size);
-
- return position;
- }
-
- /**
- * Reads only the data 'position' of the index entry and returns it. Note that this left 'in' in the middle
- * of reading an entry, so this is only useful if you know what you are doing and in most case 'deserialize'
- * should be used instead.
- */
- public static long readPosition(DataInputPlus in) throws IOException
- {
- return in.readUnsignedVInt();
- }
-
- public static void skip(DataInputPlus in, Version version) throws IOException
- {
- readPosition(in);
- skipPromotedIndex(in);
- }
-
- private static void skipPromotedIndex(DataInputPlus in) throws IOException
- {
- int size = (int)in.readUnsignedVInt();
- if (size <= 0)
- return;
-
- in.skipBytesFully(size);
- }
-
- public static void serializeOffsets(DataOutputBuffer out, int[] indexOffsets, int columnIndexCount) throws IOException
- {
- for (int i = 0; i < columnIndexCount; i++)
- out.writeInt(indexOffsets[i]);
- }
- }
-
- private static int serializedSize(DeletionTime deletionTime, long headerLength, int columnIndexCount)
- {
- return TypeSizes.sizeofUnsignedVInt(headerLength)
- + (int) DeletionTime.serializer.serializedSize(deletionTime)
- + TypeSizes.sizeofUnsignedVInt(columnIndexCount);
- }
-
- public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
- {
- out.writeUnsignedVInt(position);
-
- out.writeUnsignedVInt(0);
- }
-
- public void serializeForCache(DataOutputPlus out) throws IOException
- {
- out.writeUnsignedVInt(position);
-
- out.writeByte(CACHE_NOT_INDEXED);
- }
-
- /**
- * An entry in the row index for a row whose columns are indexed - used for both legacy and current formats.
- */
- private static final class IndexedEntry extends RowIndexEntry<IndexInfo>
- {
- private static final long BASE_SIZE;
-
- static
- {
- BASE_SIZE = ObjectSizes.measure(new IndexedEntry(0, DeletionTime.LIVE, 0, null, null, 0, null));
- }
-
- private final DeletionTime deletionTime;
- private final long headerLength;
-
- private final IndexInfo[] columnsIndex;
- private final int[] offsets;
- private final int indexedPartSize;
- @Unmetered
- private final ISerializer<IndexInfo> idxInfoSerializer;
-
- private IndexedEntry(long dataFilePosition, DeletionTime deletionTime, long headerLength,
- IndexInfo[] columnsIndex, int[] offsets,
- int indexedPartSize, ISerializer<IndexInfo> idxInfoSerializer)
- {
- super(dataFilePosition);
-
- this.headerLength = headerLength;
- this.deletionTime = deletionTime;
-
- this.columnsIndex = columnsIndex;
- this.offsets = offsets;
- this.indexedPartSize = indexedPartSize;
- this.idxInfoSerializer = idxInfoSerializer;
- }
-
- private IndexedEntry(long dataFilePosition, DataInputPlus in,
- DeletionTime deletionTime, long headerLength, int columnIndexCount,
- IndexInfo.Serializer idxInfoSerializer, int indexedPartSize) throws IOException
- {
- super(dataFilePosition);
-
- this.headerLength = headerLength;
- this.deletionTime = deletionTime;
- int columnsIndexCount = columnIndexCount;
-
- this.columnsIndex = new IndexInfo[columnsIndexCount];
- for (int i = 0; i < columnsIndexCount; i++)
- this.columnsIndex[i] = idxInfoSerializer.deserialize(in);
-
- this.offsets = new int[this.columnsIndex.length];
- for (int i = 0; i < offsets.length; i++)
- offsets[i] = in.readInt();
-
- this.indexedPartSize = indexedPartSize;
-
- this.idxInfoSerializer = idxInfoSerializer;
- }
-
- /**
- * Constructor called from {@link Serializer#deserializeForCache(org.apache.cassandra.io.util.DataInputPlus)}.
- */
- private IndexedEntry(long dataFilePosition, DataInputPlus in, ISerializer<IndexInfo> idxInfoSerializer) throws IOException
- {
- super(dataFilePosition);
-
- this.headerLength = in.readUnsignedVInt();
- this.deletionTime = DeletionTime.serializer.deserialize(in);
- int columnsIndexCount = (int) in.readUnsignedVInt();
-
- TrackedDataInputPlus trackedIn = new TrackedDataInputPlus(in);
-
- this.columnsIndex = new IndexInfo[columnsIndexCount];
- for (int i = 0; i < columnsIndexCount; i++)
- this.columnsIndex[i] = idxInfoSerializer.deserialize(trackedIn);
-
- this.offsets = null;
-
- this.indexedPartSize = (int) trackedIn.getBytesRead();
-
- this.idxInfoSerializer = idxInfoSerializer;
- }
-
- @Override
- public boolean indexOnHeap()
- {
- return true;
- }
-
- @Override
- public int columnsIndexCount()
- {
- return columnsIndex.length;
- }
-
- @Override
- public DeletionTime deletionTime()
- {
- return deletionTime;
- }
-
- @Override
- public IndexInfoRetriever openWithIndex(FileHandle indexFile)
- {
- indexEntrySizeHistogram.update(serializedSize(deletionTime, headerLength, columnsIndex.length) + indexedPartSize);
- indexInfoCountHistogram.update(columnsIndex.length);
- return new IndexInfoRetriever()
- {
- private int retrievals;
-
- @Override
- public IndexInfo columnsIndex(int index)
- {
- retrievals++;
- return columnsIndex[index];
- }
-
- public void close()
- {
- indexInfoGetsHistogram.update(retrievals);
- }
- };
- }
-
- @Override
- public long unsharedHeapSize()
- {
- long entrySize = 0;
- for (IndexInfo idx : columnsIndex)
- entrySize += idx.unsharedHeapSize();
- return BASE_SIZE
- + entrySize
- + ObjectSizes.sizeOfReferenceArray(columnsIndex.length);
- }
-
- public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
- {
- assert indexedPartSize != Integer.MIN_VALUE;
-
- out.writeUnsignedVInt(position);
-
- out.writeUnsignedVInt(serializedSize(deletionTime, headerLength, columnsIndex.length) + indexedPartSize);
-
- out.writeUnsignedVInt(headerLength);
- DeletionTime.serializer.serialize(deletionTime, out);
- out.writeUnsignedVInt(columnsIndex.length);
- for (IndexInfo info : columnsIndex)
- idxInfoSerializer.serialize(info, out);
- for (int offset : offsets)
- out.writeInt(offset);
- }
-
- @Override
- public void serializeForCache(DataOutputPlus out) throws IOException
- {
- out.writeUnsignedVInt(position);
- out.writeByte(CACHE_INDEXED);
-
- out.writeUnsignedVInt(headerLength);
- DeletionTime.serializer.serialize(deletionTime, out);
- out.writeUnsignedVInt(columnsIndexCount());
-
- for (IndexInfo indexInfo : columnsIndex)
- idxInfoSerializer.serialize(indexInfo, out);
- }
-
- static void skipForCache(DataInputPlus in) throws IOException
- {
- in.readUnsignedVInt();
- DeletionTime.serializer.skip(in);
- in.readUnsignedVInt();
-
- in.readUnsignedVInt();
- }
- }
-
- /**
- * An entry in the row index for a row whose columns are indexed and the {@link IndexInfo} objects
- * are not read into the key cache.
- */
- private static final class ShallowIndexedEntry extends RowIndexEntry<IndexInfo>
- {
- private static final long BASE_SIZE;
-
- static
- {
- BASE_SIZE = ObjectSizes.measure(new ShallowIndexedEntry(0, 0, DeletionTime.LIVE, 0, 10, 0, null));
- }
-
- private final long indexFilePosition;
-
- private final DeletionTime deletionTime;
- private final long headerLength;
- private final int columnsIndexCount;
-
- private final int indexedPartSize;
- private final int offsetsOffset;
- @Unmetered
- private final ISerializer<IndexInfo> idxInfoSerializer;
- private final int fieldsSerializedSize;
-
- /**
- * See {@link #create(long, long, DeletionTime, long, int, int, List, int[], ISerializer)} for a description
- * of the parameters.
- */
- private ShallowIndexedEntry(long dataFilePosition, long indexFilePosition,
- DeletionTime deletionTime, long headerLength, int columnIndexCount,
- int indexedPartSize, ISerializer<IndexInfo> idxInfoSerializer)
- {
- super(dataFilePosition);
-
- assert columnIndexCount > 1;
-
- this.indexFilePosition = indexFilePosition;
- this.headerLength = headerLength;
- this.deletionTime = deletionTime;
- this.columnsIndexCount = columnIndexCount;
-
- this.indexedPartSize = indexedPartSize;
- this.idxInfoSerializer = idxInfoSerializer;
-
- this.fieldsSerializedSize = serializedSize(deletionTime, headerLength, columnIndexCount);
- this.offsetsOffset = indexedPartSize + fieldsSerializedSize - columnsIndexCount * TypeSizes.INT_SIZE;
- }
-
- /**
- * Constructor for key-cache deserialization
- */
- private ShallowIndexedEntry(long dataFilePosition, DataInputPlus in, IndexInfo.Serializer idxInfoSerializer) throws IOException
- {
- super(dataFilePosition);
-
- this.indexFilePosition = in.readUnsignedVInt();
-
- this.headerLength = in.readUnsignedVInt();
- this.deletionTime = DeletionTime.serializer.deserialize(in);
- this.columnsIndexCount = (int) in.readUnsignedVInt();
-
- this.indexedPartSize = (int) in.readUnsignedVInt();
-
- this.idxInfoSerializer = idxInfoSerializer;
-
- this.fieldsSerializedSize = serializedSize(deletionTime, headerLength, columnsIndexCount);
- this.offsetsOffset = indexedPartSize + fieldsSerializedSize - columnsIndexCount * TypeSizes.INT_SIZE;
- }
-
- @Override
- public int columnsIndexCount()
- {
- return columnsIndexCount;
- }
-
- @Override
- public DeletionTime deletionTime()
- {
- return deletionTime;
- }
-
- @Override
- public IndexInfoRetriever openWithIndex(FileHandle indexFile)
- {
- indexEntrySizeHistogram.update(indexedPartSize + fieldsSerializedSize);
- indexInfoCountHistogram.update(columnsIndexCount);
- return new ShallowInfoRetriever(indexFilePosition +
- VIntCoding.computeUnsignedVIntSize(position) +
- VIntCoding.computeUnsignedVIntSize(indexedPartSize + fieldsSerializedSize) +
- fieldsSerializedSize,
- offsetsOffset - fieldsSerializedSize,
- indexFile.createReader(), idxInfoSerializer);
- }
-
- @Override
- public long unsharedHeapSize()
- {
- return BASE_SIZE;
- }
-
- @Override
- public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
- {
- out.writeUnsignedVInt(position);
-
- out.writeUnsignedVInt(fieldsSerializedSize + indexInfo.limit());
-
- out.writeUnsignedVInt(headerLength);
- DeletionTime.serializer.serialize(deletionTime, out);
- out.writeUnsignedVInt(columnsIndexCount);
-
- out.write(indexInfo);
- }
-
- @Override
- public void serializeForCache(DataOutputPlus out) throws IOException
- {
- out.writeUnsignedVInt(position);
- out.writeByte(CACHE_INDEXED_SHALLOW);
-
- out.writeUnsignedVInt(indexFilePosition);
-
- out.writeUnsignedVInt(headerLength);
- DeletionTime.serializer.serialize(deletionTime, out);
- out.writeUnsignedVInt(columnsIndexCount);
-
- out.writeUnsignedVInt(indexedPartSize);
- }
-
- static void skipForCache(DataInputPlus in) throws IOException
- {
- in.readUnsignedVInt();
-
- in.readUnsignedVInt();
- DeletionTime.serializer.skip(in);
- in.readUnsignedVInt();
-
- in.readUnsignedVInt();
- }
- }
-
- private static final class ShallowInfoRetriever extends FileIndexInfoRetriever
- {
- private final int offsetsOffset;
-
- private ShallowInfoRetriever(long indexInfoFilePosition, int offsetsOffset,
- FileDataInput indexReader, ISerializer<IndexInfo> idxInfoSerializer)
- {
- super(indexInfoFilePosition, indexReader, idxInfoSerializer);
- this.offsetsOffset = offsetsOffset;
- }
-
- IndexInfo fetchIndex(int index) throws IOException
- {
- // seek to position in "offsets to IndexInfo" table
- indexReader.seek(indexInfoFilePosition + offsetsOffset + index * TypeSizes.INT_SIZE);
-
- // read offset of IndexInfo
- int indexInfoPos = indexReader.readInt();
-
- // seek to posision of IndexInfo
- indexReader.seek(indexInfoFilePosition + indexInfoPos);
-
- // finally, deserialize IndexInfo
- return idxInfoSerializer.deserialize(indexReader);
- }
- }
-
- /**
- * Base class to access {@link IndexInfo} objects.
- */
- public interface IndexInfoRetriever extends AutoCloseable
- {
- IndexInfo columnsIndex(int index) throws IOException;
-
- void close() throws IOException;
- }
-
- /**
- * Base class to access {@link IndexInfo} objects on disk that keeps already
- * read {@link IndexInfo} on heap.
- */
- private abstract static class FileIndexInfoRetriever implements IndexInfoRetriever
- {
- final long indexInfoFilePosition;
- final ISerializer<IndexInfo> idxInfoSerializer;
- final FileDataInput indexReader;
- int retrievals;
-
- /**
- *
- * @param indexInfoFilePosition offset of first serialized {@link IndexInfo} object
- * @param indexReader file data input to access the index file, closed by this instance
- * @param idxInfoSerializer the index serializer to deserialize {@link IndexInfo} objects
- */
- FileIndexInfoRetriever(long indexInfoFilePosition, FileDataInput indexReader, ISerializer<IndexInfo> idxInfoSerializer)
- {
- this.indexInfoFilePosition = indexInfoFilePosition;
- this.idxInfoSerializer = idxInfoSerializer;
- this.indexReader = indexReader;
- }
-
- public final IndexInfo columnsIndex(int index) throws IOException
- {
- retrievals++;
- return fetchIndex(index);
- }
-
- abstract IndexInfo fetchIndex(int index) throws IOException;
-
- public void close() throws IOException
- {
- indexReader.close();
-
- indexInfoGetsHistogram.update(retrievals);
- indexInfoReadsHistogram.update(retrievals);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java
index 5949559..66a56dc 100644
--- a/src/java/org/apache/cassandra/db/SSTableImporter.java
+++ b/src/java/org/apache/cassandra/db/SSTableImporter.java
@@ -25,21 +25,24 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IVerifier;
import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Refs;
@@ -69,8 +72,8 @@
@VisibleForTesting
synchronized List<String> importNewSSTables(Options options)
{
- logger.info("Loading new SSTables for {}/{}: {}",
- cfs.keyspace.getName(), cfs.getTableName(), options);
+ UUID importID = UUID.randomUUID();
+ logger.info("[{}] Loading new SSTables for {}/{}: {}", importID, cfs.keyspace.getName(), cfs.getTableName(), options);
List<Pair<Directories.SSTableLister, String>> listers = getSSTableListers(options.srcPaths);
@@ -93,19 +96,20 @@
{
try
{
+ abortIfDraining();
verifySSTableForImport(descriptor, entry.getValue(), options.verifyTokens, options.verifySSTables, options.extendedVerify);
}
catch (Throwable t)
{
if (dir != null)
{
- logger.error("Failed verifying sstable {} in directory {}", descriptor, dir, t);
+ logger.error("[{}] Failed verifying SSTable {} in directory {}", importID, descriptor, dir, t);
failedDirectories.add(dir);
}
else
{
- logger.error("Failed verifying sstable {}", descriptor, t);
- throw new RuntimeException("Failed verifying sstable "+descriptor, t);
+ logger.error("[{}] Failed verifying SSTable {}", importID, descriptor, t);
+ throw new RuntimeException("Failed verifying SSTable " + descriptor, t);
}
break;
}
@@ -128,6 +132,7 @@
{
try
{
+ abortIfDraining();
Descriptor oldDescriptor = entry.getKey();
if (currentDescriptors.contains(oldDescriptor))
continue;
@@ -144,7 +149,7 @@
newSSTablesPerDirectory.forEach(s -> s.selfRef().release());
if (dir != null)
{
- logger.error("Failed importing sstables in directory {}", dir, t);
+ logger.error("[{}] Failed importing sstables in directory {}", importID, dir, t);
failedDirectories.add(dir);
if (options.copyData)
{
@@ -160,8 +165,8 @@
}
else
{
- logger.error("Failed importing sstables from data directory - renamed sstables are: {}", movedSSTables);
- throw new RuntimeException("Failed importing sstables", t);
+ logger.error("[{}] Failed importing sstables from data directory - renamed SSTables are: {}", importID, movedSSTables, t);
+ throw new RuntimeException("Failed importing SSTables", t);
}
}
}
@@ -170,28 +175,68 @@
if (newSSTables.isEmpty())
{
- logger.info("No new SSTables were found for {}/{}", cfs.keyspace.getName(), cfs.getTableName());
+ logger.info("[{}] No new SSTables were found for {}/{}", importID, cfs.keyspace.getName(), cfs.getTableName());
return failedDirectories;
}
- logger.info("Loading new SSTables and building secondary indexes for {}/{}: {}", cfs.keyspace.getName(), cfs.getTableName(), newSSTables);
+ logger.info("[{}] Loading new SSTables and building secondary indexes for {}/{}: {}", importID, cfs.keyspace.getName(), cfs.getTableName(), newSSTables);
+ if (logger.isTraceEnabled())
+ logLeveling(importID, newSSTables);
try (Refs<SSTableReader> refs = Refs.ref(newSSTables))
{
+ abortIfDraining();
cfs.getTracker().addSSTables(newSSTables);
for (SSTableReader reader : newSSTables)
{
if (options.invalidateCaches && cfs.isRowCacheEnabled())
- invalidateCachesForSSTable(reader.descriptor);
+ invalidateCachesForSSTable(reader);
}
-
+ }
+ catch (Throwable t)
+ {
+ logger.error("[{}] Failed adding SSTables", importID, t);
+ throw new RuntimeException("Failed adding SSTables", t);
}
- logger.info("Done loading load new SSTables for {}/{}", cfs.keyspace.getName(), cfs.getTableName());
+ logger.info("[{}] Done loading load new SSTables for {}/{}", importID, cfs.keyspace.getName(), cfs.getTableName());
return failedDirectories;
}
/**
+ * Check the state of this node and throws an {@link InterruptedException} if it is currently draining
+ *
+ * @throws InterruptedException if the node is draining
+ */
+ private static void abortIfDraining() throws InterruptedException
+ {
+ if (StorageService.instance.isDraining())
+ throw new InterruptedException("SSTables import has been aborted");
+ }
+
+ private void logLeveling(UUID importID, Set<SSTableReader> newSSTables)
+ {
+ StringBuilder sb = new StringBuilder();
+ for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
+ sb.append(formatMetadata(sstable));
+ logger.debug("[{}] Current sstables: {}", importID, sb);
+ sb = new StringBuilder();
+ for (SSTableReader sstable : newSSTables)
+ sb.append(formatMetadata(sstable));
+ logger.debug("[{}] New sstables: {}", importID, sb);
+ }
+
+ private static String formatMetadata(SSTableReader sstable)
+ {
+ return String.format("{[%s, %s], %d, %s, %d}",
+ sstable.getFirst().getToken(),
+ sstable.getLast().getToken(),
+ sstable.getSSTableLevel(),
+ sstable.isRepaired(),
+ sstable.onDiskLength());
+ }
+
+ /**
* Opens the sstablereader described by descriptor and figures out the correct directory for it based
* on the first token
*
@@ -208,7 +253,7 @@
SSTableReader sstable = null;
try
{
- sstable = SSTableReader.open(descriptor, components, cfs.metadata);
+ sstable = SSTableReader.open(cfs, descriptor, components, cfs.metadata);
targetDirectory = cfs.getDirectories().getLocationForDisk(cfs.diskBoundaryManager.getDiskBoundaries(cfs).getCorrectDiskForSSTable(sstable));
}
finally
@@ -216,7 +261,7 @@
if (sstable != null)
sstable.selfRef().release();
}
- return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(new File(descriptor.baseFilename())) : targetDirectory;
+ return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(descriptor.baseFile()) : targetDirectory;
}
/**
@@ -279,11 +324,11 @@
{
for (MovedSSTable movedSSTable : movedSSTables)
{
- if (new File(movedSSTable.newDescriptor.filenameFor(Component.DATA)).exists())
+ if (movedSSTable.newDescriptor.fileFor(Components.DATA).exists())
{
- logger.debug("Moving sstable {} back to {}", movedSSTable.newDescriptor.filenameFor(Component.DATA)
- , movedSSTable.oldDescriptor.filenameFor(Component.DATA));
- SSTableWriter.rename(movedSSTable.newDescriptor, movedSSTable.oldDescriptor, movedSSTable.components);
+ logger.debug("Moving sstable {} back to {}", movedSSTable.newDescriptor.fileFor(Components.DATA)
+ , movedSSTable.oldDescriptor.fileFor(Components.DATA));
+ SSTable.rename(movedSSTable.newDescriptor, movedSSTable.oldDescriptor, movedSSTable.components);
}
}
}
@@ -299,11 +344,8 @@
logger.debug("Removing copied SSTables which were left in data directories after failed SSTable import.");
for (MovedSSTable movedSSTable : movedSSTables)
{
- if (new File(movedSSTable.newDescriptor.filenameFor(Component.DATA)).exists())
- {
- // no logging here as for moveSSTablesBack case above as logging is done in delete method
- SSTableWriter.delete(movedSSTable.newDescriptor, movedSSTable.components);
- }
+ // no logging here as for moveSSTablesBack case above as logging is done in delete method
+ movedSSTable.newDescriptor.getFormat().delete(movedSSTable.newDescriptor);
}
}
@@ -311,9 +353,9 @@
* Iterates over all keys in the sstable index and invalidates the row cache
*/
@VisibleForTesting
- void invalidateCachesForSSTable(Descriptor desc)
+ void invalidateCachesForSSTable(SSTableReader reader)
{
- try (KeyIterator iter = new KeyIterator(desc, cfs.metadata()))
+ try (KeyIterator iter = reader.keyIterator())
{
while (iter.hasNext())
{
@@ -321,6 +363,10 @@
cfs.invalidateCachedPartition(decoratedKey);
}
}
+ catch (IOException ex)
+ {
+ throw new RuntimeException("Failed to import sstable " + reader.getFilename(), ex);
+ }
}
/**
@@ -335,14 +381,15 @@
SSTableReader reader = null;
try
{
- reader = SSTableReader.open(descriptor, components, cfs.metadata);
- Verifier.Options verifierOptions = Verifier.options()
- .extendedVerification(extendedVerify)
- .checkOwnsTokens(verifyTokens)
- .quick(!verifySSTables)
- .invokeDiskFailurePolicy(false)
- .mutateRepairStatus(false).build();
- try (Verifier verifier = new Verifier(cfs, reader, false, verifierOptions))
+ reader = SSTableReader.open(cfs, descriptor, components, cfs.metadata);
+ IVerifier.Options verifierOptions = IVerifier.options()
+ .extendedVerification(extendedVerify)
+ .checkOwnsTokens(verifyTokens)
+ .quick(!verifySSTables)
+ .invokeDiskFailurePolicy(false)
+ .mutateRepairStatus(false).build();
+
+ try (IVerifier verifier = reader.getVerifier(cfs, new OutputHandler.LogOutput(), false, verifierOptions))
{
verifier.verify();
}
@@ -364,7 +411,7 @@
*/
private void maybeMutateMetadata(Descriptor descriptor, Options options) throws IOException
{
- if (new File(descriptor.filenameFor(Component.STATS)).exists())
+ if (descriptor.fileFor(Components.STATS).exists())
{
if (options.resetLevel)
{
diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java
index 11239d8..d75adfd 100644
--- a/src/java/org/apache/cassandra/db/SerializationHeader.java
+++ b/src/java/org/apache/cassandra/db/SerializationHeader.java
@@ -25,9 +25,8 @@
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
@@ -38,6 +37,7 @@
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.serializers.AbstractTypeSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
public class SerializationHeader
@@ -169,12 +169,12 @@
public void writeLocalDeletionTime(int localDeletionTime, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(localDeletionTime - stats.minLocalDeletionTime);
+ out.writeUnsignedVInt32(localDeletionTime - stats.minLocalDeletionTime);
}
public void writeTTL(int ttl, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(ttl - stats.minTTL);
+ out.writeUnsignedVInt32(ttl - stats.minTTL);
}
public void writeDeletionTime(DeletionTime dt, DataOutputPlus out) throws IOException
@@ -190,12 +190,12 @@
public int readLocalDeletionTime(DataInputPlus in) throws IOException
{
- return (int)in.readUnsignedVInt() + stats.minLocalDeletionTime;
+ return in.readUnsignedVInt32() + stats.minLocalDeletionTime;
}
public int readTTL(DataInputPlus in) throws IOException
{
- return (int)in.readUnsignedVInt() + stats.minTTL;
+ return in.readUnsignedVInt32() + stats.minTTL;
}
public DeletionTime readDeletionTime(DataInputPlus in) throws IOException
@@ -398,6 +398,8 @@
public static class Serializer implements IMetadataComponentSerializer<Component>
{
+ private final AbstractTypeSerializer typeSerializer = new AbstractTypeSerializer();
+
public void serializeForMessaging(SerializationHeader header, ColumnFilter selection, DataOutputPlus out, boolean hasStatic) throws IOException
{
EncodingStats.serializer.serialize(header.stats, out);
@@ -462,10 +464,8 @@
{
EncodingStats.serializer.serialize(header.stats, out);
- writeType(header.keyType, out);
- out.writeUnsignedVInt(header.clusteringTypes.size());
- for (AbstractType<?> type : header.clusteringTypes)
- writeType(type, out);
+ typeSerializer.serialize(header.keyType, out);
+ typeSerializer.serializeList(header.clusteringTypes, out);
writeColumnsWithTypes(header.staticColumns, out);
writeColumnsWithTypes(header.regularColumns, out);
@@ -476,17 +476,11 @@
{
EncodingStats stats = EncodingStats.serializer.deserialize(in);
- AbstractType<?> keyType = readType(in);
- int size = (int)in.readUnsignedVInt();
- List<AbstractType<?>> clusteringTypes = new ArrayList<>(size);
- for (int i = 0; i < size; i++)
- clusteringTypes.add(readType(in));
+ AbstractType<?> keyType = typeSerializer.deserialize(in);
+ List<AbstractType<?>> clusteringTypes = typeSerializer.deserializeList(in);
- Map<ByteBuffer, AbstractType<?>> staticColumns = new LinkedHashMap<>();
- Map<ByteBuffer, AbstractType<?>> regularColumns = new LinkedHashMap<>();
-
- readColumnsWithType(in, staticColumns);
- readColumnsWithType(in, regularColumns);
+ Map<ByteBuffer, AbstractType<?>> staticColumns = readColumnsWithType(in);
+ Map<ByteBuffer, AbstractType<?>> regularColumns = readColumnsWithType(in);
return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats);
}
@@ -496,10 +490,8 @@
{
int size = EncodingStats.serializer.serializedSize(header.stats);
- size += sizeofType(header.keyType);
- size += TypeSizes.sizeofUnsignedVInt(header.clusteringTypes.size());
- for (AbstractType<?> type : header.clusteringTypes)
- size += sizeofType(type);
+ size += typeSerializer.serializedSize(header.keyType);
+ size += typeSerializer.serializedListSize(header.clusteringTypes);
size += sizeofColumnsWithTypes(header.staticColumns);
size += sizeofColumnsWithTypes(header.regularColumns);
@@ -508,11 +500,11 @@
private void writeColumnsWithTypes(Map<ByteBuffer, AbstractType<?>> columns, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(columns.size());
+ out.writeUnsignedVInt32(columns.size());
for (Map.Entry<ByteBuffer, AbstractType<?>> entry : columns.entrySet())
{
ByteBufferUtil.writeWithVIntLength(entry.getKey(), out);
- writeType(entry.getValue(), out);
+ typeSerializer.serialize(entry.getValue(), out);
}
}
@@ -522,36 +514,21 @@
for (Map.Entry<ByteBuffer, AbstractType<?>> entry : columns.entrySet())
{
size += ByteBufferUtil.serializedSizeWithVIntLength(entry.getKey());
- size += sizeofType(entry.getValue());
+ size += typeSerializer.serializedSize(entry.getValue());
}
return size;
}
- private void readColumnsWithType(DataInputPlus in, Map<ByteBuffer, AbstractType<?>> typeMap) throws IOException
+ private Map<ByteBuffer, AbstractType<?>> readColumnsWithType(DataInputPlus in) throws IOException
{
- int length = (int)in.readUnsignedVInt();
+ int length = in.readUnsignedVInt32();
+ Map<ByteBuffer, AbstractType<?>> typeMap = new LinkedHashMap<>(length);
for (int i = 0; i < length; i++)
{
ByteBuffer name = ByteBufferUtil.readWithVIntLength(in);
- typeMap.put(name, readType(in));
+ typeMap.put(name, typeSerializer.deserialize(in));
}
- }
-
- private void writeType(AbstractType<?> type, DataOutputPlus out) throws IOException
- {
- // TODO: we should have a terser serializaion format. Not a big deal though
- ByteBufferUtil.writeWithVIntLength(UTF8Type.instance.decompose(type.toString()), out);
- }
-
- private AbstractType<?> readType(DataInputPlus in) throws IOException
- {
- ByteBuffer raw = ByteBufferUtil.readWithVIntLength(in);
- return TypeParser.parse(UTF8Type.instance.compose(raw));
- }
-
- private int sizeofType(AbstractType<?> type)
- {
- return ByteBufferUtil.serializedSizeWithVIntLength(UTF8Type.instance.decompose(type.toString()));
+ return typeMap;
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index 963b9fe..d85e2fe 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@ -19,7 +19,12 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -30,18 +35,37 @@
import org.apache.cassandra.cache.RowCacheKey;
import org.apache.cassandra.cache.RowCacheSentinel;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.lifecycle.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.CachedBTreePartition;
+import org.apache.cassandra.db.partitions.CachedPartition;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
+import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator;
import org.apache.cassandra.db.transform.RTBoundValidator;
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.metrics.TableMetrics;
@@ -49,7 +73,9 @@
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.*;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.btree.BTreeSet;
@@ -538,20 +564,26 @@
try
{
// Use a custom iterator instead of DataLimits to avoid stopping the original iterator
- UnfilteredRowIterator toCacheIterator = new WrappingUnfilteredRowIterator(iter)
+ UnfilteredRowIterator toCacheIterator = new WrappingUnfilteredRowIterator()
{
private int rowsCounted = 0;
@Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return iter;
+ }
+
+ @Override
public boolean hasNext()
{
- return rowsCounted < rowsToCache && super.hasNext();
+ return rowsCounted < rowsToCache && iter.hasNext();
}
@Override
public Unfiltered next()
{
- Unfiltered unfiltered = super.next();
+ Unfiltered unfiltered = iter.next();
if (unfiltered.isRow())
{
Row row = (Row) unfiltered;
@@ -713,14 +745,26 @@
break;
}
- if (shouldInclude(sstable))
+ boolean intersects = intersects(sstable);
+ boolean hasRequiredStatics = hasRequiredStatics(sstable);
+ boolean hasPartitionLevelDeletions = hasPartitionLevelDeletions(sstable);
+
+ if (!intersects && !hasRequiredStatics && !hasPartitionLevelDeletions)
+ {
+ nonIntersectingSSTables++;
+ continue;
+ }
+
+ if (intersects || hasRequiredStatics)
{
if (!sstable.isRepaired())
controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime());
// 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
@SuppressWarnings("resource")
- UnfilteredRowIteratorWithLowerBound iter = makeIterator(cfs, sstable, metricsCollector);
+ UnfilteredRowIterator iter = intersects ? makeRowIteratorWithLowerBound(cfs, sstable, metricsCollector)
+ : makeRowIteratorWithSkippedNonStaticContent(cfs, sstable, metricsCollector);
+
inputCollector.addSSTableIterator(sstable, iter);
mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
iter.partitionLevelDeletion().markedForDeleteAt());
@@ -728,27 +772,30 @@
else
{
nonIntersectingSSTables++;
- // sstable contains no tombstone if maxLocalDeletionTime == Integer.MAX_VALUE, so we can safely skip those entirely
- if (sstable.mayHaveTombstones())
+
+ // if the sstable contained range or cell tombstones, it would intersect; since we are here, it means
+ // that there are no cell or range tombstones we are interested in (due to the filter)
+ // however, we know that there are partition level deletions in this sstable and we need to make
+ // an iterator figure out that (see `StatsMetadata.hasPartitionLevelDeletions`)
+
+ // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
+ @SuppressWarnings("resource")
+ UnfilteredRowIterator iter = makeRowIteratorWithSkippedNonStaticContent(cfs, sstable, metricsCollector);
+
+ // if the sstable contains a partition delete, then we must include it regardless of whether it
+ // shadows any other data seen locally as we can't guarantee that other replicas have seen it
+ if (!iter.partitionLevelDeletion().isLive())
{
- // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
- @SuppressWarnings("resource")
- UnfilteredRowIteratorWithLowerBound iter = makeIterator(cfs, sstable, metricsCollector);
- // if the sstable contains a partition delete, then we must include it regardless of whether it
- // shadows any other data seen locally as we can't guarantee that other replicas have seen it
- if (!iter.partitionLevelDeletion().isLive())
- {
- if (!sstable.isRepaired())
- controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime());
- inputCollector.addSSTableIterator(sstable, iter);
- includedDueToTombstones++;
- mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
- iter.partitionLevelDeletion().markedForDeleteAt());
- }
- else
- {
- iter.close();
- }
+ if (!sstable.isRepaired())
+ controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime());
+ inputCollector.addSSTableIterator(sstable, iter);
+ includedDueToTombstones++;
+ mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
+ iter.partitionLevelDeletion().markedForDeleteAt());
+ }
+ else
+ {
+ iter.close();
}
}
}
@@ -779,30 +826,52 @@
}
}
- private boolean shouldInclude(SSTableReader sstable)
+ @Override
+ protected boolean intersects(SSTableReader sstable)
{
- // If some static columns are queried, we should always include the sstable: the clustering values stats of the sstable
- // don't tell us if the sstable contains static values in particular.
- // TODO: we could record if a sstable contains any static value at all.
- if (!columnFilter().fetchedColumns().statics.isEmpty())
- return true;
-
- return clusteringIndexFilter().shouldInclude(sstable);
+ return clusteringIndexFilter().intersects(sstable.metadata().comparator, sstable.getSSTableMetadata().coveredClustering);
}
- private UnfilteredRowIteratorWithLowerBound makeIterator(ColumnFamilyStore cfs,
- SSTableReader sstable,
- SSTableReadsListener listener)
+ private UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs,
+ SSTableReader sstable,
+ SSTableReadsListener listener)
{
return StorageHook.instance.makeRowIteratorWithLowerBound(cfs,
- partitionKey(),
sstable,
+ partitionKey(),
clusteringIndexFilter(),
columnFilter(),
listener);
}
+ private UnfilteredRowIterator makeRowIterator(ColumnFamilyStore cfs,
+ SSTableReader sstable,
+ ClusteringIndexNamesFilter clusteringIndexFilter,
+ SSTableReadsListener listener)
+ {
+ return StorageHook.instance.makeRowIterator(cfs,
+ sstable,
+ partitionKey(),
+ clusteringIndexFilter.getSlices(cfs.metadata()),
+ columnFilter(),
+ clusteringIndexFilter.isReversed(),
+ listener);
+ }
+
+ private UnfilteredRowIterator makeRowIteratorWithSkippedNonStaticContent(ColumnFamilyStore cfs,
+ SSTableReader sstable,
+ SSTableReadsListener listener)
+ {
+ return StorageHook.instance.makeRowIterator(cfs,
+ sstable,
+ partitionKey(),
+ Slices.NONE,
+ columnFilter(),
+ clusteringIndexFilter().isReversed(),
+ listener);
+ }
+
/**
* Return a wrapped iterator that when closed will update the sstables iterated and READ sample metrics.
* Note that we cannot use the Transformations framework because they greedily get the static row, which
@@ -820,6 +889,7 @@
{
DecoratedKey key = merged.partitionKey();
metrics.topReadPartitionFrequency.addSample(key.getKey(), 1);
+ metrics.topReadPartitionSSTableCount.addSample(key.getKey(), metricsCollector.getMergedSSTables());
}
class UpdateSstablesIterated extends Transformation<UnfilteredRowIterator>
@@ -894,23 +964,21 @@
if (filter == null)
break;
- if (!shouldInclude(sstable))
+ boolean intersects = intersects(sstable);
+ boolean hasRequiredStatics = hasRequiredStatics(sstable);
+ boolean hasPartitionLevelDeletions = hasPartitionLevelDeletions(sstable);
+
+ if (!intersects && !hasRequiredStatics)
{
// This mean that nothing queried by the filter can be in the sstable. One exception is the top-level partition deletion
// however: if it is set, it impacts everything and must be included. Getting that top-level partition deletion costs us
// some seek in general however (unless the partition is indexed and is in the key cache), so we first check if the sstable
// has any tombstone at all as a shortcut.
- if (!sstable.mayHaveTombstones())
+ if (!hasPartitionLevelDeletions)
continue; // no tombstone at all, we can skip that sstable
// We need to get the partition deletion and include it if it's live. In any case though, we're done with that sstable.
- try (UnfilteredRowIterator iter = StorageHook.instance.makeRowIterator(cfs,
- sstable,
- partitionKey(),
- filter.getSlices(metadata()),
- columnFilter(),
- filter.isReversed(),
- metricsCollector))
+ try (UnfilteredRowIterator iter = makeRowIteratorWithSkippedNonStaticContent(cfs, sstable, metricsCollector))
{
if (!iter.partitionLevelDeletion().isLive())
{
@@ -937,13 +1005,7 @@
continue;
}
- try (UnfilteredRowIterator iter = StorageHook.instance.makeRowIterator(cfs,
- sstable,
- partitionKey(),
- filter.getSlices(metadata()),
- columnFilter(),
- filter.isReversed(),
- metricsCollector))
+ try (UnfilteredRowIterator iter = makeRowIterator(cfs, sstable, filter, metricsCollector))
{
if (iter.isEmpty())
continue;
@@ -963,6 +1025,7 @@
DecoratedKey key = result.partitionKey();
cfs.metric.topReadPartitionFrequency.addSample(key.getKey(), 1);
+ cfs.metric.topReadPartitionSSTableCount.addSample(key.getKey(), metricsCollector.getMergedSSTables());
StorageHook.instance.reportRead(cfs.metadata.id, partitionKey());
return result.unfilteredIterator(columnFilter(), Slices.ALL, clusteringIndexFilter().isReversed());
@@ -1023,7 +1086,7 @@
toRemove = new TreeSet<>(result.metadata().comparator);
toRemove.add(clustering);
}
- }
+ }
}
try (UnfilteredRowIterator iterator = result.unfilteredIterator(columnFilter(), clusterings, false))
@@ -1276,7 +1339,7 @@
private int mergedSSTables;
@Override
- public void onSSTableSelected(SSTableReader sstable, RowIndexEntry<?> indexEntry, SelectionReason reason)
+ public void onSSTableSelected(SSTableReader sstable, SelectionReason reason)
{
sstable.incrementReadCount();
mergedSSTables++;
@@ -1337,4 +1400,4 @@
return executionController();
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/Slice.java b/src/java/org/apache/cassandra/db/Slice.java
index e2c787d..1fc60ba 100644
--- a/src/java/org/apache/cassandra/db/Slice.java
+++ b/src/java/org/apache/cassandra/db/Slice.java
@@ -19,24 +19,28 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.List;
+import java.util.Objects;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.Comparables;
/**
* A slice represents the selection of a range of rows.
* <p>
* A slice has a start and an end bound that are both (potentially full) clustering prefixes.
- * A slice selects every rows whose clustering is bigger than the slice start prefix but smaller
- * than the end prefix. Both start and end can be either inclusive or exclusive.
+ * A slice selects every row whose clustering is included within its start and end bounds.
+ * Both start and end can be either inclusive or exclusive.
*/
public class Slice
{
public static final Serializer serializer = new Serializer();
- /** The slice selecting all rows (of a given partition) */
+ /**
+ * The slice selecting all rows (of a given partition)
+ */
public static final Slice ALL = new Slice(BufferClusteringBound.BOTTOM, BufferClusteringBound.TOP)
{
@Override
@@ -46,9 +50,9 @@
}
@Override
- public boolean intersects(ClusteringComparator comparator, List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues)
+ public boolean intersects(ClusteringComparator comparator, Slice other)
{
- return true;
+ return !other.isEmpty(comparator);
}
@Override
@@ -91,6 +95,9 @@
return new Slice(builder.buildBound(true, true), builder.buildBound(false, true));
}
+ /**
+ * Makes a slice covering a single clustering
+ */
public static Slice make(Clustering<?> clustering)
{
// This doesn't give us what we want with the clustering prefix
@@ -98,14 +105,25 @@
return new Slice(ClusteringBound.inclusiveStartOf(clustering), ClusteringBound.inclusiveEndOf(clustering));
}
+ /**
+ * Makes a slice covering a range from start to end clusterings, with both start and end included
+ */
public static Slice make(Clustering<?> start, Clustering<?> end)
{
// This doesn't give us what we want with the clustering prefix
assert start != Clustering.STATIC_CLUSTERING && end != Clustering.STATIC_CLUSTERING;
-
return new Slice(ClusteringBound.inclusiveStartOf(start), ClusteringBound.inclusiveEndOf(end));
}
+ /**
+ * Makes a slice for the given bounds
+ */
+ public static Slice make(ClusteringBoundOrBoundary<?> start, ClusteringBoundOrBoundary<?> end)
+ {
+ // This doesn't give us what we want with the clustering prefix
+ return make(start.asStartBound(), end.asEndBound());
+ }
+
public ClusteringBound<?> start()
{
return start;
@@ -141,31 +159,23 @@
* Return whether the slice formed by the two provided bound is empty or not.
*
* @param comparator the comparator to compare the bounds.
- * @param start the start for the slice to consider. This must be a start bound.
- * @param end the end for the slice to consider. This must be an end bound.
+ * @param start the start for the slice to consider. This must be a start bound.
+ * @param end the end for the slice to consider. This must be an end bound.
* @return whether the slice formed by {@code start} and {@code end} is
* empty or not.
*/
public static boolean isEmpty(ClusteringComparator comparator, ClusteringBound<?> start, ClusteringBound<?> end)
{
assert start.isStart() && end.isEnd();
-
- int cmp = comparator.compare(start, end);
-
- if (cmp < 0)
- return false;
- else if (cmp > 0)
- return true;
- else
- return start.isExclusive() || end.isExclusive();
+ // Note: the comparator orders inclusive starts and exclusive ends as equal, and inclusive ends as being greater than starts.
+ return comparator.compare(start, end) >= 0;
}
/**
* Returns whether a given clustering or bound is included in this slice.
*
* @param comparator the comparator for the table this is a slice of.
- * @param bound the bound to test inclusion of.
- *
+ * @param bound the bound to test inclusion of.
* @return whether {@code bound} is within the bounds of this slice.
*/
public boolean includes(ClusteringComparator comparator, ClusteringPrefix<?> bound)
@@ -176,13 +186,12 @@
/**
* Returns a slice for continuing paging from the last returned clustering prefix.
*
- * @param comparator the comparator for the table this is a filter for.
+ * @param comparator the comparator for the table this is a filter for.
* @param lastReturned the last clustering that was returned for the query we are paging for. The
- * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned
- * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise).
- * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results.
- * @param reversed whether the query we're paging for is reversed or not.
- *
+ * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned
+ * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise).
+ * @param inclusive whether we want to include the {@code lastReturned} in the newly returned page of results.
+ * @param reversed whether the query we're paging for is reversed or not.
* @return a new slice that selects results coming after {@code lastReturned}, or {@code null} if paging
* the resulting slice selects nothing (i.e. if it originally selects nothing coming after {@code lastReturned}).
*/
@@ -229,20 +238,18 @@
}
/**
- * Given the per-clustering column minimum and maximum value a sstable contains, whether or not this slice potentially
- * intersects that sstable or not.
+ * Whether this slice and the provided slice intersects.
*
* @param comparator the comparator for the table this is a slice of.
- * @param minClusteringValues the smallest values for each clustering column that a sstable contains.
- * @param maxClusteringValues the biggest values for each clustering column that a sstable contains.
- *
- * @return whether the slice might intersects with the sstable having {@code minClusteringValues} and
- * {@code maxClusteringValues}.
+ * @param other the other slice to check intersection with.
+ * @return whether this slice intersects {@code other}.
*/
- public boolean intersects(ClusteringComparator comparator, List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues)
+ public boolean intersects(ClusteringComparator comparator, Slice other)
{
- // If this slice starts after max clustering or ends before min clustering, it can't intersect
- return start.compareTo(comparator, maxClusteringValues) <= 0 && end.compareTo(comparator, minClusteringValues) >= 0;
+ // Construct the intersection of the two slices and check if it is non-empty.
+ // This also works correctly when one or more of the inputs are be empty (i.e. with end <= start).
+ return comparator.compare(Comparables.max(start, other.start, comparator),
+ Comparables.min(end, other.end, comparator)) < 0;
}
public String toString(ClusteringComparator comparator)
@@ -269,12 +276,12 @@
@Override
public boolean equals(Object other)
{
- if(!(other instanceof Slice))
+ if (!(other instanceof Slice))
return false;
- Slice that = (Slice)other;
+ Slice that = (Slice) other;
return this.start().equals(that.start())
- && this.end().equals(that.end());
+ && this.end().equals(that.end());
}
@Override
@@ -294,7 +301,7 @@
public long serializedSize(Slice slice, int version, List<AbstractType<?>> types)
{
return ClusteringBound.serializer.serializedSize(slice.start, version, types)
- + ClusteringBound.serializer.serializedSize(slice.end, version, types);
+ + ClusteringBound.serializer.serializedSize(slice.end, version, types);
}
public Slice deserialize(DataInputPlus in, int version, List<AbstractType<?>> types) throws IOException
@@ -304,4 +311,4 @@
return new Slice(start, end);
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/Slices.java b/src/java/org/apache/cassandra/db/Slices.java
index b3f5681..bae83d5 100644
--- a/src/java/org/apache/cassandra/db/Slices.java
+++ b/src/java/org/apache/cassandra/db/Slices.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
@@ -55,7 +56,7 @@
* Creates a {@code Slices} object that contains a single slice.
*
* @param comparator the comparator for the table {@code slice} is a slice of.
- * @param slice the single slice that the return object should contains.
+ * @param slice the single slice that the return object should contain.
*
* @return the newly created {@code Slices} object.
*/
@@ -69,16 +70,16 @@
}
/**
- * Whether the slices has a lower bound, that is whether it's first slice start is {@code Slice.BOTTOM}.
+ * Whether the slices instance has a lower bound, that is whether it's first slice start is {@code Slice.BOTTOM}.
*
- * @return whether the slices has a lower bound.
+ * @return whether this slices instance has a lower bound.
*/
public abstract boolean hasLowerBound();
/**
- * Whether the slices has an upper bound, that is whether it's last slice end is {@code Slice.TOP}.
+ * Whether the slices instance has an upper bound, that is whether it's last slice end is {@code Slice.TOP}.
*
- * @return whether the slices has an upper bound.
+ * @return whether this slices instance has an upper bound.
*/
public abstract boolean hasUpperBound();
@@ -96,6 +97,16 @@
*/
public abstract Slice get(int i);
+ public ClusteringBound<?> start()
+ {
+ return get(0).start();
+ }
+
+ public ClusteringBound<?> end()
+ {
+ return get(size() - 1).end();
+ }
+
/**
* Returns slices for continuing the paging of those slices given the last returned clustering prefix.
*
@@ -103,7 +114,7 @@
* @param lastReturned the last clustering that was returned for the query we are paging for. The
* resulting slices will be such that only results coming stricly after {@code lastReturned} are returned
* (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise).
- * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results.
+ * @param inclusive whether we want to include the {@code lastReturned} in the newly returned page of results.
* @param reversed whether the query we're paging for is reversed or not.
*
* @return new slices that select results coming after {@code lastReturned}.
@@ -130,18 +141,13 @@
*/
public abstract boolean selects(Clustering<?> clustering);
-
/**
- * Given the per-clustering column minimum and maximum value a sstable contains, whether or not this slices potentially
- * intersects that sstable or not.
+ * Checks whether any of the slices intersects witht the given one.
*
- * @param minClusteringValues the smallest values for each clustering column that a sstable contains.
- * @param maxClusteringValues the biggest values for each clustering column that a sstable contains.
- *
- * @return whether the slices might intersects with the sstable having {@code minClusteringValues} and
- * {@code maxClusteringValues}.
+ * @return {@code true} if there exists a slice which ({@link Slice#intersects(ClusteringComparator, Slice)}) with
+ * the provided slice
*/
- public abstract boolean intersects(List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues);
+ public abstract boolean intersects(Slice slice);
public abstract String toCQLString(TableMetadata metadata, RowFilter rowFilter);
@@ -157,12 +163,12 @@
/**
* In simple object that allows to test the inclusion of rows in those slices assuming those rows
* are passed (to {@link #includes}) in clustering order (or reverse clustering ordered, depending
- * of the argument passed to {@link #inOrderTester}).
+ * on the argument passed to {@link #inOrderTester}).
*/
public interface InOrderTester
{
- public boolean includes(Clustering<?> value);
- public boolean isDone();
+ boolean includes(Clustering<?> value);
+ boolean isDone();
}
/**
@@ -243,17 +249,12 @@
if (slices.size() <= 1)
return slices;
- Collections.sort(slices, new Comparator<Slice>()
- {
- @Override
- public int compare(Slice s1, Slice s2)
- {
- int c = comparator.compare(s1.start(), s2.start());
- if (c != 0)
- return c;
+ slices.sort((s1, s2) -> {
+ int c = comparator.compare(s1.start(), s2.start());
+ if (c != 0)
+ return c;
- return comparator.compare(s1.end(), s2.end());
- }
+ return comparator.compare(s2.end(), s1.end());
});
List<Slice> slicesCopy = new ArrayList<>(slices.size());
@@ -278,12 +279,7 @@
}
if (includesStart)
- {
last = Slice.make(last.start(), s2.end());
- continue;
- }
-
- assert !includesFinish;
}
slicesCopy.add(last);
@@ -296,13 +292,13 @@
public void serialize(Slices slices, DataOutputPlus out, int version) throws IOException
{
int size = slices.size();
- out.writeUnsignedVInt(size);
+ out.writeUnsignedVInt32(size);
if (size == 0)
return;
List<AbstractType<?>> types = slices == ALL
- ? Collections.<AbstractType<?>>emptyList()
+ ? Collections.emptyList()
: ((ArrayBackedSlices)slices).comparator.subtypes();
for (Slice slice : slices)
@@ -317,7 +313,7 @@
return size;
List<AbstractType<?>> types = slices instanceof SelectAllSlices
- ? Collections.<AbstractType<?>>emptyList()
+ ? Collections.emptyList()
: ((ArrayBackedSlices)slices).comparator.subtypes();
for (Slice slice : slices)
@@ -328,7 +324,7 @@
public Slices deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
- int size = (int)in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
if (size == 0)
return NONE;
@@ -441,11 +437,12 @@
return Slices.NONE;
}
- public boolean intersects(List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues)
+ @Override
+ public boolean intersects(Slice slice)
{
- for (Slice slice : this)
+ for (Slice s : this)
{
- if (slice.intersects(comparator, minClusteringValues, maxClusteringValues))
+ if (s.intersects(comparator, slice))
return true;
}
return false;
@@ -540,15 +537,7 @@
@Override
public String toString()
{
- StringBuilder sb = new StringBuilder();
- sb.append("{");
- for (int i = 0; i < slices.length; i++)
- {
- if (i > 0)
- sb.append(", ");
- sb.append(slices[i].toString(comparator));
- }
- return sb.append("}").toString();
+ return Arrays.stream(slices).map(s -> s.toString(comparator)).collect(Collectors.joining(", ", "{", "}"));
}
@Override
@@ -636,7 +625,7 @@
operator = first.startInclusive ? Operator.LTE : Operator.LT;
else
operator = first.startInclusive ? Operator.GTE : Operator.GT;
- sb.append(' ').append(operator.toString()).append(' ')
+ sb.append(' ').append(operator).append(' ')
.append(column.type.toCQLString(first.startValue));
rowFilter = rowFilter.without(column, operator, first.startValue);
}
@@ -650,7 +639,7 @@
operator = first.endInclusive ? Operator.GTE : Operator.GT;
else
operator = first.endInclusive ? Operator.LTE : Operator.LT;
- sb.append(' ').append(operator.toString()).append(' ')
+ sb.append(' ').append(operator).append(' ')
.append(column.type.toCQLString(first.endValue));
rowFilter = rowFilter.without(column, operator, first.endValue);
}
@@ -667,7 +656,7 @@
return sb.toString();
}
- // An somewhat adhoc utility class only used by nameAsCQLString
+ // Somewhat adhoc utility class only used by nameAsCQLString
private static class ComponentOfSlice
{
public final boolean startInclusive;
@@ -768,7 +757,8 @@
return trivialTester;
}
- public boolean intersects(List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues)
+ @Override
+ public boolean intersects(Slice slice)
{
return true;
}
@@ -844,7 +834,8 @@
return trivialTester;
}
- public boolean intersects(List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues)
+ @Override
+ public boolean intersects(Slice slice)
{
return false;
}
@@ -866,4 +857,4 @@
return "";
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java b/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
index 4e6ab11..c5debe4 100644
--- a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
+++ b/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
@@ -19,7 +19,6 @@
import javax.management.openmbean.*;
-import com.google.common.base.Throwables;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.snapshot.TableSnapshot;
@@ -32,7 +31,8 @@
"True size",
"Size on disk",
"Creation time",
- "Expiration time",};
+ "Expiration time",
+ "Ephemeral"};
private static final String[] ITEM_DESCS = new String[]{"snapshot_name",
"keyspace_name",
@@ -40,7 +40,8 @@
"TrueDiskSpaceUsed",
"TotalDiskSpaceUsed",
"created_at",
- "expires_at",};
+ "expires_at",
+ "ephemeral"};
private static final String TYPE_NAME = "SnapshotDetails";
@@ -56,7 +57,7 @@
{
try
{
- ITEM_TYPES = new OpenType[]{ SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING };
+ ITEM_TYPES = new OpenType[]{ SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING };
COMPOSITE_TYPE = new CompositeType(TYPE_NAME, ROW_DESC, ITEM_NAMES, ITEM_DESCS, ITEM_TYPES);
@@ -64,7 +65,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -77,8 +78,9 @@
final String liveSize = FileUtils.stringifyFileSize(details.computeTrueSizeBytes());
String createdAt = safeToString(details.getCreatedAt());
String expiresAt = safeToString(details.getExpiresAt());
+ String ephemeral = Boolean.toString(details.isEphemeral());
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
- new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt }));
+ new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt, ephemeral }));
}
catch (OpenDataException e)
{
diff --git a/src/java/org/apache/cassandra/db/StorageHook.java b/src/java/org/apache/cassandra/db/StorageHook.java
index c998338..f5fdec6 100644
--- a/src/java/org/apache/cassandra/db/StorageHook.java
+++ b/src/java/org/apache/cassandra/db/StorageHook.java
@@ -23,11 +23,13 @@
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.STORAGE_HOOK;
+
public interface StorageHook
{
public static final StorageHook instance = createHook();
@@ -35,11 +37,11 @@
public void reportWrite(TableId tableId, PartitionUpdate partitionUpdate);
public void reportRead(TableId tableId, DecoratedKey key);
public UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs,
- DecoratedKey partitionKey,
- SSTableReader sstable,
- ClusteringIndexFilter filter,
- ColumnFilter selectedColumns,
- SSTableReadsListener listener);
+ SSTableReader sstable,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter filter,
+ ColumnFilter selectedColumns,
+ SSTableReadsListener listener);
public UnfilteredRowIterator makeRowIterator(ColumnFamilyStore cfs,
SSTableReader sstable,
DecoratedKey key,
@@ -50,7 +52,7 @@
static StorageHook createHook()
{
- String className = System.getProperty("cassandra.storage_hook");
+ String className = STORAGE_HOOK.getString();
if (className != null)
{
return FBUtilities.construct(className, StorageHook.class.getSimpleName());
@@ -63,8 +65,7 @@
public void reportRead(TableId tableId, DecoratedKey key) {}
public UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs,
- DecoratedKey partitionKey,
- SSTableReader sstable,
+ SSTableReader sstable, DecoratedKey partitionKey,
ClusteringIndexFilter filter,
ColumnFilter selectedColumns,
SSTableReadsListener listener)
@@ -88,4 +89,4 @@
}
};
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index fd2145b..eb0e508 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -58,16 +58,11 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.cql3.functions.AggregateFcts;
-import org.apache.cassandra.cql3.functions.BytesConversionFcts;
-import org.apache.cassandra.cql3.functions.CastFcts;
-import org.apache.cassandra.cql3.functions.OperationFcts;
-import org.apache.cassandra.cql3.functions.TimeFcts;
-import org.apache.cassandra.cql3.functions.UuidFcts;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.CompactionHistoryTabularData;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.TimeUUIDType;
@@ -94,7 +89,6 @@
import org.apache.cassandra.metrics.TopPartitionTracker;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.schema.Functions;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
@@ -103,6 +97,7 @@
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
import org.apache.cassandra.schema.Views;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.paxos.Ballot;
@@ -152,6 +147,7 @@
public static final String BATCHES = "batches";
public static final String PAXOS = "paxos";
public static final String PAXOS_REPAIR_HISTORY = "paxos_repair_history";
+ public static final String PAXOS_REPAIR_STATE = "_paxos_repair_state";
public static final String BUILT_INDEXES = "IndexInfo";
public static final String LOCAL = "local";
public static final String PEERS_V2 = "peers_v2";
@@ -187,6 +183,14 @@
@Deprecated public static final String LEGACY_SIZE_ESTIMATES = "size_estimates";
@Deprecated public static final String LEGACY_SSTABLE_ACTIVITY = "sstable_activity";
+ // Names of all tables that could have been a part of a system keyspace. Useful for pre-flight checks.
+ // For details, see CASSANDRA-17777
+ public static final Set<String> ALL_TABLE_NAMES = ImmutableSet.of(
+ BATCHES, PAXOS, PAXOS_REPAIR_HISTORY, PAXOS_REPAIR_STATE, BUILT_INDEXES, LOCAL, PEERS_V2, PEER_EVENTS_V2,
+ COMPACTION_HISTORY, SSTABLE_ACTIVITY_V2, TABLE_ESTIMATES, TABLE_ESTIMATES_TYPE_PRIMARY,
+ TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY, AVAILABLE_RANGES_V2, TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS,
+ BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS, LEGACY_PEERS, LEGACY_PEER_EVENTS,
+ LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY);
public static final TableMetadata Batches =
parse(BATCHES,
@@ -308,6 +312,7 @@
+ "compacted_at timestamp,"
+ "keyspace_name text,"
+ "rows_merged map<int, bigint>,"
+ + "compaction_properties frozen<map<text, text>>,"
+ "PRIMARY KEY ((id)))")
.defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7))
.build();
@@ -509,7 +514,7 @@
public static KeyspaceMetadata metadata()
{
- return KeyspaceMetadata.create(SchemaConstants.SYSTEM_KEYSPACE_NAME, KeyspaceParams.local(), tables(), Views.none(), Types.none(), functions());
+ return KeyspaceMetadata.create(SchemaConstants.SYSTEM_KEYSPACE_NAME, KeyspaceParams.local(), tables(), Views.none(), Types.none(), UserFunctions.none());
}
private static Tables tables()
@@ -539,18 +544,6 @@
TopPartitions);
}
- private static Functions functions()
- {
- return Functions.builder()
- .add(UuidFcts.all())
- .add(TimeFcts.all())
- .add(BytesConversionFcts.all())
- .add(AggregateFcts.all())
- .add(CastFcts.all())
- .add(OperationFcts.all())
- .build();
- }
-
private static volatile Map<TableId, Pair<CommitLogPosition, Long>> truncationRecords;
public enum BootstrapState
@@ -615,12 +608,13 @@
long compactedAt,
long bytesIn,
long bytesOut,
- Map<Integer, Long> rowsMerged)
+ Map<Integer, Long> rowsMerged,
+ Map<String, String> compactionProperties)
{
// don't write anything when the history table itself is compacted, since that would in turn cause new compactions
if (ksname.equals("system") && cfname.equals(COMPACTION_HISTORY))
return;
- String req = "INSERT INTO system.%s (id, keyspace_name, columnfamily_name, compacted_at, bytes_in, bytes_out, rows_merged) VALUES (?, ?, ?, ?, ?, ?, ?)";
+ String req = "INSERT INTO system.%s (id, keyspace_name, columnfamily_name, compacted_at, bytes_in, bytes_out, rows_merged, compaction_properties) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
executeInternal(format(req, COMPACTION_HISTORY),
nextTimeUUID(),
ksname,
@@ -628,7 +622,8 @@
ByteBufferUtil.bytes(compactedAt),
bytesIn,
bytesOut,
- rowsMerged);
+ rowsMerged,
+ compactionProperties);
}
public static TabularData getCompactionHistory() throws OpenDataException
@@ -1487,8 +1482,7 @@
*/
public static RestorableMeter getSSTableReadMeter(String keyspace, String table, SSTableId id)
{
- String cql = "SELECT * FROM system.%s WHERE keyspace_name=? and table_name=? and id=?";
- UntypedResultSet results = executeInternal(format(cql, SSTABLE_ACTIVITY_V2), keyspace, table, id.toString());
+ UntypedResultSet results = readSSTableActivity(keyspace, table, id);
if (results.isEmpty())
return new RestorableMeter();
@@ -1499,6 +1493,13 @@
return new RestorableMeter(m15rate, m120rate);
}
+ @VisibleForTesting
+ public static UntypedResultSet readSSTableActivity(String keyspace, String table, SSTableId id)
+ {
+ String cql = "SELECT * FROM system.%s WHERE keyspace_name=? and table_name=? and id=?";
+ return executeInternal(format(cql, SSTABLE_ACTIVITY_V2), keyspace, table, id.toString());
+ }
+
/**
* Writes the current read rates for a given SSTable to system.sstable_activity
*/
@@ -1665,12 +1666,18 @@
}
}
- public static void resetAvailableRanges()
+ public static void resetAvailableStreamedRanges()
{
ColumnFamilyStore availableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(AVAILABLE_RANGES_V2);
availableRanges.truncateBlockingWithoutSnapshot();
}
+ public static void resetAvailableStreamedRangesForKeyspace(String keyspace)
+ {
+ String cql = "DELETE FROM %s.%s WHERE keyspace_name = ?";
+ executeInternal(format(cql, SchemaConstants.SYSTEM_KEYSPACE_NAME, AVAILABLE_RANGES_V2), keyspace);
+ }
+
public static synchronized void updateTransferredRanges(StreamOperation streamOperation,
InetAddressAndPort peer,
String keyspace,
@@ -1780,7 +1787,8 @@
return rawRanges.stream().map(buf -> byteBufferToRange(buf, partitioner)).collect(Collectors.toSet());
}
- static ByteBuffer rangeToBytes(Range<Token> range)
+ @VisibleForTesting
+ public static ByteBuffer rangeToBytes(Range<Token> range)
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
@@ -1899,7 +1907,7 @@
TupleType tupleType = new TupleType(Lists.newArrayList(UTF8Type.instance, LongType.instance));
for (ByteBuffer bb : top)
{
- ByteBuffer[] components = tupleType.split(bb);
+ ByteBuffer[] components = tupleType.split(ByteBufferAccessor.instance, bb);
String keyStr = UTF8Type.instance.compose(components[0]);
long value = LongType.instance.compose(components[1]);
topPartitions.add(new TopPartitionTracker.TopPartition(metadata.partitioner.decorateKey(metadata.partitionKeyType.fromString(keyStr)), value));
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java
index bfce780..ab9f01f 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java
@@ -24,6 +24,7 @@
import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +34,7 @@
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
@@ -62,6 +64,7 @@
migrateTransferredRanges();
migrateAvailableRanges();
migrateSSTableActivity();
+ migrateCompactionHistory();
}
@VisibleForTesting
@@ -159,12 +162,38 @@
})
);
}
+
+ @VisibleForTesting
+ static void migrateCompactionHistory()
+ {
+ migrateTable(false,
+ SystemKeyspace.COMPACTION_HISTORY,
+ SystemKeyspace.COMPACTION_HISTORY,
+ new String[]{ "id",
+ "bytes_in",
+ "bytes_out",
+ "columnfamily_name",
+ "compacted_at",
+ "keyspace_name",
+ "rows_merged",
+ "compaction_properties" },
+ row -> Collections.singletonList(new Object[]{ row.getTimeUUID("id") ,
+ row.has("bytes_in") ? row.getLong("bytes_in") : null,
+ row.has("bytes_out") ? row.getLong("bytes_out") : null,
+ row.has("columnfamily_name") ? row.getString("columnfamily_name") : null,
+ row.has("compacted_at") ? row.getTimestamp("compacted_at") : null,
+ row.has("keyspace_name") ? row.getString("keyspace_name") : null,
+ row.has("rows_merged") ? row.getMap("rows_merged", Int32Type.instance, LongType.instance) : null,
+ row.has("compaction_properties") ? row.getMap("compaction_properties", UTF8Type.instance, UTF8Type.instance) : ImmutableMap.of() })
+ );
+ }
/**
* Perform table migration by reading data from the old table, converting it, and adding to the new table.
- *
+ * If oldName and newName are same, it means data in the table will be refreshed.
+ *
* @param truncateIfExists truncate the existing table if it exists before migration; if it is disabled
- * and the new table is not empty, no migration is performed
+ * and the new table is not empty and oldName is not equal to newName, no migration is performed
* @param oldName old table name
* @param newName new table name
* @param columns columns to fill in the new table in the same order as returned by the transformation
@@ -175,7 +204,7 @@
{
ColumnFamilyStore newTable = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(newName);
- if (!newTable.isEmpty() && !truncateIfExists)
+ if (!newTable.isEmpty() && !truncateIfExists && !oldName.equals(newName))
return;
if (truncateIfExists)
@@ -189,6 +218,9 @@
StringUtils.join(columns, ", "), StringUtils.repeat("?", ", ", columns.length));
UntypedResultSet rows = QueryProcessor.executeInternal(query);
+
+ assert rows != null : String.format("Migrating rows from legacy %s to %s was not done as returned rows from %s are null!", oldName, newName, oldName);
+
int transferred = 0;
logger.info("Migrating rows from legacy {} to {}", oldName, newName);
for (UntypedResultSet.Row row : rows)
diff --git a/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java b/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java
index 0d6c0ee..a4a1c57 100644
--- a/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java
+++ b/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.aggregation;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import org.apache.cassandra.cql3.QueryOptions;
@@ -27,6 +28,7 @@
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
/**
@@ -61,7 +63,7 @@
/**
* The <code>AggregationSpecification</code> kinds.
*/
- public static enum Kind
+ public enum Kind
{
AGGREGATE_EVERYTHING, AGGREGATE_BY_PK_PREFIX, AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR
}
@@ -115,7 +117,8 @@
public static AggregationSpecification.Factory aggregatePkPrefixFactoryWithSelector(final ClusteringComparator comparator,
final int clusteringPrefixSize,
- final Selector.Factory factory)
+ final Selector.Factory factory,
+ final List<ColumnMetadata> columns)
{
return new Factory()
{
@@ -131,8 +134,9 @@
Selector selector = factory.newInstance(options);
selector.validateForGroupBy();
return new AggregateByPkPrefixWithSelector(comparator,
- clusteringPrefixSize,
- selector);
+ clusteringPrefixSize,
+ selector,
+ columns);
}
};
}
@@ -200,18 +204,25 @@
*/
private final Selector selector;
+ /**
+ * The columns used by the selector.
+ */
+ private final List<ColumnMetadata> columns;
+
public AggregateByPkPrefixWithSelector(ClusteringComparator comparator,
int clusteringPrefixSize,
- Selector selector)
+ Selector selector,
+ List<ColumnMetadata> columns)
{
super(Kind.AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR, comparator, clusteringPrefixSize);
this.selector = selector;
+ this.columns = columns;
}
@Override
public GroupMaker newGroupMaker(GroupingState state)
{
- return GroupMaker.newSelectorGroupMaker(comparator, clusteringPrefixSize, selector, state);
+ return GroupMaker.newSelectorGroupMaker(comparator, clusteringPrefixSize, selector, columns, state);
}
}
@@ -225,12 +236,15 @@
case AGGREGATE_EVERYTHING:
break;
case AGGREGATE_BY_PK_PREFIX:
- out.writeUnsignedVInt(((AggregateByPkPrefix) aggregationSpec).clusteringPrefixSize);
+ out.writeUnsignedVInt32(((AggregateByPkPrefix) aggregationSpec).clusteringPrefixSize);
break;
case AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR:
AggregateByPkPrefixWithSelector spec = (AggregateByPkPrefixWithSelector) aggregationSpec;
- out.writeUnsignedVInt(spec.clusteringPrefixSize);
+ out.writeUnsignedVInt32(spec.clusteringPrefixSize);
Selector.serializer.serialize(spec.selector, out, version);
+ // Ideally we should serialize the columns but that will break backward compatibility.
+ // So for the moment we can rebuild the list from the prefix size as we know that there will be
+ // only one column and that its indice will be: clusteringPrefixSize - 1.
break;
default:
throw new AssertionError("Unknow aggregation kind: " + aggregationSpec.kind());
@@ -245,13 +259,15 @@
case AGGREGATE_EVERYTHING:
return AggregationSpecification.AGGREGATE_EVERYTHING;
case AGGREGATE_BY_PK_PREFIX:
- return new AggregateByPkPrefix(metadata.comparator, (int) in.readUnsignedVInt());
+ return new AggregateByPkPrefix(metadata.comparator, in.readUnsignedVInt32());
case AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR:
- int clusteringPrefixSize = (int) in.readUnsignedVInt();
+ int clusteringPrefixSize = in.readUnsignedVInt32();
Selector selector = Selector.serializer.deserialize(in, version, metadata);
+ ColumnMetadata functionArgument = metadata.clusteringColumns().get(clusteringPrefixSize - 1);
return new AggregateByPkPrefixWithSelector(metadata.comparator,
clusteringPrefixSize,
- selector);
+ selector,
+ Collections.singletonList(functionArgument));
default:
throw new AssertionError("Unknow aggregation kind: " + kind);
}
@@ -270,9 +286,7 @@
case AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR:
AggregateByPkPrefixWithSelector spec = (AggregateByPkPrefixWithSelector) aggregationSpec;
size += TypeSizes.sizeofUnsignedVInt(spec.clusteringPrefixSize);
- size += Selector.serializer.serializedSize(spec.selector, version
-
- );
+ size += Selector.serializer.serializedSize(spec.selector, version);
break;
default:
throw new AssertionError("Unknow aggregation kind: " + aggregationSpec.kind());
diff --git a/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java b/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java
index 968219f..21bb438 100644
--- a/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java
+++ b/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java
@@ -18,11 +18,13 @@
package org.apache.cassandra.db.aggregation;
import java.nio.ByteBuffer;
+import java.util.List;
import org.apache.cassandra.cql3.selection.Selector;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.transport.ProtocolVersion;
/**
@@ -61,16 +63,18 @@
public static GroupMaker newSelectorGroupMaker(ClusteringComparator comparator,
int clusteringPrefixSize,
Selector selector,
+ List<ColumnMetadata> columns,
GroupingState state)
{
- return new SelectorGroupMaker(comparator, clusteringPrefixSize, selector, state);
+ return new SelectorGroupMaker(comparator, clusteringPrefixSize, selector, columns, state);
}
public static GroupMaker newSelectorGroupMaker(ClusteringComparator comparator,
int clusteringPrefixSize,
- Selector selector)
+ Selector selector,
+ List<ColumnMetadata> columns)
{
- return new SelectorGroupMaker(comparator, clusteringPrefixSize, selector);
+ return new SelectorGroupMaker(comparator, clusteringPrefixSize, selector, columns);
}
/**
@@ -158,25 +162,29 @@
*/
private ByteBuffer lastOutput;
- private final Selector.InputRow input = new Selector.InputRow(1, false, false);
+ private final Selector.InputRow input;
public SelectorGroupMaker(ClusteringComparator comparator,
int clusteringPrefixSize,
Selector selector,
+ List<ColumnMetadata> columns,
GroupingState state)
{
super(comparator, clusteringPrefixSize, state);
this.selector = selector;
+ this.input = new Selector.InputRow(ProtocolVersion.CURRENT, columns, false);
this.lastOutput = lastClustering == null ? null :
executeSelector(lastClustering.bufferAt(clusteringPrefixSize - 1));
}
public SelectorGroupMaker(ClusteringComparator comparator,
int clusteringPrefixSize,
- Selector selector)
+ Selector selector,
+ List<ColumnMetadata> columns)
{
super(comparator, clusteringPrefixSize);
this.selector = selector;
+ this.input = new Selector.InputRow(ProtocolVersion.CURRENT, columns, false);
}
@Override
@@ -217,7 +225,7 @@
input.add(argument);
// For computing groups we do not need to use the client protocol version.
- selector.addInput(ProtocolVersion.CURRENT, input);
+ selector.addInput(input);
ByteBuffer output = selector.getOutput(ProtocolVersion.CURRENT);
selector.reset();
input.reset(false);
diff --git a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
deleted file mode 100644
index fee45c2..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.sstable.IndexInfo;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.DataPosition;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
-{
- protected final SSTableReader sstable;
- // We could use sstable.metadata(), but that can change during execution so it's good hygiene to grab an immutable instance
- protected final TableMetadata metadata;
-
- protected final DecoratedKey key;
- protected final DeletionTime partitionLevelDeletion;
- protected final ColumnFilter columns;
- protected final DeserializationHelper helper;
-
- protected final Row staticRow;
- protected final Reader reader;
-
- protected final FileHandle ifile;
-
- private boolean isClosed;
-
- protected final Slices slices;
-
- @SuppressWarnings("resource") // We need this because the analysis is not able to determine that we do close
- // file on every path where we created it.
- protected AbstractSSTableIterator(SSTableReader sstable,
- FileDataInput file,
- DecoratedKey key,
- RowIndexEntry indexEntry,
- Slices slices,
- ColumnFilter columnFilter,
- FileHandle ifile)
- {
- this.sstable = sstable;
- this.metadata = sstable.metadata();
- this.ifile = ifile;
- this.key = key;
- this.columns = columnFilter;
- this.slices = slices;
- this.helper = new DeserializationHelper(metadata, sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL, columnFilter);
-
- if (indexEntry == null)
- {
- this.partitionLevelDeletion = DeletionTime.LIVE;
- this.reader = null;
- this.staticRow = Rows.EMPTY_STATIC_ROW;
- }
- else
- {
- boolean shouldCloseFile = file == null;
- try
- {
- // We seek to the beginning to the partition if either:
- // - the partition is not indexed; we then have a single block to read anyway
- // (and we need to read the partition deletion time).
- // - we're querying static columns.
- boolean needSeekAtPartitionStart = !indexEntry.isIndexed() || !columns.fetchedColumns().statics.isEmpty();
-
- if (needSeekAtPartitionStart)
- {
- // Not indexed (or is reading static), set to the beginning of the partition and read partition level deletion there
- if (file == null)
- file = sstable.getFileDataInput(indexEntry.position);
- else
- file.seek(indexEntry.position);
-
- ByteBufferUtil.skipShortLength(file); // Skip partition key
- this.partitionLevelDeletion = DeletionTime.serializer.deserialize(file);
-
- // Note that this needs to be called after file != null and after the partitionDeletion has been set, but before readStaticRow
- // (since it uses it) so we can't move that up (but we'll be able to simplify as soon as we drop support for the old file format).
- this.reader = createReader(indexEntry, file, shouldCloseFile);
- this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics);
- }
- else
- {
- this.partitionLevelDeletion = indexEntry.deletionTime();
- this.staticRow = Rows.EMPTY_STATIC_ROW;
- this.reader = createReader(indexEntry, file, shouldCloseFile);
- }
- if (!partitionLevelDeletion.validate())
- UnfilteredValidation.handleInvalid(metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
-
- if (reader != null && !slices.isEmpty())
- reader.setForSlice(nextSlice());
-
- if (reader == null && file != null && shouldCloseFile)
- file.close();
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- String filePath = file.getPath();
- if (shouldCloseFile)
- {
- try
- {
- file.close();
- }
- catch (IOException suppressed)
- {
- e.addSuppressed(suppressed);
- }
- }
- throw new CorruptSSTableException(e, filePath);
- }
- }
- }
-
- private Slice nextSlice()
- {
- return slices.get(nextSliceIndex());
- }
-
- /**
- * Returns the index of the next slice to process.
- * @return the index of the next slice to process
- */
- protected abstract int nextSliceIndex();
-
- /**
- * Checks if there are more slice to process.
- * @return {@code true} if there are more slice to process, {@code false} otherwise.
- */
- protected abstract boolean hasMoreSlices();
-
- private static Row readStaticRow(SSTableReader sstable,
- FileDataInput file,
- DeserializationHelper helper,
- Columns statics) throws IOException
- {
- if (!sstable.header.hasStatic())
- return Rows.EMPTY_STATIC_ROW;
-
- if (statics.isEmpty())
- {
- UnfilteredSerializer.serializer.skipStaticRow(file, sstable.header, helper);
- return Rows.EMPTY_STATIC_ROW;
- }
- else
- {
- return UnfilteredSerializer.serializer.deserializeStaticRow(file, sstable.header, helper);
- }
- }
-
- protected abstract Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile);
-
- private Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
- {
- return slices.isEmpty() ? new NoRowsReader(file, shouldCloseFile)
- : createReaderInternal(indexEntry, file, shouldCloseFile);
- };
-
- public TableMetadata metadata()
- {
- return metadata;
- }
-
- public RegularAndStaticColumns columns()
- {
- return columns.fetchedColumns();
- }
-
- public DecoratedKey partitionKey()
- {
- return key;
- }
-
- public DeletionTime partitionLevelDeletion()
- {
- return partitionLevelDeletion;
- }
-
- public Row staticRow()
- {
- return staticRow;
- }
-
- public EncodingStats stats()
- {
- return sstable.stats();
- }
-
- public boolean hasNext()
- {
- while (true)
- {
- if (reader == null)
- return false;
-
- if (reader.hasNext())
- return true;
-
- if (!hasMoreSlices())
- return false;
-
- slice(nextSlice());
- }
- }
-
- public Unfiltered next()
- {
- assert reader != null;
- return reader.next();
- }
-
- private void slice(Slice slice)
- {
- try
- {
- if (reader != null)
- reader.setForSlice(slice);
- }
- catch (IOException e)
- {
- try
- {
- closeInternal();
- }
- catch (IOException suppressed)
- {
- e.addSuppressed(suppressed);
- }
- sstable.markSuspect();
- throw new CorruptSSTableException(e, reader.file.getPath());
- }
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
-
- private void closeInternal() throws IOException
- {
- // It's important to make closing idempotent since it would bad to double-close 'file' as its a RandomAccessReader
- // and its close is not idemptotent in the case where we recycle it.
- if (isClosed)
- return;
-
- if (reader != null)
- reader.close();
-
- isClosed = true;
- }
-
- public void close()
- {
- try
- {
- closeInternal();
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, reader.file.getPath());
- }
- }
-
- protected abstract class Reader implements Iterator<Unfiltered>
- {
- private final boolean shouldCloseFile;
- public FileDataInput file;
-
- protected UnfilteredDeserializer deserializer;
-
- // Records the currently open range tombstone (if any)
- protected DeletionTime openMarker = null;
-
- protected Reader(FileDataInput file, boolean shouldCloseFile)
- {
- this.file = file;
- this.shouldCloseFile = shouldCloseFile;
-
- if (file != null)
- createDeserializer();
- }
-
- private void createDeserializer()
- {
- assert file != null && deserializer == null;
- deserializer = UnfilteredDeserializer.create(metadata, file, sstable.header, helper);
- }
-
- protected void seekToPosition(long position) throws IOException
- {
- // This may be the first time we're actually looking into the file
- if (file == null)
- {
- file = sstable.getFileDataInput(position);
- createDeserializer();
- }
- else
- {
- file.seek(position);
- }
- }
-
- protected void updateOpenMarker(RangeTombstoneMarker marker)
- {
- // Note that we always read index blocks in forward order so this method is always called in forward order
- openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null;
- }
-
- public boolean hasNext()
- {
- try
- {
- return hasNextInternal();
- }
- catch (IOException | IndexOutOfBoundsException e)
- {
- try
- {
- closeInternal();
- }
- catch (IOException suppressed)
- {
- e.addSuppressed(suppressed);
- }
- sstable.markSuspect();
- throw new CorruptSSTableException(e, reader.file.getPath());
- }
- }
-
- public Unfiltered next()
- {
- try
- {
- return nextInternal();
- }
- catch (IOException e)
- {
- try
- {
- closeInternal();
- }
- catch (IOException suppressed)
- {
- e.addSuppressed(suppressed);
- }
- sstable.markSuspect();
- throw new CorruptSSTableException(e, reader.file.getPath());
- }
- }
-
- // Set the reader so its hasNext/next methods return values within the provided slice
- public abstract void setForSlice(Slice slice) throws IOException;
-
- protected abstract boolean hasNextInternal() throws IOException;
- protected abstract Unfiltered nextInternal() throws IOException;
-
- public void close() throws IOException
- {
- if (shouldCloseFile && file != null)
- file.close();
- }
- }
-
- // Reader for when we have Slices.NONE but need to read static row or partition level deletion
- private class NoRowsReader extends AbstractSSTableIterator.Reader
- {
- private NoRowsReader(FileDataInput file, boolean shouldCloseFile)
- {
- super(file, shouldCloseFile);
- }
-
- public void setForSlice(Slice slice) throws IOException
- {
- return;
- }
-
- protected boolean hasNextInternal() throws IOException
- {
- return false;
- }
-
- protected Unfiltered nextInternal() throws IOException
- {
- throw new NoSuchElementException();
- }
- }
-
- // Used by indexed readers to store where they are of the index.
- public static class IndexState implements AutoCloseable
- {
- private final Reader reader;
- private final ClusteringComparator comparator;
-
- private final RowIndexEntry indexEntry;
- private final RowIndexEntry.IndexInfoRetriever indexInfoRetriever;
- private final boolean reversed;
-
- private int currentIndexIdx;
-
- // Marks the beginning of the block corresponding to currentIndexIdx.
- private DataPosition mark;
-
- public IndexState(Reader reader, ClusteringComparator comparator, RowIndexEntry indexEntry, boolean reversed, FileHandle indexFile)
- {
- this.reader = reader;
- this.comparator = comparator;
- this.indexEntry = indexEntry;
- this.indexInfoRetriever = indexEntry.openWithIndex(indexFile);
- this.reversed = reversed;
- this.currentIndexIdx = reversed ? indexEntry.columnsIndexCount() : -1;
- }
-
- public boolean isDone()
- {
- return reversed ? currentIndexIdx < 0 : currentIndexIdx >= indexEntry.columnsIndexCount();
- }
-
- // Sets the reader to the beginning of blockIdx.
- public void setToBlock(int blockIdx) throws IOException
- {
- if (blockIdx >= 0 && blockIdx < indexEntry.columnsIndexCount())
- {
- reader.seekToPosition(columnOffset(blockIdx));
- mark = reader.file.mark();
- reader.deserializer.clearState();
- }
-
- currentIndexIdx = blockIdx;
- reader.openMarker = blockIdx > 0 ? index(blockIdx - 1).endOpenMarker : null;
- }
-
- private long columnOffset(int i) throws IOException
- {
- return indexEntry.position + index(i).offset;
- }
-
- public int blocksCount()
- {
- return indexEntry.columnsIndexCount();
- }
-
- // Update the block idx based on the current reader position if we're past the current block.
- // This only makes sense for forward iteration (for reverse ones, when we reach the end of a block we
- // should seek to the previous one, not update the index state and continue).
- public void updateBlock() throws IOException
- {
- assert !reversed;
-
- // If we get here with currentBlockIdx < 0, it means setToBlock() has never been called, so it means
- // we're about to read from the beginning of the partition, but haven't "prepared" the IndexState yet.
- // Do so by setting us on the first block.
- if (currentIndexIdx < 0)
- {
- setToBlock(0);
- return;
- }
-
- while (currentIndexIdx + 1 < indexEntry.columnsIndexCount() && isPastCurrentBlock())
- {
- reader.openMarker = currentIndex().endOpenMarker;
- ++currentIndexIdx;
-
- // We have to set the mark, and we have to set it at the beginning of the block. So if we're not at the beginning of the block, this forces us to a weird seek dance.
- // This can only happen when reading old file however.
- long startOfBlock = columnOffset(currentIndexIdx);
- long currentFilePointer = reader.file.getFilePointer();
- if (startOfBlock == currentFilePointer)
- {
- mark = reader.file.mark();
- }
- else
- {
- reader.seekToPosition(startOfBlock);
- mark = reader.file.mark();
- reader.seekToPosition(currentFilePointer);
- }
- }
- }
-
- // Check if we've crossed an index boundary (based on the mark on the beginning of the index block).
- public boolean isPastCurrentBlock() throws IOException
- {
- assert reader.deserializer != null;
- return reader.file.bytesPastMark(mark) >= currentIndex().width;
- }
-
- public int currentBlockIdx()
- {
- return currentIndexIdx;
- }
-
- public IndexInfo currentIndex() throws IOException
- {
- return index(currentIndexIdx);
- }
-
- public IndexInfo index(int i) throws IOException
- {
- return indexInfoRetriever.columnsIndex(i);
- }
-
- // Finds the index of the first block containing the provided bound, starting at the provided index.
- // Will be -1 if the bound is before any block, and blocksCount() if it is after every block.
- public int findBlockIndex(ClusteringBound<?> bound, int fromIdx) throws IOException
- {
- if (bound.isBottom())
- return -1;
- if (bound.isTop())
- return blocksCount();
-
- return indexFor(bound, fromIdx);
- }
-
- public int indexFor(ClusteringPrefix<?> name, int lastIndex) throws IOException
- {
- IndexInfo target = new IndexInfo(name, name, 0, 0, null);
- /*
- Take the example from the unit test, and say your index looks like this:
- [0..5][10..15][20..25]
- and you look for the slice [13..17].
-
- When doing forward slice, we are doing a binary search comparing 13 (the start of the query)
- to the lastName part of the index slot. You'll end up with the "first" slot, going from left to right,
- that may contain the start.
-
- When doing a reverse slice, we do the same thing, only using as a start column the end of the query,
- i.e. 17 in this example, compared to the firstName part of the index slots. bsearch will give us the
- first slot where firstName > start ([20..25] here), so we subtract an extra one to get the slot just before.
- */
- int startIdx = 0;
- int endIdx = indexEntry.columnsIndexCount() - 1;
-
- if (reversed)
- {
- if (lastIndex < endIdx)
- {
- endIdx = lastIndex;
- }
- }
- else
- {
- if (lastIndex > 0)
- {
- startIdx = lastIndex;
- }
- }
-
- int index = binarySearch(target, comparator.indexComparator(reversed), startIdx, endIdx);
- return (index < 0 ? -index - (reversed ? 2 : 1) : index);
- }
-
- private int binarySearch(IndexInfo key, Comparator<IndexInfo> c, int low, int high) throws IOException
- {
- while (low <= high)
- {
- int mid = (low + high) >>> 1;
- IndexInfo midVal = index(mid);
- int cmp = c.compare(midVal, key);
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid;
- }
- return -(low + 1);
- }
-
- @Override
- public String toString()
- {
- return String.format("IndexState(indexSize=%d, currentBlock=%d, reversed=%b)", indexEntry.columnsIndexCount(), currentIndexIdx, reversed);
- }
-
- @Override
- public void close() throws IOException
- {
- indexInfoRetriever.close();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java
deleted file mode 100644
index d4362f7..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-import java.util.NoSuchElementException;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileHandle;
-
-/**
- * A Cell Iterator over SSTable
- */
-public class SSTableIterator extends AbstractSSTableIterator
-{
- /**
- * The index of the slice being processed.
- */
- private int slice;
-
- public SSTableIterator(SSTableReader sstable,
- FileDataInput file,
- DecoratedKey key,
- RowIndexEntry indexEntry,
- Slices slices,
- ColumnFilter columns,
- FileHandle ifile)
- {
- super(sstable, file, key, indexEntry, slices, columns, ifile);
- }
-
- protected Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
- {
- return indexEntry.isIndexed()
- ? new ForwardIndexedReader(indexEntry, file, shouldCloseFile)
- : new ForwardReader(file, shouldCloseFile);
- }
-
- protected int nextSliceIndex()
- {
- int next = slice;
- slice++;
- return next;
- }
-
- protected boolean hasMoreSlices()
- {
- return slice < slices.size();
- }
-
- public boolean isReverseOrder()
- {
- return false;
- }
-
- private class ForwardReader extends Reader
- {
- // The start of the current slice. This will be null as soon as we know we've passed that bound.
- protected ClusteringBound<?> start;
- // The end of the current slice. Will never be null.
- protected ClusteringBound<?> end = BufferClusteringBound.TOP;
-
- protected Unfiltered next; // the next element to return: this is computed by hasNextInternal().
-
- protected boolean sliceDone; // set to true once we know we have no more result for the slice. This is in particular
- // used by the indexed reader when we know we can't have results based on the index.
-
- private ForwardReader(FileDataInput file, boolean shouldCloseFile)
- {
- super(file, shouldCloseFile);
- }
-
- public void setForSlice(Slice slice) throws IOException
- {
- start = slice.start().isBottom() ? null : slice.start();
- end = slice.end();
-
- sliceDone = false;
- next = null;
- }
-
- // Skip all data that comes before the currently set slice.
- // Return what should be returned at the end of this, or null if nothing should.
- private Unfiltered handlePreSliceData() throws IOException
- {
- assert deserializer != null;
-
- // Note that the following comparison is not strict. The reason is that the only cases
- // where it can be == is if the "next" is a RT start marker (either a '[' of a ')[' boundary),
- // and if we had a strict inequality and an open RT marker before this, we would issue
- // the open marker first, and then return then next later, which would send in the
- // stream both '[' (or '(') and then ')[' for the same clustering value, which is wrong.
- // By using a non-strict inequality, we avoid that problem (if we do get ')[' for the same
- // clustering value than the slice, we'll simply record it in 'openMarker').
- while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0)
- {
- if (deserializer.nextIsRow())
- deserializer.skipNext();
- else
- updateOpenMarker((RangeTombstoneMarker)deserializer.readNext());
- }
-
- ClusteringBound<?> sliceStart = start;
- start = null;
-
- // We've reached the beginning of our queried slice. If we have an open marker
- // we should return that first.
- if (openMarker != null)
- return new RangeTombstoneBoundMarker(sliceStart, openMarker);
-
- return null;
- }
-
- // Compute the next element to return, assuming we're in the middle to the slice
- // and the next element is either in the slice, or just after it. Returns null
- // if we're done with the slice.
- protected Unfiltered computeNext() throws IOException
- {
- assert deserializer != null;
-
- while (true)
- {
- // We use a same reasoning as in handlePreSliceData regarding the strictness of the inequality below.
- // We want to exclude deserialized unfiltered equal to end, because 1) we won't miss any rows since those
- // woudn't be equal to a slice bound and 2) a end bound can be equal to a start bound
- // (EXCL_END(x) == INCL_START(x) for instance) and in that case we don't want to return start bound because
- // it's fundamentally excluded. And if the bound is a end (for a range tombstone), it means it's exactly
- // our slice end, but in that case we will properly close the range tombstone anyway as part of our "close
- // an open marker" code in hasNextInterna
- if (!deserializer.hasNext() || deserializer.compareNextTo(end) >= 0)
- return null;
-
- Unfiltered next = deserializer.readNext();
- UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable);
- // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
- if (next.isEmpty())
- continue;
-
- if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
- updateOpenMarker((RangeTombstoneMarker) next);
- return next;
- }
- }
-
- protected boolean hasNextInternal() throws IOException
- {
- if (next != null)
- return true;
-
- if (sliceDone)
- return false;
-
- if (start != null)
- {
- Unfiltered unfiltered = handlePreSliceData();
- if (unfiltered != null)
- {
- next = unfiltered;
- return true;
- }
- }
-
- next = computeNext();
- if (next != null)
- return true;
-
- // for current slice, no data read from deserialization
- sliceDone = true;
- // If we have an open marker, we should not close it, there could be more slices
- if (openMarker != null)
- {
- next = new RangeTombstoneBoundMarker(end, openMarker);
- return true;
- }
- return false;
- }
-
- protected Unfiltered nextInternal() throws IOException
- {
- if (!hasNextInternal())
- throw new NoSuchElementException();
-
- Unfiltered toReturn = next;
- next = null;
- return toReturn;
- }
- }
-
- private class ForwardIndexedReader extends ForwardReader
- {
- private final IndexState indexState;
-
- private int lastBlockIdx; // the last index block that has data for the current query
-
- private ForwardIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
- {
- super(file, shouldCloseFile);
- this.indexState = new IndexState(this, metadata.comparator, indexEntry, false, ifile);
- this.lastBlockIdx = indexState.blocksCount(); // if we never call setForSlice, that's where we want to stop
- }
-
- @Override
- public void close() throws IOException
- {
- super.close();
- this.indexState.close();
- }
-
- @Override
- public void setForSlice(Slice slice) throws IOException
- {
- super.setForSlice(slice);
-
- // if our previous slicing already got us the biggest row in the sstable, we're done
- if (indexState.isDone())
- {
- sliceDone = true;
- return;
- }
-
- // Find the first index block we'll need to read for the slice.
- int startIdx = indexState.findBlockIndex(slice.start(), indexState.currentBlockIdx());
- if (startIdx >= indexState.blocksCount())
- {
- sliceDone = true;
- return;
- }
-
- // Find the last index block we'll need to read for the slice.
- lastBlockIdx = indexState.findBlockIndex(slice.end(), startIdx);
-
- // If the slice end is before the very first block, we have nothing for that slice
- if (lastBlockIdx < 0)
- {
- assert startIdx < 0;
- sliceDone = true;
- return;
- }
-
- // If we start before the very first block, just read from the first one.
- if (startIdx < 0)
- startIdx = 0;
-
- // If that's the last block we were reading, we're already where we want to be. Otherwise,
- // seek to that first block
- if (startIdx != indexState.currentBlockIdx())
- indexState.setToBlock(startIdx);
-
- // The index search is based on the last name of the index blocks, so at that point we have that:
- // 1) indexes[currentIdx - 1].lastName < slice.start <= indexes[currentIdx].lastName
- // 2) indexes[lastBlockIdx - 1].lastName < slice.end <= indexes[lastBlockIdx].lastName
- // so if currentIdx == lastBlockIdx and slice.end < indexes[currentIdx].firstName, we're guaranteed that the
- // whole slice is between the previous block end and this block start, and thus has no corresponding
- // data. One exception is if the previous block ends with an openMarker as it will cover our slice
- // and we need to return it.
- if (indexState.currentBlockIdx() == lastBlockIdx
- && metadata().comparator.compare(slice.end(), indexState.currentIndex().firstName) < 0
- && openMarker == null)
- {
- sliceDone = true;
- }
- }
-
- @Override
- protected Unfiltered computeNext() throws IOException
- {
- while (true)
- {
- // Our previous read might have made us cross an index block boundary. If so, update our informations.
- // If we read from the beginning of the partition, this is also what will initialize the index state.
- indexState.updateBlock();
-
- // Return the next unfiltered unless we've reached the end, or we're beyond our slice
- // end (note that unless we're on the last block for the slice, there is no point
- // in checking the slice end).
- if (indexState.isDone()
- || indexState.currentBlockIdx() > lastBlockIdx
- || !deserializer.hasNext()
- || (indexState.currentBlockIdx() == lastBlockIdx && deserializer.compareNextTo(end) >= 0))
- return null;
-
-
- Unfiltered next = deserializer.readNext();
- UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable);
- // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
- if (next.isEmpty())
- continue;
-
- if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
- updateOpenMarker((RangeTombstoneMarker) next);
- return next;
- }
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java
deleted file mode 100644
index a60aafa..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-import java.util.*;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.utils.btree.BTree;
-
-/**
- * A Cell Iterator in reversed clustering order over SSTable
- */
-public class SSTableReversedIterator extends AbstractSSTableIterator
-{
- /**
- * The index of the slice being processed.
- */
- private int slice;
-
- public SSTableReversedIterator(SSTableReader sstable,
- FileDataInput file,
- DecoratedKey key,
- RowIndexEntry indexEntry,
- Slices slices,
- ColumnFilter columns,
- FileHandle ifile)
- {
- super(sstable, file, key, indexEntry, slices, columns, ifile);
- }
-
- protected Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
- {
- return indexEntry.isIndexed()
- ? new ReverseIndexedReader(indexEntry, file, shouldCloseFile)
- : new ReverseReader(file, shouldCloseFile);
- }
-
- public boolean isReverseOrder()
- {
- return true;
- }
-
- protected int nextSliceIndex()
- {
- int next = slice;
- slice++;
- return slices.size() - (next + 1);
- }
-
- protected boolean hasMoreSlices()
- {
- return slice < slices.size();
- }
-
- private class ReverseReader extends Reader
- {
- protected ReusablePartitionData buffer;
- protected Iterator<Unfiltered> iterator;
-
- // Set in loadFromDisk () and used in setIterator to handle range tombstone extending on multiple index block. See
- // loadFromDisk for details. Note that those are always false for non-indexed readers.
- protected boolean skipFirstIteratedItem;
- protected boolean skipLastIteratedItem;
-
- private ReverseReader(FileDataInput file, boolean shouldCloseFile)
- {
- super(file, shouldCloseFile);
- }
-
- protected ReusablePartitionData createBuffer(int blocksCount)
- {
- int estimatedRowCount = 16;
- int columnCount = metadata().regularColumns().size();
- if (columnCount == 0 || metadata().clusteringColumns().isEmpty())
- {
- estimatedRowCount = 1;
- }
- else
- {
- try
- {
- // To avoid wasted resizing we guess-estimate the number of rows we're likely to read. For that
- // we use the stats on the number of rows per partition for that sstable.
- // FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows,
- // we divide by the number of regular columns the table has. We should fix once we collect the
- // stats on rows
- int estimatedRowsPerPartition = (int)(sstable.getEstimatedCellPerPartitionCount().percentile(0.75) / columnCount);
- estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1);
- }
- catch (IllegalStateException e)
- {
- // The EstimatedHistogram mean() method can throw this (if it overflows). While such overflow
- // shouldn't happen, it's not worth taking the risk of letting the exception bubble up.
- }
- }
- return new ReusablePartitionData(metadata(), partitionKey(), columns(), estimatedRowCount);
- }
-
- public void setForSlice(Slice slice) throws IOException
- {
- // If we have read the data, just create the iterator for the slice. Otherwise, read the data.
- if (buffer == null)
- {
- buffer = createBuffer(1);
- // Note that we can reuse that buffer between slices (we could alternatively re-read from disk
- // every time, but that feels more wasteful) so we want to include everything from the beginning.
- // We can stop at the slice end however since any following slice will be before that.
- loadFromDisk(null, slice.end(), false, false);
- }
- setIterator(slice);
- }
-
- protected void setIterator(Slice slice)
- {
- assert buffer != null;
- iterator = buffer.built.unfilteredIterator(columns, Slices.with(metadata().comparator, slice), true);
-
- if (!iterator.hasNext())
- return;
-
- if (skipFirstIteratedItem)
- iterator.next();
-
- if (skipLastIteratedItem)
- iterator = new SkipLastIterator(iterator);
- }
-
- protected boolean hasNextInternal() throws IOException
- {
- // If we've never called setForSlice, we're reading everything
- if (iterator == null)
- setForSlice(Slice.ALL);
-
- return iterator.hasNext();
- }
-
- protected Unfiltered nextInternal() throws IOException
- {
- if (!hasNext())
- throw new NoSuchElementException();
- return iterator.next();
- }
-
- protected boolean stopReadingDisk() throws IOException
- {
- return false;
- }
-
- // Reads the unfiltered from disk and load them into the reader buffer. It stops reading when either the partition
- // is fully read, or when stopReadingDisk() returns true.
- protected void loadFromDisk(ClusteringBound<?> start,
- ClusteringBound<?> end,
- boolean hasPreviousBlock,
- boolean hasNextBlock) throws IOException
- {
- // start != null means it's the block covering the beginning of the slice, so it has to be the last block for this slice.
- assert start == null || !hasNextBlock;
-
- buffer.reset();
- skipFirstIteratedItem = false;
- skipLastIteratedItem = false;
-
- // If the start might be in this block, skip everything that comes before it.
- if (start != null)
- {
- while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0 && !stopReadingDisk())
- {
- if (deserializer.nextIsRow())
- deserializer.skipNext();
- else
- updateOpenMarker((RangeTombstoneMarker)deserializer.readNext());
- }
- }
-
- // If we have an open marker, it's either one from what we just skipped or it's one that open in the next (or
- // one of the next) index block (if openMarker == openMarkerAtStartOfBlock).
- if (openMarker != null)
- {
- // We have to feed a marker to the buffer, because that marker is likely to be close later and ImmtableBTreePartition
- // doesn't take kindly to marker that comes without their counterpart. If that's the last block we're gonna read (for
- // the current slice at least) it's easy because we'll want to return that open marker at the end of the data in this
- // block anyway, so we have nothing more to do than adding it to the buffer.
- // If it's not the last block however, in which case we know we'll have start == null, it means this marker is really
- // open in a next block and so while we do need to add it the buffer for the reason mentioned above, we don't
- // want to "return" it just yet, we'll wait until we reach it in the next blocks. That's why we trigger
- // skipLastIteratedItem in that case (this is first item of the block, but we're iterating in reverse order
- // so it will be last returned by the iterator).
- ClusteringBound<?> markerStart = start == null ? BufferClusteringBound.BOTTOM : start;
- buffer.add(new RangeTombstoneBoundMarker(markerStart, openMarker));
- if (hasNextBlock)
- skipLastIteratedItem = true;
- }
-
- // Now deserialize everything until we reach our requested end (if we have one)
- // See SSTableIterator.ForwardRead.computeNext() for why this is a strict inequality below: this is the same
- // reasoning here.
- while (deserializer.hasNext()
- && (end == null || deserializer.compareNextTo(end) < 0)
- && !stopReadingDisk())
- {
- Unfiltered unfiltered = deserializer.readNext();
- UnfilteredValidation.maybeValidateUnfiltered(unfiltered, metadata(), key, sstable);
- // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
- if (!unfiltered.isEmpty())
- buffer.add(unfiltered);
-
- if (unfiltered.isRangeTombstoneMarker())
- updateOpenMarker((RangeTombstoneMarker)unfiltered);
- }
-
- // If we have an open marker, we should close it before finishing
- if (openMarker != null)
- {
- // This is the reverse problem than the one at the start of the block. Namely, if it's the first block
- // we deserialize for the slice (the one covering the slice end basically), then it's easy, we just want
- // to add the close marker to the buffer and return it normally.
- // If it's note our first block (for the slice) however, it means that marker closed in a previously read
- // block and we have already returned it. So while we should still add it to the buffer for the sake of
- // not breaking ImmutableBTreePartition, we should skip it when returning from the iterator, hence the
- // skipFirstIteratedItem (this is the last item of the block, but we're iterating in reverse order so it will
- // be the first returned by the iterator).
- ClusteringBound<?> markerEnd = end == null ? BufferClusteringBound.TOP : end;
- buffer.add(new RangeTombstoneBoundMarker(markerEnd, openMarker));
- if (hasPreviousBlock)
- skipFirstIteratedItem = true;
- }
-
- buffer.build();
- }
- }
-
- private class ReverseIndexedReader extends ReverseReader
- {
- private final IndexState indexState;
-
- // The slice we're currently iterating over
- private Slice slice;
- // The last index block to consider for the slice
- private int lastBlockIdx;
-
- private ReverseIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
- {
- super(file, shouldCloseFile);
- this.indexState = new IndexState(this, metadata.comparator, indexEntry, true, ifile);
- }
-
- @Override
- public void close() throws IOException
- {
- super.close();
- this.indexState.close();
- }
-
- @Override
- public void setForSlice(Slice slice) throws IOException
- {
- this.slice = slice;
-
- // if our previous slicing already got us past the beginning of the sstable, we're done
- if (indexState.isDone())
- {
- iterator = Collections.emptyIterator();
- return;
- }
-
- // Find the first index block we'll need to read for the slice.
- int startIdx = indexState.findBlockIndex(slice.end(), indexState.currentBlockIdx());
- if (startIdx < 0)
- {
- iterator = Collections.emptyIterator();
- indexState.setToBlock(startIdx);
- return;
- }
-
- lastBlockIdx = indexState.findBlockIndex(slice.start(), startIdx);
-
- // If the last block to look (in reverse order) is after the very last block, we have nothing for that slice
- if (lastBlockIdx >= indexState.blocksCount())
- {
- assert startIdx >= indexState.blocksCount();
- iterator = Collections.emptyIterator();
- return;
- }
-
- // If we start (in reverse order) after the very last block, just read from the last one.
- if (startIdx >= indexState.blocksCount())
- startIdx = indexState.blocksCount() - 1;
-
- // Note that even if we were already set on the proper block (which would happen if the previous slice
- // requested ended on the same block this one start), we can't reuse it because when reading the previous
- // slice we've only read that block from the previous slice start. Re-reading also handles
- // skipFirstIteratedItem/skipLastIteratedItem that we would need to handle otherwise.
- indexState.setToBlock(startIdx);
-
- readCurrentBlock(false, startIdx != lastBlockIdx);
- }
-
- @Override
- protected boolean hasNextInternal() throws IOException
- {
- if (super.hasNextInternal())
- return true;
-
- while (true)
- {
- // We have nothing more for our current block, move the next one (so the one before on disk).
- int nextBlockIdx = indexState.currentBlockIdx() - 1;
- if (nextBlockIdx < 0 || nextBlockIdx < lastBlockIdx)
- return false;
-
- // The slice start can be in
- indexState.setToBlock(nextBlockIdx);
- readCurrentBlock(true, nextBlockIdx != lastBlockIdx);
-
- // If an indexed block only contains data for a dropped column, the iterator will be empty, even
- // though we may still have data to read in subsequent blocks
-
- // also, for pre-3.0 storage formats, index blocks that only contain a single row and that row crosses
- // index boundaries, the iterator will be empty even though we haven't read everything we're intending
- // to read. In that case, we want to read the next index block. This shouldn't be possible in 3.0+
- // formats (see next comment)
- if (!iterator.hasNext() && nextBlockIdx > lastBlockIdx)
- {
- continue;
- }
-
- return iterator.hasNext();
- }
- }
-
- /**
- * Reads the current block, the last one we've set.
- *
- * @param hasPreviousBlock is whether we have already read a previous block for the current slice.
- * @param hasNextBlock is whether we have more blocks to read for the current slice.
- */
- private void readCurrentBlock(boolean hasPreviousBlock, boolean hasNextBlock) throws IOException
- {
- if (buffer == null)
- buffer = createBuffer(indexState.blocksCount());
-
- // The slice start (resp. slice end) is only meaningful on the last (resp. first) block read (since again,
- // we read blocks in reverse order).
- boolean canIncludeSliceStart = !hasNextBlock;
- boolean canIncludeSliceEnd = !hasPreviousBlock;
-
- loadFromDisk(canIncludeSliceStart ? slice.start() : null,
- canIncludeSliceEnd ? slice.end() : null,
- hasPreviousBlock,
- hasNextBlock);
- setIterator(slice);
- }
-
- @Override
- protected boolean stopReadingDisk() throws IOException
- {
- return indexState.isPastCurrentBlock();
- }
- }
-
- private class ReusablePartitionData
- {
- private final TableMetadata metadata;
- private final DecoratedKey partitionKey;
- private final RegularAndStaticColumns columns;
-
- private MutableDeletionInfo.Builder deletionBuilder;
- private MutableDeletionInfo deletionInfo;
- private BTree.Builder<Row> rowBuilder;
- private ImmutableBTreePartition built;
-
- private ReusablePartitionData(TableMetadata metadata,
- DecoratedKey partitionKey,
- RegularAndStaticColumns columns,
- int initialRowCapacity)
- {
- this.metadata = metadata;
- this.partitionKey = partitionKey;
- this.columns = columns;
- this.rowBuilder = BTree.builder(metadata.comparator, initialRowCapacity);
- }
-
-
- public void add(Unfiltered unfiltered)
- {
- if (unfiltered.isRow())
- rowBuilder.add((Row)unfiltered);
- else
- deletionBuilder.add((RangeTombstoneMarker)unfiltered);
- }
-
- public void reset()
- {
- built = null;
- rowBuilder.reuse();
- deletionBuilder = MutableDeletionInfo.builder(partitionLevelDeletion, metadata().comparator, false);
- }
-
- public void build()
- {
- deletionInfo = deletionBuilder.build();
- built = new ImmutableBTreePartition(metadata, partitionKey, columns, Rows.EMPTY_STATIC_ROW, rowBuilder.build(),
- deletionInfo, EncodingStats.NO_STATS);
- deletionBuilder = null;
- }
- }
-
- private static class SkipLastIterator extends AbstractIterator<Unfiltered>
- {
- private final Iterator<Unfiltered> iterator;
-
- private SkipLastIterator(Iterator<Unfiltered> iterator)
- {
- this.iterator = iterator;
- }
-
- protected Unfiltered computeNext()
- {
- if (!iterator.hasNext())
- return endOfData();
-
- Unfiltered next = iterator.next();
- return iterator.hasNext() ? next : endOfData();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
index d8eb0e7..e6cc2fa 100644
--- a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java
@@ -195,7 +195,7 @@
interrupted = true;
}
}
-
+
if (interrupted)
{
discardAvailableSegment();
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
index 6195b1b..ca5077e 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
@@ -314,7 +314,7 @@
buffer.putInt((int) checksum.getValue());
// checksummed mutation
- dos.write(dob.getData(), 0, size);
+ dos.write(dob.unsafeGetBufferAndFlip());
updateChecksum(checksum, buffer, buffer.position() - size, size);
buffer.putInt((int) checksum.getValue());
}
@@ -453,11 +453,7 @@
@Override
public void setCDCBlockWrites(boolean val)
{
- Preconditions.checkState(DatabaseDescriptor.isCDCEnabled(),
- "Unable to set block_writes (%s): CDC is not enabled.", val);
- Preconditions.checkState(segmentManager instanceof CommitLogSegmentManagerCDC,
- "CDC is enabled but we have the wrong CommitLogSegmentManager type: %s. " +
- "Please report this as bug.", segmentManager.getClass().getName());
+ ensureCDCEnabled("Unable to set block_writes.");
boolean oldVal = DatabaseDescriptor.getCDCBlockWrites();
CommitLogSegment currentSegment = segmentManager.allocatingFrom();
// Update the current segment CDC state to PERMITTED if block_writes is disabled now, and it was in FORBIDDEN state
@@ -467,6 +463,29 @@
logger.info("Updated CDC block_writes from {} to {}", oldVal, val);
}
+
+ @Override
+ public boolean isCDCOnRepairEnabled()
+ {
+ return DatabaseDescriptor.isCDCOnRepairEnabled();
+ }
+
+ @Override
+ public void setCDCOnRepairEnabled(boolean value)
+ {
+ ensureCDCEnabled("Unable to set cdc_on_repair_enabled.");
+ DatabaseDescriptor.setCDCOnRepairEnabled(value);
+ logger.info("Set cdc_on_repair_enabled to {}", value);
+ }
+
+ private void ensureCDCEnabled(String hint)
+ {
+ Preconditions.checkState(DatabaseDescriptor.isCDCEnabled(), "CDC is not enabled. %s", hint);
+ Preconditions.checkState(segmentManager instanceof CommitLogSegmentManagerCDC,
+ "CDC is enabled but we have the wrong CommitLogSegmentManager type: %s. " +
+ "Please report this as bug.", segmentManager.getClass().getName());
+ }
+
/**
* Shuts down the threads used by the commit log, blocking until completion.
* TODO this should accept a timeout, and throw TimeoutException
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
index ed2af1b..4d91f51 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
@@ -43,7 +43,7 @@
import org.apache.cassandra.io.util.FileInputStreamPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.security.EncryptionContext;
-import org.json.simple.JSONValue;
+import org.apache.cassandra.utils.JsonUtils;
import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
@@ -128,7 +128,7 @@
if (encryptionContext != null)
params.putAll(encryptionContext.toHeaderParameters());
params.putAll(additionalHeaders);
- return JSONValue.toJSONString(params);
+ return JsonUtils.writeAsJsonString(params);
}
public static CommitLogDescriptor fromHeader(File file, EncryptionContext encryptionContext)
@@ -169,7 +169,7 @@
if (crc == (int) checkcrc.getValue())
{
- Map<?, ?> map = (Map<?, ?>) JSONValue.parse(new String(parametersBytes, StandardCharsets.UTF_8));
+ Map<?, ?> map = (Map<?, ?>) JsonUtils.decodeJson(parametersBytes);
return new CommitLogDescriptor(version, id, parseCompression(map), EncryptionContext.createFromMap(map, encryptionContext));
}
return null;
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java
index 7e8deca..189916c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java
@@ -88,4 +88,10 @@
public boolean getCDCBlockWrites();
public void setCDCBlockWrites(boolean val);
+
+ /** Returns true if internodes streaming of CDC data should go through write path */
+ boolean isCDCOnRepairEnabled();
+
+ /** Set whether enable write path for CDC data during internodes streaming, e.g. repair */
+ void setCDCOnRepairEnabled(boolean value);
}
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 74aa67d..a50273c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -38,7 +38,6 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@@ -55,15 +54,20 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
+import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_IGNORE_REPLAY_ERRORS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_MAX_OUTSTANDING_REPLAY_BYTES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_MAX_OUTSTANDING_REPLAY_COUNT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMIT_LOG_REPLAY_LIST;
+
public class CommitLogReplayer implements CommitLogReadHandler
{
@VisibleForTesting
- public static long MAX_OUTSTANDING_REPLAY_BYTES = Long.getLong("cassandra.commitlog_max_outstanding_replay_bytes", 1024 * 1024 * 64);
+ public static long MAX_OUTSTANDING_REPLAY_BYTES = COMMITLOG_MAX_OUTSTANDING_REPLAY_BYTES.getLong();
@VisibleForTesting
public static MutationInitiator mutationInitiator = new MutationInitiator();
- static final String IGNORE_REPLAY_ERRORS_PROPERTY = Config.PROPERTY_PREFIX + "commitlog.ignorereplayerrors";
private static final Logger logger = LoggerFactory.getLogger(CommitLogReplayer.class);
- private static final int MAX_OUTSTANDING_REPLAY_COUNT = Integer.getInteger(Config.PROPERTY_PREFIX + "commitlog_max_outstanding_replay_count", 1024);
+ private static final int MAX_OUTSTANDING_REPLAY_COUNT = COMMITLOG_MAX_OUTSTANDING_REPLAY_COUNT.getInt();
private final Set<Keyspace> keyspacesReplayed;
private final Queue<Future<Integer>> futures;
@@ -373,28 +377,60 @@
public abstract boolean includes(TableMetadataRef metadata);
+ /**
+ * Creates filter for entities to replay mutations for upon commit log replay.
+ *
+ * @see org.apache.cassandra.config.CassandraRelevantProperties#COMMIT_LOG_REPLAY_LIST
+ * */
public static ReplayFilter create()
{
- // If no replaylist is supplied an empty array of strings is used to replay everything.
- if (System.getProperty("cassandra.replayList") == null)
+ String replayList = COMMIT_LOG_REPLAY_LIST.getString();
+
+ if (replayList == null)
return new AlwaysReplayFilter();
Multimap<String, String> toReplay = HashMultimap.create();
- for (String rawPair : System.getProperty("cassandra.replayList").split(","))
+ for (String rawPair : replayList.split(","))
{
- String[] pair = StringUtils.split(rawPair.trim(), '.');
- if (pair.length != 2)
- throw new IllegalArgumentException("Each table to be replayed must be fully qualified with keyspace name, e.g., 'system.peers'");
+ String trimmedRawPair = rawPair.trim();
+ if (trimmedRawPair.isEmpty() || trimmedRawPair.endsWith("."))
+ throw new IllegalArgumentException(format("Invalid pair: '%s'", trimmedRawPair));
- Keyspace ks = Schema.instance.getKeyspaceInstance(pair[0]);
+ String[] pair = StringUtils.split(trimmedRawPair, '.');
+
+ if (pair.length > 2)
+ throw new IllegalArgumentException(format("%s property contains an item which " +
+ "is not in format 'keyspace' or 'keyspace.table' " +
+ "but it is '%s'",
+ COMMIT_LOG_REPLAY_LIST.getKey(),
+ String.join(".", pair)));
+
+ String keyspaceName = pair[0];
+
+ Keyspace ks = Schema.instance.getKeyspaceInstance(keyspaceName);
if (ks == null)
- throw new IllegalArgumentException("Unknown keyspace " + pair[0]);
- ColumnFamilyStore cfs = ks.getColumnFamilyStore(pair[1]);
- if (cfs == null)
- throw new IllegalArgumentException(String.format("Unknown table %s.%s", pair[0], pair[1]));
+ throw new IllegalArgumentException("Unknown keyspace " + keyspaceName);
- toReplay.put(pair[0], pair[1]);
+ if (pair.length == 1)
+ {
+ for (ColumnFamilyStore cfs : ks.getColumnFamilyStores())
+ toReplay.put(keyspaceName, cfs.name);
+ }
+ else
+ {
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(pair[1]);
+ if (cfs == null)
+ throw new IllegalArgumentException(format("Unknown table %s.%s", keyspaceName, pair[1]));
+
+ toReplay.put(keyspaceName, pair[1]);
+ }
}
+
+ if (toReplay.isEmpty())
+ logger.info("All tables will be included in commit log replay.");
+ else
+ logger.info("Tables to be replayed: {}", toReplay.asMap().toString());
+
return new CustomReplayFilter(toReplay);
}
}
@@ -490,13 +526,13 @@
{
if (exception.permissible)
logger.error("Ignoring commit log replay error likely due to incomplete flush to disk", exception);
- else if (Boolean.getBoolean(IGNORE_REPLAY_ERRORS_PROPERTY))
+ else if (COMMITLOG_IGNORE_REPLAY_ERRORS.getBoolean())
logger.error("Ignoring commit log replay error", exception);
else if (!CommitLog.handleCommitError("Failed commit log replay", exception))
{
logger.error("Replay stopped. If you wish to override this error and continue starting the node ignoring " +
- "commit log replay problems, specify -D" + IGNORE_REPLAY_ERRORS_PROPERTY + "=true " +
- "on the command line");
+ "commit log replay problems, specify -D{}=true on the command line",
+ COMMITLOG_IGNORE_REPLAY_ERRORS.getKey());
throw new CommitLogReplayException(exception.getMessage(), exception);
}
return false;
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java
index 33e70c1..964fe49 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentReader.java
@@ -29,7 +29,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.commitlog.EncryptedFileSegmentInputStream.ChunkProvider;
import org.apache.cassandra.db.commitlog.CommitLogReadHandler.*;
import org.apache.cassandra.io.FSReadError;
@@ -42,6 +41,7 @@
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_ALLOW_IGNORE_SYNC_CRC;
import static org.apache.cassandra.db.commitlog.CommitLogSegment.SYNC_MARKER_SIZE;
import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
@@ -50,8 +50,7 @@
*/
public class CommitLogSegmentReader implements Iterable<CommitLogSegmentReader.SyncSegment>
{
- public static final String ALLOW_IGNORE_SYNC_CRC = Config.PROPERTY_PREFIX + "commitlog.allow_ignore_sync_crc";
- private static volatile boolean allowSkipSyncMarkerCrc = Boolean.getBoolean(ALLOW_IGNORE_SYNC_CRC);
+ private static volatile boolean allowSkipSyncMarkerCrc = COMMITLOG_ALLOW_IGNORE_SYNC_CRC.getBoolean();
private static final Logger logger = LoggerFactory.getLogger(CommitLogSegmentReader.class);
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 5fe1df7..2eb119d 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -17,34 +17,39 @@
*/
package org.apache.cassandra.db.compaction;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.utils.TimeUUID;
-import static org.apache.cassandra.io.sstable.Component.DATA;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
@@ -207,6 +212,14 @@
public abstract int getEstimatedRemainingTasks();
/**
+ * @return the estimated number of background tasks needed, assuming an additional number of SSTables
+ */
+ int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
+ {
+ return getEstimatedRemainingTasks() + (int)Math.ceil((double)additionalSSTables / cfs.getMaximumCompactionThreshold());
+ }
+
+ /**
* @return size in bytes of the largest sstables for this strategy
*/
public abstract long getMaxSSTableBytes();
@@ -386,12 +399,12 @@
*/
protected boolean worthDroppingTombstones(SSTableReader sstable, int gcBefore)
{
- if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
+ if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || cfs.getNeverPurgeTombstones())
return false;
// since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually.
// if that happens we will end up in infinite compaction loop, so first we check enough if enough time has
// elapsed since SSTable created.
- if (currentTimeMillis() < sstable.getCreationTimeFor(DATA) + tombstoneCompactionInterval * 1000)
+ if (currentTimeMillis() < sstable.getDataCreationTime() + tombstoneCompactionInterval * 1000)
return false;
double droppableRatio = sstable.getEstimatedDroppableTombstoneRatio(gcBefore);
@@ -415,16 +428,16 @@
else
{
// what percentage of columns do we expect to compact outside of overlap?
- if (sstable.getIndexSummarySize() < 2)
+ if (!sstable.isEstimationInformative())
{
// we have too few samples to estimate correct percentage
return false;
}
// first, calculate estimated keys that do not overlap
long keys = sstable.estimatedKeys();
- Set<Range<Token>> ranges = new HashSet<Range<Token>>(overlaps.size());
+ Set<Range<Token>> ranges = new HashSet<>(overlaps.size());
for (SSTableReader overlap : overlaps)
- ranges.add(new Range<>(overlap.first.getToken(), overlap.last.getToken()));
+ ranges.add(new Range<>(overlap.getFirst().getToken(), overlap.getLast().getToken()));
long remainingKeys = keys - sstable.estimatedKeysForRanges(ranges);
// next, calculate what percentage of columns we have within those keys
long columns = sstable.getEstimatedCellPerPartitionCount().mean() * remainingKeys;
@@ -552,7 +565,7 @@
Collection<Index> indexes,
LifecycleNewTracker lifecycleNewTracker)
{
- return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker);
+ return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker, cfs);
}
public boolean supportsEarlyOpen()
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
index de6ff71..3421123 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
@@ -206,4 +206,6 @@
public abstract int getStrategyIndex(AbstractCompactionStrategy strategy);
public abstract boolean containsSSTable(SSTableReader sstable);
+
+ public abstract int getEstimatedRemainingTasks();
}
diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
index 7b6b5bf..4e238ad 100644
--- a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
+++ b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
@@ -21,11 +21,14 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
public class ActiveCompactions implements ActiveCompactionsTracker
{
@@ -50,6 +53,28 @@
}
/**
+ * Get the estimated number of bytes remaining to write per sstable directory
+ */
+ public Map<File, Long> estimatedRemainingWriteBytes()
+ {
+ synchronized (compactions)
+ {
+ Map<File, Long> writeBytesPerSSTableDir = new HashMap<>();
+ for (CompactionInfo.Holder holder : compactions)
+ {
+ CompactionInfo compactionInfo = holder.getCompactionInfo();
+ List<File> directories = compactionInfo.getTargetDirectories();
+ if (directories == null || directories.isEmpty())
+ continue;
+ long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteBytes() / directories.size();
+ for (File directory : directories)
+ writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum);
+ }
+ return writeBytesPerSSTableDir;
+ }
+ }
+
+ /**
* Iterates over the active compactions and tries to find CompactionInfos with the given compactionType for the given sstable
*
* Number of entries in compactions should be small (< 10) but avoid calling in any time-sensitive context
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index 6480631..1117a5d 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -17,7 +17,14 @@
*/
package org.apache.cassandra.db.compaction;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.function.LongPredicate;
import com.google.common.base.Predicates;
@@ -26,8 +33,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.AbstractCompactionController;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -37,6 +46,7 @@
import org.apache.cassandra.utils.OverlapIterator;
import org.apache.cassandra.utils.concurrent.Refs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NEVER_PURGE_TOMBSTONES;
import static org.apache.cassandra.db.lifecycle.SSTableIntervalTree.buildIntervals;
/**
@@ -45,8 +55,7 @@
public class CompactionController extends AbstractCompactionController
{
private static final Logger logger = LoggerFactory.getLogger(CompactionController.class);
- private static final String NEVER_PURGE_TOMBSTONES_PROPERTY = Config.PROPERTY_PREFIX + "never_purge_tombstones";
- static final boolean NEVER_PURGE_TOMBSTONES = Boolean.getBoolean(NEVER_PURGE_TOMBSTONES_PROPERTY);
+ static final boolean NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE = NEVER_PURGE_TOMBSTONES.getBoolean();
private final boolean compactingRepaired;
// note that overlapIterator and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES is set - this is a
@@ -80,16 +89,15 @@
? compacting.stream().mapToLong(SSTableReader::getMinTimestamp).min().getAsLong()
: 0;
refreshOverlaps();
- if (NEVER_PURGE_TOMBSTONES)
- logger.warn("You are running with -Dcassandra.never_purge_tombstones=true, this is dangerous!");
+ if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE)
+ logger.warn("You are running with -D{}=true, this is dangerous!", NEVER_PURGE_TOMBSTONES.getKey());
}
public void maybeRefreshOverlaps()
{
- if (NEVER_PURGE_TOMBSTONES)
+ if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE)
{
- logger.debug("not refreshing overlaps - running with -D{}=true",
- NEVER_PURGE_TOMBSTONES_PROPERTY);
+ logger.debug("not refreshing overlaps - running with -D{}=true", NEVER_PURGE_TOMBSTONES.getKey());
return;
}
@@ -105,19 +113,13 @@
return;
}
- for (SSTableReader reader : overlappingSSTables)
- {
- if (reader.isMarkedCompacted())
- {
- refreshOverlaps();
- return;
- }
- }
+ if (overlappingSSTables == null || overlappingSSTables.stream().anyMatch(SSTableReader::isMarkedCompacted))
+ refreshOverlaps();
}
void refreshOverlaps()
{
- if (NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
+ if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || cfs.getNeverPurgeTombstones())
return;
if (this.overlappingSSTables != null)
@@ -160,8 +162,8 @@
{
logger.trace("Checking droppable sstables in {}", cfStore);
- if (NEVER_PURGE_TOMBSTONES || compacting == null || cfStore.getNeverPurgeTombstones())
- return Collections.<SSTableReader>emptySet();
+ if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || compacting == null || cfStore.getNeverPurgeTombstones() || overlapping == null)
+ return Collections.emptySet();
if (cfStore.getCompactionStrategyManager().onlyPurgeRepairedTombstones() && !Iterables.all(compacting, SSTableReader::isRepaired))
return Collections.emptySet();
@@ -246,7 +248,7 @@
@Override
public LongPredicate getPurgeEvaluator(DecoratedKey key)
{
- if (NEVER_PURGE_TOMBSTONES || !compactingRepaired() || cfs.getNeverPurgeTombstones())
+ if (NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || !compactingRepaired() || cfs.getNeverPurgeTombstones() || overlapIterator == null)
return time -> false;
overlapIterator.update(key);
@@ -257,7 +259,7 @@
for (SSTableReader sstable: filteredSSTables)
{
- if (sstable.maybePresent(key))
+ if (sstable.mayContainAssumingKeyIsInRange(key))
{
minTimestampSeen = Math.min(minTimestampSeen, sstable.getMinTimestamp());
hasTimestamp = true;
@@ -307,7 +309,7 @@
// caller must close iterators
public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly)
{
- if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
+ if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES_PROPERTY_VALUE || cfs.getNeverPurgeTombstones())
return null;
overlapIterator.update(key);
return Iterables.filter(Iterables.transform(overlapIterator.overlaps(),
@@ -322,8 +324,8 @@
reader.getMaxTimestamp() <= minTimestamp ||
tombstoneOnly && !reader.mayHaveTombstones())
return null;
- RowIndexEntry<?> position = reader.getPosition(key, SSTableReader.Operator.EQ);
- if (position == null)
+ long position = reader.getPosition(key, SSTableReader.Operator.EQ);
+ if (position < 0)
return null;
FileDataInput dfile = openDataFiles.computeIfAbsent(reader, this::openDataFile);
return reader.simpleIterator(dfile, key, position, tombstoneOnly);
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionHistoryTabularData.java b/src/java/org/apache/cassandra/db/compaction/CompactionHistoryTabularData.java
index 485f1a0..182f348 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionHistoryTabularData.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionHistoryTabularData.java
@@ -21,20 +21,20 @@
import java.util.Map;
import java.util.UUID;
-import com.google.common.base.Throwables;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.utils.FBUtilities;
public class CompactionHistoryTabularData
{
private static final String[] ITEM_NAMES = new String[]{ "id", "keyspace_name", "columnfamily_name", "compacted_at",
- "bytes_in", "bytes_out", "rows_merged" };
+ "bytes_in", "bytes_out", "rows_merged", "compaction_properties" };
private static final String[] ITEM_DESCS = new String[]{ "time uuid", "keyspace name",
"column family name", "compaction finished at",
- "total bytes in", "total bytes out", "total rows merged" };
+ "total bytes in", "total bytes out", "total rows merged", "compaction properties" };
private static final String TYPE_NAME = "CompactionHistory";
@@ -45,13 +45,15 @@
private static final CompositeType COMPOSITE_TYPE;
private static final TabularType TABULAR_TYPE;
-
+
+ public static final String COMPACTION_TYPE_PROPERTY = "compaction_type";
+
static
{
try
{
ITEM_TYPES = new OpenType[]{ SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.LONG,
- SimpleType.LONG, SimpleType.LONG, SimpleType.STRING };
+ SimpleType.LONG, SimpleType.LONG, SimpleType.STRING, SimpleType.STRING };
COMPOSITE_TYPE = new CompositeType(TYPE_NAME, ROW_DESC, ITEM_NAMES, ITEM_DESCS, ITEM_TYPES);
@@ -59,7 +61,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -75,10 +77,11 @@
long bytesIn = row.getLong(ITEM_NAMES[4]);
long bytesOut = row.getLong(ITEM_NAMES[5]);
Map<Integer, Long> rowMerged = row.getMap(ITEM_NAMES[6], Int32Type.instance, LongType.instance);
-
+ Map<String, String> compactionProperties = row.getMap(ITEM_NAMES[7], UTF8Type.instance, UTF8Type.instance);
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
new Object[]{ id.toString(), ksName, cfName, compactedAt, bytesIn, bytesOut,
- "{" + FBUtilities.toString(rowMerged) + "}" }));
+ '{' + FBUtilities.toString(rowMerged) + '}',
+ '{' + FBUtilities.toString(compactionProperties) + '}' }));
}
return result;
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
index 513adfa..0bfc925 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
@@ -18,7 +18,9 @@
package org.apache.cassandra.db.compaction;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -27,7 +29,9 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.TimeUUID;
@@ -42,6 +46,7 @@
public static final String UNIT = "unit";
public static final String COMPACTION_ID = "compactionId";
public static final String SSTABLES = "sstables";
+ public static final String TARGET_DIRECTORY = "targetDirectory";
private final TableMetadata metadata;
private final OperationType tasktype;
@@ -50,13 +55,9 @@
private final Unit unit;
private final TimeUUID compactionId;
private final ImmutableSet<SSTableReader> sstables;
+ private final String targetDirectory;
- public CompactionInfo(TableMetadata metadata, OperationType tasktype, long bytesComplete, long totalBytes, TimeUUID compactionId, Collection<SSTableReader> sstables)
- {
- this(metadata, tasktype, bytesComplete, totalBytes, Unit.BYTES, compactionId, sstables);
- }
-
- private CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection<SSTableReader> sstables)
+ public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection<? extends SSTableReader> sstables, String targetDirectory)
{
this.tasktype = tasktype;
this.completed = completed;
@@ -65,21 +66,41 @@
this.unit = unit;
this.compactionId = compactionId;
this.sstables = ImmutableSet.copyOf(sstables);
+ this.targetDirectory = targetDirectory;
+ }
+
+ public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<SSTableReader> sstables, String targetDirectory)
+ {
+ this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, targetDirectory);
+ }
+
+ public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<? extends SSTableReader> sstables)
+ {
+ this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, null);
}
/**
- * Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask and AutoSavingCache where we don't know
+ * Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask where we don't know
* the sstables at construction
*/
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId)
{
- return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of());
+ return withoutSSTables(metadata, tasktype, completed, total, unit, compactionId, null);
+ }
+
+ /**
+ * Special compaction info where we always need to cancel the compaction - for example AutoSavingCache where we don't know
+ * the sstables at construction
+ */
+ public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, String targetDirectory)
+ {
+ return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of(), targetDirectory);
}
/** @return A copy of this CompactionInfo with updated progress. */
public CompactionInfo forProgress(long complete, long total)
{
- return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables);
+ return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables, targetDirectory);
}
public Optional<String> getKeyspace()
@@ -127,6 +148,50 @@
return sstables;
}
+ /**
+ * Get the directories this compaction could possibly write to.
+ *
+ * @return the directories that we might write to, or empty list if we don't know the metadata
+ * (like for index summary redistribution), or null if we don't have any disk boundaries
+ */
+ public List<File> getTargetDirectories()
+ {
+ if (metadata != null && !metadata.isIndex())
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(metadata.id);
+ if (cfs != null)
+ return cfs.getDirectoriesForFiles(sstables);
+ }
+ return Collections.emptyList();
+ }
+
+ public String targetDirectory()
+ {
+ if (targetDirectory == null)
+ return "";
+
+ try
+ {
+ return new File(targetDirectory).canonicalPath();
+ }
+ catch (Throwable t)
+ {
+ throw new RuntimeException("Unable to resolve canonical path for " + targetDirectory);
+ }
+ }
+
+ /**
+ * Note that this estimate is based on the amount of data we have left to read - it assumes input
+ * size == output size for a compaction, which is not really true, but should most often provide a worst case
+ * remaining write size.
+ */
+ public long estimatedRemainingWriteBytes()
+ {
+ if (unit == Unit.BYTES && tasktype.writesData)
+ return getTotal() - getCompleted();
+ return 0;
+ }
+
@Override
public String toString()
{
@@ -155,6 +220,7 @@
ret.put(UNIT, unit.toString());
ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString());
ret.put(SSTABLES, Joiner.on(',').join(sstables));
+ ret.put(TARGET_DIRECTORY, targetDirectory());
return ret;
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
index a0dc087..5cde41e 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
@@ -17,33 +17,51 @@
*/
package org.apache.cassandra.db.compaction;
-import java.util.*;
-import java.util.function.LongPredicate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.function.LongPredicate;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Ordering;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.schema.TableMetadata;
-
-import org.apache.cassandra.db.transform.DuplicateRowChecker;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.AbstractCompactionController;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.PurgeFunction;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator;
+import org.apache.cassandra.db.transform.DuplicateRowChecker;
import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.transactions.CompactionTransaction;
import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.TopPartitionTracker;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
import org.apache.cassandra.utils.TimeUUID;
@@ -78,11 +96,13 @@
private final ImmutableSet<SSTableReader> sstables;
private final int nowInSec;
private final TimeUUID compactionId;
-
private final long totalBytes;
private long bytesRead;
private long totalSourceCQLRows;
+ // Keep targetDirectory for compactions, needed for `nodetool compactionstats`
+ private String targetDirectory;
+
/*
* counters for merged rows.
* array index represents (number of merged rows - 1), so index 0 is counter for no merge (1 row),
@@ -151,7 +171,8 @@
bytesRead,
totalBytes,
compactionId,
- sstables);
+ sstables,
+ targetDirectory);
}
public boolean isGlobal()
@@ -159,6 +180,11 @@
return false;
}
+ public void setTargetDirectory(final String targetDirectory)
+ {
+ this.targetDirectory = targetDirectory;
+ }
+
private void updateCounterFor(int rows)
{
assert rows > 0 && rows - 1 < mergeCounters.length;
@@ -194,9 +220,12 @@
CompactionIterator.this.updateCounterFor(merged);
- if (type != OperationType.COMPACTION || !controller.cfs.indexManager.hasIndexes())
+ if ( (type != OperationType.COMPACTION && type != OperationType.MAJOR_COMPACTION)
+ || !controller.cfs.indexManager.hasIndexes() )
+ {
return null;
-
+ }
+
Columns statics = Columns.NONE;
Columns regulars = Columns.NONE;
for (int i=0, isize=versions.size(); i<isize; i++)
@@ -342,6 +371,18 @@
}
/*
+ * Called at the beginning of each new partition
+ * Return true if the current partitionKey ignores the gc_grace_seconds during compaction.
+ * Note that this method should be called after the onNewPartition because it depends on the currentKey
+ * which is set in the onNewPartition
+ */
+ @Override
+ protected boolean shouldIgnoreGcGrace()
+ {
+ return controller.cfs.shouldIgnoreGcGraceForKey(currentKey);
+ }
+
+ /*
* Evaluates whether a tombstone with the given deletion timestamp can be purged. This is the minimum
* timestamp for any sstable containing `currentKey` outside of the set of sstables involved in this compaction.
* This is computed lazily on demand as we only need this if there is tombstones and this a bit expensive
@@ -362,8 +403,10 @@
* The result produced by this iterator is such that when merged with tombSource it produces the same output
* as the merge of dataSource and tombSource.
*/
- private static class GarbageSkippingUnfilteredRowIterator extends WrappingUnfilteredRowIterator
+ private static class GarbageSkippingUnfilteredRowIterator implements WrappingUnfilteredRowIterator
{
+ private final UnfilteredRowIterator wrapped;
+
final UnfilteredRowIterator tombSource;
final DeletionTime partitionLevelDeletion;
final Row staticRow;
@@ -390,7 +433,7 @@
*/
protected GarbageSkippingUnfilteredRowIterator(UnfilteredRowIterator dataSource, UnfilteredRowIterator tombSource, boolean cellLevelGC)
{
- super(dataSource);
+ this.wrapped = dataSource;
this.tombSource = tombSource;
this.cellLevelGC = cellLevelGC;
metadata = dataSource.metadata();
@@ -410,6 +453,12 @@
dataNext = advance(dataSource);
}
+ @Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return wrapped;
+ }
+
private static Unfiltered advance(UnfilteredRowIterator source)
{
return source.hasNext() ? source.next() : null;
@@ -423,7 +472,7 @@
public void close()
{
- super.close();
+ wrapped.close();
tombSource.close();
}
@@ -696,4 +745,4 @@
{
return cfs.name.equals(SystemKeyspace.PAXOS) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
index 9bf063b..88a7705 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
@@ -21,13 +21,16 @@
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.lang.ref.WeakReference;
-import java.nio.file.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
@@ -49,6 +52,7 @@
import org.apache.cassandra.utils.NoSpamLogger;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOG_DIR;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class CompactionLogger
@@ -172,7 +176,7 @@
{
ObjectNode node = json.objectNode();
node.put("generation", sstable.descriptor.id.toString());
- node.put("version", sstable.descriptor.version.getVersion());
+ node.put("version", sstable.descriptor.version.version);
node.put("size", sstable.onDiskLength());
JsonNode logResult = strategy.strategyLogger().sstable(sstable);
if (logResult != null)
@@ -300,7 +304,7 @@
private static class CompactionLogSerializer implements Writer
{
- private static final String logDirectory = System.getProperty("cassandra.logdir", ".");
+ private static final String logDirectory = LOG_DIR.getString();
private final ExecutorPlus loggerService = executorFactory().sequential("CompactionLogger");
// This is only accessed on the logger service thread, so it does not need to be thread safe
private final Set<Object> rolled = new HashSet<>();
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 347f9b3..a972415 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -43,6 +43,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Collections2;
import com.google.common.collect.ConcurrentHashMultiset;
@@ -57,6 +58,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import net.openhft.chronicle.core.util.ThrowingSupplier;
import org.apache.cassandra.cache.AutoSavingCache;
import org.apache.cassandra.concurrent.ExecutorFactory;
import org.apache.cassandra.concurrent.WrappedExecutorPlus;
@@ -78,16 +80,16 @@
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.view.ViewBuilderTask;
import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.SecondaryIndexBuilder;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.IndexSummaryRedistribution;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.IVerifier;
import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -107,6 +109,7 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.WrappedRunnable;
@@ -156,7 +159,13 @@
private final CompactionExecutor cacheCleanupExecutor = new CacheCleanupExecutor();
private final CompactionExecutor viewBuildExecutor = new ViewBuildExecutor();
- private final CompactionMetrics metrics = new CompactionMetrics(executor, validationExecutor, viewBuildExecutor);
+ // We can't house 2i builds in SecondaryIndexManagement because it could cause deadlocks with itself, and can cause
+ // massive to indefinite pauses if prioritized either before or after normal compactions so we instead put it in its
+ // own pool to prevent either scenario.
+ private final SecondaryIndexExecutor secondaryIndexExecutor = new SecondaryIndexExecutor();
+
+ private final CompactionMetrics metrics = new CompactionMetrics(executor, validationExecutor, viewBuildExecutor, secondaryIndexExecutor);
+
@VisibleForTesting
final Multiset<ColumnFamilyStore> compactingCF = ConcurrentHashMultiset.create();
@@ -269,14 +278,16 @@
int pendingTasks = executor.getPendingTaskCount() +
validationExecutor.getPendingTaskCount() +
viewBuildExecutor.getPendingTaskCount() +
- cacheCleanupExecutor.getPendingTaskCount();
+ cacheCleanupExecutor.getPendingTaskCount() +
+ secondaryIndexExecutor.getPendingTaskCount();
if (pendingTasks > 0)
return true;
int activeTasks = executor.getActiveTaskCount() +
validationExecutor.getActiveTaskCount() +
viewBuildExecutor.getActiveTaskCount() +
- cacheCleanupExecutor.getActiveTaskCount();
+ cacheCleanupExecutor.getActiveTaskCount() +
+ secondaryIndexExecutor.getActiveTaskCount();
return activeTasks > 0;
}
@@ -292,6 +303,7 @@
validationExecutor.shutdown();
viewBuildExecutor.shutdown();
cacheCleanupExecutor.shutdown();
+ secondaryIndexExecutor.shutdown();
// interrupt compactions and validations
for (Holder compactionHolder : active.getCompactions())
@@ -302,7 +314,8 @@
// wait for tasks to terminate
// compaction tasks are interrupted above, so it shuold be fairy quick
// until not interrupted tasks to complete.
- for (ExecutorService exec : Arrays.asList(executor, validationExecutor, viewBuildExecutor, cacheCleanupExecutor))
+ for (ExecutorService exec : Arrays.asList(executor, validationExecutor, viewBuildExecutor,
+ cacheCleanupExecutor, secondaryIndexExecutor))
{
try
{
@@ -405,72 +418,78 @@
* @param operation the operation to run
* @param jobs the number of threads to use - 0 means use all available. It never uses more than concurrent_compactors threads
* @return status of the operation
- * @throws ExecutionException
- * @throws InterruptedException
*/
@SuppressWarnings("resource")
- private AllSSTableOpStatus parallelAllSSTableOperation(final ColumnFamilyStore cfs, final OneSSTableOperation operation, int jobs, OperationType operationType) throws ExecutionException, InterruptedException
+ private AllSSTableOpStatus parallelAllSSTableOperation(final ColumnFamilyStore cfs,
+ final OneSSTableOperation operation,
+ int jobs,
+ OperationType operationType)
{
- logger.info("Starting {} for {}.{}", operationType, cfs.keyspace.getName(), cfs.getTableName());
- List<LifecycleTransaction> transactions = new ArrayList<>();
- List<Future<?>> futures = new ArrayList<>();
- try (LifecycleTransaction compacting = cfs.markAllCompacting(operationType))
- {
- if (compacting == null)
- return AllSSTableOpStatus.UNABLE_TO_CANCEL;
-
- Iterable<SSTableReader> sstables = Lists.newArrayList(operation.filterSSTables(compacting));
- if (Iterables.isEmpty(sstables))
- {
- logger.info("No sstables to {} for {}.{}", operationType.name(), cfs.keyspace.getName(), cfs.name);
- return AllSSTableOpStatus.SUCCESSFUL;
- }
-
- for (final SSTableReader sstable : sstables)
- {
- final LifecycleTransaction txn = compacting.split(singleton(sstable));
- transactions.add(txn);
- Callable<Object> callable = new Callable<Object>()
- {
- @Override
- public Object call() throws Exception
- {
- operation.execute(txn);
- return this;
- }
- };
- Future<?> fut = executor.submitIfRunning(callable, "paralell sstable operation");
- if (!fut.isCancelled())
- futures.add(fut);
- else
- return AllSSTableOpStatus.ABORTED;
-
- if (jobs > 0 && futures.size() == jobs)
- {
- Future<?> f = FBUtilities.waitOnFirstFuture(futures);
- futures.remove(f);
- }
- }
- FBUtilities.waitOnFutures(futures);
- assert compacting.originals().isEmpty();
- logger.info("Finished {} for {}.{} successfully", operationType, cfs.keyspace.getName(), cfs.getTableName());
- return AllSSTableOpStatus.SUCCESSFUL;
- }
- finally
- {
- // wait on any unfinished futures to make sure we don't close an ongoing transaction
+ String operationName = operationType.name();
+ String keyspace = cfs.keyspace.getName();
+ String table = cfs.getTableName();
+ return cfs.withAllSSTables(operationType, (compacting) -> {
+ logger.info("Starting {} for {}.{}", operationType, cfs.keyspace.getName(), cfs.getTableName());
+ List<LifecycleTransaction> transactions = new ArrayList<>();
+ List<Future<?>> futures = new ArrayList<>();
try
{
+ if (compacting == null)
+ return AllSSTableOpStatus.UNABLE_TO_CANCEL;
+
+ Iterable<SSTableReader> sstables = Lists.newArrayList(operation.filterSSTables(compacting));
+ if (Iterables.isEmpty(sstables))
+ {
+ logger.info("No sstables to {} for {}.{}", operationName, keyspace, table);
+ return AllSSTableOpStatus.SUCCESSFUL;
+ }
+
+ for (final SSTableReader sstable : sstables)
+ {
+ final LifecycleTransaction txn = compacting.split(singleton(sstable));
+ transactions.add(txn);
+ Callable<Object> callable = new Callable<Object>()
+ {
+ @Override
+ public Object call() throws Exception
+ {
+ operation.execute(txn);
+ return this;
+ }
+ };
+ Future<?> fut = executor.submitIfRunning(callable, "parallel SSTable operation");
+ if (!fut.isCancelled())
+ futures.add(fut);
+ else
+ return AllSSTableOpStatus.ABORTED;
+
+ if (jobs > 0 && futures.size() == jobs)
+ {
+ Future<?> f = FBUtilities.waitOnFirstFuture(futures);
+ futures.remove(f);
+ }
+ }
FBUtilities.waitOnFutures(futures);
+ assert compacting.originals().isEmpty();
+ logger.info("Finished {} for {}.{} successfully", operationType, keyspace, table);
+ return AllSSTableOpStatus.SUCCESSFUL;
}
- catch (Throwable t)
+ finally
{
- // these are handled/logged in CompactionExecutor#afterExecute
+ // wait on any unfinished futures to make sure we don't close an ongoing transaction
+ try
+ {
+ FBUtilities.waitOnFutures(futures);
+ }
+ catch (Throwable t)
+ {
+ // these are handled/logged in CompactionExecutor#afterExecute
+ }
+ Throwable fail = Throwables.close(null, transactions);
+ if (fail != null)
+ logger.error("Failed to cleanup lifecycle transactions ({} for {}.{})", operationType, keyspace, table, fail);
}
- Throwable fail = Throwables.close(null, transactions);
- if (fail != null)
- logger.error("Failed to cleanup lifecycle transactions ({} for {}.{})", operationType, cfs.keyspace.getName(), cfs.getTableName(), fail);
- }
+ });
}
private static interface OneSSTableOperation
@@ -493,16 +512,7 @@
}
}
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData,
- int jobs)
- throws InterruptedException, ExecutionException
- {
- return performScrub(cfs, skipCorrupted, checkData, false, jobs);
- }
-
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData,
- final boolean reinsertOverflowedTTL, int jobs)
- throws InterruptedException, ExecutionException
+ public AllSSTableOpStatus performScrub(ColumnFamilyStore cfs, IScrubber.Options options, int jobs)
{
return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
{
@@ -515,12 +525,12 @@
@Override
public void execute(LifecycleTransaction input)
{
- scrubOne(cfs, input, skipCorrupted, checkData, reinsertOverflowedTTL, active);
+ scrubOne(cfs, input, options, active);
}
}, jobs, OperationType.SCRUB);
}
- public AllSSTableOpStatus performVerify(ColumnFamilyStore cfs, Verifier.Options options) throws InterruptedException, ExecutionException
+ public AllSSTableOpStatus performVerify(ColumnFamilyStore cfs, IVerifier.Options options) throws InterruptedException, ExecutionException
{
assert !cfs.isIndex();
return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
@@ -551,7 +561,7 @@
return false;
// Skip if SSTable creation time is past given timestamp
- if (sstable.getCreationTimeFor(Component.DATA) > skipIfOlderThanTimestamp)
+ if (sstable.getDataCreationTime() > skipIfOlderThanTimestamp)
return false;
TableMetadata metadata = cfs.metadata.get();
@@ -636,8 +646,8 @@
{
logger.debug("Skipping {} ([{}, {}]) for cleanup; all rows should be kept. Needs cleanup full ranges: {} Needs cleanup transient ranges: {} Repaired: {}",
sstable,
- sstable.first.getToken(),
- sstable.last.getToken(),
+ sstable.getFirst().getToken(),
+ sstable.getLast().getToken(),
needsCleanupFull,
needsCleanupTransient,
sstable.isRepaired());
@@ -900,7 +910,7 @@
List<Range<Token>> normalizedRanges = Range.normalize(ranges.ranges());
for (SSTableReader sstable : sstables)
{
- Bounds<Token> bounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken());
+ AbstractBounds<Token> bounds = sstable.getBounds();
if (!Iterables.any(normalizedRanges, r -> (r.contains(bounds.left) && r.contains(bounds.right)) || r.intersects(bounds)))
{
@@ -922,12 +932,12 @@
{
SSTableReader sstable = sstableIterator.next();
- Bounds<Token> sstableBounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken());
+ AbstractBounds<Token> sstableBounds = sstable.getBounds();
for (Range<Token> r : normalizedRanges)
{
// ranges are normalized - no wrap around - if first and last are contained we know that all tokens are contained in the range
- if (r.contains(sstable.first.getToken()) && r.contains(sstable.last.getToken()))
+ if (r.contains(sstable.getFirst().getToken()) && r.contains(sstable.getLast().getToken()))
{
logger.info("{} SSTable {} fully contained in range {}, mutating repairedAt instead of anticompacting", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, r);
fullyContainedSSTables.add(sstable);
@@ -951,10 +961,16 @@
@SuppressWarnings("resource") // the tasks are executed in parallel on the executor, making sure that they get closed
public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final int gcBefore, boolean splitOutput)
{
+ return submitMaximal(cfStore, gcBefore, splitOutput, OperationType.MAJOR_COMPACTION);
+ }
+
+ @SuppressWarnings("resource")
+ public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final int gcBefore, boolean splitOutput, OperationType operationType)
+ {
// here we compute the task off the compaction executor, so having that present doesn't
// confuse runWithCompactionsDisabled -- i.e., we don't want to deadlock ourselves, waiting
// for ourselves to finish/acknowledge cancellation before continuing.
- CompactionTasks tasks = cfStore.getCompactionStrategyManager().getMaximalTasks(gcBefore, splitOutput);
+ CompactionTasks tasks = cfStore.getCompactionStrategyManager().getMaximalTasks(gcBefore, splitOutput, operationType);
if (tasks.isEmpty())
return Collections.emptyList();
@@ -999,6 +1015,7 @@
try (CompactionTasks tasks = cfStore.runWithCompactionsDisabled(taskCreator,
sstablesPredicate,
+ OperationType.MAJOR_COMPACTION,
false,
false,
false))
@@ -1012,7 +1029,10 @@
{
for (AbstractCompactionTask task : tasks)
if (task != null)
+ {
+ task.setCompactionType(OperationType.MAJOR_COMPACTION);
task.execute(active);
+ }
}
};
@@ -1033,7 +1053,7 @@
{
forceCompaction(cfStore,
() -> sstablesInBounds(cfStore, ranges),
- sstable -> new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges));
+ sstable -> sstable.getBounds().intersects(ranges));
}
/**
@@ -1071,7 +1091,12 @@
public void forceCompactionForKey(ColumnFamilyStore cfStore, DecoratedKey key)
{
- forceCompaction(cfStore, () -> sstablesWithKey(cfStore, key), sstable -> sstable.maybePresent(key));
+ forceCompaction(cfStore, () -> sstablesWithKey(cfStore, key), Predicates.alwaysTrue());
+ }
+
+ public void forceCompactionForKeys(ColumnFamilyStore cfStore, Collection<DecoratedKey> keys)
+ {
+ forceCompaction(cfStore, () -> sstablesWithKeys(cfStore, keys), Predicates.alwaysTrue());
}
private static Collection<SSTableReader> sstablesWithKey(ColumnFamilyStore cfs, DecoratedKey key)
@@ -1081,12 +1106,24 @@
key.getToken().maxKeyBound());
for (SSTableReader sstable : liveTables)
{
- if (sstable.maybePresent(key))
+ if (sstable.mayContainAssumingKeyIsInRange(key))
sstables.add(sstable);
}
return sstables.isEmpty() ? Collections.emptyList() : sstables;
}
+ private static Collection<SSTableReader> sstablesWithKeys(ColumnFamilyStore cfs, Collection<DecoratedKey> decoratedKeys)
+ {
+ final Set<SSTableReader> sstables = new HashSet<>();
+
+ for (DecoratedKey decoratedKey : decoratedKeys)
+ {
+ sstables.addAll(sstablesWithKey(cfs, decoratedKey));
+ }
+
+ return sstables;
+ }
+
public void forceUserDefinedCompaction(String dataFiles)
{
String[] filenames = dataFiles.split(",");
@@ -1095,7 +1132,7 @@
for (String filename : filenames)
{
// extract keyspace and columnfamily name from filename
- Descriptor desc = Descriptor.fromFilename(filename.trim());
+ Descriptor desc = Descriptor.fromFileWithComponent(new File(filename.trim()), false).left;
if (Schema.instance.getTableMetadataRef(desc) == null)
{
logger.warn("Schema does not exist for file {}. Skipping.", filename);
@@ -1121,7 +1158,7 @@
for (String filename : filenames)
{
// extract keyspace and columnfamily name from filename
- Descriptor desc = Descriptor.fromFilename(filename.trim());
+ Descriptor desc = Descriptor.fromFileWithComponent(new File(filename.trim()), false).left;
if (Schema.instance.getTableMetadataRef(desc) == null)
{
logger.warn("Schema does not exist for file {}. Skipping.", filename);
@@ -1242,11 +1279,11 @@
}
@VisibleForTesting
- void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, ActiveCompactionsTracker activeCompactions)
+ void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, IScrubber.Options options, ActiveCompactionsTracker activeCompactions)
{
CompactionInfo.Holder scrubInfo = null;
-
- try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, checkData, reinsertOverflowedTTL))
+ SSTableFormat format = modifier.onlyOne().descriptor.getFormat();
+ try (IScrubber scrubber = format.getScrubber(cfs, modifier, new OutputHandler.LogOutput(), options))
{
scrubInfo = scrubber.getScrubInfo();
activeCompactions.beginCompaction(scrubInfo);
@@ -1260,11 +1297,10 @@
}
@VisibleForTesting
- void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, Verifier.Options options, ActiveCompactionsTracker activeCompactions)
+ void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, IVerifier.Options options, ActiveCompactionsTracker activeCompactions)
{
CompactionInfo.Holder verifyInfo = null;
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, options))
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, options))
{
verifyInfo = verifier.getVerifyInfo();
activeCompactions.beginCompaction(verifyInfo);
@@ -1295,7 +1331,7 @@
// see if there are any keys LTE the token for the start of the first range
// (token range ownership is exclusive on the LHS.)
Range<Token> firstRange = sortedRanges.get(0);
- if (sstable.first.getToken().compareTo(firstRange.left) <= 0)
+ if (sstable.getFirst().getToken().compareTo(firstRange.left) <= 0)
return true;
// then, iterate over all owned ranges and see if the next key beyond the end of the owned
@@ -1351,11 +1387,11 @@
SSTableReader sstable = txn.onlyOne();
// if ranges is empty and no index, entire sstable is discarded
- if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(allRanges))
+ if (!hasIndexes && !sstable.getBounds().intersects(allRanges))
{
txn.obsoleteOriginals();
txn.finish();
- logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.first.getToken(), sstable.last.getToken(), allRanges);
+ logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.getFirst().getToken(), sstable.getLast().getToken(), allRanges);
return;
}
@@ -1391,6 +1427,7 @@
while (ci.hasNext())
{
+ ci.setTargetDirectory(writer.currentWriter().getFilename());
try (UnfilteredRowIterator partition = ci.next();
UnfilteredRowIterator notCleaned = cleanupStrategy.cleanup(partition))
{
@@ -1554,16 +1591,18 @@
{
FileUtils.createDirectory(compactionFileLocation);
- return SSTableWriter.create(cfs.metadata,
- cfs.newSSTableDescriptor(compactionFileLocation),
- expectedBloomFilterSize,
- repairedAt,
- pendingRepair,
- isTransient,
- sstable.getSSTableLevel(),
- sstable.header,
- cfs.indexManager.listIndexes(),
- txn);
+ Descriptor descriptor = cfs.newSSTableDescriptor(compactionFileLocation);
+ return descriptor.getFormat().getWriterFactory().builder(descriptor)
+ .setKeyCount(expectedBloomFilterSize)
+ .setRepairedAt(repairedAt)
+ .setPendingRepair(pendingRepair)
+ .setTransientSSTable(isTransient)
+ .setTableMetadataRef(cfs.metadata)
+ .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel()))
+ .setSerializationHeader(sstable.header)
+ .addDefaultComponents()
+ .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+ .build(txn, cfs);
}
public static SSTableWriter createWriterForAntiCompaction(ColumnFamilyStore cfs,
@@ -1591,16 +1630,19 @@
break;
}
}
- return SSTableWriter.create(cfs.newSSTableDescriptor(compactionFileLocation),
- (long) expectedBloomFilterSize,
- repairedAt,
- pendingRepair,
- isTransient,
- cfs.metadata,
- new MetadataCollector(sstables, cfs.metadata().comparator, minLevel),
- SerializationHeader.make(cfs.metadata(), sstables),
- cfs.indexManager.listIndexes(),
- txn);
+
+ Descriptor descriptor = cfs.newSSTableDescriptor(compactionFileLocation);
+ return descriptor.getFormat().getWriterFactory().builder(descriptor)
+ .setKeyCount(expectedBloomFilterSize)
+ .setRepairedAt(repairedAt)
+ .setPendingRepair(pendingRepair)
+ .setTransientSSTable(isTransient)
+ .setTableMetadataRef(cfs.metadata)
+ .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator, minLevel))
+ .setSerializationHeader(SerializationHeader.make(cfs.metadata(), sstables))
+ .addDefaultComponents()
+ .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+ .build(txn, cfs);
}
/**
@@ -1745,15 +1787,18 @@
if (fullChecker.test(token))
{
fullWriter.append(partition);
+ ci.setTargetDirectory(fullWriter.currentWriter().getFilename());
}
else if (transChecker.test(token))
{
transWriter.append(partition);
+ ci.setTargetDirectory(transWriter.currentWriter().getFilename());
}
else
{
// otherwise, append it to the unrepaired sstable
unrepairedWriter.append(partition);
+ ci.setTargetDirectory(unrepairedWriter.currentWriter().getFilename());
}
long bytesScanned = scanners.getTotalBytesScanned();
compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
@@ -1788,10 +1833,17 @@
}
catch (Throwable e)
{
- if (e instanceof CompactionInterruptedException && isCancelled.getAsBoolean())
+ if (e instanceof CompactionInterruptedException)
{
- logger.info("Anticompaction has been canceled for session {}", pendingRepair);
- logger.trace(e.getMessage(), e);
+ if (isCancelled.getAsBoolean())
+ {
+ logger.info("Anticompaction has been canceled for session {}", pendingRepair);
+ logger.trace(e.getMessage(), e);
+ }
+ else
+ {
+ logger.info("Anticompaction for session {} has been stopped by request.", pendingRepair);
+ }
}
else
{
@@ -1833,7 +1885,7 @@
}
};
- return executor.submitIfRunning(runnable, "index build");
+ return secondaryIndexExecutor.submitIfRunning(runnable, "index build");
}
/**
@@ -1882,22 +1934,16 @@
return executor.submitIfRunning(runnable, "cache write");
}
- public List<SSTableReader> runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution) throws IOException
+ public <T, E extends Throwable> T runAsActiveCompaction(Holder activeCompactionInfo, ThrowingSupplier<T, E> callable) throws E
{
- return runIndexSummaryRedistribution(redistribution, active);
- }
-
- @VisibleForTesting
- List<SSTableReader> runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution, ActiveCompactionsTracker activeCompactions) throws IOException
- {
- activeCompactions.beginCompaction(redistribution);
+ active.beginCompaction(activeCompactionInfo);
try
{
- return redistribution.redistributeSummaries();
+ return callable.get();
}
finally
{
- activeCompactions.finishCompaction(redistribution);
+ active.finishCompaction(activeCompactionInfo);
}
}
@@ -2076,6 +2122,13 @@
metrics.sstablesDropppedFromCompactions.inc(num);
}
+ private static class SecondaryIndexExecutor extends CompactionExecutor
+ {
+ public SecondaryIndexExecutor()
+ {
+ super(DatabaseDescriptor.getConcurrentIndexBuilders(), "SecondaryIndexExecutor", Integer.MAX_VALUE);
+ }
+ }
public List<Map<String, String>> getCompactions()
{
@@ -2283,6 +2336,24 @@
}
}
+ public List<Holder> getCompactionsMatching(Iterable<TableMetadata> columnFamilies, Predicate<CompactionInfo> predicate)
+ {
+ Preconditions.checkArgument(columnFamilies != null, "Attempted to getCompactionsMatching in CompactionManager with no columnFamilies specified.");
+
+ List<Holder> matched = new ArrayList<>();
+ // consider all in-progress compactions
+ for (Holder holder : active.getCompactions())
+ {
+ CompactionInfo info = holder.getCompactionInfo();
+ if (info.getTableMetadata() == null || Iterables.contains(columnFamilies, info.getTableMetadata()))
+ {
+ if (predicate.test(info))
+ matched.add(holder);
+ }
+ }
+ return matched;
+ }
+
/**
* Try to stop all of the compactions for given ColumnFamilies.
*
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
index cab0af0..29e8c29 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
@@ -261,4 +261,14 @@
{
return Iterables.any(strategies, acs -> acs.getSSTables().contains(sstable));
}
+
+ public int getEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ for (AbstractCompactionStrategy strategy : strategies)
+ {
+ tasks += strategy.getEstimatedRemainingTasks();
+ }
+ return tasks;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index 5e1d561..ea312bc 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@ -23,21 +23,25 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.ConcurrentModificationException;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.primitives.Longs;
-import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,14 +58,15 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.Index;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.notifications.INotification;
import org.apache.cassandra.notifications.INotificationConsumer;
import org.apache.cassandra.notifications.SSTableAddedNotification;
@@ -71,7 +76,6 @@
import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.TimeUUID;
@@ -141,6 +145,8 @@
private volatile long maxSSTableSizeBytes;
private volatile String name;
+ public static int TWCS_BUCKET_COUNT_MAX = 128;
+
public CompactionStrategyManager(ColumnFamilyStore cfs)
{
this(cfs, cfs::getDiskBoundaries, cfs.getPartitioner().splitter().isPresent());
@@ -174,9 +180,12 @@
this.compactionLogger = new CompactionLogger(cfs, this);
this.boundariesSupplier = boundariesSupplier;
this.partitionSSTablesByTokenRange = partitionSSTablesByTokenRange;
- params = cfs.metadata().params.compaction;
+
+ currentBoundaries = boundariesSupplier.get();
+ params = schemaCompactionParams = cfs.metadata().params.compaction;
enabled = params.isEnabled();
- reload(cfs.metadata().params.compaction);
+ setStrategy(schemaCompactionParams);
+ startup();
}
/**
@@ -243,8 +252,8 @@
.stream()
.filter(s -> !compacting.contains(s) && !s.descriptor.version.isLatestVersion())
.sorted((o1, o2) -> {
- File f1 = new File(o1.descriptor.filenameFor(Component.DATA));
- File f2 = new File(o2.descriptor.filenameFor(Component.DATA));
+ File f1 = o1.descriptor.fileFor(Components.DATA);
+ File f2 = o2.descriptor.fileFor(Components.DATA);
return Longs.compare(f1.lastModified(), f2.lastModified());
}).collect(Collectors.toList());
for (SSTableReader sstable : potentialUpgrade)
@@ -449,19 +458,20 @@
}
}
- public void maybeReload(TableMetadata metadata)
+ /**
+ * Maybe reload the compaction strategies. Called after changing configuration.
+ */
+ public void maybeReloadParamsFromSchema(CompactionParams params)
{
// compare the old schema configuration to the new one, ignore any locally set changes.
- if (metadata.params.compaction.equals(schemaCompactionParams))
+ if (params.equals(schemaCompactionParams))
return;
writeLock.lock();
try
{
- // compare the old schema configuration to the new one, ignore any locally set changes.
- if (metadata.params.compaction.equals(schemaCompactionParams))
- return;
- reload(metadata.params.compaction);
+ if (!params.equals(schemaCompactionParams))
+ reloadParamsFromSchema(params);
}
finally
{
@@ -470,17 +480,84 @@
}
/**
+ * @param newParams new CompactionParams set in via CQL
+ */
+ private void reloadParamsFromSchema(CompactionParams newParams)
+ {
+ logger.debug("Recreating compaction strategy for {}.{} - compaction parameters changed via CQL",
+ cfs.keyspace.getName(), cfs.getTableName());
+
+ /*
+ * It's possible for compaction to be explicitly enabled/disabled
+ * via JMX when already enabled/disabled via params. In that case,
+ * if we now toggle enabled/disabled via params, we'll technically
+ * be overriding JMX-set value with params-set value.
+ */
+ boolean enabledWithJMX = enabled && !shouldBeEnabled();
+ boolean disabledWithJMX = !enabled && shouldBeEnabled();
+
+ schemaCompactionParams = newParams;
+ setStrategy(newParams);
+
+ // enable/disable via JMX overrides CQL params, but please see the comment above
+ if (enabled && !shouldBeEnabled() && !enabledWithJMX)
+ disable();
+ else if (!enabled && shouldBeEnabled() && !disabledWithJMX)
+ enable();
+
+ startup();
+ }
+
+ private void maybeReloadParamsFromJMX(CompactionParams params)
+ {
+ // compare the old local configuration to the new one, ignoring schema
+ if (params.equals(this.params))
+ return;
+
+ writeLock.lock();
+ try
+ {
+ if (!params.equals(this.params))
+ reloadParamsFromJMX(params);
+ }
+ finally
+ {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * @param newParams new CompactionParams set via JMX
+ */
+ private void reloadParamsFromJMX(CompactionParams newParams)
+ {
+ logger.debug("Recreating compaction strategy for {}.{} - compaction parameters changed via JMX",
+ cfs.keyspace.getName(), cfs.getTableName());
+
+ setStrategy(newParams);
+
+ // compaction params set via JMX override enable/disable via JMX
+ if (enabled && !shouldBeEnabled())
+ disable();
+ else if (!enabled && shouldBeEnabled())
+ enable();
+
+ startup();
+ }
+
+ /**
* Checks if the disk boundaries changed and reloads the compaction strategies
* to reflect the most up-to-date disk boundaries.
- *
+ * <p>
* This is typically called before acquiring the {@link this#readLock} to ensure the most up-to-date
* disk locations and boundaries are used.
- *
+ * <p>
* This should *never* be called inside by a thread holding the {@link this#readLock}, since it
* will potentially acquire the {@link this#writeLock} to update the compaction strategies
* what can cause a deadlock.
+ * <p>
+ * TODO: improve this to reload after receiving a notification rather than trying to reload on every operation
*/
- //TODO improve this to reload after receiving a notification rather than trying to reload on every operation
@VisibleForTesting
protected void maybeReloadDiskBoundaries()
{
@@ -490,9 +567,8 @@
writeLock.lock();
try
{
- if (!currentBoundaries.isOutOfDate())
- return;
- reload(params);
+ if (currentBoundaries.isOutOfDate())
+ reloadDiskBoundaries(boundariesSupplier.get());
}
finally
{
@@ -501,34 +577,23 @@
}
/**
- * Reload the compaction strategies
- *
- * Called after changing configuration and at startup.
- * @param newCompactionParams
+ * @param newBoundaries new DiskBoundaries - potentially functionally equivalent to current ones
*/
- private void reload(CompactionParams newCompactionParams)
+ private void reloadDiskBoundaries(DiskBoundaries newBoundaries)
{
- boolean enabledWithJMX = enabled && !shouldBeEnabled();
- boolean disabledWithJMX = !enabled && shouldBeEnabled();
+ DiskBoundaries oldBoundaries = currentBoundaries;
+ currentBoundaries = newBoundaries;
- if (currentBoundaries != null)
+ if (newBoundaries.isEquivalentTo(oldBoundaries))
{
- if (!newCompactionParams.equals(schemaCompactionParams))
- logger.debug("Recreating compaction strategy - compaction parameters changed for {}.{}", cfs.keyspace.getName(), cfs.getTableName());
- else if (currentBoundaries.isOutOfDate())
- logger.debug("Recreating compaction strategy - disk boundaries are out of date for {}.{}.", cfs.keyspace.getName(), cfs.getTableName());
+ logger.debug("Not recreating compaction strategy for {}.{} - disk boundaries are equivalent",
+ cfs.keyspace.getName(), cfs.getTableName());
+ return;
}
- if (currentBoundaries == null || currentBoundaries.isOutOfDate())
- currentBoundaries = boundariesSupplier.get();
-
- setStrategy(newCompactionParams);
- schemaCompactionParams = cfs.metadata().params.compaction;
-
- if (disabledWithJMX || !shouldBeEnabled() && !enabledWithJMX)
- disable();
- else
- enable();
+ logger.debug("Recreating compaction strategy for {}.{} - disk boundaries are out of date",
+ cfs.keyspace.getName(), cfs.getTableName());
+ setStrategy(params);
startup();
}
@@ -610,6 +675,44 @@
}
}
+ public boolean isLeveledCompaction()
+ {
+ readLock.lock();
+ try
+ {
+ return repaired.first() instanceof LeveledCompactionStrategy;
+ } finally
+ {
+ readLock.unlock();
+ }
+ }
+
+ public int[] getSSTableCountPerTWCSBucket()
+ {
+ readLock.lock();
+ try
+ {
+ List<Map<Long, Integer>> countsByBucket = Stream.concat(
+ StreamSupport.stream(repaired.allStrategies().spliterator(), false),
+ StreamSupport.stream(unrepaired.allStrategies().spliterator(), false))
+ .filter((TimeWindowCompactionStrategy.class)::isInstance)
+ .map(s -> ((TimeWindowCompactionStrategy)s).getSSTableCountByBuckets())
+ .collect(Collectors.toList());
+ return countsByBucket.isEmpty() ? null : sumCountsByBucket(countsByBucket, TWCS_BUCKET_COUNT_MAX);
+ }
+ finally
+ {
+ readLock.unlock();
+ }
+ }
+
+ static int[] sumCountsByBucket(List<Map<Long, Integer>> countsByBucket, int max)
+ {
+ TreeMap<Long, Integer> merged = new TreeMap<>(Comparator.reverseOrder());
+ countsByBucket.stream().flatMap(e -> e.entrySet().stream()).forEach(e -> merged.merge(e.getKey(), e.getValue(), Integer::sum));
+ return merged.values().stream().limit(max).mapToInt(i -> i).toArray();
+ }
+
static int[] sumArrays(int[] a, int[] b)
{
int[] res = new int[Math.max(a.length, b.length)];
@@ -957,7 +1060,7 @@
}
}
- public CompactionTasks getMaximalTasks(final int gcBefore, final boolean splitOutput)
+ public CompactionTasks getMaximalTasks(final int gcBefore, final boolean splitOutput, OperationType operationType)
{
maybeReloadDiskBoundaries();
// runWithCompactionsDisabled cancels active compactions and disables them, then we are able
@@ -970,7 +1073,10 @@
{
for (AbstractStrategyHolder holder : holders)
{
- tasks.addAll(holder.getMaximalTasks(gcBefore, splitOutput));
+ for (AbstractCompactionTask task: holder.getMaximalTasks(gcBefore, splitOutput))
+ {
+ tasks.add(task.setCompactionType(operationType));
+ }
}
}
finally
@@ -978,7 +1084,7 @@
readLock.unlock();
}
return CompactionTasks.create(tasks);
- }, false, false);
+ }, operationType, false, false);
}
/**
@@ -1027,6 +1133,46 @@
return tasks;
}
+ public int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes, boolean isIncremental)
+ {
+ if (additionalBytes == 0 || additionalSSTables == 0)
+ return getEstimatedRemainingTasks();
+
+ maybeReloadDiskBoundaries();
+ readLock.lock();
+ try
+ {
+ int tasks = pendingRepairs.getEstimatedRemainingTasks();
+
+ Iterable<AbstractCompactionStrategy> strategies;
+ if (isIncremental)
+ {
+ // Note that it is unlikely that we are behind in the pending strategies (as they only have a small fraction
+ // of the total data), so we assume here that any pending sstables go directly to the repaired bucket.
+ strategies = repaired.allStrategies();
+ tasks += unrepaired.getEstimatedRemainingTasks();
+ }
+ else
+ {
+ // Here we assume that all sstables go to unrepaired, which can be wrong if we are running
+ // both incremental and full repairs.
+ strategies = unrepaired.allStrategies();
+ tasks += repaired.getEstimatedRemainingTasks();
+
+ }
+ int strategyCount = Math.max(1, Iterables.size(strategies));
+ int sstablesPerStrategy = additionalSSTables / strategyCount;
+ long bytesPerStrategy = additionalBytes / strategyCount;
+ for (AbstractCompactionStrategy strategy : strategies)
+ tasks += strategy.getEstimatedRemainingTasks(sstablesPerStrategy, bytesPerStrategy);
+ return tasks;
+ }
+ finally
+ {
+ readLock.unlock();
+ }
+ }
+
public boolean shouldBeEnabled()
{
return params.isEnabled();
@@ -1053,23 +1199,10 @@
}
}
- public void setNewLocalCompactionStrategy(CompactionParams params)
+ public void overrideLocalParams(CompactionParams params)
{
- logger.info("Switching local compaction strategy from {} to {}}", this.params, params);
- writeLock.lock();
- try
- {
- setStrategy(params);
- if (shouldBeEnabled())
- enable();
- else
- disable();
- startup();
- }
- finally
- {
- writeLock.unlock();
- }
+ logger.info("Switching local compaction strategy from {} to {}", this.params, params);
+ maybeReloadParamsFromJMX(params);
}
private int getNumTokenPartitions()
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 5fc8031..b62bae5 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -21,11 +21,13 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
@@ -42,11 +44,13 @@
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Refs;
+import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.FBUtilities.now;
@@ -131,8 +135,9 @@
final Set<SSTableReader> fullyExpiredSSTables = controller.getFullyExpiredSSTables();
+ TimeUUID taskId = transaction.opId();
// select SSTables to compact based on available disk space.
- if (!buildCompactionCandidatesForAvailableDiskSpace(fullyExpiredSSTables))
+ if (!buildCompactionCandidatesForAvailableDiskSpace(fullyExpiredSSTables, taskId))
{
// The set of sstables has changed (one or more were excluded due to limited available disk space).
// We need to recompute the overlaps between sstables.
@@ -149,8 +154,6 @@
}
});
- TimeUUID taskId = transaction.opId();
-
// new sstables from flush can be added during a compaction, but only the compaction can remove them,
// so in our single-threaded compaction world this is a valid way of determining if we're compacting
// all the sstables (that existed when we started)
@@ -205,6 +208,7 @@
if (writer.append(ci.next()))
totalKeysWritten++;
+ ci.setTargetDirectory(writer.getSStableDirectory().path());
long bytesScanned = scanners.getTotalBytesScanned();
// Rate limit the scanners, and account for compression
@@ -243,12 +247,13 @@
StringBuilder newSSTableNames = new StringBuilder();
for (SSTableReader reader : newSStables)
- newSSTableNames.append(reader.descriptor.baseFilename()).append(",");
+ newSSTableNames.append(reader.descriptor.baseFile()).append(",");
long totalSourceRows = 0;
for (int i = 0; i < mergedRowCounts.length; i++)
totalSourceRows += mergedRowCounts[i] * (i + 1);
- String mergeSummary = updateCompactionHistory(cfs.keyspace.getName(), cfs.getTableName(), mergedRowCounts, startsize, endsize);
+ String mergeSummary = updateCompactionHistory(cfs.keyspace.getName(), cfs.getTableName(), mergedRowCounts, startsize, endsize,
+ ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType.type));
logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d. %s to %s (~%d%% of original) in %,dms. Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s. %,d total partitions merged to %,d. Partition merge counts were {%s}. Time spent writing keys = %,dms",
taskId,
@@ -287,7 +292,7 @@
return new DefaultCompactionWriter(cfs, directories, transaction, nonExpiredSSTables, keepOriginals, getLevel());
}
- public static String updateCompactionHistory(String keyspaceName, String columnFamilyName, long[] mergedRowCounts, long startSize, long endSize)
+ public static String updateCompactionHistory(String keyspaceName, String columnFamilyName, long[] mergedRowCounts, long startSize, long endSize, Map<String, String> compactionProperties)
{
StringBuilder mergeSummary = new StringBuilder(mergedRowCounts.length * 10);
Map<Integer, Long> mergedRows = new HashMap<>();
@@ -301,7 +306,7 @@
mergeSummary.append(String.format("%d:%d, ", rows, count));
mergedRows.put(rows, count);
}
- SystemKeyspace.updateCompactionHistory(keyspaceName, columnFamilyName, currentTimeMillis(), startSize, endSize, mergedRows);
+ SystemKeyspace.updateCompactionHistory(keyspaceName, columnFamilyName, currentTimeMillis(), startSize, endSize, mergedRows, compactionProperties);
return mergeSummary.toString();
}
@@ -356,12 +361,11 @@
/*
* Checks if we have enough disk space to execute the compaction. Drops the largest sstable out of the Task until
- * there's enough space (in theory) to handle the compaction. Does not take into account space that will be taken by
- * other compactions.
+ * there's enough space (in theory) to handle the compaction.
*
* @return true if there is enough disk space to execute the complete compaction, false if some sstables are excluded.
*/
- protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTableReader> fullyExpiredSSTables)
+ protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set<SSTableReader> fullyExpiredSSTables, TimeUUID taskId)
{
if(!cfs.isCompactionDiskSpaceCheckEnabled() && compactionType == OperationType.COMPACTION)
{
@@ -376,13 +380,29 @@
while(!nonExpiredSSTables.isEmpty())
{
// Only consider write size of non expired SSTables
- long expectedWriteSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
- long estimatedSSTables = Math.max(1, expectedWriteSize / strategy.getMaxSSTableBytes());
+ long writeSize;
+ try
+ {
+ writeSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
+ Map<File, Long> expectedNewWriteSize = new HashMap<>();
+ List<File> newCompactionDatadirs = cfs.getDirectoriesForFiles(nonExpiredSSTables);
+ long writeSizePerOutputDatadir = writeSize / Math.max(newCompactionDatadirs.size(), 1);
+ for (File directory : newCompactionDatadirs)
+ expectedNewWriteSize.put(directory, writeSizePerOutputDatadir);
- if(cfs.getDirectories().hasAvailableDiskSpace(estimatedSSTables, expectedWriteSize))
+ Map<File, Long> expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteBytes();
+
+ // todo: abort streams if they block compactions
+ if (cfs.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize))
+ break;
+ }
+ catch (Exception e)
+ {
+ logger.error("Could not check if there is enough disk space for compaction {}", taskId, e);
break;
+ }
- if (!reduceScopeForLimitedSpace(nonExpiredSSTables, expectedWriteSize))
+ if (!reduceScopeForLimitedSpace(nonExpiredSSTables, writeSize))
{
// we end up here if we can't take any more sstables out of the compaction.
// usually means we've run out of disk space
@@ -395,15 +415,20 @@
break;
}
- String msg = String.format("Not enough space for compaction, estimated sstables = %d, expected write size = %d", estimatedSSTables, expectedWriteSize);
+ String msg = String.format("Not enough space for compaction (%s) of %s.%s, estimated sstables = %d, expected write size = %d",
+ taskId,
+ cfs.keyspace.getName(),
+ cfs.name,
+ Math.max(1, writeSize / strategy.getMaxSSTableBytes()),
+ writeSize);
logger.warn(msg);
CompactionManager.instance.incrementAborted();
throw new RuntimeException(msg);
}
sstablesRemoved++;
- logger.warn("Not enough space for compaction, {}MiB estimated. Reducing scope.",
- (float) expectedWriteSize / 1024 / 1024);
+ logger.warn("Not enough space for compaction {}, {}MiB estimated. Reducing scope.",
+ taskId, (float) writeSize / 1024 / 1024);
}
if(sstablesRemoved > 0)
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
deleted file mode 100644
index 1e18bf9..0000000
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ /dev/null
@@ -1,519 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.compaction;
-
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicate;
-import com.google.common.collect.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.CompactionParams;
-import org.apache.cassandra.utils.Pair;
-
-import static com.google.common.collect.Iterables.filter;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-
-/**
- * @deprecated in favour of {@link TimeWindowCompactionStrategy}
- */
-@Deprecated
-public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
-{
- private static final Logger logger = LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
-
- private final DateTieredCompactionStrategyOptions options;
- protected volatile int estimatedRemainingTasks;
- private final Set<SSTableReader> sstables = new HashSet<>();
- private long lastExpiredCheck;
- private final SizeTieredCompactionStrategyOptions stcsOptions;
-
- public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
- {
- super(cfs, options);
- this.estimatedRemainingTasks = 0;
- this.options = new DateTieredCompactionStrategyOptions(options);
- if (!options.containsKey(AbstractCompactionStrategy.TOMBSTONE_COMPACTION_INTERVAL_OPTION) && !options.containsKey(AbstractCompactionStrategy.TOMBSTONE_THRESHOLD_OPTION))
- {
- disableTombstoneCompactions = true;
- logger.trace("Disabling tombstone compactions for DTCS");
- }
- else
- logger.trace("Enabling tombstone compactions for DTCS");
-
- this.stcsOptions = new SizeTieredCompactionStrategyOptions(options);
- }
-
- @Override
- @SuppressWarnings("resource")
- public AbstractCompactionTask getNextBackgroundTask(int gcBefore)
- {
- List<SSTableReader> previousCandidate = null;
- while (true)
- {
- List<SSTableReader> latestBucket = getNextBackgroundSSTables(gcBefore);
-
- if (latestBucket.isEmpty())
- return null;
-
- // Already tried acquiring references without success. It means there is a race with
- // the tracker but candidate SSTables were not yet replaced in the compaction strategy manager
- if (latestBucket.equals(previousCandidate))
- {
- logger.warn("Could not acquire references for compacting SSTables {} which is not a problem per se," +
- "unless it happens frequently, in which case it must be reported. Will retry later.",
- latestBucket);
- return null;
- }
-
- LifecycleTransaction modifier = cfs.getTracker().tryModify(latestBucket, OperationType.COMPACTION);
- if (modifier != null)
- return new CompactionTask(cfs, modifier, gcBefore);
- previousCandidate = latestBucket;
- }
- }
-
- /**
- *
- * @param gcBefore
- * @return
- */
- private synchronized List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
- {
- Set<SSTableReader> uncompacting;
- synchronized (sstables)
- {
- if (sstables.isEmpty())
- return Collections.emptyList();
-
- uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains));
- }
-
- Set<SSTableReader> expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation
- if (currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
- {
- // Find fully expired SSTables. Those will be included no matter what.
- expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingLiveSSTables(uncompacting), gcBefore);
- lastExpiredCheck = currentTimeMillis();
- }
- Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting));
-
- List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore));
- if (!expired.isEmpty())
- {
- logger.trace("Including expired sstables: {}", expired);
- compactionCandidates.addAll(expired);
- }
- return compactionCandidates;
- }
-
- private List<SSTableReader> getNextNonExpiredSSTables(Iterable<SSTableReader> nonExpiringSSTables, final int gcBefore)
- {
- int base = cfs.getMinimumCompactionThreshold();
- long now = getNow();
- List<SSTableReader> mostInteresting = getCompactionCandidates(nonExpiringSSTables, now, base);
- if (mostInteresting != null)
- {
- return mostInteresting;
- }
-
- // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
- // ratio is greater than threshold.
- List<SSTableReader> sstablesWithTombstones = Lists.newArrayList();
- for (SSTableReader sstable : nonExpiringSSTables)
- {
- if (worthDroppingTombstones(sstable, gcBefore))
- sstablesWithTombstones.add(sstable);
- }
- if (sstablesWithTombstones.isEmpty())
- return Collections.emptyList();
-
- return Collections.singletonList(Collections.min(sstablesWithTombstones, SSTableReader.sizeComparator));
- }
-
- private List<SSTableReader> getCompactionCandidates(Iterable<SSTableReader> candidateSSTables, long now, int base)
- {
- Iterable<SSTableReader> candidates = filterOldSSTables(Lists.newArrayList(candidateSSTables), options.maxSSTableAge, now);
-
- List<List<SSTableReader>> buckets = getBuckets(createSSTableAndMinTimestampPairs(candidates), options.baseTime, base, now, options.maxWindowSize);
- logger.debug("Compaction buckets are {}", buckets);
- updateEstimatedCompactionsByTasks(buckets);
- List<SSTableReader> mostInteresting = newestBucket(buckets,
- cfs.getMinimumCompactionThreshold(),
- cfs.getMaximumCompactionThreshold(),
- now,
- options.baseTime,
- options.maxWindowSize,
- stcsOptions);
- if (!mostInteresting.isEmpty())
- return mostInteresting;
- return null;
- }
-
- /**
- * Gets the timestamp that DateTieredCompactionStrategy considers to be the "current time".
- * @return the maximum timestamp across all SSTables.
- * @throws java.util.NoSuchElementException if there are no SSTables.
- */
- private long getNow()
- {
- // no need to convert to collection if had an Iterables.max(), but not present in standard toolkit, and not worth adding
- List<SSTableReader> list = new ArrayList<>();
- Iterables.addAll(list, cfs.getSSTables(SSTableSet.LIVE));
- if (list.isEmpty())
- return 0;
- return Collections.max(list, (o1, o2) -> Long.compare(o1.getMaxTimestamp(), o2.getMaxTimestamp()))
- .getMaxTimestamp();
- }
-
- /**
- * Removes all sstables with max timestamp older than maxSSTableAge.
- * @param sstables all sstables to consider
- * @param maxSSTableAge the age in milliseconds when an SSTable stops participating in compactions
- * @param now current time. SSTables with max timestamp less than (now - maxSSTableAge) are filtered.
- * @return a list of sstables with the oldest sstables excluded
- */
- @VisibleForTesting
- static Iterable<SSTableReader> filterOldSSTables(List<SSTableReader> sstables, long maxSSTableAge, long now)
- {
- if (maxSSTableAge == 0)
- return sstables;
- final long cutoff = now - maxSSTableAge;
- return filter(sstables, new Predicate<SSTableReader>()
- {
- @Override
- public boolean apply(SSTableReader sstable)
- {
- return sstable.getMaxTimestamp() >= cutoff;
- }
- });
- }
-
- public static List<Pair<SSTableReader, Long>> createSSTableAndMinTimestampPairs(Iterable<SSTableReader> sstables)
- {
- List<Pair<SSTableReader, Long>> sstableMinTimestampPairs = Lists.newArrayListWithCapacity(Iterables.size(sstables));
- for (SSTableReader sstable : sstables)
- sstableMinTimestampPairs.add(Pair.create(sstable, sstable.getMinTimestamp()));
- return sstableMinTimestampPairs;
- }
-
- @Override
- public synchronized void addSSTable(SSTableReader sstable)
- {
- sstables.add(sstable);
- }
-
- @Override
- public synchronized void removeSSTable(SSTableReader sstable)
- {
- sstables.remove(sstable);
- }
-
- @Override
- protected synchronized Set<SSTableReader> getSSTables()
- {
- return ImmutableSet.copyOf(sstables);
- }
-
- /**
- * A target time span used for bucketing SSTables based on timestamps.
- */
- private static class Target
- {
- // How big a range of timestamps fit inside the target.
- public final long size;
- // A timestamp t hits the target iff t / size == divPosition.
- public final long divPosition;
-
- public final long maxWindowSize;
-
- public Target(long size, long divPosition, long maxWindowSize)
- {
- this.size = size;
- this.divPosition = divPosition;
- this.maxWindowSize = maxWindowSize;
- }
-
- /**
- * Compares the target to a timestamp.
- * @param timestamp the timestamp to compare.
- * @return a negative integer, zero, or a positive integer as the target lies before, covering, or after than the timestamp.
- */
- public int compareToTimestamp(long timestamp)
- {
- return Long.compare(divPosition, timestamp / size);
- }
-
- /**
- * Tells if the timestamp hits the target.
- * @param timestamp the timestamp to test.
- * @return <code>true</code> iff timestamp / size == divPosition.
- */
- public boolean onTarget(long timestamp)
- {
- return compareToTimestamp(timestamp) == 0;
- }
-
- /**
- * Gets the next target, which represents an earlier time span.
- * @param base The number of contiguous targets that will have the same size. Targets following those will be <code>base</code> times as big.
- * @return
- */
- public Target nextTarget(int base)
- {
- if (divPosition % base > 0 || size * base > maxWindowSize)
- return new Target(size, divPosition - 1, maxWindowSize);
- else
- return new Target(size * base, divPosition / base - 1, maxWindowSize);
- }
- }
-
-
- /**
- * Group files with similar min timestamp into buckets. Files with recent min timestamps are grouped together into
- * buckets designated to short timespans while files with older timestamps are grouped into buckets representing
- * longer timespans.
- * @param files pairs consisting of a file and its min timestamp
- * @param timeUnit
- * @param base
- * @param now
- * @return a list of buckets of files. The list is ordered such that the files with newest timestamps come first.
- * Each bucket is also a list of files ordered from newest to oldest.
- */
- @VisibleForTesting
- static <T> List<List<T>> getBuckets(Collection<Pair<T, Long>> files, long timeUnit, int base, long now, long maxWindowSize)
- {
- // Sort files by age. Newest first.
- final List<Pair<T, Long>> sortedFiles = Lists.newArrayList(files);
- Collections.sort(sortedFiles, Collections.reverseOrder(new Comparator<Pair<T, Long>>()
- {
- public int compare(Pair<T, Long> p1, Pair<T, Long> p2)
- {
- return p1.right.compareTo(p2.right);
- }
- }));
-
- List<List<T>> buckets = Lists.newArrayList();
- Target target = getInitialTarget(now, timeUnit, maxWindowSize);
- PeekingIterator<Pair<T, Long>> it = Iterators.peekingIterator(sortedFiles.iterator());
-
- outerLoop:
- while (it.hasNext())
- {
- while (!target.onTarget(it.peek().right))
- {
- // If the file is too new for the target, skip it.
- if (target.compareToTimestamp(it.peek().right) < 0)
- {
- it.next();
-
- if (!it.hasNext())
- break outerLoop;
- }
- else // If the file is too old for the target, switch targets.
- target = target.nextTarget(base);
- }
- List<T> bucket = Lists.newArrayList();
- while (target.onTarget(it.peek().right))
- {
- bucket.add(it.next().left);
-
- if (!it.hasNext())
- break;
- }
- buckets.add(bucket);
- }
-
- return buckets;
- }
-
- @VisibleForTesting
- static Target getInitialTarget(long now, long timeUnit, long maxWindowSize)
- {
- return new Target(timeUnit, now / timeUnit, maxWindowSize);
- }
-
-
- private void updateEstimatedCompactionsByTasks(List<List<SSTableReader>> tasks)
- {
- int n = 0;
- for (List<SSTableReader> bucket : tasks)
- {
- for (List<SSTableReader> stcsBucket : getSTCSBuckets(bucket, stcsOptions))
- if (stcsBucket.size() >= cfs.getMinimumCompactionThreshold())
- n += Math.ceil((double)stcsBucket.size() / cfs.getMaximumCompactionThreshold());
- }
- estimatedRemainingTasks = n;
- cfs.getCompactionStrategyManager().compactionLogger.pending(this, n);
- }
-
-
- /**
- * @param buckets list of buckets, sorted from newest to oldest, from which to return the newest bucket within thresholds.
- * @param minThreshold minimum number of sstables in a bucket to qualify.
- * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this).
- * @return a bucket (list) of sstables to compact.
- */
- @VisibleForTesting
- static List<SSTableReader> newestBucket(List<List<SSTableReader>> buckets, int minThreshold, int maxThreshold, long now, long baseTime, long maxWindowSize, SizeTieredCompactionStrategyOptions stcsOptions)
- {
- // If the "incoming window" has at least minThreshold SSTables, choose that one.
- // For any other bucket, at least 2 SSTables is enough.
- // In any case, limit to maxThreshold SSTables.
- Target incomingWindow = getInitialTarget(now, baseTime, maxWindowSize);
- for (List<SSTableReader> bucket : buckets)
- {
- boolean inFirstWindow = incomingWindow.onTarget(bucket.get(0).getMinTimestamp());
- if (bucket.size() >= minThreshold || (bucket.size() >= 2 && !inFirstWindow))
- {
- List<SSTableReader> stcsSSTables = getSSTablesForSTCS(bucket, inFirstWindow ? minThreshold : 2, maxThreshold, stcsOptions);
- if (!stcsSSTables.isEmpty())
- return stcsSSTables;
- }
- }
- return Collections.emptyList();
- }
-
- private static List<SSTableReader> getSSTablesForSTCS(Collection<SSTableReader> sstables, int minThreshold, int maxThreshold, SizeTieredCompactionStrategyOptions stcsOptions)
- {
- List<SSTableReader> s = SizeTieredCompactionStrategy.mostInterestingBucket(getSTCSBuckets(sstables, stcsOptions), minThreshold, maxThreshold);
- logger.debug("Got sstables {} for STCS from {}", s, sstables);
- return s;
- }
-
- private static List<List<SSTableReader>> getSTCSBuckets(Collection<SSTableReader> sstables, SizeTieredCompactionStrategyOptions stcsOptions)
- {
- List<Pair<SSTableReader,Long>> pairs = SizeTieredCompactionStrategy.createSSTableAndLengthPairs(AbstractCompactionStrategy.filterSuspectSSTables(sstables));
- return SizeTieredCompactionStrategy.getBuckets(pairs,
- stcsOptions.bucketHigh,
- stcsOptions.bucketLow,
- stcsOptions.minSSTableSize);
- }
-
- @Override
- @SuppressWarnings("resource")
- public synchronized Collection<AbstractCompactionTask> getMaximalTask(int gcBefore, boolean splitOutput)
- {
- Iterable<SSTableReader> filteredSSTables = filterSuspectSSTables(sstables);
- if (Iterables.isEmpty(filteredSSTables))
- return null;
- LifecycleTransaction txn = cfs.getTracker().tryModify(filteredSSTables, OperationType.COMPACTION);
- if (txn == null)
- return null;
- return Collections.<AbstractCompactionTask>singleton(new CompactionTask(cfs, txn, gcBefore));
- }
-
- @Override
- @SuppressWarnings("resource")
- public synchronized AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
- {
- assert !sstables.isEmpty(); // checked for by CM.submitUserDefined
-
- LifecycleTransaction modifier = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
- if (modifier == null)
- {
- logger.trace("Unable to mark {} for compaction; probably a background compaction got to it first. You can disable background compactions temporarily if this is a problem", sstables);
- return null;
- }
-
- return new CompactionTask(cfs, modifier, gcBefore).setUserDefined(true);
- }
-
- public int getEstimatedRemainingTasks()
- {
- return estimatedRemainingTasks;
- }
-
- public long getMaxSSTableBytes()
- {
- return Long.MAX_VALUE;
- }
-
- /**
- * DTCS should not group sstables for anticompaction - this can mix new and old data
- */
- @Override
- public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
- {
- Collection<Collection<SSTableReader>> groups = new ArrayList<>(sstablesToGroup.size());
- for (SSTableReader sstable : sstablesToGroup)
- {
- groups.add(Collections.singleton(sstable));
- }
- return groups;
- }
-
- public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
- {
- Map<String, String> uncheckedOptions = AbstractCompactionStrategy.validateOptions(options);
- uncheckedOptions = DateTieredCompactionStrategyOptions.validateOptions(options, uncheckedOptions);
-
- uncheckedOptions.remove(CompactionParams.Option.MIN_THRESHOLD.toString());
- uncheckedOptions.remove(CompactionParams.Option.MAX_THRESHOLD.toString());
-
- uncheckedOptions = SizeTieredCompactionStrategyOptions.validateOptions(options, uncheckedOptions);
-
- return uncheckedOptions;
- }
-
- public CompactionLogger.Strategy strategyLogger()
- {
- return new CompactionLogger.Strategy()
- {
- public JsonNode sstable(SSTableReader sstable)
- {
- ObjectNode node = JsonNodeFactory.instance.objectNode();
- node.put("min_timestamp", sstable.getMinTimestamp());
- node.put("max_timestamp", sstable.getMaxTimestamp());
- return node;
- }
-
- public JsonNode options()
- {
- ObjectNode node = JsonNodeFactory.instance.objectNode();
- TimeUnit resolution = DateTieredCompactionStrategy.this.options.timestampResolution;
- node.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
- resolution.toString());
- node.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY,
- resolution.toSeconds(DateTieredCompactionStrategy.this.options.baseTime));
- node.put(DateTieredCompactionStrategyOptions.MAX_WINDOW_SIZE_KEY,
- resolution.toSeconds(DateTieredCompactionStrategy.this.options.maxWindowSize));
- return node;
- }
- };
- }
-
- public String toString()
- {
- return String.format("DateTieredCompactionStrategy[%s/%s]",
- cfs.getMinimumCompactionThreshold(),
- cfs.getMaximumCompactionThreshold());
- }
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
deleted file mode 100644
index 7604bbc..0000000
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.compaction;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-
-public final class DateTieredCompactionStrategyOptions
-{
- private static final Logger logger = LoggerFactory.getLogger(DateTieredCompactionStrategyOptions.class);
- protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION = TimeUnit.MICROSECONDS;
- @Deprecated
- protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365*1000;
- protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
- protected static final long DEFAULT_MAX_WINDOW_SIZE_SECONDS = TimeUnit.SECONDS.convert(1, TimeUnit.DAYS);
-
- protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
- protected static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution";
- @Deprecated
- protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
- protected static final String BASE_TIME_KEY = "base_time_seconds";
- protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds";
- protected static final String MAX_WINDOW_SIZE_KEY = "max_window_size_seconds";
-
- @Deprecated
- protected final long maxSSTableAge;
- protected final TimeUnit timestampResolution;
- protected final long baseTime;
- protected final long expiredSSTableCheckFrequency;
- protected final long maxWindowSize;
-
- public DateTieredCompactionStrategyOptions(Map<String, String> options)
- {
- String optionValue = options.get(TIMESTAMP_RESOLUTION_KEY);
- timestampResolution = optionValue == null ? DEFAULT_TIMESTAMP_RESOLUTION : TimeUnit.valueOf(optionValue);
- if (timestampResolution != DEFAULT_TIMESTAMP_RESOLUTION)
- logger.warn("Using a non-default timestamp_resolution {} - are you really doing inserts with USING TIMESTAMP <non_microsecond_timestamp> (or driver equivalent)?", timestampResolution);
- optionValue = options.get(MAX_SSTABLE_AGE_KEY);
- double fractionalDays = optionValue == null ? DEFAULT_MAX_SSTABLE_AGE_DAYS : Double.parseDouble(optionValue);
- maxSSTableAge = Math.round(fractionalDays * timestampResolution.convert(1, TimeUnit.DAYS));
- optionValue = options.get(BASE_TIME_KEY);
- baseTime = timestampResolution.convert(optionValue == null ? DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
- optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
- expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
- optionValue = options.get(MAX_WINDOW_SIZE_KEY);
- maxWindowSize = timestampResolution.convert(optionValue == null ? DEFAULT_MAX_WINDOW_SIZE_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
- }
-
- public DateTieredCompactionStrategyOptions()
- {
- maxSSTableAge = Math.round(DEFAULT_MAX_SSTABLE_AGE_DAYS * DEFAULT_TIMESTAMP_RESOLUTION.convert((long) DEFAULT_MAX_SSTABLE_AGE_DAYS, TimeUnit.DAYS));
- timestampResolution = DEFAULT_TIMESTAMP_RESOLUTION;
- baseTime = timestampResolution.convert(DEFAULT_BASE_TIME_SECONDS, TimeUnit.SECONDS);
- expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS, TimeUnit.SECONDS);
- maxWindowSize = timestampResolution.convert(1, TimeUnit.DAYS);
- }
-
- public static Map<String, String> validateOptions(Map<String, String> options, Map<String, String> uncheckedOptions) throws ConfigurationException
- {
- String optionValue = options.get(TIMESTAMP_RESOLUTION_KEY);
- try
- {
- if (optionValue != null)
- TimeUnit.valueOf(optionValue);
- }
- catch (IllegalArgumentException e)
- {
- throw new ConfigurationException(String.format("timestamp_resolution %s is not valid", optionValue));
- }
-
- optionValue = options.get(MAX_SSTABLE_AGE_KEY);
- try
- {
- double maxSStableAge = optionValue == null ? DEFAULT_MAX_SSTABLE_AGE_DAYS : Double.parseDouble(optionValue);
- if (maxSStableAge < 0)
- {
- throw new ConfigurationException(String.format("%s must be non-negative: %.2f", MAX_SSTABLE_AGE_KEY, maxSStableAge));
- }
- }
- catch (NumberFormatException e)
- {
- throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, MAX_SSTABLE_AGE_KEY), e);
- }
-
- optionValue = options.get(BASE_TIME_KEY);
- try
- {
- long baseTime = optionValue == null ? DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue);
- if (baseTime <= 0)
- {
- throw new ConfigurationException(String.format("%s must be greater than 0, but was %d", BASE_TIME_KEY, baseTime));
- }
- }
- catch (NumberFormatException e)
- {
- throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, BASE_TIME_KEY), e);
- }
-
- optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
- try
- {
- long expiredCheckFrequency = optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue);
- if (expiredCheckFrequency < 0)
- {
- throw new ConfigurationException(String.format("%s must not be negative, but was %d", EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, expiredCheckFrequency));
- }
- }
- catch (NumberFormatException e)
- {
- throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY), e);
- }
-
- optionValue = options.get(MAX_WINDOW_SIZE_KEY);
- try
- {
- long maxWindowSize = optionValue == null ? DEFAULT_MAX_WINDOW_SIZE_SECONDS : Long.parseLong(optionValue);
- if (maxWindowSize < 0)
- {
- throw new ConfigurationException(String.format("%s must not be negative, but was %d", MAX_WINDOW_SIZE_KEY, maxWindowSize));
- }
- }
- catch (NumberFormatException e)
- {
- throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, MAX_WINDOW_SIZE_KEY), e);
- }
-
-
- uncheckedOptions.remove(MAX_SSTABLE_AGE_KEY);
- uncheckedOptions.remove(BASE_TIME_KEY);
- uncheckedOptions.remove(TIMESTAMP_RESOLUTION_KEY);
- uncheckedOptions.remove(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
- uncheckedOptions.remove(MAX_WINDOW_SIZE_KEY);
-
- return uncheckedOptions;
- }
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index 54953e4..bbdd13c 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -33,7 +33,6 @@
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -43,11 +42,13 @@
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TOLERATE_SSTABLE_SIZE;
+
public class LeveledCompactionStrategy extends AbstractCompactionStrategy
{
private static final Logger logger = LoggerFactory.getLogger(LeveledCompactionStrategy.class);
private static final String SSTABLE_SIZE_OPTION = "sstable_size_in_mb";
- private static final boolean tolerateSstableSize = Boolean.getBoolean(Config.PROPERTY_PREFIX + "tolerate_sstable_size");
+ private static final boolean tolerateSstableSize = TOLERATE_SSTABLE_SIZE.getBoolean();
private static final String LEVEL_FANOUT_SIZE_OPTION = "fanout_size";
private static final String SINGLE_SSTABLE_UPLEVEL_OPTION = "single_sstable_uplevel";
public static final int DEFAULT_LEVEL_FANOUT_SIZE = 10;
@@ -280,6 +281,12 @@
return n;
}
+ @Override
+ int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
+ {
+ return manifest.getEstimatedTasks(additionalBytes);
+ }
+
public long getMaxSSTableBytes()
{
return maxSSTableSizeInMiB * 1024L * 1024L;
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
index 36b5600..08cda89 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
@@ -37,11 +37,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_STRICT_LCS_CHECKS;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
/**
@@ -52,7 +52,7 @@
class LeveledGenerations
{
private static final Logger logger = LoggerFactory.getLogger(LeveledGenerations.class);
- private final boolean strictLCSChecksTest = Boolean.getBoolean(Config.PROPERTY_PREFIX + "test.strict_lcs_checks");
+ private final boolean strictLCSChecksTest = TEST_STRICT_LCS_CHECKS.getBoolean();
// It includes L0, i.e. we support [L0 - L8] levels
static final int MAX_LEVEL_COUNT = 9;
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 2972d7d..4339181 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -17,7 +17,17 @@
*/
package org.apache.cassandra.db.compaction;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicates;
@@ -26,18 +36,16 @@
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
-
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Pair;
@@ -114,7 +122,7 @@
long maxModificationTime = Long.MIN_VALUE;
for (SSTableReader ssTableReader : level)
{
- long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA);
+ long modificationTime = ssTableReader.getDataCreationTime();
if (modificationTime >= maxModificationTime)
{
sstableWithMaxModificationTime = ssTableReader;
@@ -622,16 +630,25 @@
return 0;
}
- public synchronized int getEstimatedTasks()
+ public int getEstimatedTasks()
+ {
+ return getEstimatedTasks(0);
+ }
+
+ int getEstimatedTasks(long additionalLevel0Bytes)
+ {
+ return getEstimatedTasks((level) -> SSTableReader.getTotalBytes(getLevel(level)) + (level == 0 ? additionalLevel0Bytes : 0));
+ }
+
+ private synchronized int getEstimatedTasks(Function<Integer,Long> fnTotalSizeBytesByLevel)
{
long tasks = 0;
long[] estimated = new long[generations.levelCount()];
for (int i = generations.levelCount() - 1; i >= 0; i--)
{
- Set<SSTableReader> sstables = generations.get(i);
// If there is 1 byte over TBL - (MBL * 1.001), there is still a task left, so we need to round up.
- estimated[i] = (long)Math.ceil((double)Math.max(0L, SSTableReader.getTotalBytes(sstables) - (long)(maxBytesForLevel(i, maxSSTableSizeInBytes) * 1.001)) / (double)maxSSTableSizeInBytes);
+ estimated[i] = (long)Math.ceil((double)Math.max(0L, fnTotalSizeBytesByLevel.apply(i) - (long)(maxBytesForLevel(i, maxSSTableSizeInBytes) * 1.001)) / (double)maxSSTableSizeInBytes);
tasks += estimated[i];
}
diff --git a/src/java/org/apache/cassandra/db/compaction/OperationType.java b/src/java/org/apache/cassandra/db/compaction/OperationType.java
index e957e42..2a5ffc6 100644
--- a/src/java/org/apache/cassandra/db/compaction/OperationType.java
+++ b/src/java/org/apache/cassandra/db/compaction/OperationType.java
@@ -20,35 +20,58 @@
public enum OperationType
{
/** Each modification here should be also applied to {@link org.apache.cassandra.tools.nodetool.Stop#compactionType} */
- COMPACTION("Compaction"),
- VALIDATION("Validation"),
- KEY_CACHE_SAVE("Key cache save"),
- ROW_CACHE_SAVE("Row cache save"),
- COUNTER_CACHE_SAVE("Counter cache save"),
- CLEANUP("Cleanup"),
- SCRUB("Scrub"),
- UPGRADE_SSTABLES("Upgrade sstables"),
- INDEX_BUILD("Secondary index build"),
- /** Compaction for tombstone removal */
- TOMBSTONE_COMPACTION("Tombstone Compaction"),
- UNKNOWN("Unknown compaction type"),
- ANTICOMPACTION("Anticompaction after repair"),
- VERIFY("Verify"),
- FLUSH("Flush"),
- STREAM("Stream"),
- WRITE("Write"),
- VIEW_BUILD("View build"),
- INDEX_SUMMARY("Index summary redistribution"),
- RELOCATE("Relocate sstables to correct disk"),
- GARBAGE_COLLECT("Remove deleted data");
+ P0("Cancel all operations", false, 0),
+
+ // Automation or operator-driven tasks
+ CLEANUP("Cleanup", true, 1),
+ SCRUB("Scrub", true, 1),
+ UPGRADE_SSTABLES("Upgrade sstables", true, 1),
+ VERIFY("Verify", false, 1),
+ MAJOR_COMPACTION("Major compaction", true, 1),
+ RELOCATE("Relocate sstables to correct disk", false, 1),
+ GARBAGE_COLLECT("Remove deleted data", true, 1),
+
+ // Internal SSTable writing
+ FLUSH("Flush", true, 1),
+ WRITE("Write", true, 1),
+
+ ANTICOMPACTION("Anticompaction after repair", true, 2),
+ VALIDATION("Validation", false, 3),
+
+ INDEX_BUILD("Secondary index build", false, 4),
+ VIEW_BUILD("View build", false, 4),
+
+ COMPACTION("Compaction", true, 5),
+ TOMBSTONE_COMPACTION("Tombstone Compaction", true, 5), // Compaction for tombstone removal
+ UNKNOWN("Unknown compaction type", false, 5),
+
+ STREAM("Stream", true, 6),
+ KEY_CACHE_SAVE("Key cache save", false, 6),
+ ROW_CACHE_SAVE("Row cache save", false, 6),
+ COUNTER_CACHE_SAVE("Counter cache save", false, 6),
+ INDEX_SUMMARY("Index summary redistribution", false, 6);
public final String type;
public final String fileName;
- OperationType(String type)
+ /**
+ * For purposes of calculating space for interim compactions in flight, whether or not this OperationType is expected
+ * to write data to disk
+ */
+ public final boolean writesData;
+
+ // As of now, priority takes part only for interrupting tasks to give way to operator-driven tasks.
+ // Operation types that have a smaller number will be allowed to cancel ones that have larger numbers.
+ //
+ // Submitted tasks may be prioritised differently when forming a queue, if/when CASSANDRA-11218 is implemented.
+ public final int priority;
+
+ OperationType(String type, boolean writesData, int priority)
{
this.type = type;
this.fileName = type.toLowerCase().replace(" ", "");
+ this.writesData = writesData;
+ this.priority = priority;
}
public static OperationType fromFileName(String fileName)
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
index 314df9e..bf6c497 100644
--- a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
@@ -281,4 +281,13 @@
{
return Iterables.any(managers, prm -> prm.containsSSTable(sstable));
}
+
+ @Override
+ public int getEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ for (PendingRepairManager manager : managers)
+ tasks += manager.getEstimatedRemainingTasks();
+ return tasks;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
index 11d6fe8..85a4483 100644
--- a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
@@ -228,22 +228,25 @@
private int getEstimatedRemainingTasks(TimeUUID sessionID, AbstractCompactionStrategy strategy)
{
- if (canCleanup(sessionID))
- {
- return 0;
- }
- else
- {
- return strategy.getEstimatedRemainingTasks();
- }
+ return getEstimatedRemainingTasks(sessionID, strategy, 0, 0);
+ }
+
+ private int getEstimatedRemainingTasks(TimeUUID sessionID, AbstractCompactionStrategy strategy, int additionalSSTables, long additionalBytes)
+ {
+ return canCleanup(sessionID) ? 0 : strategy.getEstimatedRemainingTasks();
}
int getEstimatedRemainingTasks()
{
+ return getEstimatedRemainingTasks(0, 0);
+ }
+
+ int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes)
+ {
int tasks = 0;
for (Map.Entry<TimeUUID, AbstractCompactionStrategy> entry : strategies.entrySet())
{
- tasks += getEstimatedRemainingTasks(entry.getKey(), entry.getValue());
+ tasks += getEstimatedRemainingTasks(entry.getKey(), entry.getValue(), additionalSSTables, additionalBytes);
}
return tasks;
}
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
deleted file mode 100644
index 56825d0..0000000
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ /dev/null
@@ -1,853 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.compaction;
-
-import java.io.IOError;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import java.util.*;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.Refs;
-import org.apache.cassandra.utils.memory.HeapCloner;
-
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-
-public class Scrubber implements Closeable
-{
- private final ColumnFamilyStore cfs;
- private final SSTableReader sstable;
- private final LifecycleTransaction transaction;
- private final File destination;
- private final boolean skipCorrupted;
- private final boolean reinsertOverflowedTTLRows;
-
- private final boolean isCommutative;
- private final boolean isIndex;
- private final boolean checkData;
- private final long expectedBloomFilterSize;
-
- private final ReadWriteLock fileAccessLock;
- private final RandomAccessReader dataFile;
- private final RandomAccessReader indexFile;
- private final ScrubInfo scrubInfo;
- private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
-
- private int goodPartitions;
- private int badPartitions;
- private int emptyPartitions;
-
- private ByteBuffer currentIndexKey;
- private ByteBuffer nextIndexKey;
- private long currentPartitionPositionFromIndex;
- private long nextPartitionPositionFromIndex;
-
- private NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics = new NegativeLocalDeletionInfoMetrics();
-
- private final OutputHandler outputHandler;
-
- private static final Comparator<Partition> partitionComparator = Comparator.comparing(Partition::partitionKey);
- private final SortedSet<Partition> outOfOrder = new TreeSet<>(partitionComparator);
-
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData)
- {
- this(cfs, transaction, skipCorrupted, checkData, false);
- }
-
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData,
- boolean reinsertOverflowedTTLRows)
- {
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData, reinsertOverflowedTTLRows);
- }
-
- @SuppressWarnings("resource")
- public Scrubber(ColumnFamilyStore cfs,
- LifecycleTransaction transaction,
- boolean skipCorrupted,
- OutputHandler outputHandler,
- boolean checkData,
- boolean reinsertOverflowedTTLRows)
- {
- this.cfs = cfs;
- this.transaction = transaction;
- this.sstable = transaction.onlyOne();
- this.outputHandler = outputHandler;
- this.skipCorrupted = skipCorrupted;
- this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(cfs.metadata(),
- sstable.descriptor.version,
- sstable.header);
-
- List<SSTableReader> toScrub = Collections.singletonList(sstable);
-
- this.destination = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForSSTable(sstable));
- this.isCommutative = cfs.metadata().isCounter();
-
- boolean hasIndexFile = (new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))).exists();
- this.isIndex = cfs.isIndex();
- if (!hasIndexFile)
- {
- // if there's any corruption in the -Data.db then partitions can't be skipped over. but it's worth a shot.
- outputHandler.warn("Missing component: " + sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
- }
- this.checkData = checkData && !this.isIndex; //LocalByPartitionerType does not support validation
- this.expectedBloomFilterSize = Math.max(
- cfs.metadata().params.minIndexInterval,
- hasIndexFile ? SSTableReader.getApproximateKeyCount(toScrub) : 0);
-
- this.fileAccessLock = new ReentrantReadWriteLock();
- // loop through each partition, deserializing to check for damage.
- // We'll also loop through the index at the same time, using the position from the index to recover if the
- // partition header (key or data size) is corrupt. (This means our position in the index file will be one
- // partition "ahead" of the data file.)
- this.dataFile = transaction.isOffline()
- ? sstable.openDataReader()
- : sstable.openDataReader(CompactionManager.instance.getRateLimiter());
-
- this.indexFile = hasIndexFile
- ? RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)))
- : null;
-
- this.scrubInfo = new ScrubInfo(dataFile, sstable, fileAccessLock.readLock());
-
- this.currentPartitionPositionFromIndex = 0;
- this.nextPartitionPositionFromIndex = 0;
-
- if (reinsertOverflowedTTLRows)
- outputHandler.output("Starting scrub with reinsert overflowed TTL option");
- }
-
- private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
- {
- return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
- }
-
- private String keyString(DecoratedKey key)
- {
- if (key == null)
- return "(unknown)";
-
- try
- {
- return cfs.metadata().partitionKeyType.getString(key.getKey());
- }
- catch (Exception e)
- {
- return String.format("(corrupted; hex value: %s)", ByteBufferUtil.bytesToHex(key.getKey()));
- }
- }
-
- public void scrub()
- {
- List<SSTableReader> finished = new ArrayList<>();
- outputHandler.output(String.format("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge);
- Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
- {
- nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
- if (indexAvailable())
- {
- // throw away variable so we don't have a side effect in the assert
- long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
- assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
- }
-
- StatsMetadata metadata = sstable.getSSTableMetadata();
- writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction));
-
- DecoratedKey prevKey = null;
-
- while (!dataFile.isEOF())
- {
- if (scrubInfo.isStopRequested())
- throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
-
- long partitionStart = dataFile.getFilePointer();
- outputHandler.debug("Reading row at " + partitionStart);
-
- DecoratedKey key = null;
- try
- {
- ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile);
- if (!cfs.metadata.getLocal().isIndex())
- cfs.metadata.getLocal().partitionKeyType.validate(raw);
- key = sstable.decorateKey(raw);
- }
- catch (Throwable th)
- {
- throwIfFatal(th);
- // check for null key below
- }
-
- updateIndexKey();
-
- long dataStart = dataFile.getFilePointer();
-
- long dataStartFromIndex = -1;
- long dataSizeFromIndex = -1;
- if (currentIndexKey != null)
- {
- dataStartFromIndex = currentPartitionPositionFromIndex + 2 + currentIndexKey.remaining();
- dataSizeFromIndex = nextPartitionPositionFromIndex - dataStartFromIndex;
- }
-
- String keyName = key == null ? "(unreadable key)" : keyString(key);
- outputHandler.debug(String.format("partition %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSizeFromIndex)));
- assert currentIndexKey != null || !indexAvailable();
-
- try
- {
- if (key == null)
- throw new IOError(new IOException("Unable to read partition key from data file"));
-
- if (currentIndexKey != null && !key.getKey().equals(currentIndexKey))
- {
- throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)",
- //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
- "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey))));
- }
-
- if (indexFile != null && dataSizeFromIndex > dataFile.length())
- throw new IOError(new IOException("Impossible partition size (greater than file length): " + dataSizeFromIndex));
-
- if (indexFile != null && dataStart != dataStartFromIndex)
- outputHandler.warn(String.format("Data file partition position %d differs from index file row position %d", dataStart, dataStartFromIndex));
-
- if (tryAppend(prevKey, key, writer))
- prevKey = key;
- }
- catch (Throwable th)
- {
- throwIfFatal(th);
- outputHandler.warn(String.format("Error reading partition %s (stacktrace follows):", keyName), th);
-
- if (currentIndexKey != null
- && (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex))
- {
-
- outputHandler.output(String.format("Retrying from partition index; data is %s bytes starting at %s",
- dataSizeFromIndex, dataStartFromIndex));
- key = sstable.decorateKey(currentIndexKey);
- try
- {
- if (!cfs.metadata.getLocal().isIndex())
- cfs.metadata.getLocal().partitionKeyType.validate(key.getKey());
- dataFile.seek(dataStartFromIndex);
-
- if (tryAppend(prevKey, key, writer))
- prevKey = key;
- }
- catch (Throwable th2)
- {
- throwIfFatal(th2);
- throwIfCannotContinue(key, th2);
-
- outputHandler.warn("Retry failed too. Skipping to next partition (retry's stacktrace follows)", th2);
- badPartitions++;
- if (!seekToNextPartition())
- break;
- }
- }
- else
- {
- throwIfCannotContinue(key, th);
-
- outputHandler.warn("Partition starting at position " + dataStart + " is unreadable; skipping to next");
- badPartitions++;
- if (currentIndexKey != null)
- if (!seekToNextPartition())
- break;
- }
- }
- }
-
- if (!outOfOrder.isEmpty())
- {
- // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time
- long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
- SSTableReader newInOrderSstable;
- try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction))
- {
- for (Partition partition : outOfOrder)
- inOrderWriter.append(partition.unfilteredIterator());
- newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true);
- }
- transaction.update(newInOrderSstable, false);
- finished.add(newInOrderSstable);
- outputHandler.warn(String.format("%d out of order partition (or partitions with out of order rows) found while scrubbing %s; " +
- "Those have been written (in order) to a new sstable (%s)", outOfOrder.size(), sstable, newInOrderSstable));
- }
-
- // finish obsoletes the old sstable
- finished.addAll(writer.setRepairedAt(badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt).finish());
- }
- catch (IOException e)
- {
- throw Throwables.propagate(e);
- }
- finally
- {
- if (transaction.isOffline())
- finished.forEach(sstable -> sstable.selfRef().release());
- }
-
- if (!finished.isEmpty())
- {
- outputHandler.output("Scrub of " + sstable + " complete: " + goodPartitions + " partitions in new sstable and " + emptyPartitions + " empty (tombstoned) partitions dropped");
- if (negativeLocalDeletionInfoMetrics.fixedRows > 0)
- outputHandler.output("Fixed " + negativeLocalDeletionInfoMetrics.fixedRows + " rows with overflowed local deletion time.");
- if (badPartitions > 0)
- outputHandler.warn("Unable to recover " + badPartitions + " partitions that were skipped. You can attempt manual recovery from the pre-scrub snapshot. You can also run nodetool repair to transfer the data from a healthy replica, if any");
- }
- else
- {
- if (badPartitions > 0)
- outputHandler.warn("No valid partitions found while scrubbing " + sstable + "; it is marked for deletion now. If you want to attempt manual recovery, you can find a copy in the pre-scrub snapshot");
- else
- outputHandler.output("Scrub of " + sstable + " complete; looks like all " + emptyPartitions + " partitions were tombstoned");
- }
- }
-
- @SuppressWarnings("resource")
- private boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewriter writer)
- {
- // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects
- // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added
- // to the outOfOrder set that will be later written to a new SSTable.
- OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key),
- cfs.metadata().comparator);
-
- try (UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath()))
- {
- if (prevKey != null && prevKey.compareTo(key) > 0)
- {
- saveOutOfOrderPartition(prevKey, key, iterator);
- return false;
- }
-
- if (writer.tryAppend(iterator) == null)
- emptyPartitions++;
- else
- goodPartitions++;
- }
-
- if (sstableIterator.hasRowsOutOfOrder())
- {
- outputHandler.warn(String.format("Out of order rows found in partition: %s", keyString(key)));
- outOfOrder.add(sstableIterator.getRowsOutOfOrder());
- }
-
- return true;
- }
-
- /**
- * Only wrap with {@link FixNegativeLocalDeletionTimeIterator} if {@link #reinsertOverflowedTTLRows} option
- * is specified
- */
- @SuppressWarnings("resource")
- private UnfilteredRowIterator getIterator(DecoratedKey key)
- {
- RowMergingSSTableIterator rowMergingIterator = new RowMergingSSTableIterator(SSTableIdentityIterator.create(sstable, dataFile, key));
- return reinsertOverflowedTTLRows ? new FixNegativeLocalDeletionTimeIterator(rowMergingIterator,
- outputHandler,
- negativeLocalDeletionInfoMetrics) : rowMergingIterator;
- }
-
- private void updateIndexKey()
- {
- currentIndexKey = nextIndexKey;
- currentPartitionPositionFromIndex = nextPartitionPositionFromIndex;
- try
- {
- nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile);
-
- nextPartitionPositionFromIndex = !indexAvailable()
- ? dataFile.length()
- : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
- }
- catch (Throwable th)
- {
- JVMStabilityInspector.inspectThrowable(th);
- outputHandler.warn("Error reading index file", th);
- nextIndexKey = null;
- nextPartitionPositionFromIndex = dataFile.length();
- }
- }
-
- private boolean indexAvailable()
- {
- return indexFile != null && !indexFile.isEOF();
- }
-
- private boolean seekToNextPartition()
- {
- while(nextPartitionPositionFromIndex < dataFile.length())
- {
- try
- {
- dataFile.seek(nextPartitionPositionFromIndex);
- return true;
- }
- catch (Throwable th)
- {
- throwIfFatal(th);
- outputHandler.warn(String.format("Failed to seek to next partition position %d", nextPartitionPositionFromIndex), th);
- badPartitions++;
- }
-
- updateIndexKey();
- }
-
- return false;
- }
-
- private void saveOutOfOrderPartition(DecoratedKey prevKey, DecoratedKey key, UnfilteredRowIterator iterator)
- {
- // TODO bitch if the row is too large? if it is there's not much we can do ...
- outputHandler.warn(String.format("Out of order partition detected (%s found after %s)",
- keyString(key), keyString(prevKey)));
- outOfOrder.add(ImmutableBTreePartition.create(iterator));
- }
-
- private void throwIfFatal(Throwable th)
- {
- if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError))
- throw (Error) th;
- }
-
- private void throwIfCannotContinue(DecoratedKey key, Throwable th)
- {
- if (isIndex)
- {
- outputHandler.warn(String.format("An error occurred while scrubbing the partition with key '%s' for an index table. " +
- "Scrubbing will abort for this table and the index will be rebuilt.", keyString(key)));
- throw new IOError(th);
- }
-
- if (isCommutative && !skipCorrupted)
- {
- outputHandler.warn(String.format("An error occurred while scrubbing the partition with key '%s'. Skipping corrupt " +
- "data in counter tables will result in undercounts for the affected " +
- "counters (see CASSANDRA-2759 for more details), so by default the scrub will " +
- "stop at this point. If you would like to skip the row anyway and continue " +
- "scrubbing, re-run the scrub with the --skip-corrupted option.",
- keyString(key)));
- throw new IOError(th);
- }
- }
-
- public void close()
- {
- fileAccessLock.writeLock().lock();
- try
- {
- FileUtils.closeQuietly(dataFile);
- FileUtils.closeQuietly(indexFile);
- }
- finally
- {
- fileAccessLock.writeLock().unlock();
- }
- }
-
- public CompactionInfo.Holder getScrubInfo()
- {
- return scrubInfo;
- }
-
- private static class ScrubInfo extends CompactionInfo.Holder
- {
- private final RandomAccessReader dataFile;
- private final SSTableReader sstable;
- private final TimeUUID scrubCompactionId;
- private final Lock fileReadLock;
-
- public ScrubInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock)
- {
- this.dataFile = dataFile;
- this.sstable = sstable;
- this.fileReadLock = fileReadLock;
- scrubCompactionId = nextTimeUUID();
- }
-
- public CompactionInfo getCompactionInfo()
- {
- fileReadLock.lock();
- try
- {
- return new CompactionInfo(sstable.metadata(),
- OperationType.SCRUB,
- dataFile.getFilePointer(),
- dataFile.length(),
- scrubCompactionId,
- ImmutableSet.of(sstable));
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- finally
- {
- fileReadLock.unlock();
- }
- }
-
- public boolean isGlobal()
- {
- return false;
- }
- }
-
- @VisibleForTesting
- public ScrubResult scrubWithResult()
- {
- scrub();
- return new ScrubResult(this);
- }
-
- public static final class ScrubResult
- {
- public final int goodPartitions;
- public final int badPartitions;
- public final int emptyPartitions;
-
- public ScrubResult(Scrubber scrubber)
- {
- this.goodPartitions = scrubber.goodPartitions;
- this.badPartitions = scrubber.badPartitions;
- this.emptyPartitions = scrubber.emptyPartitions;
- }
- }
-
- public class NegativeLocalDeletionInfoMetrics
- {
- public volatile int fixedRows = 0;
- }
-
- /**
- * During 2.x migration, under some circumstances rows might have gotten duplicated.
- * Merging iterator merges rows with same clustering.
- *
- * For more details, refer to CASSANDRA-12144.
- */
- private static class RowMergingSSTableIterator extends WrappingUnfilteredRowIterator
- {
- Unfiltered nextToOffer = null;
-
- RowMergingSSTableIterator(UnfilteredRowIterator source)
- {
- super(source);
- }
-
- @Override
- public boolean hasNext()
- {
- return nextToOffer != null || wrapped.hasNext();
- }
-
- @Override
- public Unfiltered next()
- {
- Unfiltered next = nextToOffer != null ? nextToOffer : wrapped.next();
-
- if (next.isRow())
- {
- while (wrapped.hasNext())
- {
- Unfiltered peek = wrapped.next();
- if (!peek.isRow() || !next.clustering().equals(peek.clustering()))
- {
- nextToOffer = peek; // Offer peek in next call
- return next;
- }
-
- // Duplicate row, merge it.
- next = Rows.merge((Row) next, (Row) peek);
- }
- }
-
- nextToOffer = null;
- return next;
- }
- }
-
- /**
- * In some case like CASSANDRA-12127 the cells might have been stored in the wrong order. This decorator check the
- * cells order and collect the out of order cells to correct the problem.
- */
- private static final class OrderCheckerIterator extends AbstractIterator<Unfiltered> implements UnfilteredRowIterator
- {
- /**
- * The decorated iterator.
- */
- private final UnfilteredRowIterator iterator;
-
- private final ClusteringComparator comparator;
-
- private Unfiltered previous;
-
- /**
- * The partition containing the rows which are out of order.
- */
- private Partition rowsOutOfOrder;
-
- public OrderCheckerIterator(UnfilteredRowIterator iterator, ClusteringComparator comparator)
- {
- this.iterator = iterator;
- this.comparator = comparator;
- }
-
- public TableMetadata metadata()
- {
- return iterator.metadata();
- }
-
- public boolean isReverseOrder()
- {
- return iterator.isReverseOrder();
- }
-
- public RegularAndStaticColumns columns()
- {
- return iterator.columns();
- }
-
- public DecoratedKey partitionKey()
- {
- return iterator.partitionKey();
- }
-
- public Row staticRow()
- {
- return iterator.staticRow();
- }
-
- @Override
- public boolean isEmpty()
- {
- return iterator.isEmpty();
- }
-
- public void close()
- {
- iterator.close();
- }
-
- public DeletionTime partitionLevelDeletion()
- {
- return iterator.partitionLevelDeletion();
- }
-
- public EncodingStats stats()
- {
- return iterator.stats();
- }
-
- public boolean hasRowsOutOfOrder()
- {
- return rowsOutOfOrder != null;
- }
-
- public Partition getRowsOutOfOrder()
- {
- return rowsOutOfOrder;
- }
-
- protected Unfiltered computeNext()
- {
- if (!iterator.hasNext())
- return endOfData();
-
- Unfiltered next = iterator.next();
-
- // If we detect that some rows are out of order we will store and sort the remaining ones to insert them
- // in a separate SSTable.
- if (previous != null && comparator.compare(next, previous) < 0)
- {
- rowsOutOfOrder = ImmutableBTreePartition.create(UnfilteredRowIterators.concat(next, iterator), false);
- return endOfData();
- }
- previous = next;
- return next;
- }
- }
-
- /**
- * This iterator converts negative {@link AbstractCell#localDeletionTime()} into {@link AbstractCell#MAX_DELETION_TIME}
- *
- * This is to recover entries with overflowed localExpirationTime due to CASSANDRA-14092
- */
- private static final class FixNegativeLocalDeletionTimeIterator extends AbstractIterator<Unfiltered> implements UnfilteredRowIterator
- {
- /**
- * The decorated iterator.
- */
- private final UnfilteredRowIterator iterator;
-
- private final OutputHandler outputHandler;
- private final NegativeLocalDeletionInfoMetrics negativeLocalExpirationTimeMetrics;
-
- public FixNegativeLocalDeletionTimeIterator(UnfilteredRowIterator iterator, OutputHandler outputHandler,
- NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics)
- {
- this.iterator = iterator;
- this.outputHandler = outputHandler;
- this.negativeLocalExpirationTimeMetrics = negativeLocalDeletionInfoMetrics;
- }
-
- public TableMetadata metadata()
- {
- return iterator.metadata();
- }
-
- public boolean isReverseOrder()
- {
- return iterator.isReverseOrder();
- }
-
- public RegularAndStaticColumns columns()
- {
- return iterator.columns();
- }
-
- public DecoratedKey partitionKey()
- {
- return iterator.partitionKey();
- }
-
- public Row staticRow()
- {
- return iterator.staticRow();
- }
-
- @Override
- public boolean isEmpty()
- {
- return iterator.isEmpty();
- }
-
- public void close()
- {
- iterator.close();
- }
-
- public DeletionTime partitionLevelDeletion()
- {
- return iterator.partitionLevelDeletion();
- }
-
- public EncodingStats stats()
- {
- return iterator.stats();
- }
-
- protected Unfiltered computeNext()
- {
- if (!iterator.hasNext())
- return endOfData();
-
- Unfiltered next = iterator.next();
- if (!next.isRow())
- return next;
-
- if (hasNegativeLocalExpirationTime((Row) next))
- {
- outputHandler.debug(String.format("Found row with negative local expiration time: %s", next.toString(metadata(), false)));
- negativeLocalExpirationTimeMetrics.fixedRows++;
- return fixNegativeLocalExpirationTime((Row) next);
- }
-
- return next;
- }
-
- private boolean hasNegativeLocalExpirationTime(Row next)
- {
- Row row = next;
- if (row.primaryKeyLivenessInfo().isExpiring() && row.primaryKeyLivenessInfo().localExpirationTime() < 0)
- {
- return true;
- }
-
- for (ColumnData cd : row)
- {
- if (cd.column().isSimple())
- {
- Cell<?> cell = (Cell<?>)cd;
- if (cell.isExpiring() && cell.localDeletionTime() < 0)
- return true;
- }
- else
- {
- ComplexColumnData complexData = (ComplexColumnData)cd;
- for (Cell<?> cell : complexData)
- {
- if (cell.isExpiring() && cell.localDeletionTime() < 0)
- return true;
- }
- }
- }
-
- return false;
- }
-
- private Unfiltered fixNegativeLocalExpirationTime(Row row)
- {
- LivenessInfo livenessInfo = row.primaryKeyLivenessInfo();
- if (livenessInfo.isExpiring() && livenessInfo.localExpirationTime() < 0)
- livenessInfo = livenessInfo.withUpdatedTimestampAndLocalDeletionTime(livenessInfo.timestamp() + 1, AbstractCell.MAX_DELETION_TIME);
-
- return row.transformAndFilter(livenessInfo, row.deletion(), cd -> {
- if (cd.column().isSimple())
- {
- Cell cell = (Cell)cd;
- return cell.isExpiring() && cell.localDeletionTime() < 0
- ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME)
- : cell;
- }
- else
- {
- ComplexColumnData complexData = (ComplexColumnData)cd;
- return complexData.transformAndFilter(cell -> cell.isExpiring() && cell.localDeletionTime() < 0
- ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME)
- : cell);
- }
- }).clone(HeapCloner.instance);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
index 2e1dffc..1f73c4c 100644
--- a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java
@@ -27,8 +27,8 @@
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
@@ -82,7 +82,7 @@
catch (Throwable t)
{
transaction.abort();
- throw new CorruptSSTableException(t, sstable.descriptor.filenameFor(Component.DATA));
+ throw new CorruptSSTableException(t, sstable.descriptor.fileFor(Components.DATA));
}
cfs.getTracker().notifySSTableMetadataChanged(sstable, metadataBefore);
}
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
index 555d86a..c10d3a6 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategy.java
@@ -30,6 +30,8 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.*;
@@ -57,6 +59,9 @@
private long lastExpiredCheck;
private long highestWindowSeen;
+ // This is accessed in both the threading context of compaction / repair and also JMX
+ private volatile Map<Long, Integer> sstableCountByBuckets = Collections.emptyMap();
+
public TimeWindowCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
{
super(cfs, options);
@@ -179,6 +184,7 @@
this.highestWindowSeen);
this.estimatedRemainingTasks = mostInteresting.estimatedRemainingTasks;
+ this.sstableCountByBuckets = buckets.left.keySet().stream().collect(Collectors.toMap(Function.identity(), k -> buckets.left.get(k).size()));
if (!mostInteresting.sstables.isEmpty())
return mostInteresting.sstables;
return null;
@@ -426,6 +432,10 @@
return Long.MAX_VALUE;
}
+ public Map<Long, Integer> getSSTableCountByBuckets()
+ {
+ return sstableCountByBuckets;
+ }
public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
{
diff --git a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java
index 8b2ba23..b9312a0 100644
--- a/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java
+++ b/src/java/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyOptions.java
@@ -26,9 +26,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.exceptions.ConfigurationException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION;
+
public final class TimeWindowCompactionStrategyOptions
{
private static final Logger logger = LoggerFactory.getLogger(TimeWindowCompactionStrategyOptions.class);
@@ -39,13 +40,13 @@
protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
protected static final Boolean DEFAULT_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION = false;
- protected static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution";
- protected static final String COMPACTION_WINDOW_UNIT_KEY = "compaction_window_unit";
- protected static final String COMPACTION_WINDOW_SIZE_KEY = "compaction_window_size";
- protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds";
- protected static final String UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY = "unsafe_aggressive_sstable_expiration";
+ public static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution";
+ public static final String COMPACTION_WINDOW_UNIT_KEY = "compaction_window_unit";
+ public static final String COMPACTION_WINDOW_SIZE_KEY = "compaction_window_size";
+ public static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds";
+ public static final String UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY = "unsafe_aggressive_sstable_expiration";
- static final String UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY = Config.PROPERTY_PREFIX + "allow_unsafe_aggressive_sstable_expiration";
+ static final boolean UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_ENABLED = ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.getBoolean();
protected final int sstableWindowSize;
protected final TimeUnit sstableWindowUnit;
@@ -75,7 +76,7 @@
expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
optionValue = options.get(UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY);
- ignoreOverlaps = optionValue == null ? DEFAULT_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION : (Boolean.getBoolean(UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY) && Boolean.parseBoolean(optionValue));
+ ignoreOverlaps = optionValue == null ? DEFAULT_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION : (UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_ENABLED && Boolean.parseBoolean(optionValue));
stcsOptions = new SizeTieredCompactionStrategyOptions(options);
}
@@ -153,8 +154,9 @@
if (!(optionValue.equalsIgnoreCase("true") || optionValue.equalsIgnoreCase("false")))
throw new ConfigurationException(String.format("%s is not 'true' or 'false' (%s)", UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, optionValue));
- if(optionValue.equalsIgnoreCase("true") && !Boolean.getBoolean(UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY))
- throw new ConfigurationException(String.format("%s is requested but not allowed, restart cassandra with -D%s=true to allow it", UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY));
+ if (optionValue.equalsIgnoreCase("true") && !UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_ENABLED)
+ throw new ConfigurationException(String.format("%s is requested but not allowed, restart cassandra with -D%s=true to allow it",
+ UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_KEY, ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.getKey()));
}
uncheckedOptions.remove(COMPACTION_WINDOW_SIZE_KEY);
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index 87bf5b8..f18cc49 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.db.compaction;
-import java.util.*;
+import java.util.Arrays;
import java.util.function.LongPredicate;
import com.google.common.base.Throwables;
@@ -25,9 +25,10 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableRewriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -72,16 +73,19 @@
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator());
sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
- return SSTableWriter.create(cfs.newSSTableDescriptor(directory),
- estimatedRows,
- metadata.repairedAt,
- metadata.pendingRepair,
- metadata.isTransient,
- cfs.metadata,
- sstableMetadataCollector,
- SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)),
- cfs.indexManager.listIndexes(),
- transaction);
+
+ Descriptor descriptor = cfs.newSSTableDescriptor(directory);
+ return descriptor.getFormat().getWriterFactory().builder(descriptor)
+ .setKeyCount(estimatedRows)
+ .setRepairedAt(metadata.repairedAt)
+ .setPendingRepair(metadata.pendingRepair)
+ .setTransientSSTable(metadata.isTransient)
+ .setTableMetadataRef(cfs.metadata)
+ .setMetadataCollector(sstableMetadataCollector)
+ .setSerializationHeader(SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)))
+ .addDefaultComponents()
+ .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), transaction.opType())
+ .build(transaction, cfs);
}
public void upgrade(boolean keepOriginals)
@@ -93,6 +97,7 @@
CompactionIterator iter = new CompactionIterator(transaction.opType(), scanners.scanners, controller, nowInSec, nextTimeUUID()))
{
writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata()));
+ iter.setTargetDirectory(writer.currentWriter().getFilename());
while (iter.hasNext())
writer.append(iter.next());
@@ -101,7 +106,8 @@
}
catch (Exception e)
{
- Throwables.propagate(e);
+ Throwables.throwIfUnchecked(e);
+ throw new RuntimeException(e);
}
finally
{
diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java
deleted file mode 100644
index 29eb299..0000000
--- a/src/java/org/apache/cassandra/db/compaction/Verifier.java
+++ /dev/null
@@ -1,664 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.compaction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-
-import org.apache.cassandra.dht.LocalPartitioner;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.IndexSummary;
-import org.apache.cassandra.io.sstable.KeyIterator;
-import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
-import org.apache.cassandra.io.util.DataIntegrityMetadata;
-import org.apache.cassandra.io.util.DataIntegrityMetadata.FileDigestValidator;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.BloomFilterSerializer;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.IFilter;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.cassandra.utils.TimeUUID;
-
-import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.IOError;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.*;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.Function;
-import java.util.function.LongPredicate;
-
-import org.apache.cassandra.io.util.File;
-
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-
-public class Verifier implements Closeable
-{
- private final ColumnFamilyStore cfs;
- private final SSTableReader sstable;
-
- private final CompactionController controller;
-
- private final ReadWriteLock fileAccessLock;
- private final RandomAccessReader dataFile;
- private final RandomAccessReader indexFile;
- private final VerifyInfo verifyInfo;
- private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final Options options;
- private final boolean isOffline;
- /**
- * Given a keyspace, return the set of local and pending token ranges. By default {@link StorageService#getLocalAndPendingRanges(String)}
- * is expected, but for the standalone verifier case we can't use that, so this is here to allow the CLI to provide
- * the token ranges.
- */
- private final Function<String, ? extends Collection<Range<Token>>> tokenLookup;
-
- private int goodRows;
-
- private final OutputHandler outputHandler;
- private FileDigestValidator validator;
-
- public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, boolean isOffline, Options options)
- {
- this(cfs, sstable, new OutputHandler.LogOutput(), isOffline, options);
- }
-
- public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, OutputHandler outputHandler, boolean isOffline, Options options)
- {
- this.cfs = cfs;
- this.sstable = sstable;
- this.outputHandler = outputHandler;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(cfs.metadata(), sstable.descriptor.version, sstable.header);
-
- this.controller = new VerifyController(cfs);
-
- this.fileAccessLock = new ReentrantReadWriteLock();
- this.dataFile = isOffline
- ? sstable.openDataReader()
- : sstable.openDataReader(CompactionManager.instance.getRateLimiter());
- this.indexFile = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)));
- this.verifyInfo = new VerifyInfo(dataFile, sstable, fileAccessLock.readLock());
- this.options = options;
- this.isOffline = isOffline;
- this.tokenLookup = options.tokenLookup;
- }
-
- public void verify()
- {
- boolean extended = options.extendedVerification;
- long rowStart = 0;
-
- outputHandler.output(String.format("Verifying %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())));
- if (options.checkVersion && !sstable.descriptor.version.isLatestVersion())
- {
- String msg = String.format("%s is not the latest version, run upgradesstables", sstable);
- outputHandler.output(msg);
- // don't use markAndThrow here because we don't want a CorruptSSTableException for this.
- throw new RuntimeException(msg);
- }
-
- outputHandler.output(String.format("Deserializing sstable metadata for %s ", sstable));
- try
- {
- EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, types);
- if (sstableMetadata.containsKey(MetadataType.VALIDATION) &&
- !((ValidationMetadata)sstableMetadata.get(MetadataType.VALIDATION)).partitioner.equals(sstable.getPartitioner().getClass().getCanonicalName()))
- throw new IOException("Partitioner does not match validation metadata");
- }
- catch (Throwable t)
- {
- outputHandler.warn(t);
- markAndThrow(t, false);
- }
-
- try
- {
- outputHandler.debug("Deserializing index for "+sstable);
- deserializeIndex(sstable);
- }
- catch (Throwable t)
- {
- outputHandler.warn(t);
- markAndThrow(t);
- }
-
- try
- {
- outputHandler.debug("Deserializing index summary for "+sstable);
- deserializeIndexSummary(sstable);
- }
- catch (Throwable t)
- {
- outputHandler.output("Index summary is corrupt - if it is removed it will get rebuilt on startup "+sstable.descriptor.filenameFor(Component.SUMMARY));
- outputHandler.warn(t);
- markAndThrow(t, false);
- }
-
- try
- {
- outputHandler.debug("Deserializing bloom filter for "+sstable);
- deserializeBloomFilter(sstable);
-
- }
- catch (Throwable t)
- {
- outputHandler.warn(t);
- markAndThrow(t);
- }
-
- if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner() instanceof LocalPartitioner))
- {
- outputHandler.debug("Checking that all tokens are owned by the current node");
- try (KeyIterator iter = new KeyIterator(sstable.descriptor, sstable.metadata()))
- {
- List<Range<Token>> ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace));
- if (ownedRanges.isEmpty())
- return;
- RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
- while (iter.hasNext())
- {
- DecoratedKey key = iter.next();
- rangeOwnHelper.validate(key);
- }
- }
- catch (Throwable t)
- {
- outputHandler.warn(t);
- markAndThrow(t);
- }
- }
-
- if (options.quick)
- return;
-
- // Verify will use the Digest files, which works for both compressed and uncompressed sstables
- outputHandler.output(String.format("Checking computed hash of %s ", sstable));
- try
- {
- validator = null;
-
- if (new File(sstable.descriptor.filenameFor(Component.DIGEST)).exists())
- {
- validator = DataIntegrityMetadata.fileDigestValidator(sstable.descriptor);
- validator.validate();
- }
- else
- {
- outputHandler.output("Data digest missing, assuming extended verification of disk values");
- extended = true;
- }
- }
- catch (IOException e)
- {
- outputHandler.warn(e);
- markAndThrow(e);
- }
- finally
- {
- FileUtils.closeQuietly(validator);
- }
-
- if (!extended)
- return;
-
- outputHandler.output("Extended Verify requested, proceeding to inspect values");
-
- try
- {
- ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile);
- {
- long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
- if (firstRowPositionFromIndex != 0)
- markAndThrow(new RuntimeException("firstRowPositionFromIndex != 0: "+firstRowPositionFromIndex));
- }
-
- List<Range<Token>> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace));
- RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
- DecoratedKey prevKey = null;
-
- while (!dataFile.isEOF())
- {
-
- if (verifyInfo.isStopRequested())
- throw new CompactionInterruptedException(verifyInfo.getCompactionInfo());
-
- rowStart = dataFile.getFilePointer();
- outputHandler.debug("Reading row at " + rowStart);
-
- DecoratedKey key = null;
- try
- {
- key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dataFile));
- }
- catch (Throwable th)
- {
- throwIfFatal(th);
- // check for null key below
- }
-
- if (options.checkOwnsTokens && ownedRanges.size() > 0 && !(cfs.getPartitioner() instanceof LocalPartitioner))
- {
- try
- {
- rangeOwnHelper.validate(key);
- }
- catch (Throwable t)
- {
- outputHandler.warn(String.format("Key %s in sstable %s not owned by local ranges %s", key, sstable, ownedRanges), t);
- markAndThrow(t);
- }
- }
-
- ByteBuffer currentIndexKey = nextIndexKey;
- long nextRowPositionFromIndex = 0;
- try
- {
- nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile);
- nextRowPositionFromIndex = indexFile.isEOF()
- ? dataFile.length()
- : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
- }
- catch (Throwable th)
- {
- markAndThrow(th);
- }
-
- long dataStart = dataFile.getFilePointer();
- long dataStartFromIndex = currentIndexKey == null
- ? -1
- : rowStart + 2 + currentIndexKey.remaining();
-
- long dataSize = nextRowPositionFromIndex - dataStartFromIndex;
- // avoid an NPE if key is null
- String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey());
- outputHandler.debug(String.format("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize)));
-
- assert currentIndexKey != null || indexFile.isEOF();
-
- try
- {
- if (key == null || dataSize > dataFile.length())
- markAndThrow(new RuntimeException(String.format("key = %s, dataSize=%d, dataFile.length() = %d", key, dataSize, dataFile.length())));
-
- //mimic the scrub read path, intentionally unused
- try (UnfilteredRowIterator iterator = SSTableIdentityIterator.create(sstable, dataFile, key))
- {
- }
-
- if ( (prevKey != null && prevKey.compareTo(key) > 0) || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex )
- markAndThrow(new RuntimeException("Key out of order: previous = "+prevKey + " : current = " + key));
-
- goodRows++;
- prevKey = key;
-
-
- outputHandler.debug(String.format("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex));
- dataFile.seek(nextRowPositionFromIndex);
- }
- catch (Throwable th)
- {
- markAndThrow(th);
- }
- }
- }
- catch (Throwable t)
- {
- throw Throwables.propagate(t);
- }
- finally
- {
- controller.close();
- }
-
- outputHandler.output("Verify of " + sstable + " succeeded. All " + goodRows + " rows read successfully");
- }
-
- /**
- * Use the fact that check(..) is called with sorted tokens - we keep a pointer in to the normalized ranges
- * and only bump the pointer if the key given is out of range. This is done to avoid calling .contains(..) many
- * times for each key (with vnodes for example)
- */
- @VisibleForTesting
- public static class RangeOwnHelper
- {
- private final List<Range<Token>> normalizedRanges;
- private int rangeIndex = 0;
- private DecoratedKey lastKey;
-
- public RangeOwnHelper(List<Range<Token>> normalizedRanges)
- {
- this.normalizedRanges = normalizedRanges;
- Range.assertNormalized(normalizedRanges);
- }
-
- /**
- * check if the given key is contained in any of the given ranges
- *
- * Must be called in sorted order - key should be increasing
- *
- * @param key the key
- * @throws RuntimeException if the key is not contained
- */
- public void validate(DecoratedKey key)
- {
- if (!check(key))
- throw new RuntimeException("Key " + key + " is not contained in the given ranges");
- }
-
- /**
- * check if the given key is contained in any of the given ranges
- *
- * Must be called in sorted order - key should be increasing
- *
- * @param key the key
- * @return boolean
- */
- public boolean check(DecoratedKey key)
- {
- assert lastKey == null || key.compareTo(lastKey) > 0;
- lastKey = key;
-
- if (normalizedRanges.isEmpty()) // handle tests etc where we don't have any ranges
- return true;
-
- if (rangeIndex > normalizedRanges.size() - 1)
- throw new IllegalStateException("RangeOwnHelper can only be used to find the first out-of-range-token");
-
- while (!normalizedRanges.get(rangeIndex).contains(key.getToken()))
- {
- rangeIndex++;
- if (rangeIndex > normalizedRanges.size() - 1)
- return false;
- }
-
- return true;
- }
- }
-
- private void deserializeIndex(SSTableReader sstable) throws IOException
- {
- try (RandomAccessReader primaryIndex = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))))
- {
- long indexSize = primaryIndex.length();
-
- while ((primaryIndex.getFilePointer()) != indexSize)
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex);
- RowIndexEntry.Serializer.skip(primaryIndex, sstable.descriptor.version);
- }
- }
- }
-
- private void deserializeIndexSummary(SSTableReader sstable) throws IOException
- {
- File file = new File(sstable.descriptor.filenameFor(Component.SUMMARY));
- TableMetadata metadata = cfs.metadata();
- try (DataInputStream iStream = new DataInputStream(Files.newInputStream(file.toPath())))
- {
- try (IndexSummary indexSummary = IndexSummary.serializer.deserialize(iStream,
- cfs.getPartitioner(),
- metadata.params.minIndexInterval,
- metadata.params.maxIndexInterval))
- {
- ByteBufferUtil.readWithLength(iStream);
- ByteBufferUtil.readWithLength(iStream);
- }
- }
- }
-
- private void deserializeBloomFilter(SSTableReader sstable) throws IOException
- {
- File bfPath = new File(sstable.descriptor.filenameFor(Component.FILTER));
- if (bfPath.exists())
- {
- try (FileInputStreamPlus stream = bfPath.newInputStream();
- IFilter bf = BloomFilterSerializer.deserialize(stream, sstable.descriptor.version.hasOldBfFormat()))
- {
- }
- }
- }
-
- public void close()
- {
- fileAccessLock.writeLock().lock();
- try
- {
- FileUtils.closeQuietly(dataFile);
- FileUtils.closeQuietly(indexFile);
- }
- finally
- {
- fileAccessLock.writeLock().unlock();
- }
- }
-
- private void throwIfFatal(Throwable th)
- {
- if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError))
- throw (Error) th;
- }
-
- private void markAndThrow(Throwable cause)
- {
- markAndThrow(cause, true);
- }
-
- private void markAndThrow(Throwable cause, boolean mutateRepaired)
- {
- if (mutateRepaired && options.mutateRepairStatus) // if we are able to mutate repaired flag, an incremental repair should be enough
- {
- try
- {
- sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient());
- cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
- }
- catch(IOException ioe)
- {
- outputHandler.output("Error mutating repairedAt for SSTable " + sstable.getFilename() + ", as part of markAndThrow");
- }
- }
- Exception e = new Exception(String.format("Invalid SSTable %s, please force %srepair", sstable.getFilename(), (mutateRepaired && options.mutateRepairStatus) ? "" : "a full "), cause);
- if (options.invokeDiskFailurePolicy)
- throw new CorruptSSTableException(e, sstable.getFilename());
- else
- throw new RuntimeException(e);
- }
-
- public CompactionInfo.Holder getVerifyInfo()
- {
- return verifyInfo;
- }
-
- private static class VerifyInfo extends CompactionInfo.Holder
- {
- private final RandomAccessReader dataFile;
- private final SSTableReader sstable;
- private final TimeUUID verificationCompactionId;
- private final Lock fileReadLock;
-
- public VerifyInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock)
- {
- this.dataFile = dataFile;
- this.sstable = sstable;
- this.fileReadLock = fileReadLock;
- verificationCompactionId = nextTimeUUID();
- }
-
- public CompactionInfo getCompactionInfo()
- {
- fileReadLock.lock();
- try
- {
- return new CompactionInfo(sstable.metadata(),
- OperationType.VERIFY,
- dataFile.getFilePointer(),
- dataFile.length(),
- verificationCompactionId,
- ImmutableSet.of(sstable));
- }
- catch (Exception e)
- {
- throw new RuntimeException();
- }
- finally
- {
- fileReadLock.unlock();
- }
- }
-
- public boolean isGlobal()
- {
- return false;
- }
- }
-
- private static class VerifyController extends CompactionController
- {
- public VerifyController(ColumnFamilyStore cfs)
- {
- super(cfs, Integer.MAX_VALUE);
- }
-
- @Override
- public LongPredicate getPurgeEvaluator(DecoratedKey key)
- {
- return time -> false;
- }
- }
-
- public static Options.Builder options()
- {
- return new Options.Builder();
- }
-
- public static class Options
- {
- public final boolean invokeDiskFailurePolicy;
- public final boolean extendedVerification;
- public final boolean checkVersion;
- public final boolean mutateRepairStatus;
- public final boolean checkOwnsTokens;
- public final boolean quick;
- public final Function<String, ? extends Collection<Range<Token>>> tokenLookup;
-
- private Options(boolean invokeDiskFailurePolicy, boolean extendedVerification, boolean checkVersion, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, Function<String, ? extends Collection<Range<Token>>> tokenLookup)
- {
- this.invokeDiskFailurePolicy = invokeDiskFailurePolicy;
- this.extendedVerification = extendedVerification;
- this.checkVersion = checkVersion;
- this.mutateRepairStatus = mutateRepairStatus;
- this.checkOwnsTokens = checkOwnsTokens;
- this.quick = quick;
- this.tokenLookup = tokenLookup;
- }
-
- @Override
- public String toString()
- {
- return "Options{" +
- "invokeDiskFailurePolicy=" + invokeDiskFailurePolicy +
- ", extendedVerification=" + extendedVerification +
- ", checkVersion=" + checkVersion +
- ", mutateRepairStatus=" + mutateRepairStatus +
- ", checkOwnsTokens=" + checkOwnsTokens +
- ", quick=" + quick +
- '}';
- }
-
- public static class Builder
- {
- private boolean invokeDiskFailurePolicy = false; // invoking disk failure policy can stop the node if we find a corrupt stable
- private boolean extendedVerification = false;
- private boolean checkVersion = false;
- private boolean mutateRepairStatus = false; // mutating repair status can be dangerous
- private boolean checkOwnsTokens = false;
- private boolean quick = false;
- private Function<String, ? extends Collection<Range<Token>>> tokenLookup = StorageService.instance::getLocalAndPendingRanges;
-
- public Builder invokeDiskFailurePolicy(boolean param)
- {
- this.invokeDiskFailurePolicy = param;
- return this;
- }
-
- public Builder extendedVerification(boolean param)
- {
- this.extendedVerification = param;
- return this;
- }
-
- public Builder checkVersion(boolean param)
- {
- this.checkVersion = param;
- return this;
- }
-
- public Builder mutateRepairStatus(boolean param)
- {
- this.mutateRepairStatus = param;
- return this;
- }
-
- public Builder checkOwnsTokens(boolean param)
- {
- this.checkOwnsTokens = param;
- return this;
- }
-
- public Builder quick(boolean param)
- {
- this.quick = param;
- return this;
- }
-
- public Builder tokenLookup(Function<String, ? extends Collection<Range<Token>>> tokenLookup)
- {
- this.tokenLookup = tokenLookup;
- return this;
- }
-
- public Options build()
- {
- return new Options(invokeDiskFailurePolicy, extendedVerification, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup);
- }
-
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
index 0f4fe2d..451a78d 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.compaction.writers;
+import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@@ -30,16 +31,21 @@
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.DiskBoundaries;
import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.CompactionTask;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableRewriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Transactional;
+
/**
* Class that abstracts away the actual writing of files to make it possible to use CompactionTask for more
* use cases.
@@ -63,6 +69,9 @@
private final List<PartitionPosition> diskBoundaries;
private int locationIndex;
+ // Keep targetDirectory for compactions, needed for `nodetool compactionstats`
+ protected Directories.DataDirectory sstableDirectory;
+
public CompactionAwareWriter(ColumnFamilyStore cfs,
Directories directories,
LifecycleTransaction txn,
@@ -134,6 +143,11 @@
return realAppend(partition);
}
+ public final File getSStableDirectory() throws IOException
+ {
+ return getDirectories().getLocationForDisk(sstableDirectory);
+ }
+
@Override
protected Throwable doPostCleanup(Throwable accumulate)
{
@@ -235,4 +249,24 @@
{
return cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
}
+
+ /**
+ * It is up to the caller to set the following fields:
+ * - {@link SSTableWriter.Builder#setKeyCount(long)},
+ * - {@link SSTableWriter.Builder#setSerializationHeader(SerializationHeader)} and,
+ * - {@link SSTableWriter.Builder#setMetadataCollector(MetadataCollector)}
+ *
+ * @param descriptor
+ * @return
+ */
+ protected SSTableWriter.Builder<?, ?> newWriterBuilder(Descriptor descriptor)
+ {
+ return descriptor.getFormat().getWriterFactory().builder(descriptor)
+ .setTableMetadataRef(cfs.metadata)
+ .setTransientSSTable(isTransient)
+ .setRepairedAt(minRepairedAt)
+ .setPendingRepair(pendingRepair)
+ .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+ .addDefaultComponents();
+ }
}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
index 6180f96..f985847 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
@@ -26,8 +26,9 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -67,17 +68,17 @@
@Override
public void switchCompactionLocation(Directories.DataDirectory directory)
{
+ sstableDirectory = directory;
+
+ Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+ MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
+
@SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory)),
- estimatedTotalKeys,
- minRepairedAt,
- pendingRepair,
- isTransient,
- cfs.metadata,
- new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel),
- SerializationHeader.make(cfs.metadata(), nonExpiredSSTables),
- cfs.indexManager.listIndexes(),
- txn);
+ SSTableWriter writer = newWriterBuilder(descriptor).setMetadataCollector(collector)
+ .setSerializationHeader(header)
+ .setKeyCount(estimatedTotalKeys)
+ .build(txn, cfs);
sstableWriter.switchWriter(writer);
}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index b7fb881..38e2d2d 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -21,11 +21,12 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -39,7 +40,6 @@
private long totalWrittenInLevel = 0;
private int sstablesWritten = 0;
private final long keysPerSSTable;
- private Directories.DataDirectory sstableDirectory;
private final int levelFanoutSize;
public MajorLeveledCompactionWriter(ColumnFamilyStore cfs,
@@ -70,7 +70,7 @@
@SuppressWarnings("resource")
public boolean realAppend(UnfilteredRowIterator partition)
{
- RowIndexEntry rie = sstableWriter.append(partition);
+ AbstractRowIndexEntry rie = sstableWriter.append(partition);
partitionsWritten++;
long totalWrittenInCurrentWriter = sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten();
if (totalWrittenInCurrentWriter > maxSSTableSize)
@@ -90,18 +90,20 @@
@Override
public void switchCompactionLocation(Directories.DataDirectory location)
{
- this.sstableDirectory = location;
+ sstableDirectory = location;
averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1));
- sstableWriter.switchWriter(SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)),
- keysPerSSTable,
- minRepairedAt,
- pendingRepair,
- isTransient,
- cfs.metadata,
- new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel),
- SerializationHeader.make(cfs.metadata(), txn.originals()),
- cfs.indexManager.listIndexes(),
- txn));
+
+ Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory));
+ MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel);
+ SerializationHeader serializationHeader = SerializationHeader.make(cfs.metadata(), txn.originals());
+
+ @SuppressWarnings("resource")
+ SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(keysPerSSTable)
+ .setSerializationHeader(serializationHeader)
+ .setMetadataCollector(collector)
+ .build(txn, cfs);
+
+ sstableWriter.switchWriter(writer);
partitionsWritten = 0;
sstablesWritten = 0;
}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index df918bc..834bda9 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -21,11 +21,12 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -36,7 +37,6 @@
private final int level;
private final long estimatedSSTables;
private final Set<SSTableReader> allSSTables;
- private Directories.DataDirectory sstableDirectory;
public MaxSSTableSizeWriter(ColumnFamilyStore cfs,
Directories directories,
@@ -81,7 +81,7 @@
protected boolean realAppend(UnfilteredRowIterator partition)
{
- RowIndexEntry rie = sstableWriter.append(partition);
+ AbstractRowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize)
{
switchCompactionLocation(sstableDirectory);
@@ -93,18 +93,16 @@
public void switchCompactionLocation(Directories.DataDirectory location)
{
sstableDirectory = location;
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- pendingRepair,
- isTransient,
- cfs.metadata,
- new MetadataCollector(allSSTables, cfs.metadata().comparator, level),
- SerializationHeader.make(cfs.metadata(), nonExpiredSSTables),
- cfs.indexManager.listIndexes(),
- txn);
+ Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory));
+ MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, level);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
+
+ @SuppressWarnings("resource")
+ SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(estimatedTotalKeys / estimatedSSTables)
+ .setMetadataCollector(collector)
+ .setSerializationHeader(header)
+ .build(txn, cfs);
sstableWriter.switchWriter(writer);
}
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 264e19c..458d80a 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -25,10 +25,11 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -49,7 +50,6 @@
private final Set<SSTableReader> allSSTables;
private long currentBytesToWrite;
private int currentRatioIndex = 0;
- private Directories.DataDirectory location;
public SplittingSizeTieredCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables)
{
@@ -86,12 +86,13 @@
@Override
public boolean realAppend(UnfilteredRowIterator partition)
{
- RowIndexEntry rie = sstableWriter.append(partition);
+ AbstractRowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect
{
currentRatioIndex++;
+
currentBytesToWrite = getExpectedWriteSize();
- switchCompactionLocation(location);
+ switchCompactionLocation(sstableDirectory);
logger.debug("Switching writer, currentBytesToWrite = {}", currentBytesToWrite);
}
return rie != null;
@@ -100,19 +101,17 @@
@Override
public void switchCompactionLocation(Directories.DataDirectory location)
{
- this.location = location;
+ sstableDirectory = location;
long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys);
+ Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location));
+ MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, 0);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
+
@SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location)),
- currentPartitionsToWrite,
- minRepairedAt,
- pendingRepair,
- isTransient,
- cfs.metadata,
- new MetadataCollector(allSSTables, cfs.metadata().comparator, 0),
- SerializationHeader.make(cfs.metadata(), nonExpiredSSTables),
- cfs.indexManager.listIndexes(),
- txn);
+ SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(currentPartitionsToWrite)
+ .setMetadataCollector(collector)
+ .setSerializationHeader(header)
+ .build(txn, cfs);
logger.trace("Switching writer, currentPartitionsToWrite = {}", currentPartitionsToWrite);
sstableWriter.switchWriter(writer);
}
diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java
index 924ff21..564ca1a 100644
--- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java
@@ -23,7 +23,6 @@
import org.apache.cassandra.db.partitions.CachedPartition;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableMetadata;
@@ -142,13 +141,14 @@
public UnfilteredRowIterator getUnfilteredRowIterator(ColumnFilter columnFilter, Partition partition);
/**
- * Whether the provided sstable may contain data that is selected by this filter (based on the sstable metadata).
+ * Whether the data selected by this filter intersects with the provided slice.
*
- * @param sstable the sstable for which we want to test the need for inclusion.
+ * @param comparator the comparator of the table this if a filter on.
+ * @param slice the slice to check intersection with,
*
- * @return whether {@code sstable} should be included to answer this filter.
+ * @return whether the data selected by this filter intersects with {@code slice}.
*/
- public boolean shouldInclude(SSTableReader sstable);
+ public boolean intersects(ClusteringComparator comparator, Slice slice);
public Kind kind();
@@ -161,4 +161,4 @@
public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException;
public long serializedSize(ClusteringIndexFilter filter, int version);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java
index 18dc471..a98e3bd 100644
--- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.db.filter;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.*;
import org.apache.cassandra.cql3.Operator;
@@ -26,7 +25,6 @@
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.ColumnMetadata;
@@ -142,16 +140,11 @@
return partition.unfilteredIterator(columnFilter, clusteringsInQueryOrder, isReversed());
}
- public boolean shouldInclude(SSTableReader sstable)
+ public boolean intersects(ClusteringComparator comparator, Slice slice)
{
- ClusteringComparator comparator = sstable.metadata().comparator;
- List<ByteBuffer> minClusteringValues = sstable.getSSTableMetadata().minClusteringValues;
- List<ByteBuffer> maxClusteringValues = sstable.getSSTableMetadata().maxClusteringValues;
-
- // If any of the requested clustering is within the bounds covered by the sstable, we need to include the sstable
for (Clustering<?> clustering : clusterings)
{
- if (Slice.make(clustering).intersects(comparator, minClusteringValues, maxClusteringValues))
+ if (slice.includes(comparator, clustering))
return true;
}
return false;
@@ -226,7 +219,7 @@
protected void serializeInternal(DataOutputPlus out, int version) throws IOException
{
ClusteringComparator comparator = (ClusteringComparator)clusterings.comparator();
- out.writeUnsignedVInt(clusterings.size());
+ out.writeUnsignedVInt32(clusterings.size());
for (Clustering<?> clustering : clusterings)
Clustering.serializer.serialize(clustering, out, version, comparator.subtypes());
}
@@ -245,7 +238,7 @@
public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata, boolean reversed) throws IOException
{
ClusteringComparator comparator = metadata.comparator;
- int size = (int)in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
try (BTree.FastBuilder<Clustering<?>> builder = BTree.fastBuilder())
{
for (int i = 0; i < size; i++)
@@ -255,4 +248,4 @@
}
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java
index 178c96b..67aeeb7 100644
--- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.db.filter;
import java.io.IOException;
-import java.util.List;
-import java.nio.ByteBuffer;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.CachedPartition;
import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.transform.Transformation;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
/**
* A filter over a single partition.
@@ -126,15 +123,9 @@
return partition.unfilteredIterator(columnFilter, slices, reversed);
}
- public boolean shouldInclude(SSTableReader sstable)
+ public boolean intersects(ClusteringComparator comparator, Slice slice)
{
- List<ByteBuffer> minClusteringValues = sstable.getSSTableMetadata().minClusteringValues;
- List<ByteBuffer> maxClusteringValues = sstable.getSSTableMetadata().maxClusteringValues;
-
- if (minClusteringValues.isEmpty() || maxClusteringValues.isEmpty())
- return true;
-
- return slices.intersects(minClusteringValues, maxClusteringValues);
+ return slices.intersects(slice);
}
public String toString(TableMetadata metadata)
@@ -176,4 +167,4 @@
return new ClusteringIndexSliceFilter(slices, reversed);
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
index 0ed6237..48ba738 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java
@@ -988,7 +988,7 @@
{
if (subSelections != null)
{
- out.writeUnsignedVInt(subSelections.size());
+ out.writeUnsignedVInt32(subSelections.size());
for (ColumnSubselection subSel : subSelections.values())
ColumnSubselection.serializer.serialize(subSel, out, version);
}
@@ -1078,7 +1078,7 @@
TableMetadata metadata) throws IOException
{
SortedSetMultimap<ColumnIdentifier, ColumnSubselection> subSelections = TreeMultimap.create(Comparator.naturalOrder(), Comparator.naturalOrder());
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
for (int i = 0; i < size; i++)
{
ColumnSubselection subSel = ColumnSubselection.serializer.deserialize(in, version, metadata);
diff --git a/src/java/org/apache/cassandra/db/filter/DataLimits.java b/src/java/org/apache/cassandra/db/filter/DataLimits.java
index f988cb3..649df22 100644
--- a/src/java/org/apache/cassandra/db/filter/DataLimits.java
+++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java
@@ -1144,22 +1144,22 @@
case CQL_LIMIT:
case CQL_PAGING_LIMIT:
CQLLimits cqlLimits = (CQLLimits)limits;
- out.writeUnsignedVInt(cqlLimits.rowLimit);
- out.writeUnsignedVInt(cqlLimits.perPartitionLimit);
+ out.writeUnsignedVInt32(cqlLimits.rowLimit);
+ out.writeUnsignedVInt32(cqlLimits.perPartitionLimit);
out.writeBoolean(cqlLimits.isDistinct);
if (limits.kind() == Kind.CQL_PAGING_LIMIT)
{
CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits;
ByteBufferUtil.writeWithVIntLength(pagingLimits.lastReturnedKey, out);
- out.writeUnsignedVInt(pagingLimits.lastReturnedKeyRemaining);
+ out.writeUnsignedVInt32(pagingLimits.lastReturnedKeyRemaining);
}
break;
case CQL_GROUP_BY_LIMIT:
case CQL_GROUP_BY_PAGING_LIMIT:
CQLGroupByLimits groupByLimits = (CQLGroupByLimits) limits;
- out.writeUnsignedVInt(groupByLimits.groupLimit);
- out.writeUnsignedVInt(groupByLimits.groupPerPartitionLimit);
- out.writeUnsignedVInt(groupByLimits.rowLimit);
+ out.writeUnsignedVInt32(groupByLimits.groupLimit);
+ out.writeUnsignedVInt32(groupByLimits.groupPerPartitionLimit);
+ out.writeUnsignedVInt32(groupByLimits.rowLimit);
AggregationSpecification groupBySpec = groupByLimits.groupBySpec;
AggregationSpecification.serializer.serialize(groupBySpec, out, version);
@@ -1170,7 +1170,7 @@
{
CQLGroupByPagingLimits pagingLimits = (CQLGroupByPagingLimits) groupByLimits;
ByteBufferUtil.writeWithVIntLength(pagingLimits.lastReturnedKey, out);
- out.writeUnsignedVInt(pagingLimits.lastReturnedKeyRemaining);
+ out.writeUnsignedVInt32(pagingLimits.lastReturnedKeyRemaining);
}
break;
}
@@ -1184,21 +1184,21 @@
case CQL_LIMIT:
case CQL_PAGING_LIMIT:
{
- int rowLimit = (int) in.readUnsignedVInt();
- int perPartitionLimit = (int) in.readUnsignedVInt();
+ int rowLimit = in.readUnsignedVInt32();
+ int perPartitionLimit = in.readUnsignedVInt32();
boolean isDistinct = in.readBoolean();
if (kind == Kind.CQL_LIMIT)
return cqlLimits(rowLimit, perPartitionLimit, isDistinct);
ByteBuffer lastKey = ByteBufferUtil.readWithVIntLength(in);
- int lastRemaining = (int) in.readUnsignedVInt();
+ int lastRemaining = in.readUnsignedVInt32();
return new CQLPagingLimits(rowLimit, perPartitionLimit, isDistinct, lastKey, lastRemaining);
}
case CQL_GROUP_BY_LIMIT:
case CQL_GROUP_BY_PAGING_LIMIT:
{
- int groupLimit = (int) in.readUnsignedVInt();
- int groupPerPartitionLimit = (int) in.readUnsignedVInt();
- int rowLimit = (int) in.readUnsignedVInt();
+ int groupLimit = in.readUnsignedVInt32();
+ int groupPerPartitionLimit = in.readUnsignedVInt32();
+ int rowLimit = in.readUnsignedVInt32();
AggregationSpecification groupBySpec = AggregationSpecification.serializer.deserialize(in, version, metadata);
@@ -1212,7 +1212,7 @@
state);
ByteBuffer lastKey = ByteBufferUtil.readWithVIntLength(in);
- int lastRemaining = (int) in.readUnsignedVInt();
+ int lastRemaining = in.readUnsignedVInt32();
return new CQLGroupByPagingLimits(groupLimit,
groupPerPartitionLimit,
rowLimit,
diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java
index 5e0fb51..6d71701 100644
--- a/src/java/org/apache/cassandra/db/filter/RowFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java
@@ -1038,7 +1038,7 @@
public void serialize(RowFilter filter, DataOutputPlus out, int version) throws IOException
{
out.writeBoolean(false); // Old "is for thrift" boolean
- out.writeUnsignedVInt(filter.expressions.size());
+ out.writeUnsignedVInt32(filter.expressions.size());
for (Expression expr : filter.expressions)
Expression.serializer.serialize(expr, out, version);
@@ -1047,7 +1047,7 @@
public RowFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException
{
in.readBoolean(); // Unused
- int size = (int)in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
List<Expression> expressions = new ArrayList<>(size);
for (int i = 0; i < size; i++)
expressions.add(Expression.serializer.deserialize(in, version, metadata));
diff --git a/src/java/org/apache/cassandra/db/filter/RowIndexEntryReadSizeTooLargeException.java b/src/java/org/apache/cassandra/db/filter/RowIndexEntryReadSizeTooLargeException.java
deleted file mode 100644
index 20f3f8f..0000000
--- a/src/java/org/apache/cassandra/db/filter/RowIndexEntryReadSizeTooLargeException.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.filter;
-
-import org.apache.cassandra.db.RejectException;
-
-public class RowIndexEntryReadSizeTooLargeException extends RejectException
-{
- public RowIndexEntryReadSizeTooLargeException(String message)
- {
- super(message);
- }
-}
diff --git a/src/java/org/apache/cassandra/db/guardrails/DisableFlag.java b/src/java/org/apache/cassandra/db/guardrails/DisableFlag.java
deleted file mode 100644
index 9ec1951..0000000
--- a/src/java/org/apache/cassandra/db/guardrails/DisableFlag.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db.guardrails;
-
-import java.util.function.Predicate;
-import javax.annotation.Nullable;
-
-import org.apache.cassandra.service.ClientState;
-
-/**
- * A guardrail that completely disables the use of a particular feature.
- *
- * <p>Note that this guardrail only aborts operations (if the feature is disabled) so is only meant for
- * query-based guardrails (we're happy to reject queries deemed dangerous, but we don't want to create a guardrail
- * that breaks compaction for instance).
- */
-public class DisableFlag extends Guardrail
-{
- private final Predicate<ClientState> disabled;
- private final String what;
-
- /**
- * Creates a new {@link DisableFlag} guardrail.
- *
- * @param name the identifying name of the guardrail
- * @param disabled a {@link ClientState}-based supplier of boolean indicating whether the feature guarded by this
- * guardrail must be disabled.
- * @param what The feature that is guarded by this guardrail (for reporting in error messages),
- * {@link DisableFlag#ensureEnabled(String, ClientState)} can specify a different {@code what}.
- */
- public DisableFlag(String name, Predicate<ClientState> disabled, String what)
- {
- super(name);
- this.disabled = disabled;
- this.what = what;
- }
-
- /**
- * Aborts the operation if this guardrail is disabled.
- *
- * <p>This must be called when the feature guarded by this guardrail is used to ensure such use is in fact
- * allowed.
- *
- * @param state The client state, used to skip the check if the query is internal or is done by a superuser.
- * A {@code null} value means that the check should be done regardless of the query.
- */
- public void ensureEnabled(@Nullable ClientState state)
- {
- ensureEnabled(what, state);
- }
-
- /**
- * Aborts the operation if this guardrail is disabled.
- *
- * <p>This must be called when the feature guarded by this guardrail is used to ensure such use is in fact
- * allowed.
- *
- * @param what The feature that is guarded by this guardrail (for reporting in error messages).
- * @param state The client state, used to skip the check if the query is internal or is done by a superuser.
- * A {@code null} value means that the check should be done regardless of the query, although it won't
- * throw any exception if the failure threshold is exceeded. This is so because checks without an
- * associated client come from asynchronous processes such as compaction, and we don't want to
- * interrupt such processes.
- */
- public void ensureEnabled(String what, @Nullable ClientState state)
- {
- if (enabled(state) && disabled.test(state))
- fail(what + " is not allowed", state);
- }
-}
diff --git a/src/java/org/apache/cassandra/db/guardrails/EnableFlag.java b/src/java/org/apache/cassandra/db/guardrails/EnableFlag.java
new file mode 100644
index 0000000..dde09be
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/guardrails/EnableFlag.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.function.Predicate;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.service.ClientState;
+
+/**
+ * A guardrail that enables the use of a particular feature.
+ *
+ * <p>Note that this guardrail only aborts operations (if the feature is not enabled) so is only meant for query-based
+ * guardrails (we're happy to reject queries deemed dangerous, but we don't want to create a guardrail that breaks
+ * compaction for instance).
+ */
+public class EnableFlag extends Guardrail
+{
+ private final Predicate<ClientState> warned;
+ private final Predicate<ClientState> enabled;
+ private final String featureName;
+
+ /**
+ * Creates a new {@link EnableFlag} guardrail.
+ *
+ * @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
+ * @param enabled a {@link ClientState}-based supplier of boolean indicating whether the feature guarded by this
+ * guardrail is enabled.
+ * @param featureName The feature that is guarded by this guardrail (for reporting in error messages), {@link
+ * EnableFlag#ensureEnabled(String, ClientState)} can specify a different {@code featureName}.
+ */
+ public EnableFlag(String name, @Nullable String reason, Predicate<ClientState> enabled, String featureName)
+ {
+ this(name, reason, (state) -> false, enabled, featureName);
+ }
+
+ /**
+ * Creates a new {@link EnableFlag} guardrail.
+ *
+ * @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
+ * @param warned a {@link ClientState}-based supplier of boolean indicating whether warning should be
+ * emitted even guardrail as such has passed. If guardrail fails, the warning will not be
+ * emitted. This might be used for cases when we want to warn a user regardless of successful
+ * guardrail execution.
+ * @param enabled a {@link ClientState}-based supplier of boolean indicating whether the feature guarded by this
+ * guardrail is enabled.
+ * @param featureName The feature that is guarded by this guardrail (for reporting in error messages), {@link
+ * EnableFlag#ensureEnabled(String, ClientState)} can specify a different {@code featureName}.
+ */
+ public EnableFlag(String name,
+ @Nullable String reason,
+ Predicate<ClientState> warned,
+ Predicate<ClientState> enabled,
+ String featureName)
+ {
+ super(name, reason);
+ this.warned = warned;
+ this.enabled = enabled;
+ this.featureName = featureName;
+ }
+
+ /**
+ * Returns whether the guarded feature is enabled or not.
+ *
+ * @param state The client state, used to skip the check if the query is internal or is done by a superuser.
+ * A {@code null} value means that the check should be done regardless of the query.
+ * @return {@code true} is the feature is enabled, {@code false} otherwise.
+ */
+ public boolean isEnabled(@Nullable ClientState state)
+ {
+ return !enabled(state) || enabled.test(state);
+ }
+
+ /**
+ * Aborts the operation if this guardrail is not enabled.
+ *
+ * <p>This must be called when the feature guarded by this guardrail is used to ensure such use is in fact
+ * allowed.
+ *
+ * @param state The client state, used to skip the check if the query is internal or is done by a superuser.
+ * A {@code null} value means that the check should be done regardless of the query.
+ */
+ public void ensureEnabled(@Nullable ClientState state)
+ {
+ ensureEnabled(featureName, state);
+ }
+
+ /**
+ * Aborts the operation if this guardrail is not enabled.
+ *
+ * <p>This must be called when the feature guarded by this guardrail is used to ensure such use is in fact
+ * allowed.
+ *
+ * @param featureName The feature that is guarded by this guardrail (for reporting in error messages).
+ * @param state The client state, used to skip the check if the query is internal or is done by a superuser. A
+ * {@code null} value means that the check should be done regardless of the query, although it
+ * won't throw any exception if the failure threshold is exceeded. This is so because checks
+ * without an associated client come from asynchronous processes such as compaction, and we don't
+ * want to interrupt such processes.
+ */
+ public void ensureEnabled(String featureName, @Nullable ClientState state)
+ {
+ if (!enabled(state))
+ return;
+
+ if (!enabled.test(state))
+ {
+ fail(featureName + " is not allowed", state);
+ return;
+ }
+
+ if (warned.test(state))
+ warn(featureName + " is not recommended");
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/guardrails/Guardrail.java b/src/java/org/apache/cassandra/db/guardrails/Guardrail.java
index c058f10..fbd1a5b 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Guardrail.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Guardrail.java
@@ -51,6 +51,10 @@
/** A name identifying the guardrail (mainly for shipping with diagnostic events). */
public final String name;
+ /** An optional description of the reason for guarding the operation. */
+ @Nullable
+ public final String reason;
+
/** Minimum logging and triggering interval to avoid spamming downstream. */
private long minNotifyIntervalInMs = 0;
@@ -60,9 +64,10 @@
/** Time of last failure in milliseconds. */
private volatile long lastFailInMs = 0;
- Guardrail(String name)
+ Guardrail(String name, @Nullable String reason)
{
this.name = name;
+ this.reason = reason;
}
/**
@@ -138,8 +143,16 @@
@VisibleForTesting
String decorateMessage(String message)
{
- // Add a prefix to error message so user knows what threw the warning or cause the failure
- return String.format("Guardrail %s violated: %s", name, message);
+ // Add a prefix to error message so user knows what threw the warning or cause the failure.
+ String decoratedMessage = String.format("Guardrail %s violated: %s", name, message);
+
+ // Add the reason for the guardrail triggering, if there is any.
+ if (reason != null)
+ {
+ decoratedMessage += (message.endsWith(".") ? ' ' : ". ") + reason;
+ }
+
+ return decoratedMessage;
}
/**
diff --git a/src/java/org/apache/cassandra/db/guardrails/Guardrails.java b/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
index 9d08ab0..d2289b4 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
@@ -31,9 +31,12 @@
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.config.GuardrailsOptions;
import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster;
import org.apache.cassandra.utils.MBeanWrapper;
@@ -50,13 +53,14 @@
private static final GuardrailsOptions DEFAULT_CONFIG = DatabaseDescriptor.getGuardrailsConfig();
@VisibleForTesting
- static final Guardrails instance = new Guardrails();
+ public static final Guardrails instance = new Guardrails();
/**
* Guardrail on the total number of user keyspaces.
*/
public static final MaxThreshold keyspaces =
new MaxThreshold("keyspaces",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getKeyspacesWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getKeyspacesFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -70,6 +74,7 @@
*/
public static final MaxThreshold tables =
new MaxThreshold("tables",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getTablesWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getTablesFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -83,6 +88,7 @@
*/
public static final MaxThreshold columnsPerTable =
new MaxThreshold("columns_per_table",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getColumnsPerTableWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getColumnsPerTableFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -93,6 +99,7 @@
public static final MaxThreshold secondaryIndexesPerTable =
new MaxThreshold("secondary_indexes_per_table",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getSecondaryIndexesPerTableWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getSecondaryIndexesPerTableFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -104,16 +111,18 @@
/**
* Guardrail disabling user's ability to create secondary indexes
*/
- public static final DisableFlag createSecondaryIndexesEnabled =
- new DisableFlag("secondary_indexes",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getSecondaryIndexesEnabled(),
- "User creation of secondary indexes");
+ public static final EnableFlag createSecondaryIndexesEnabled =
+ new EnableFlag("secondary_indexes",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getSecondaryIndexesEnabled(),
+ "User creation of secondary indexes");
/**
* Guardrail on the number of materialized views per table.
*/
public static final MaxThreshold materializedViewsPerTable =
new MaxThreshold("materialized_views_per_table",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getMaterializedViewsPerTableWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getMaterializedViewsPerTableFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -127,6 +136,7 @@
*/
public static final Values<String> tableProperties =
new Values<>("table_properties",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getTablePropertiesWarned(),
state -> CONFIG_PROVIDER.getOrCreate(state).getTablePropertiesIgnored(),
state -> CONFIG_PROVIDER.getOrCreate(state).getTablePropertiesDisallowed(),
@@ -135,42 +145,83 @@
/**
* Guardrail disabling user-provided timestamps.
*/
- public static final DisableFlag userTimestampsEnabled =
- new DisableFlag("user_timestamps",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getUserTimestampsEnabled(),
- "User provided timestamps (USING TIMESTAMP)");
+ public static final EnableFlag userTimestampsEnabled =
+ new EnableFlag("user_timestamps",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getUserTimestampsEnabled(),
+ "User provided timestamps (USING TIMESTAMP)");
- public static final DisableFlag groupByEnabled =
- new DisableFlag("group_by",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getGroupByEnabled(),
- "GROUP BY functionality");
+ public static final EnableFlag groupByEnabled =
+ new EnableFlag("group_by",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getGroupByEnabled(),
+ "GROUP BY functionality");
- public static final DisableFlag dropTruncateTableEnabled =
- new DisableFlag("drop_truncate_table_enabled",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getDropTruncateTableEnabled(),
- "DROP and TRUNCATE TABLE functionality");
+ /**
+ * Guardrail disabling ALTER TABLE column mutation access.
+ */
+ public static final EnableFlag alterTableEnabled =
+ new EnableFlag("alter_table",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getAlterTableEnabled(),
+ "User access to ALTER TABLE statement for column mutation");
+
+ /**
+ * Guardrail disabling DROP / TRUNCATE TABLE behavior
+ */
+ public static final EnableFlag dropTruncateTableEnabled =
+ new EnableFlag("drop_truncate_table_enabled",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getDropTruncateTableEnabled(),
+ "DROP and TRUNCATE TABLE functionality");
+
+ /**
+ * Guardrail disabling DROP KEYSPACE behavior
+ */
+ public static final EnableFlag dropKeyspaceEnabled =
+ new EnableFlag("drop_keyspace_enabled",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getDropKeyspaceEnabled(),
+ "DROP KEYSPACE functionality");
/**
* Guardrail disabling user's ability to turn off compression
*/
- public static final DisableFlag uncompressedTablesEnabled =
- new DisableFlag("uncompressed_tables_enabled",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getUncompressedTablesEnabled(),
- "Uncompressed table");
+ public static final EnableFlag uncompressedTablesEnabled =
+ new EnableFlag("uncompressed_tables_enabled",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getUncompressedTablesEnabled(),
+ "Uncompressed table");
/**
* Guardrail disabling the creation of new COMPACT STORAGE tables
*/
- public static final DisableFlag compactTablesEnabled =
- new DisableFlag("compact_tables",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getCompactTablesEnabled(),
- "Creation of new COMPACT STORAGE tables");
+ public static final EnableFlag compactTablesEnabled =
+ new EnableFlag("compact_tables",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getCompactTablesEnabled(),
+ "Creation of new COMPACT STORAGE tables");
+
+ /**
+ * Guardrail to warn or fail a CREATE or ALTER TABLE statement when default_time_to_live is set to 0 and
+ * the table is using TimeWindowCompactionStrategy compaction or a subclass of it.
+ */
+ public static final EnableFlag zeroTTLOnTWCSEnabled =
+ new EnableFlag("zero_ttl_on_twcs",
+ "It is suspicious to use default_time_to_live set to 0 with such compaction strategy. " +
+ "Please keep in mind that data will not start to automatically expire after they are older " +
+ "than a respective compaction window unit of a certain size. Please set TTL for your INSERT or UPDATE " +
+ "statements if you expect data to be expired as table settings will not do it. ",
+ state -> CONFIG_PROVIDER.getOrCreate(state).getZeroTTLOnTWCSWarned(),
+ state -> CONFIG_PROVIDER.getOrCreate(state).getZeroTTLOnTWCSEnabled(),
+ "0 default_time_to_live on a table with " + TimeWindowCompactionStrategy.class.getSimpleName() + " compaction strategy");
/**
* Guardrail on the number of elements returned within page.
*/
public static final MaxThreshold pageSize =
new MaxThreshold("page_size",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getPageSizeWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getPageSizeFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -184,6 +235,7 @@
*/
public static final MaxThreshold partitionKeysInSelect =
new MaxThreshold("partition_keys_in_select",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getPartitionKeysInSelectWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getPartitionKeysInSelectFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -197,24 +249,36 @@
/**
* Guardrail disabling operations on lists that require read before write.
*/
- public static final DisableFlag readBeforeWriteListOperationsEnabled =
- new DisableFlag("read_before_write_list_operations",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getReadBeforeWriteListOperationsEnabled(),
- "List operation requiring read before write");
+ public static final EnableFlag readBeforeWriteListOperationsEnabled =
+ new EnableFlag("read_before_write_list_operations",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getReadBeforeWriteListOperationsEnabled(),
+ "List operation requiring read before write");
/**
* Guardrail disabling ALLOW FILTERING statement within a query
*/
- public static final DisableFlag allowFilteringEnabled =
- new DisableFlag("allow_filtering",
- state -> !CONFIG_PROVIDER.getOrCreate(state).getAllowFilteringEnabled(),
- "Querying with ALLOW FILTERING");
+ public static final EnableFlag allowFilteringEnabled =
+ new EnableFlag("allow_filtering",
+ "ALLOW FILTERING can potentially visit all the data in the table and have unpredictable performance.",
+ state -> CONFIG_PROVIDER.getOrCreate(state).getAllowFilteringEnabled(),
+ "Querying with ALLOW FILTERING");
+
+ /**
+ * Guardrail disabling setting SimpleStrategy via keyspace creation or alteration
+ */
+ public static final EnableFlag simpleStrategyEnabled =
+ new EnableFlag("simplestrategy",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getSimpleStrategyEnabled(),
+ "SimpleStrategy");
/**
* Guardrail on the number of restrictions created by a cartesian product of a CQL's {@code IN} query.
*/
public static final MaxThreshold inSelectCartesianProduct =
new MaxThreshold("in_select_cartesian_product",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getInSelectCartesianProductWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getInSelectCartesianProductFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -230,6 +294,7 @@
*/
public static final Values<ConsistencyLevel> readConsistencyLevels =
new Values<>("read_consistency_levels",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getReadConsistencyLevelsWarned(),
state -> Collections.emptySet(),
state -> CONFIG_PROVIDER.getOrCreate(state).getReadConsistencyLevelsDisallowed(),
@@ -240,42 +305,66 @@
*/
public static final Values<ConsistencyLevel> writeConsistencyLevels =
new Values<>("write_consistency_levels",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getWriteConsistencyLevelsWarned(),
state -> Collections.emptySet(),
state -> CONFIG_PROVIDER.getOrCreate(state).getWriteConsistencyLevelsDisallowed(),
"write consistency levels");
/**
+ * Guardrail on the size of a partition.
+ */
+ public static final MaxThreshold partitionSize =
+ new MaxThreshold("partition_size",
+ "Too large partitions can cause performance problems. ",
+ state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getPartitionSizeWarnThreshold()),
+ state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getPartitionSizeFailThreshold()),
+ (isWarning, what, value, threshold) ->
+ format("Partition %s has size %s, this exceeds the %s threshold of %s.",
+ what, value, isWarning ? "warning" : "failure", threshold));
+
+ /**
+ * Guardrail on the size of a collection.
+ */
+ public static final MaxThreshold columnValueSize =
+ new MaxThreshold("column_value_size",
+ null,
+ state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getColumnValueSizeWarnThreshold()),
+ state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getColumnValueSizeFailThreshold()),
+ (isWarning, what, value, threshold) ->
+ format("Value of column %s has size %s, this exceeds the %s threshold of %s.",
+ what, value, isWarning ? "warning" : "failure", threshold));
+
+ /**
* Guardrail on the size of a collection.
*/
public static final MaxThreshold collectionSize =
new MaxThreshold("collection_size",
+ null,
state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getCollectionSizeWarnThreshold()),
state -> sizeToBytes(CONFIG_PROVIDER.getOrCreate(state).getCollectionSizeFailThreshold()),
(isWarning, what, value, threshold) ->
- isWarning ? format("Detected collection %s of size %s, this exceeds the warning threshold of %s.",
- what, value, threshold)
- : format("Detected collection %s of size %s, this exceeds the failure threshold of %s.",
- what, value, threshold));
+ format("Detected collection %s of size %s, this exceeds the %s threshold of %s.",
+ what, value, isWarning ? "warning" : "failure", threshold));
/**
* Guardrail on the number of items of a collection.
*/
public static final MaxThreshold itemsPerCollection =
new MaxThreshold("items_per_collection",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getItemsPerCollectionWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getItemsPerCollectionFailThreshold(),
(isWarning, what, value, threshold) ->
- isWarning ? format("Detected collection %s with %s items, this exceeds the warning threshold of %s.",
- what, value, threshold)
- : format("Detected collection %s with %s items, this exceeds the failure threshold of %s.",
- what, value, threshold));
+ format("Detected collection %s with %s items, this exceeds the %s threshold of %s.",
+ what, value, isWarning ? "warning" : "failure", threshold));
/**
* Guardrail on the number of fields on each UDT.
*/
public static final MaxThreshold fieldsPerUDT =
new MaxThreshold("fields_per_udt",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getFieldsPerUDTWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getFieldsPerUDTFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -290,6 +379,7 @@
*/
public static final PercentageThreshold localDataDiskUsage =
new PercentageThreshold("local_data_disk_usage",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getDataDiskUsagePercentageWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getDataDiskUsagePercentageFailThreshold(),
(isWarning, what, value, threshold) ->
@@ -305,6 +395,7 @@
*/
public static final Predicates<InetAddressAndPort> replicaDiskUsage =
new Predicates<>("replica_disk_usage",
+ null,
state -> DiskUsageBroadcaster.instance::isStuffed,
state -> DiskUsageBroadcaster.instance::isFull,
// not using `value` because it represents replica address which should be hidden from client.
@@ -325,13 +416,42 @@
*/
public static final MinThreshold minimumReplicationFactor =
new MinThreshold("minimum_replication_factor",
+ null,
state -> CONFIG_PROVIDER.getOrCreate(state).getMinimumReplicationFactorWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getMinimumReplicationFactorFailThreshold(),
(isWarning, what, value, threshold) ->
- isWarning ? format("The keyspace %s has a replication factor of %s, below the warning threshold of %s.",
- what, value, threshold)
- : format("The keyspace %s has a replication factor of %s, below the failure threshold of %s.",
- what, value, threshold));
+ format("The keyspace %s has a replication factor of %s, below the %s threshold of %s.",
+ what, value, isWarning ? "warning" : "failure", threshold));
+
+ /**
+ * Guardrail on the maximum replication factor.
+ */
+ public static final MaxThreshold maximumReplicationFactor =
+ new MaxThreshold("maximum_replication_factor",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getMaximumReplicationFactorWarnThreshold(),
+ state -> CONFIG_PROVIDER.getOrCreate(state).getMaximumReplicationFactorFailThreshold(),
+ (isWarning, what, value, threshold) ->
+ format("The keyspace %s has a replication factor of %s, above the %s threshold of %s.",
+ what, value, isWarning ? "warning" : "failure", threshold));
+
+ public static final MaxThreshold maximumAllowableTimestamp =
+ new MaxThreshold("maximum_timestamp",
+ "Timestamps too far in the future can lead to data that can't be easily overwritten",
+ state -> maximumTimestampAsRelativeMicros(CONFIG_PROVIDER.getOrCreate(state).getMaximumTimestampWarnThreshold()),
+ state -> maximumTimestampAsRelativeMicros(CONFIG_PROVIDER.getOrCreate(state).getMaximumTimestampFailThreshold()),
+ (isWarning, what, value, threshold) ->
+ format("The modification to table %s has a timestamp %s after the maximum allowable %s threshold %s",
+ what, value, isWarning ? "warning" : "failure", threshold));
+
+ public static final MinThreshold minimumAllowableTimestamp =
+ new MinThreshold("minimum_timestamp",
+ "Timestamps too far in the past can cause writes can be unexpectedly lost",
+ state -> minimumTimestampAsRelativeMicros(CONFIG_PROVIDER.getOrCreate(state).getMinimumTimestampWarnThreshold()),
+ state -> minimumTimestampAsRelativeMicros(CONFIG_PROVIDER.getOrCreate(state).getMinimumTimestampFailThreshold()),
+ (isWarning, what, value, threshold) ->
+ format("The modification to table %s has a timestamp %s before the minimum allowable %s threshold %s",
+ what, value, isWarning ? "warning" : "failure", threshold));
private Guardrails()
{
@@ -540,6 +660,18 @@
}
@Override
+ public boolean getAlterTableEnabled()
+ {
+ return DEFAULT_CONFIG.getAlterTableEnabled();
+ }
+
+ @Override
+ public void setAlterTableEnabled(boolean enabled)
+ {
+ DEFAULT_CONFIG.setAlterTableEnabled(enabled);
+ }
+
+ @Override
public boolean getAllowFilteringEnabled()
{
return DEFAULT_CONFIG.getAllowFilteringEnabled();
@@ -552,6 +684,18 @@
}
@Override
+ public boolean getSimpleStrategyEnabled()
+ {
+ return DEFAULT_CONFIG.getSimpleStrategyEnabled();
+ }
+
+ @Override
+ public void setSimpleStrategyEnabled(boolean enabled)
+ {
+ DEFAULT_CONFIG.setSimpleStrategyEnabled(enabled);
+ }
+
+ @Override
public boolean getUncompressedTablesEnabled()
{
return DEFAULT_CONFIG.getUncompressedTablesEnabled();
@@ -600,6 +744,18 @@
}
@Override
+ public boolean getDropKeyspaceEnabled()
+ {
+ return DEFAULT_CONFIG.getDropKeyspaceEnabled();
+ }
+
+ @Override
+ public void setDropKeyspaceEnabled(boolean enabled)
+ {
+ DEFAULT_CONFIG.setDropKeyspaceEnabled(enabled);
+ }
+
+ @Override
public int getPageSizeWarnThreshold()
{
return DEFAULT_CONFIG.getPageSizeWarnThreshold();
@@ -649,6 +805,46 @@
@Override
@Nullable
+ public String getPartitionSizeWarnThreshold()
+ {
+ return sizeToString(DEFAULT_CONFIG.getPartitionSizeWarnThreshold());
+ }
+
+ @Override
+ @Nullable
+ public String getPartitionSizeFailThreshold()
+ {
+ return sizeToString(DEFAULT_CONFIG.getPartitionSizeFailThreshold());
+ }
+
+ @Override
+ public void setPartitionSizeThreshold(@Nullable String warnSize, @Nullable String failSize)
+ {
+ DEFAULT_CONFIG.setPartitionSizeThreshold(sizeFromString(warnSize), sizeFromString(failSize));
+ }
+
+ @Override
+ @Nullable
+ public String getColumnValueSizeWarnThreshold()
+ {
+ return sizeToString(DEFAULT_CONFIG.getColumnValueSizeWarnThreshold());
+ }
+
+ @Override
+ @Nullable
+ public String getColumnValueSizeFailThreshold()
+ {
+ return sizeToString(DEFAULT_CONFIG.getColumnValueSizeFailThreshold());
+ }
+
+ @Override
+ public void setColumnValueSizeThreshold(@Nullable String warnSize, @Nullable String failSize)
+ {
+ DEFAULT_CONFIG.setColumnValueSizeThreshold(sizeFromString(warnSize), sizeFromString(failSize));
+ }
+
+ @Override
+ @Nullable
public String getCollectionSizeWarnThreshold()
{
return sizeToString(DEFAULT_CONFIG.getCollectionSizeWarnThreshold());
@@ -818,6 +1014,24 @@
}
@Override
+ public int getMaximumReplicationFactorWarnThreshold()
+ {
+ return DEFAULT_CONFIG.getMaximumReplicationFactorWarnThreshold();
+ }
+
+ @Override
+ public int getMaximumReplicationFactorFailThreshold()
+ {
+ return DEFAULT_CONFIG.getMaximumReplicationFactorFailThreshold();
+ }
+
+ @Override
+ public void setMaximumReplicationFactorThreshold (int warn, int fail)
+ {
+ DEFAULT_CONFIG.setMaximumReplicationFactorThreshold(warn, fail);
+ }
+
+ @Override
public int getDataDiskUsagePercentageWarnThreshold()
{
return DEFAULT_CONFIG.getDataDiskUsagePercentageWarnThreshold();
@@ -866,6 +1080,66 @@
DEFAULT_CONFIG.setMinimumReplicationFactorThreshold(warn, fail);
}
+ @Override
+ public boolean getZeroTTLOnTWCSEnabled()
+ {
+ return DEFAULT_CONFIG.getZeroTTLOnTWCSEnabled();
+ }
+
+ @Override
+ public void setZeroTTLOnTWCSEnabled(boolean value)
+ {
+ DEFAULT_CONFIG.setZeroTTLOnTWCSEnabled(value);
+ }
+
+ @Override
+ public boolean getZeroTTLOnTWCSWarned()
+ {
+ return DEFAULT_CONFIG.getZeroTTLOnTWCSWarned();
+ }
+
+ @Override
+ public void setZeroTTLOnTWCSWarned(boolean value)
+ {
+ DEFAULT_CONFIG.setZeroTTLOnTWCSWarned(value);
+ }
+
+ @Override
+ public String getMaximumTimestampWarnThreshold()
+ {
+ return durationToString(DEFAULT_CONFIG.getMaximumTimestampWarnThreshold());
+ }
+
+ @Override
+ public String getMaximumTimestampFailThreshold()
+ {
+ return durationToString(DEFAULT_CONFIG.getMaximumTimestampFailThreshold());
+ }
+
+ @Override
+ public void setMaximumTimestampThreshold(String warnSeconds, String failSeconds)
+ {
+ DEFAULT_CONFIG.setMaximumTimestampThreshold(durationFromString(warnSeconds), durationFromString(failSeconds));
+ }
+
+ @Override
+ public String getMinimumTimestampWarnThreshold()
+ {
+ return durationToString(DEFAULT_CONFIG.getMinimumTimestampWarnThreshold());
+ }
+
+ @Override
+ public String getMinimumTimestampFailThreshold()
+ {
+ return durationToString(DEFAULT_CONFIG.getMinimumTimestampFailThreshold());
+ }
+
+ @Override
+ public void setMinimumTimestampThreshold(String warnSeconds, String failSeconds)
+ {
+ DEFAULT_CONFIG.setMinimumTimestampThreshold(durationFromString(warnSeconds), durationFromString(failSeconds));
+ }
+
private static String toCSV(Set<String> values)
{
return values == null || values.isEmpty() ? "" : String.join(",", values);
@@ -914,4 +1188,28 @@
{
return StringUtils.isEmpty(size) ? null : new DataStorageSpec.LongBytesBound(size);
}
+
+ private static String durationToString(@Nullable DurationSpec duration)
+ {
+ return duration == null ? null : duration.toString();
+ }
+
+ private static DurationSpec.LongMicrosecondsBound durationFromString(@Nullable String duration)
+ {
+ return StringUtils.isEmpty(duration) ? null : new DurationSpec.LongMicrosecondsBound(duration);
+ }
+
+ private static long maximumTimestampAsRelativeMicros(@Nullable DurationSpec.LongMicrosecondsBound duration)
+ {
+ return duration == null
+ ? Long.MAX_VALUE
+ : (ClientState.getLastTimestampMicros() + duration.toMicroseconds());
+ }
+
+ private static long minimumTimestampAsRelativeMicros(@Nullable DurationSpec.LongMicrosecondsBound duration)
+ {
+ return duration == null
+ ? Long.MIN_VALUE
+ : (ClientState.getLastTimestampMicros() - duration.toMicroseconds());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
index a52eeb0..af1b5b4 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
@@ -23,6 +23,7 @@
import javax.annotation.Nullable;
import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.ConsistencyLevel;
/**
@@ -133,6 +134,13 @@
boolean getUserTimestampsEnabled();
/**
+ * Returns whether users are allowed access to the ALTER TABLE statement to mutate columns or not
+ *
+ * @return {@code true} if ALTER TABLE ADD/REMOVE/RENAME is allowed, {@code false} otherwise.
+ */
+ boolean getAlterTableEnabled();
+
+ /**
* Returns whether tables can be uncompressed
*
* @return {@code true} if user's can disable compression, {@code false} otherwise.
@@ -161,6 +169,13 @@
boolean getDropTruncateTableEnabled();
/**
+ * Returns whether DROP on keyspaces is allowed
+ *
+ * @return {@code true} if allowed, {@code false} otherwise.
+ */
+ boolean getDropKeyspaceEnabled();
+
+ /**
* @return The threshold to warn when page size exceeds given size.
*/
int getPageSizeWarnThreshold();
@@ -185,6 +200,13 @@
boolean getAllowFilteringEnabled();
/**
+ * Returns whether setting SimpleStrategy via keyspace creation or alteration is enabled
+ *
+ * @return {@code true} if SimpleStrategy is allowed, {@code false} otherwise.
+ */
+ boolean getSimpleStrategyEnabled();
+
+ /**
* @return The threshold to warn when an IN query creates a cartesian product with a size exceeding threshold.
* -1 means disabled.
*/
@@ -217,6 +239,30 @@
Set<ConsistencyLevel> getWriteConsistencyLevelsDisallowed();
/**
+ * @return The threshold to warn when writing partitions larger than threshold.
+ */
+ @Nullable
+ DataStorageSpec.LongBytesBound getPartitionSizeWarnThreshold();
+
+ /**
+ * @return The threshold to fail when writing partitions larger than threshold.
+ */
+ @Nullable
+ DataStorageSpec.LongBytesBound getPartitionSizeFailThreshold();
+
+ /**
+ * @return The threshold to warn when writing column values larger than threshold.
+ */
+ @Nullable
+ DataStorageSpec.LongBytesBound getColumnValueSizeWarnThreshold();
+
+ /**
+ * @return The threshold to prevent writing column values larger than threshold.
+ */
+ @Nullable
+ DataStorageSpec.LongBytesBound getColumnValueSizeFailThreshold();
+
+ /**
* @return The threshold to warn when encountering a collection with larger data size than threshold.
*/
@Nullable
@@ -277,4 +323,90 @@
*/
int getMinimumReplicationFactorFailThreshold();
+ /**
+ * @return The threshold to warn when replication factor is greater than threshold.
+ */
+ int getMaximumReplicationFactorWarnThreshold();
+
+ /**
+ * @return The threshold to fail when replication factor is greater than threshold.
+ */
+ int getMaximumReplicationFactorFailThreshold();
+
+ /**
+ * Returns whether warnings will be emitted when usage of 0 default TTL on a
+ * table with TimeWindowCompactionStrategy is detected.
+ *
+ * @return {@code true} if warnings will be emitted, {@code false} otherwise.
+ */
+ boolean getZeroTTLOnTWCSWarned();
+
+ /**
+ * Sets whether warnings will be emitted when usage of 0 default TTL on a
+ * table with TimeWindowCompactionStrategy is detected.
+ *
+ * @param value {@code true} if warning will be emitted, {@code false} otherwise.
+ */
+ void setZeroTTLOnTWCSWarned(boolean value);
+
+ /**
+ * Returns whether it is allowed to create or alter table to use 0 default TTL with TimeWindowCompactionStrategy.
+ * If it is not, such query will fail.
+ *
+ * @return {@code true} if 0 default TTL is allowed on TWCS table, {@code false} otherwise.
+ */
+ boolean getZeroTTLOnTWCSEnabled();
+
+ /**
+ * Sets whether users can use 0 default TTL on a table with TimeWindowCompactionStrategy.
+ *
+ * @param value {@code true} if 0 default TTL on TWCS tables is allowed, {@code false} otherwise.
+ */
+ void setZeroTTLOnTWCSEnabled(boolean value);
+
+ /**
+ * @return A timestamp that if a user supplied timestamp is after will trigger a warning
+ */
+ @Nullable
+ DurationSpec.LongMicrosecondsBound getMaximumTimestampWarnThreshold();
+
+ /**
+ * @return A timestamp that if a user supplied timestamp is after will cause a failure
+ */
+ @Nullable
+ DurationSpec.LongMicrosecondsBound getMaximumTimestampFailThreshold();
+
+ /**
+ * Sets the warning upper bound for user supplied timestamps
+ *
+ * @param warn The highest acceptable difference between now and the written value timestamp before triggering a
+ * warning. {@code null} means disabled.
+ * @param fail The highest acceptable difference between now and the written value timestamp before triggering a
+ * failure. {@code null} means disabled.
+ */
+ void setMaximumTimestampThreshold(@Nullable DurationSpec.LongMicrosecondsBound warn,
+ @Nullable DurationSpec.LongMicrosecondsBound fail);
+
+ /**
+ * @return A timestamp that if a user supplied timestamp is before will trigger a warning
+ */
+ @Nullable
+ DurationSpec.LongMicrosecondsBound getMinimumTimestampWarnThreshold();
+
+ /**
+ * @return A timestamp that if a user supplied timestamp is after will trigger a warning
+ */
+ @Nullable
+ DurationSpec.LongMicrosecondsBound getMinimumTimestampFailThreshold();
+
+ /**
+ * Sets the warning lower bound for user supplied timestamps
+ *
+ * @param warn The lowest acceptable difference between now and the written value timestamp before triggering a
+ * warning. {@code null} means disabled.
+ * @param fail The lowest acceptable difference between now and the written value timestamp before triggering a
+ * failure. {@code null} means disabled.
+ */
+ void setMinimumTimestampThreshold(@Nullable DurationSpec.LongMicrosecondsBound warn,
+ @Nullable DurationSpec.LongMicrosecondsBound fail);
}
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfigProvider.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfigProvider.java
index 6128764..280467e 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfigProvider.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfigProvider.java
@@ -24,6 +24,8 @@
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS;
+
/**
* Provider of {@link GuardrailsConfig}s for a {@link ClientState}.
* <p>
@@ -37,11 +39,14 @@
*/
public interface GuardrailsConfigProvider
{
- public static final String CUSTOM_IMPLEMENTATION_PROPERTY = "cassandra.custom_guardrails_config_provider_class";
+ /**
+ * @deprecated CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS.getKey() must be used instead.
+ */
+ @Deprecated
+ public static final String CUSTOM_IMPLEMENTATION_PROPERTY = CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS.getKey();
- static final GuardrailsConfigProvider instance = System.getProperty(CUSTOM_IMPLEMENTATION_PROPERTY) == null
- ? new Default()
- : build(System.getProperty(CUSTOM_IMPLEMENTATION_PROPERTY));
+ GuardrailsConfigProvider instance = CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS.getString() == null ?
+ new Default() : build(CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS.getString());
/**
* Returns the {@link GuardrailsConfig} to be used for the specified {@link ClientState}.
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
index ad2edda..cabbe0c 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
@@ -223,6 +223,20 @@
void setAllowFilteringEnabled(boolean enabled);
/**
+ * Returns whether SimpleStrategy is allowed on keyspace creation or alteration
+ *
+ * @return {@code true} if SimpleStrategy is allowed; {@code false} otherwise
+ */
+ boolean getSimpleStrategyEnabled();
+
+ /**
+ * Sets whether SimpleStrategy is allowed on keyspace creation or alteration
+ *
+ * @param enabled {@code true} if SimpleStrategy is allowed, {@code false} otherwise.
+ */
+ void setSimpleStrategyEnabled(boolean enabled);
+
+ /**
* Returns whether users can disable compression on tables
*
* @return {@code true} if users can disable compression on a table, {@code false} otherwise.
@@ -251,6 +265,20 @@
void setCompactTablesEnabled(boolean enabled);
/**
+ * Gets whether users can use the ALTER TABLE statement to change columns
+ *
+ * @return {@code true} if ALTER TABLE is allowed, {@code false} otherwise.
+ */
+ boolean getAlterTableEnabled();
+
+ /**
+ * Sets whether users can use the ALTER TABLE statement to change columns
+ *
+ * @param enabled {@code true} if changing columns is allowed, {@code false} otherwise.
+ */
+ void setAlterTableEnabled(boolean enabled);
+
+ /**
* Returns whether GROUP BY queries are allowed.
*
* @return {@code true} if allowed, {@code false} otherwise.
@@ -277,6 +305,18 @@
void setDropTruncateTableEnabled(boolean enabled);
/**
+ * Returns whether users can DROP a keyspace
+ *
+ * @return {@code true} if allowed, {@code false} otherwise.
+ */
+ boolean getDropKeyspaceEnabled();
+
+ /**
+ * Sets whether users can DROP a keyspace
+ */
+ void setDropKeyspaceEnabled(boolean enabled);
+
+ /**
* @return The threshold to warn when requested page size greater than threshold.
* -1 means disabled.
*/
@@ -429,6 +469,57 @@
void setWriteConsistencyLevelsDisallowedCSV(String consistencyLevels);
/**
+ * @return The threshold to warn when encountering partitions larger than threshold, as a string formatted as in,
+ * for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value means disabled.
+ */
+ @Nullable
+ String getPartitionSizeWarnThreshold();
+
+ /**
+ * @return The threshold to fail when encountering partitions larger than threshold, as a string formatted as in,
+ * for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value means disabled.
+ * Triggering a failure emits a log message and a diagnostic event, but it doesn't throw an exception interrupting
+ * the offending sstable write.
+ */
+ @Nullable
+ String getPartitionSizeFailThreshold();
+
+ /**
+ * @param warnSize The threshold to warn when encountering partitions larger than threshold, as a string formatted
+ * as in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}.
+ * A {@code null} value means disabled.
+ * @param failSize The threshold to fail when encountering partitions larger than threshold, as a string formatted
+ * as in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}.
+ * A {@code null} value means disabled. Triggering a failure emits a log message and a diagnostic
+ * event, but it desn't throw an exception interrupting the offending sstable write.
+ */
+ void setPartitionSizeThreshold(@Nullable String warnSize, @Nullable String failSize);
+
+ /**
+ * @return The threshold to warn when encountering column values larger than threshold, as a string formatted as
+ * in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value means disabled.
+ */
+ @Nullable
+ String getColumnValueSizeWarnThreshold();
+
+ /**
+ * @return The threshold to prevent column values larger than threshold, as a string formatted as in, for example,
+ * {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value means disabled.
+ */
+ @Nullable
+ String getColumnValueSizeFailThreshold();
+
+ /**
+ * @param warnSize The threshold to warn when encountering column values larger than threshold, as a string
+ * formatted as in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}.
+ * A {@code null} value means disabled.
+ * @param failSize The threshold to prevent column values larger than threshold, as a string formatted as in, for
+ * example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}.
+ * A {@code null} value means disabled.
+ */
+ void setColumnValueSizeThreshold(@Nullable String warnSize, @Nullable String failSize);
+
+ /**
* @return The threshold to warn when encountering larger size of collection data than threshold, as a string
* formatted as in, for example, {@code 10GiB}, {@code 20MiB}, {@code 30KiB} or {@code 40B}. A {@code null} value
* means that the threshold is disabled.
@@ -522,21 +613,125 @@
void setDataDiskUsageMaxDiskSize(@Nullable String size);
/**
- * @return The threshold to warn when replication factor is lesser threshold.
+ * @return The threshold to warn when replication factor is lesser than threshold.
*/
int getMinimumReplicationFactorWarnThreshold();
/**
- * @return The threshold to fail when replication factor is lesser threshold.
+ * @return The threshold to fail when replication factor is lesser than threshold.
*/
int getMinimumReplicationFactorFailThreshold();
/**
- * @param warn the threshold to warn when the minimum replication factor is lesser than
- * threshold -1 means disabled.
- * @param fail the threshold to fail when the minimum replication factor is lesser than
- * threshold -1 means disabled.
+ * @param warn The threshold to warn when the minimum replication factor is lesser than threshold.
+ * -1 means disabled.
+ * @param fail The threshold to fail when the minimum replication factor is lesser than threshold.
+ * -1 means disabled.
*/
void setMinimumReplicationFactorThreshold (int warn, int fail);
+ /**
+ * @return The threshold to fail when replication factor is greater than threshold.
+ */
+ int getMaximumReplicationFactorWarnThreshold();
+
+ /**
+ * @return The threshold to fail when replication factor is greater than threshold.
+ */
+ int getMaximumReplicationFactorFailThreshold();
+
+ /**
+ * @param warn The threshold to warn when the maximum replication factor is greater than threshold.
+ * -1 means disabled.
+ * @param fail The threshold to fail when the maximum replication factor is greater than threshold.
+ * -1 means disabled.
+ */
+ void setMaximumReplicationFactorThreshold (int warn, int fail);
+
+ /**
+ * Returns whether warnings will be emitted when usage of 0 default TTL on a
+ * table with TimeWindowCompactionStrategy is detected.
+ *
+ * @return {@code true} if warnings will be emitted, {@code false} otherwise.
+ */
+ boolean getZeroTTLOnTWCSWarned();
+
+ /**
+ * Sets whether warnings will be emitted when usage of 0 default TTL on a
+ * table with TimeWindowCompactionStrategy is detected.
+ *
+ * @param value {@code true} if warning will be emitted, {@code false} otherwise.
+ */
+ void setZeroTTLOnTWCSWarned(boolean value);
+
+ /**
+ * Returns whether it is allowed to create or alter table to use 0 default TTL with TimeWindowCompactionStrategy.
+ * If it is not, such query will fail.
+ *
+ * @return {@code true} if 0 default TTL is allowed on TWCS table, {@code false} otherwise.
+ */
+ boolean getZeroTTLOnTWCSEnabled();
+
+ /**
+ * Sets whether users can use 0 default TTL on a table with TimeWindowCompactionStrategy.
+ *
+ * @param value {@code true} if 0 default TTL on TWCS tables is allowed, {@code false} otherwise.
+ */
+ void setZeroTTLOnTWCSEnabled(boolean value);
+
+ /**
+ * @return The highest acceptable difference between now and the written value timestamp before triggering a warning.
+ * Expressed as a string formatted as in, for example, {@code 10s} {@code 20m}, {@code 30h} or {@code 40d}.
+ * A {@code null} value means disabled.
+ */
+ @Nullable
+ String getMaximumTimestampWarnThreshold();
+
+ /**
+ * @return The highest acceptable difference between now and the written value timestamp before triggering a failure.
+ * Expressed as a string formatted as in, for example, {@code 10s} {@code 20m}, {@code 30h} or {@code 40d}.
+ * A {@code null} value means disabled.
+ */
+ @Nullable
+ String getMaximumTimestampFailThreshold();
+
+ /**
+ * Sets the warning upper bound for user supplied timestamps.
+ *
+ * @param warnDuration The highest acceptable difference between now and the written value timestamp before
+ * triggering a warning. Expressed as a string formatted as in, for example, {@code 10s},
+ * {@code 20m}, {@code 30h} or {@code 40d}. A {@code null} value means disabled.
+ * @param failDuration The highest acceptable difference between now and the written value timestamp before
+ * triggering a failure. Expressed as a string formatted as in, for example, {@code 10s},
+ * {@code 20m}, {@code 30h} or {@code 40d}. A {@code null} value means disabled.
+ */
+ void setMaximumTimestampThreshold(@Nullable String warnDuration, @Nullable String failDuration);
+
+ /**
+ * @return The lowest acceptable difference between now and the written value timestamp before triggering a warning.
+ * Expressed as a string formatted as in, for example, {@code 10s} {@code 20m}, {@code 30h} or {@code 40d}.
+ * A {@code null} value means disabled.
+ */
+ @Nullable
+ String getMinimumTimestampWarnThreshold();
+
+ /**
+ * @return The lowest acceptable difference between now and the written value timestamp before triggering a failure.
+ * Expressed as a string formatted as in, for example, {@code 10s} {@code 20m}, {@code 30h} or {@code 40d}.
+ * A {@code null} value means disabled.
+ */
+ @Nullable
+ String getMinimumTimestampFailThreshold();
+
+ /**
+ * Sets the warning lower bound for user supplied timestamps.
+ *
+ * @param warnDuration The lowest acceptable difference between now and the written value timestamp before
+ * triggering a warning. Expressed as a string formatted as in, for example, {@code 10s},
+ * {@code 20m}, {@code 30h} or {@code 40d}. A {@code null} value means disabled.
+ * @param failDuration The lowest acceptable difference between now and the written value timestamp before
+ * triggering a failure. Expressed as a string formatted as in, for example, {@code 10s},
+ * {@code 20m}, {@code 30h} or {@code 40d}. A {@code null} value means disabled.
+ */
+ void setMinimumTimestampThreshold(@Nullable String warnDuration, @Nullable String failDuration);
}
diff --git a/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java b/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java
index badaff7..6203ee2 100644
--- a/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java
+++ b/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java
@@ -19,6 +19,8 @@
package org.apache.cassandra.db.guardrails;
import java.util.function.ToLongFunction;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.service.ClientState;
/**
@@ -30,16 +32,18 @@
* Creates a new threshold guardrail.
*
* @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
* @param warnThreshold a {@link ClientState}-based provider of the value above which a warning should be triggered.
* @param failThreshold a {@link ClientState}-based provider of the value above which the operation should be aborted.
* @param messageProvider a function to generate the warning or error message if the guardrail is triggered
*/
public MaxThreshold(String name,
+ @Nullable String reason,
ToLongFunction<ClientState> warnThreshold,
ToLongFunction<ClientState> failThreshold,
Threshold.ErrorMessageProvider messageProvider)
{
- super(name, warnThreshold, failThreshold, messageProvider);
+ super(name, reason, warnThreshold, failThreshold, messageProvider);
}
@Override
diff --git a/src/java/org/apache/cassandra/db/guardrails/MinThreshold.java b/src/java/org/apache/cassandra/db/guardrails/MinThreshold.java
index 427f277..72704cc 100644
--- a/src/java/org/apache/cassandra/db/guardrails/MinThreshold.java
+++ b/src/java/org/apache/cassandra/db/guardrails/MinThreshold.java
@@ -19,6 +19,8 @@
package org.apache.cassandra.db.guardrails;
import java.util.function.ToLongFunction;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.service.ClientState;
/**
@@ -30,16 +32,18 @@
* Creates a new minimum threshold guardrail.
*
* @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
* @param warnThreshold a {@link ClientState}-based provider of the value above which a warning should be triggered.
* @param failThreshold a {@link ClientState}-based provider of the value above which the operation should be aborted.
* @param messageProvider a function to generate the warning or error message if the guardrail is triggered
*/
public MinThreshold(String name,
+ @Nullable String reason,
ToLongFunction<ClientState> warnThreshold,
ToLongFunction<ClientState> failThreshold,
Threshold.ErrorMessageProvider messageProvider)
{
- super(name, warnThreshold, failThreshold, messageProvider);
+ super(name, reason, warnThreshold, failThreshold, messageProvider);
}
@Override
diff --git a/src/java/org/apache/cassandra/db/guardrails/PercentageThreshold.java b/src/java/org/apache/cassandra/db/guardrails/PercentageThreshold.java
index 6f866c6..c316276 100644
--- a/src/java/org/apache/cassandra/db/guardrails/PercentageThreshold.java
+++ b/src/java/org/apache/cassandra/db/guardrails/PercentageThreshold.java
@@ -20,6 +20,8 @@
import java.util.function.ToLongFunction;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.service.ClientState;
/**
@@ -33,16 +35,18 @@
* Creates a new threshold guardrail.
*
* @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
* @param warnThreshold a {@link ClientState}-based provider of the value above which a warning should be triggered.
* @param failThreshold a {@link ClientState}-based provider of the value above which the operation should be aborted.
* @param messageProvider a function to generate the warning or error message if the guardrail is triggered
*/
public PercentageThreshold(String name,
+ @Nullable String reason,
ToLongFunction<ClientState> warnThreshold,
ToLongFunction<ClientState> failThreshold,
ErrorMessageProvider messageProvider)
{
- super(name, warnThreshold, failThreshold, messageProvider);
+ super(name, reason, warnThreshold, failThreshold, messageProvider);
}
@Override
diff --git a/src/java/org/apache/cassandra/db/guardrails/Predicates.java b/src/java/org/apache/cassandra/db/guardrails/Predicates.java
index 13be9e9..ab08560 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Predicates.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Predicates.java
@@ -56,16 +56,18 @@
* Creates a new {@link Predicates} guardrail.
*
* @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
* @param warnPredicate a {@link ClientState}-based predicate provider that is used to check if given value should trigger a warning.
* @param failurePredicate a {@link ClientState}-based predicate provider that is used to check if given value should trigger a failure.
* @param messageProvider a function to generate the warning or error message if the guardrail is triggered
*/
Predicates(String name,
+ @Nullable String reason,
Function<ClientState, Predicate<T>> warnPredicate,
Function<ClientState, Predicate<T>> failurePredicate,
MessageProvider<T> messageProvider)
{
- super(name);
+ super(name, reason);
this.warnPredicate = warnPredicate;
this.failurePredicate = failurePredicate;
this.messageProvider = messageProvider;
diff --git a/src/java/org/apache/cassandra/db/guardrails/Threshold.java b/src/java/org/apache/cassandra/db/guardrails/Threshold.java
index b671907..257ab01 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Threshold.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Threshold.java
@@ -41,16 +41,18 @@
* Creates a new threshold guardrail.
*
* @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
* @param warnThreshold a {@link ClientState}-based provider of the value above which a warning should be triggered.
* @param failThreshold a {@link ClientState}-based provider of the value above which the operation should be aborted.
* @param messageProvider a function to generate the warning or error message if the guardrail is triggered
*/
public Threshold(String name,
+ @Nullable String reason,
ToLongFunction<ClientState> warnThreshold,
ToLongFunction<ClientState> failThreshold,
ErrorMessageProvider messageProvider)
{
- super(name);
+ super(name, reason);
this.warnThreshold = warnThreshold;
this.failThreshold = failThreshold;
this.messageProvider = messageProvider;
diff --git a/src/java/org/apache/cassandra/db/guardrails/Values.java b/src/java/org/apache/cassandra/db/guardrails/Values.java
index f46e3af..9504a3d 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Values.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Values.java
@@ -47,18 +47,20 @@
* Creates a new values guardrail.
*
* @param name the identifying name of the guardrail
+ * @param reason the optional description of the reason for guarding the operation
* @param warnedValues a {@link ClientState}-based provider of the values for which a warning is triggered.
* @param ignoredValues a {@link ClientState}-based provider of the values that are ignored.
* @param disallowedValues a {@link ClientState}-based provider of the values that are disallowed.
* @param what The feature that is guarded by this guardrail (for reporting in error messages).
*/
public Values(String name,
+ @Nullable String reason,
Function<ClientState, Set<T>> warnedValues,
Function<ClientState, Set<T>> ignoredValues,
Function<ClientState, Set<T>> disallowedValues,
String what)
{
- super(name);
+ super(name, reason);
this.warnedValues = warnedValues;
this.ignoredValues = ignoredValues;
this.disallowedValues = disallowedValues;
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
index 8e0d514..134beec 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
@@ -17,16 +17,27 @@
*/
package org.apache.cassandra.db.lifecycle;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Throwables;
-import static com.google.common.base.Predicates.*;
+import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Predicates.in;
+import static com.google.common.base.Predicates.not;
+import static com.google.common.base.Predicates.or;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.filter;
@@ -50,7 +61,7 @@
* really present, and that the items to add are not (unless we're also removing them)
* @return a new identity map with the contents of the provided one modified
*/
- static <T> Map<T, T> replace(Map<T, T> original, Set<T> remove, Iterable<T> add)
+ static <T> Map<T, T> replace(Map<T, T> original, Set<? extends T> remove, Iterable<? extends T> add)
{
// ensure the ones being removed are the exact same ones present
for (T reader : remove)
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index d3f3a1e..7cff57d 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -18,33 +18,54 @@
package org.apache.cassandra.db.lifecycle;
import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Set;
import java.util.function.BiPredicate;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
-import com.google.common.collect.*;
-
-import org.apache.cassandra.io.util.File;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReader.UniqueIdentifier;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Transactional;
import static com.google.common.base.Functions.compose;
-import static com.google.common.base.Predicates.*;
+import static com.google.common.base.Predicates.in;
import static com.google.common.collect.ImmutableSet.copyOf;
-import static com.google.common.collect.Iterables.*;
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.getFirst;
+import static com.google.common.collect.Iterables.transform;
import static java.util.Collections.singleton;
-import static org.apache.cassandra.db.lifecycle.Helpers.*;
+import static org.apache.cassandra.db.lifecycle.Helpers.abortObsoletion;
+import static org.apache.cassandra.db.lifecycle.Helpers.checkNotReplaced;
+import static org.apache.cassandra.db.lifecycle.Helpers.concatUniq;
+import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.filterIn;
+import static org.apache.cassandra.db.lifecycle.Helpers.filterOut;
+import static org.apache.cassandra.db.lifecycle.Helpers.markObsolete;
+import static org.apache.cassandra.db.lifecycle.Helpers.orIn;
+import static org.apache.cassandra.db.lifecycle.Helpers.prepareForObsoletion;
+import static org.apache.cassandra.db.lifecycle.Helpers.select;
+import static org.apache.cassandra.db.lifecycle.Helpers.selectFirst;
+import static org.apache.cassandra.db.lifecycle.Helpers.setReplaced;
import static org.apache.cassandra.db.lifecycle.View.updateCompacting;
import static org.apache.cassandra.db.lifecycle.View.updateLiveSet;
import static org.apache.cassandra.utils.Throwables.maybeFail;
@@ -160,12 +181,12 @@
}
@SuppressWarnings("resource") // log closed during postCleanup
- LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable<SSTableReader> readers)
+ LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable<? extends SSTableReader> readers)
{
this(tracker, new LogTransaction(operationType, tracker), readers);
}
- LifecycleTransaction(Tracker tracker, LogTransaction log, Iterable<SSTableReader> readers)
+ LifecycleTransaction(Tracker tracker, LogTransaction log, Iterable<? extends SSTableReader> readers)
{
this.tracker = tracker;
this.log = log;
@@ -456,7 +477,7 @@
private List<SSTableReader> restoreUpdatedOriginals()
{
Iterable<SSTableReader> torestore = filterIn(originals, logged.update, logged.obsolete);
- return ImmutableList.copyOf(transform(torestore, (reader) -> current(reader).cloneWithRestoredStart(reader.first)));
+ return ImmutableList.copyOf(transform(torestore, (reader) -> current(reader).cloneWithRestoredStart(reader.getFirst())));
}
/**
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
index 11e3ffb..abfb55a 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java
@@ -21,24 +21,34 @@
package org.apache.cassandra.db.lifecycle;
import java.nio.file.Path;
-import java.util.*;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Optional;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
-
-import org.apache.cassandra.io.util.File;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LogRecord.Type;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
@@ -61,8 +71,10 @@
static String EXT = ".log";
static char SEP = '_';
- // cc_txn_opname_id.log (where cc is one of the sstable versions defined in BigVersion)
- static Pattern FILE_REGEX = Pattern.compile(String.format("^(.{2})_txn_(.*)_(.*)%s$", EXT));
+ // Log file name format:
+ // legacy for BIG format: cc_txn_opname_id.log (where cc is one of the sstable versions defined in BigVersion)
+ // other formats: fmt-cc_txn_opname_id.log (where fmt is the format and name and cc is one of its versions)
+ static Pattern FILE_REGEX = Pattern.compile(String.format("^((?:[a-z]+-)?.{2}_)?txn_(.*)_(.*)%s$", EXT));
// A set of physical files on disk, each file is an identical replica
private final LogReplicaSet replicas = new LogReplicaSet();
@@ -77,6 +89,8 @@
// The unique id of the transaction
private final TimeUUID id;
+ private final Version version = DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion();
+
static LogFile make(File logReplica)
{
return make(logReplica.name(), Collections.singletonList(logReplica));
@@ -496,18 +510,17 @@
private String getFileName()
{
- return StringUtils.join(BigFormat.latestVersion,
- LogFile.SEP,
- "txn",
- LogFile.SEP,
- type.fileName,
- LogFile.SEP,
- id.toString(),
- LogFile.EXT);
+ // For pre-5.0 versions, only BigFormat is supported, and the file name includes only the version string.
+ // To retain the ability to downgrade to 4.x, we keep the old file naming scheme for BigFormat sstables
+ // and add format names for other formats as they are supported only in 5.0 and above.
+ return StringUtils.join(BigFormat.is(version.format) ? version.toString() : version.toFormatAndVersionString(), LogFile.SEP, // remove version and separator when downgrading to 4.x is becomes unsupported
+ "txn", LogFile.SEP,
+ type.fileName, LogFile.SEP,
+ id.toString(), LogFile.EXT);
}
public boolean isEmpty()
{
return records.isEmpty();
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
index 45653c4..34fd0da 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java
@@ -22,7 +22,16 @@
import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -152,7 +161,7 @@
public static LogRecord make(Type type, SSTable table)
{
- String absoluteTablePath = absolutePath(table.descriptor.baseFilename());
+ String absoluteTablePath = absolutePath(table.descriptor.baseFile());
return make(type, getExistingFiles(absoluteTablePath), table.getAllFilePaths().size(), absoluteTablePath);
}
@@ -161,7 +170,7 @@
// contains a mapping from sstable absolute path (everything up until the 'Data'/'Index'/etc part of the filename) to the sstable
Map<String, SSTable> absolutePaths = new HashMap<>();
for (SSTableReader table : tables)
- absolutePaths.put(absolutePath(table.descriptor.baseFilename()), table);
+ absolutePaths.put(absolutePath(table.descriptor.baseFile()), table);
// maps sstable base file name to the actual files on disk
Map<String, List<File>> existingFiles = getExistingFiles(absolutePaths.keySet());
@@ -176,9 +185,9 @@
return records;
}
- private static String absolutePath(String baseFilename)
+ private static String absolutePath(File baseFile)
{
- return FileUtils.getCanonicalPath(baseFilename + Component.separator);
+ return baseFile.withSuffix(String.valueOf(Component.separator)).canonicalPath();
}
public LogRecord withExistingFiles(List<File> existingFiles)
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java b/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java
index 1ea8b83..073ac7c 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogReplica.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.io.IOException;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,7 +59,16 @@
{
int folderFD = NativeLibrary.tryOpenDirectory(directory.path());
if (folderFD == -1 && REQUIRE_FD)
- throw new FSReadError(new IOException(String.format("Invalid folder descriptor trying to create log replica %s", directory.path())), directory.path());
+ {
+ if (DatabaseDescriptor.isClientInitialized())
+ {
+ logger.warn("Invalid folder descriptor trying to create log replica {}. Continuing without Native I/O support.", directory.path());
+ }
+ else
+ {
+ throw new FSReadError(new IOException(String.format("Invalid folder descriptor trying to create log replica %s", directory.path())), directory.path());
+ }
+ }
return new LogReplica(new File(fileName), folderFD);
}
@@ -67,7 +77,16 @@
{
int folderFD = NativeLibrary.tryOpenDirectory(file.parent().path());
if (folderFD == -1)
- throw new FSReadError(new IOException(String.format("Invalid folder descriptor trying to create log replica %s", file.parent().path())), file.parent().path());
+ {
+ if (DatabaseDescriptor.isClientInitialized())
+ {
+ logger.warn("Invalid folder descriptor trying to create log replica {}. Continuing without Native I/O support.", file.parentPath());
+ }
+ else
+ {
+ throw new FSReadError(new IOException(String.format("Invalid folder descriptor trying to create log replica %s", file.parent().path())), file.parent().path());
+ }
+ }
return new LogReplica(file, folderFD);
}
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
index f203bd8..6d24874 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
@@ -22,15 +22,20 @@
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
-import com.codahale.metrics.Counter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.Runnables;
+import com.codahale.metrics.Counter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,15 +46,17 @@
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LogRecord.Type;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.concurrent.RefCounted;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -387,18 +394,14 @@
try
{
// If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier
- File datafile = new File(desc.filenameFor(Component.DATA));
if (logger.isTraceEnabled())
- logger.trace("Tidier running for old sstable {}", desc.baseFilename());
+ logger.trace("Tidier running for old sstable {}", desc);
- if (datafile.exists())
- delete(datafile);
- else if (!wasNew)
+ if (!desc.fileFor(Components.DATA).exists() && !wasNew)
logger.error("SSTableTidier ran with no existing data file for an sstable that was not new");
- // let the remainder be cleaned up by delete
- SSTable.delete(desc, SSTable.discoverComponentsFor(desc));
+ desc.getFormat().delete(desc);
}
catch (Throwable t)
{
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index ab8a74b..c54ee83 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -17,7 +17,11 @@
*/
package org.apache.cassandra.db.lifecycle;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicReference;
@@ -25,23 +29,34 @@
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
-import com.google.common.collect.*;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.commitlog.CommitLogPosition;
-import org.apache.cassandra.io.util.File;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.InitialSSTableAddedNotification;
+import org.apache.cassandra.notifications.MemtableDiscardedNotification;
+import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.notifications.MemtableSwitchedNotification;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableDeletingNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
+import org.apache.cassandra.notifications.SSTableMetadataChanged;
+import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
+import org.apache.cassandra.notifications.TruncationNotification;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -51,7 +66,11 @@
import static com.google.common.collect.Iterables.filter;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
-import static org.apache.cassandra.db.lifecycle.Helpers.*;
+import static org.apache.cassandra.db.lifecycle.Helpers.abortObsoletion;
+import static org.apache.cassandra.db.lifecycle.Helpers.markObsolete;
+import static org.apache.cassandra.db.lifecycle.Helpers.notIn;
+import static org.apache.cassandra.db.lifecycle.Helpers.prepareForObsoletion;
+import static org.apache.cassandra.db.lifecycle.Helpers.setupOnline;
import static org.apache.cassandra.db.lifecycle.View.permitCompacting;
import static org.apache.cassandra.db.lifecycle.View.updateCompacting;
import static org.apache.cassandra.db.lifecycle.View.updateLiveSet;
@@ -99,7 +118,7 @@
/**
* @return a Transaction over the provided sstables if we are able to mark the given @param sstables as compacted, before anyone else
*/
- public LifecycleTransaction tryModify(Iterable<SSTableReader> sstables, OperationType operationType)
+ public LifecycleTransaction tryModify(Iterable<? extends SSTableReader> sstables, OperationType operationType)
{
if (Iterables.isEmpty(sstables))
return new LifecycleTransaction(this, operationType, sstables);
@@ -152,6 +171,8 @@
return accumulate;
long add = 0;
+ long addUncompressed = 0;
+
for (SSTableReader sstable : newSSTables)
{
if (logger.isTraceEnabled())
@@ -159,13 +180,17 @@
try
{
add += sstable.bytesOnDisk();
+ addUncompressed += sstable.logicalBytesOnDisk();
}
catch (Throwable t)
{
accumulate = merge(accumulate, t);
}
}
+
long subtract = 0;
+ long subtractUncompressed = 0;
+
for (SSTableReader sstable : oldSSTables)
{
if (logger.isTraceEnabled())
@@ -173,6 +198,7 @@
try
{
subtract += sstable.bytesOnDisk();
+ subtractUncompressed += sstable.logicalBytesOnDisk();
}
catch (Throwable t)
{
@@ -181,7 +207,10 @@
}
StorageMetrics.load.inc(add - subtract);
+ StorageMetrics.uncompressedLoad.inc(addUncompressed - subtractUncompressed);
+
cfstore.metric.liveDiskSpaceUsed.inc(add - subtract);
+ cfstore.metric.uncompressedLiveDiskSpaceUsed.inc(addUncompressed - subtractUncompressed);
// we don't subtract from total until the sstable is deleted, see TransactionLogs.SSTableTidier
cfstore.metric.totalDiskSpaceUsed.inc(add);
@@ -408,7 +437,7 @@
public void maybeIncrementallyBackup(final Iterable<SSTableReader> sstables)
{
- if (!DatabaseDescriptor.isIncrementalBackupsEnabled())
+ if (!cfstore.isTableIncrementalBackupsEnabled())
return;
for (SSTableReader sstable : sstables)
diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java
index 958bc0d..b238d24 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@ -17,17 +17,22 @@
*/
package org.apache.cassandra.db.lifecycle;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Interval;
@@ -136,13 +141,30 @@
case NONCOMPACTING:
return filter(sstables, (s) -> !compacting.contains(s));
case CANONICAL:
+ // When early open is not in play, the LIVE and CANONICAL sets are the same.
+ // However, when we do have early-open sstables, we will have some unfinished sources in the live set.
+ // For these sources we need to extract the originals, in their non-moved-start versions, from the
+ // compacting set.
+ // This creates a problem when the compaction completes, as then both:
+ // - the source is in the compacting set
+ // - the result is in the live set
+ // This currently causes the CANONICAL set to return both source and result when early-open is disabled,
+ // and is otherwise worked around by opening early the last sstable in the result set (which pushes it
+ // in the compacting set with EARLY openReason) and the !compacting.contains(sstable) check in the
+ // second loop below.
+ // Unfortunately there does not appear to be a way to avoid this workaround. Filtering the compacting
+ // set through having an early-open version in live does not work because sources are fully removed from
+ // the live set when they are completely exhausted.
+
+ // Add the compacting versions first because they will be the canonical versions of compaction sources.
Set<SSTableReader> canonicalSSTables = new HashSet<>(sstables.size() + compacting.size());
for (SSTableReader sstable : compacting)
if (sstable.openReason != SSTableReader.OpenReason.EARLY)
canonicalSSTables.add(sstable);
- // reason for checking if compacting contains the sstable is that if compacting has an EARLY version
- // of a NORMAL sstable, we still have the canonical version of that sstable in sstables.
- // note that the EARLY version is equal, but not == since it is a different instance of the same sstable.
+ // Add anything that is not compacting, removing any compaction result where we still have the
+ // compaction sources.
+ // note that the EARLY version is equal to the original, i.e. the set itself can guarantee early-open
+ // versions of sstables in compacting won't be added, but we also want to remove the results.
for (SSTableReader sstable : sstables)
if (!compacting.contains(sstable) && sstable.openReason != SSTableReader.OpenReason.EARLY)
canonicalSSTables.add(sstable);
@@ -241,7 +263,7 @@
// METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
// return a function to un/mark the provided readers compacting in a view
- static Function<View, View> updateCompacting(final Set<SSTableReader> unmark, final Iterable<SSTableReader> mark)
+ static Function<View, View> updateCompacting(final Set<? extends SSTableReader> unmark, final Iterable<? extends SSTableReader> mark)
{
if (unmark.isEmpty() && Iterables.isEmpty(mark))
return Functions.identity();
@@ -259,7 +281,7 @@
// construct a predicate to reject views that do not permit us to mark these readers compacting;
// i.e. one of them is either already compacting, has been compacted, or has been replaced
- static Predicate<View> permitCompacting(final Iterable<SSTableReader> readers)
+ static Predicate<View> permitCompacting(final Iterable<? extends SSTableReader> readers)
{
return new Predicate<View>()
{
@@ -353,4 +375,4 @@
}
};
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java
index 91a5b27..38af812 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractTimeUUIDType.java
@@ -26,6 +26,9 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.UUIDSerializer;
import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUIDAsBytes;
@@ -43,6 +46,7 @@
return true;
}
+ @Override
public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
{
// Compare for length
@@ -57,12 +61,12 @@
long msb1 = accessorL.getLong(left, 0);
long msb2 = accessorR.getLong(right, 0);
+ verifyVersion(msb1);
+ verifyVersion(msb2);
+
msb1 = reorderTimestampBytes(msb1);
msb2 = reorderTimestampBytes(msb2);
- assert (msb1 & topbyte(0xf0L)) == topbyte(0x10L);
- assert (msb2 & topbyte(0xf0L)) == topbyte(0x10L);
-
int c = Long.compare(msb1, msb2);
if (c != 0)
return c;
@@ -74,6 +78,40 @@
return Long.compare(lsb1, lsb2);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+
+ long hiBits = accessor.getLong(data, 0);
+ verifyVersion(hiBits);
+ ByteBuffer swizzled = ByteBuffer.allocate(16);
+ swizzled.putLong(0, TimeUUIDType.reorderTimestampBytes(hiBits));
+ swizzled.putLong(8, accessor.getLong(data, 8) ^ 0x8080808080808080L);
+
+ return ByteSource.fixedLength(swizzled);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ // Optional-style encoding of empty values as null sources
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ // The non-lexical UUID bits are stored as an unsigned fixed-length 128-bit integer.
+ long hiBits = ByteSourceInverse.getUnsignedFixedLengthAsLong(comparableBytes, 8);
+ long loBits = ByteSourceInverse.getUnsignedFixedLengthAsLong(comparableBytes, 8);
+
+ hiBits = reorderBackTimestampBytes(hiBits);
+ verifyVersion(hiBits);
+ // In addition, TimeUUIDType also touches the low bits of the UUID (see CASSANDRA-8730 and DB-1758).
+ loBits ^= 0x8080808080808080L;
+
+ return UUIDType.makeUuidBytes(accessor, hiBits, loBits);
+ }
+
// takes as input 8 signed bytes in native machine order
// returns the first byte unchanged, and the following 7 bytes converted to an unsigned representation
// which is the same as a 2's complement long in native format
@@ -82,16 +120,30 @@
return signedBytes ^ 0x0080808080808080L;
}
- private static long topbyte(long topbyte)
+ private void verifyVersion(long hiBits)
{
- return topbyte << 56;
+ long version = (hiBits >>> 12) & 0xF;
+ if (version != 1)
+ throw new MarshalException(String.format("Invalid UUID version %d for timeuuid",
+ version));
}
protected static long reorderTimestampBytes(long input)
{
- return (input << 48)
- | ((input << 16) & 0xFFFF00000000L)
- | (input >>> 32);
+ return (input << 48)
+ | ((input << 16) & 0xFFFF00000000L)
+ | (input >>> 32);
+ }
+
+ protected static long reorderBackTimestampBytes(long input)
+ {
+ // In a time-based UUID the high bits are significantly more shuffled than in other UUIDs - if [X] represents a
+ // 16-bit tuple, [1][2][3][4] should become [3][4][2][1].
+ // See the UUID Javadoc (and more specifically the high bits layout of a Leach-Salz UUID) to understand the
+ // reasoning behind this bit twiddling in the first place (in the context of comparisons).
+ return (input << 32)
+ | ((input >>> 16) & 0xFFFF0000L)
+ | (input >>> 48);
}
public ByteBuffer fromString(String source) throws MarshalException
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
index 74d4006..bc3dd01 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
@@ -40,6 +40,9 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.github.jamm.Unmetered;
import static org.apache.cassandra.db.marshal.AbstractType.ComparisonType.CUSTOM;
@@ -55,6 +58,8 @@
@Unmetered
public abstract class AbstractType<T> implements Comparator<ByteBuffer>, AssignmentTestable
{
+ private final static int VARIABLE_LENGTH = -1;
+
public final Comparator<ByteBuffer> reverseComparator;
public enum ComparisonType
@@ -449,11 +454,28 @@
}
/**
- * The length of values for this type if all values are of fixed length, -1 otherwise.
+ * The length of values for this type if all values are of fixed length, -1 otherwise. This has an impact on
+ * serialization.
+ * <lu>
+ * <li> see {@link #writeValue} </li>
+ * <li> see {@link #read} </li>
+ * <li> see {@link #writtenLength} </li>
+ * <li> see {@link #skipValue} </li>
+ * </lu>
*/
public int valueLengthIfFixed()
{
- return -1;
+ return VARIABLE_LENGTH;
+ }
+
+ /**
+ * Checks if all values are of fixed length.
+ *
+ * @return {@code true} if all values are of fixed length, {@code false} otherwise.
+ */
+ public final boolean isValueLengthFixed()
+ {
+ return valueLengthIfFixed() != VARIABLE_LENGTH;
}
// This assumes that no empty values are passed
@@ -518,7 +540,7 @@
return accessor.read(in, length);
else
{
- int l = (int)in.readUnsignedVInt();
+ int l = in.readUnsignedVInt32();
if (l < 0)
throw new IOException("Corrupt (negative) value length encountered");
@@ -599,6 +621,69 @@
}
/**
+ * Produce a byte-comparable representation of the given value, i.e. a sequence of bytes that compares the same way
+ * using lexicographical unsigned byte comparison as the original value using the type's comparator.
+ *
+ * We use a slightly stronger requirement to be able to use the types in tuples. Precisely, for any pair x, y of
+ * non-equal valid values of this type and any bytes b1, b2 between 0x10 and 0xEF,
+ * (+ stands for concatenation)
+ * compare(x, y) == compareLexicographicallyUnsigned(asByteComparable(x)+b1, asByteComparable(y)+b2)
+ * (i.e. the values compare like the original type, and an added 0x10-0xEF byte at the end does not change that) and:
+ * asByteComparable(x)+b1 is not a prefix of asByteComparable(y) (weakly prefix free)
+ * (i.e. a valid representation of a value may be a prefix of another valid representation of a value only if the
+ * following byte in the latter is smaller than 0x10 or larger than 0xEF). These properties are trivially true if
+ * the encoding compares correctly and is prefix free, but also permits a little more freedom that enables somewhat
+ * more efficient encoding of arbitrary-length byte-comparable blobs.
+ *
+ * Depending on the type, this method can be called for null or empty input, in which case the output is allowed to
+ * be null (the clustering/tuple encoding will accept and handle it).
+ */
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V value, ByteComparable.Version version)
+ {
+ if (isByteOrderComparable)
+ {
+ // When a type is byte-ordered on its own, we only need to escape it, so that we can include it in
+ // multi-component types and make the encoding weakly-prefix-free.
+ return ByteSource.of(accessor, value, version);
+ }
+ else
+ // default is only good for byte-comparables
+ throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement asComparableBytes");
+ }
+
+ public final ByteSource asComparableBytes(ByteBuffer byteBuffer, ByteComparable.Version version)
+ {
+ return asComparableBytes(ByteBufferAccessor.instance, byteBuffer, version);
+ }
+
+ /**
+ * Translates the given byte-ordered representation to the common, non-byte-ordered binary representation of a
+ * payload for this abstract type (the latter, common binary representation is what we mostly work with in the
+ * storage engine internals). If the given bytes don't correspond to the encoding of some payload value for this
+ * abstract type, an {@link IllegalArgumentException} may be thrown.
+ *
+ * @param accessor value accessor used to construct the value.
+ * @param comparableBytes A byte-ordered representation (presumably of a payload for this abstract type).
+ * @param version The byte-comparable version used to construct the representation.
+ * @return A of a payload for this abstract type, corresponding to the given byte-ordered representation,
+ * constructed using the supplied value accessor.
+ *
+ * @see #asComparableBytes
+ */
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ if (isByteOrderComparable)
+ return accessor.valueOf(ByteSourceInverse.getUnescapedBytes(comparableBytes));
+ else
+ throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement fromComparableBytes");
+ }
+
+ public final ByteBuffer fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return fromComparableBytes(ByteBufferAccessor.instance, comparableBytes, version);
+ }
+
+ /**
* This must be overriden by subclasses if necessary so that for any
* AbstractType, this == TypeParser.parse(toString()).
*
@@ -624,4 +709,18 @@
{
return testAssignment(receiver.type);
}
+
+ @Override
+ public AbstractType<?> getCompatibleTypeIfKnown(String keyspace)
+ {
+ return this;
+ }
+
+ /**
+ * @return A fixed, serialized value to be used when the column is masked, to be returned instead of the real value.
+ */
+ public ByteBuffer getMaskedValue()
+ {
+ throw new UnsupportedOperationException("There isn't a defined masked value for type " + asCQL3Type());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/AsciiType.java b/src/java/org/apache/cassandra/db/marshal/AsciiType.java
index 2d78c1a..ac585a2 100644
--- a/src/java/org/apache/cassandra/db/marshal/AsciiType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AsciiType.java
@@ -25,7 +25,6 @@
import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.cql3.Constants;
-import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Term;
@@ -34,10 +33,12 @@
import org.apache.cassandra.serializers.AsciiSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JsonUtils;
public class AsciiType extends StringType
{
public static final AsciiType instance = new AsciiType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose("****");
AsciiType() {super(ComparisonType.BYTE_ORDER);} // singleton
@@ -85,7 +86,7 @@
{
try
{
- return '"' + Json.quoteAsJsonString(ByteBufferUtil.string(buffer, StandardCharsets.US_ASCII)) + '"';
+ return '"' + JsonUtils.quoteAsJsonString(ByteBufferUtil.string(buffer, StandardCharsets.US_ASCII)) + '"';
}
catch (CharacterCodingException exc)
{
@@ -102,4 +103,10 @@
{
return AsciiSerializer.instance;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/BooleanType.java b/src/java/org/apache/cassandra/db/marshal/BooleanType.java
index 4ef5f95..99a2da1 100644
--- a/src/java/org/apache/cassandra/db/marshal/BooleanType.java
+++ b/src/java/org/apache/cassandra/db/marshal/BooleanType.java
@@ -26,16 +26,15 @@
import org.apache.cassandra.serializers.BooleanSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
public class BooleanType extends AbstractType<Boolean>
{
- private static final Logger logger = LoggerFactory.getLogger(BooleanType.class);
-
public static final BooleanType instance = new BooleanType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(false);
+
BooleanType() {super(ComparisonType.CUSTOM);} // singleton
public boolean isEmptyValueMeaningless()
@@ -54,6 +53,26 @@
return v1 - v2;
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+ byte b = accessor.toByte(data);
+ if (b != 0)
+ b = 1;
+ return ByteSource.oneByte(b);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ if (comparableBytes == null)
+ return accessor.empty();
+ int b = comparableBytes.next();
+ return accessor.valueOf(b == 1);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
@@ -99,4 +118,10 @@
{
return 1;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java
index df24a62..d710899 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java
@@ -249,6 +249,13 @@
}
@Override
+ public int putByte(byte[] dst, int offset, byte value)
+ {
+ dst[offset] = value;
+ return TypeSizes.BYTE_SIZE;
+ }
+
+ @Override
public int putShort(byte[] dst, int offset, short value)
{
ByteArrayUtil.putShort(dst, offset, value);
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java
index ea9bf11..ed1e6a6 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java
@@ -18,13 +18,7 @@
package org.apache.cassandra.db.marshal;
-import org.apache.cassandra.db.ArrayClustering;
-import org.apache.cassandra.db.ArrayClusteringBound;
-import org.apache.cassandra.db.ArrayClusteringBoundary;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ClusteringBound;
-import org.apache.cassandra.db.ClusteringBoundary;
-import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.ArrayCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
@@ -33,7 +27,7 @@
class ByteArrayObjectFactory implements ValueAccessor.ObjectFactory<byte[]>
{
- private static final Clustering<byte[]> EMPTY_CLUSTERING = new ArrayClustering()
+ private static final Clustering<byte[]> EMPTY_CLUSTERING = new ArrayClustering(AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY)
{
public String toString(TableMetadata metadata)
{
@@ -41,14 +35,44 @@
}
};
+ public static final Clustering<byte[]> STATIC_CLUSTERING = new ArrayClustering(AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY)
+ {
+ @Override
+ public Kind kind()
+ {
+ return Kind.STATIC_CLUSTERING;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "STATIC";
+ }
+
+ @Override
+ public String toString(TableMetadata metadata)
+ {
+ return toString();
+ }
+ };
+
static final ValueAccessor.ObjectFactory<byte[]> instance = new ByteArrayObjectFactory();
private ByteArrayObjectFactory() {}
/** The smallest start bound, i.e. the one that starts before any row. */
- private static final ArrayClusteringBound BOTTOM_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND, new byte[0][]);
+ private static final ArrayClusteringBound BOTTOM_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND,
+ AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY);
/** The biggest end bound, i.e. the one that ends after any row. */
- private static final ArrayClusteringBound TOP_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, new byte[0][]);
+ private static final ArrayClusteringBound TOP_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND,
+ AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY);
+
+ /** The biggest start bound, i.e. the one that starts after any row. */
+ private static final ArrayClusteringBound MAX_START_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.EXCL_START_BOUND,
+ AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY);
+ /** The smallest end bound, i.e. the one that end before any row. */
+ private static final ArrayClusteringBound MIN_END_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.EXCL_END_BOUND,
+ AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY);
public Cell<byte[]> cell(ColumnMetadata column, long timestamp, int ttl, int localDeletionTime, byte[] value, CellPath path)
{
@@ -65,6 +89,11 @@
return EMPTY_CLUSTERING;
}
+ public Clustering<byte[]> staticClustering()
+ {
+ return STATIC_CLUSTERING;
+ }
+
public ClusteringBound<byte[]> bound(ClusteringPrefix.Kind kind, byte[]... values)
{
return new ArrayClusteringBound(kind, values);
@@ -72,11 +101,19 @@
public ClusteringBound<byte[]> bound(ClusteringPrefix.Kind kind)
{
- return kind.isStart() ? BOTTOM_BOUND : TOP_BOUND;
+ switch (kind)
+ {
+ case EXCL_END_BOUND: return MIN_END_BOUND;
+ case INCL_START_BOUND: return BOTTOM_BOUND;
+ case INCL_END_BOUND: return TOP_BOUND;
+ case EXCL_START_BOUND: return MAX_START_BOUND;
+ default:
+ throw new AssertionError(String.format("Unexpected kind %s for empty bound or boundary", kind));
+ }
}
public ClusteringBoundary<byte[]> boundary(ClusteringPrefix.Kind kind, byte[]... values)
{
return new ArrayClusteringBoundary(kind, values);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java
index 40a3bf4..0712930 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java
@@ -253,6 +253,13 @@
}
@Override
+ public int putByte(ByteBuffer dst, int offset, byte value)
+ {
+ dst.put(dst.position() + offset, value);
+ return TypeSizes.BYTE_SIZE;
+ }
+
+ @Override
public int putShort(ByteBuffer dst, int offset, short value)
{
dst.putShort(dst.position() + offset, value);
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java
index 00f4646..82eba37 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java
@@ -20,35 +20,27 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.db.BufferClustering;
-import org.apache.cassandra.db.BufferClusteringBound;
-import org.apache.cassandra.db.BufferClusteringBoundary;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.ClusteringBound;
-import org.apache.cassandra.db.ClusteringBoundary;
-import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.TableMetadata;
class ByteBufferObjectFactory implements ValueAccessor.ObjectFactory<ByteBuffer>
{
- /** Empty clustering for tables having no clustering columns. */
- private static final Clustering<ByteBuffer> EMPTY_CLUSTERING = new BufferClustering()
- {
- @Override
- public String toString(TableMetadata metadata)
- {
- return "EMPTY";
- }
- };
-
/** The smallest start bound, i.e. the one that starts before any row. */
- private static final BufferClusteringBound BOTTOM_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND, new ByteBuffer[0]);
+ private static final BufferClusteringBound BOTTOM_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND,
+ AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY);
/** The biggest end bound, i.e. the one that ends after any row. */
- private static final BufferClusteringBound TOP_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, new ByteBuffer[0]);
+ private static final BufferClusteringBound TOP_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND,
+ AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY);
+
+ /** The biggest start bound, i.e. the one that starts after any row. */
+ private static final BufferClusteringBound MAX_START_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.EXCL_START_BOUND,
+ AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY);
+ /** The smallest end bound, i.e. the one that end before any row. */
+ private static final BufferClusteringBound MIN_END_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.EXCL_END_BOUND,
+ AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY);
static final ValueAccessor.ObjectFactory<ByteBuffer> instance = new ByteBufferObjectFactory();
@@ -66,7 +58,12 @@
public Clustering<ByteBuffer> clustering()
{
- return EMPTY_CLUSTERING;
+ return Clustering.EMPTY;
+ }
+
+ public Clustering<ByteBuffer> staticClustering()
+ {
+ return Clustering.STATIC_CLUSTERING;
}
public ClusteringBound<ByteBuffer> bound(ClusteringPrefix.Kind kind, ByteBuffer... values)
@@ -76,11 +73,19 @@
public ClusteringBound<ByteBuffer> bound(ClusteringPrefix.Kind kind)
{
- return kind.isStart() ? BOTTOM_BOUND : TOP_BOUND;
+ switch (kind)
+ {
+ case EXCL_END_BOUND: return MIN_END_BOUND;
+ case INCL_START_BOUND: return BOTTOM_BOUND;
+ case INCL_END_BOUND: return TOP_BOUND;
+ case EXCL_START_BOUND: return MAX_START_BOUND;
+ default:
+ throw new AssertionError(String.format("Unexpected kind %s for empty bound or boundary", kind));
+ }
}
public ClusteringBoundary<ByteBuffer> boundary(ClusteringPrefix.Kind kind, ByteBuffer... values)
{
return new BufferClusteringBoundary(kind, values);
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/marshal/ByteType.java b/src/java/org/apache/cassandra/db/marshal/ByteType.java
index f94f4bb..5aa8880 100644
--- a/src/java/org/apache/cassandra/db/marshal/ByteType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ByteType.java
@@ -27,11 +27,17 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class ByteType extends NumberType<Byte>
{
public static final ByteType instance = new ByteType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose((byte) 0);
+
ByteType()
{
super(ComparisonType.CUSTOM);
@@ -42,6 +48,19 @@
return accessorL.getByte(left, 0) - accessorR.getByte(right, 0);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+ {
+ // This type does not allow non-present values, but we do just to avoid future complexity.
+ return ByteSource.optionalSignedFixedLengthNumber(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalSignedFixedLength(accessor, comparableBytes, 1);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -136,4 +155,40 @@
{
return ByteBufferUtil.bytes((byte) -toByte(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((byte) Math.abs(toByte(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((byte) Math.exp(toByte(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((byte) Math.log(toByte(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((byte) Math.log10(toByte(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.clone(input);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/BytesType.java b/src/java/org/apache/cassandra/db/marshal/BytesType.java
index cabd007..5e742e7 100644
--- a/src/java/org/apache/cassandra/db/marshal/BytesType.java
+++ b/src/java/org/apache/cassandra/db/marshal/BytesType.java
@@ -33,6 +33,8 @@
{
public static final BytesType instance = new BytesType();
+ private static final ByteBuffer MASKED_VALUE = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+
BytesType() {super(ComparisonType.BYTE_ORDER);} // singleton
public ByteBuffer fromString(String source)
@@ -94,4 +96,10 @@
{
return BytesSerializer.instance;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
index c52cddc..e4346d5 100644
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@ -19,14 +19,17 @@
import java.nio.ByteBuffer;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Iterator;
+import java.util.function.Consumer;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.Lists;
import org.apache.cassandra.cql3.Maps;
import org.apache.cassandra.cql3.Sets;
+import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -35,6 +38,9 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
/**
* The abstract validator that is the base for maps, sets and lists (both frozen and non-frozen).
@@ -146,12 +152,12 @@
return values.size();
}
- public ByteBuffer serializeForNativeProtocol(Iterator<Cell<?>> cells, ProtocolVersion version)
+ public ByteBuffer serializeForNativeProtocol(Iterator<Cell<?>> cells)
{
assert isMultiCell();
List<ByteBuffer> values = serializedValues(cells);
int size = collectionSize(values);
- return CollectionSerializer.pack(values, ByteBufferAccessor.instance, size, version);
+ return CollectionSerializer.pack(values, ByteBufferAccessor.instance, size);
}
@Override
@@ -163,7 +169,7 @@
if (!getClass().equals(previous.getClass()))
return false;
- CollectionType tprev = (CollectionType) previous;
+ CollectionType<?> tprev = (CollectionType<?>) previous;
if (this.isMultiCell() != tprev.isMultiCell())
return false;
@@ -191,7 +197,7 @@
if (!getClass().equals(previous.getClass()))
return false;
- CollectionType tprev = (CollectionType) previous;
+ CollectionType<?> tprev = (CollectionType<?>) previous;
if (this.isMultiCell() != tprev.isMultiCell())
return false;
@@ -228,7 +234,7 @@
if (!(o instanceof CollectionType))
return false;
- CollectionType other = (CollectionType)o;
+ CollectionType<?> other = (CollectionType<?>) o;
if (kind != other.kind)
return false;
@@ -245,6 +251,91 @@
return this.toString(false);
}
+ static <VL, VR> int compareListOrSet(AbstractType<?> elementsComparator, VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
+ {
+ // Note that this is only used if the collection is frozen
+ if (accessorL.isEmpty(left) || accessorR.isEmpty(right))
+ return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left));
+
+ int sizeL = CollectionSerializer.readCollectionSize(left, accessorL);
+ int offsetL = CollectionSerializer.sizeOfCollectionSize();
+ int sizeR = CollectionSerializer.readCollectionSize(right, accessorR);
+ int offsetR = TypeSizes.INT_SIZE;
+
+ for (int i = 0; i < Math.min(sizeL, sizeR); i++)
+ {
+ VL v1 = CollectionSerializer.readValue(left, accessorL, offsetL);
+ offsetL += CollectionSerializer.sizeOfValue(v1, accessorL);
+ VR v2 = CollectionSerializer.readValue(right, accessorR, offsetR);
+ offsetR += CollectionSerializer.sizeOfValue(v2, accessorR);
+ int cmp = elementsComparator.compare(v1, accessorL, v2, accessorR);
+ if (cmp != 0)
+ return cmp;
+ }
+
+ return Integer.compare(sizeL, sizeR);
+ }
+
+ static <V> ByteSource asComparableBytesListOrSet(AbstractType<?> elementsComparator,
+ ValueAccessor<V> accessor,
+ V data,
+ ByteComparable.Version version)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+
+ int offset = 0;
+ int size = CollectionSerializer.readCollectionSize(data, accessor);
+ offset += CollectionSerializer.sizeOfCollectionSize();
+ ByteSource[] srcs = new ByteSource[size];
+ for (int i = 0; i < size; ++i)
+ {
+ V v = CollectionSerializer.readValue(data, accessor, offset);
+ offset += CollectionSerializer.sizeOfValue(v, accessor);
+ srcs[i] = elementsComparator.asComparableBytes(accessor, v, version);
+ }
+ return ByteSource.withTerminatorMaybeLegacy(version, 0x00, srcs);
+ }
+
+ static <V> V fromComparableBytesListOrSet(ValueAccessor<V> accessor,
+ ByteSource.Peekable comparableBytes,
+ ByteComparable.Version version,
+ AbstractType<?> elementType)
+ {
+ if (comparableBytes == null)
+ return accessor.empty();
+ assert version != ByteComparable.Version.LEGACY; // legacy translation is not reversible
+
+ List<V> buffers = new ArrayList<>();
+ int separator = comparableBytes.next();
+ while (separator != ByteSource.TERMINATOR)
+ {
+ if (!ByteSourceInverse.nextComponentNull(separator))
+ buffers.add(elementType.fromComparableBytes(accessor, comparableBytes, version));
+ else
+ buffers.add(null);
+ separator = comparableBytes.next();
+ }
+ return CollectionSerializer.pack(buffers, accessor, buffers.size());
+ }
+
+ public static String setOrListToJsonString(ByteBuffer buffer, AbstractType<?> elementsType, ProtocolVersion protocolVersion)
+ {
+ ByteBuffer value = buffer.duplicate();
+ StringBuilder sb = new StringBuilder("[");
+ int size = CollectionSerializer.readCollectionSize(value, ByteBufferAccessor.instance);
+ int offset = CollectionSerializer.sizeOfCollectionSize();
+ for (int i = 0; i < size; i++)
+ {
+ if (i > 0)
+ sb.append(", ");
+ ByteBuffer element = CollectionSerializer.readValue(value, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance);
+ sb.append(elementsType.toJSONString(element, protocolVersion));
+ }
+ return sb.append("]").toString();
+ }
+
private static class CollectionPathSerializer implements CellPath.Serializer
{
public void serialize(CellPath path, DataOutputPlus out) throws IOException
@@ -267,4 +358,11 @@
ByteBufferUtil.skipWithVIntLength(in);
}
}
+
+ public int size(ByteBuffer buffer)
+ {
+ return CollectionSerializer.readCollectionSize(buffer.duplicate(), ByteBufferAccessor.instance);
+ }
+
+ public abstract void forEach(ByteBuffer input, Consumer<ByteBuffer> action);
}
diff --git a/src/java/org/apache/cassandra/db/marshal/CompositeType.java b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
index bf5e914a..00cbeb5 100644
--- a/src/java/org/apache/cassandra/db/marshal/CompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
@@ -24,6 +24,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -31,6 +32,9 @@
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.transform;
@@ -165,6 +169,86 @@
return types.get(i);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+ {
+ if (data == null || accessor.isEmpty(data))
+ return null;
+
+ ByteSource[] srcs = new ByteSource[types.size() * 2 + 1];
+ int length = accessor.size(data);
+
+ // statics go first
+ boolean isStatic = readIsStaticInternal(data, accessor);
+ int offset = startingOffsetInternal(isStatic);
+ srcs[0] = isStatic ? null : ByteSource.EMPTY;
+
+ int i = 0;
+ byte lastEoc = 0;
+ while (offset < length)
+ {
+ // Only the end-of-component byte of the last component of this composite can be non-zero, so the
+ // component before can't have a non-zero end-of-component byte.
+ assert lastEoc == 0 : lastEoc;
+
+ int componentLength = accessor.getUnsignedShort(data, offset);
+ offset += 2;
+ srcs[i * 2 + 1] = types.get(i).asComparableBytes(accessor, accessor.slice(data, offset, componentLength), version);
+ offset += componentLength;
+ lastEoc = accessor.getByte(data, offset);
+ offset += 1;
+ srcs[i * 2 + 2] = ByteSource.oneByte(lastEoc & 0xFF ^ 0x80); // end-of-component also takes part in comparison as signed byte
+ ++i;
+ }
+ // A composite may be leaving some values unspecified. If this is the case, make sure we terminate early
+ // so that translations created before an extra field was added match translations that have the field but don't
+ // specify a value for it.
+ if (i * 2 + 1 < srcs.length)
+ srcs = Arrays.copyOfRange(srcs, 0, i * 2 + 1);
+
+ return ByteSource.withTerminatorMaybeLegacy(version, ByteSource.END_OF_STREAM, srcs);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, Version version)
+ {
+ // For ByteComparable.Version.LEGACY the terminator byte is ByteSource.END_OF_STREAM. The latter means that it's
+ // indistinguishable from the END_OF_STREAM byte that gets returned _after_ the terminator byte has already
+ // been consumed, when the composite is part of a multi-component sequence. So if in such a scenario we consume
+ // the ByteSource.END_OF_STREAM terminator here, this will result in actually consuming the multi-component
+ // sequence separator after it and jumping directly into the bytes of the next component, when we try to
+ // consume the (already consumed) separator.
+ // Instead of trying to find a way around the situation, we can just take advantage of the fact that we don't
+ // need to decode from Version.LEGACY, assume that we never do that, and assert it here.
+ assert version != Version.LEGACY;
+
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ int separator = comparableBytes.next();
+ boolean isStatic = ByteSourceInverse.nextComponentNull(separator);
+ int i = 0;
+ V[] buffers = accessor.createArray(types.size());
+ byte lastEoc = 0;
+
+ while ((separator = comparableBytes.next()) != ByteSource.TERMINATOR && i < types.size())
+ {
+ // Only the end-of-component byte of the last component of this composite can be non-zero, so the
+ // component before can't have a non-zero end-of-component byte.
+ assert lastEoc == 0 : lastEoc;
+
+ // Get the next type and decode its payload.
+ AbstractType<?> type = types.get(i);
+ V decoded = type.fromComparableBytes(accessor,
+ ByteSourceInverse.nextComponentSource(comparableBytes, separator),
+ version);
+ buffers[i++] = decoded;
+
+ lastEoc = ByteSourceInverse.getSignedByte(ByteSourceInverse.nextComponentSource(comparableBytes));
+ }
+ return build(accessor, isStatic, Arrays.copyOf(buffers, i), lastEoc);
+ }
+
protected ParsedComparator parseComparator(int i, String part)
{
return new StaticParsedComparator(types.get(i), part);
@@ -371,6 +455,12 @@
@SafeVarargs
public static <V> V build(ValueAccessor<V> accessor, boolean isStatic, V... values)
{
+ return build(accessor, isStatic, values, (byte) 0);
+ }
+
+ @VisibleForTesting
+ public static <V> V build(ValueAccessor<V> accessor, boolean isStatic, V[] values, byte lastEoc)
+ {
int totalLength = isStatic ? 2 : 0;
for (V v : values)
totalLength += 2 + accessor.size(v) + 1;
@@ -380,11 +470,12 @@
if (isStatic)
out.putShort((short)STATIC_MARKER);
- for (V v : values)
+ for (int i = 0; i < values.length; ++i)
{
+ V v = values[i];
ByteBufferUtil.writeShortLength(out, accessor.size(v));
accessor.write(v, out);
- out.put((byte) 0);
+ out.put(i != values.length - 1 ? (byte) 0 : lastEoc);
}
out.flip();
return accessor.valueOf(out);
diff --git a/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java b/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java
index 0dae092..ad02bfb 100644
--- a/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java
@@ -32,6 +32,8 @@
{
public static final CounterColumnType instance = new CounterColumnType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(0L);
+
CounterColumnType() {super(ComparisonType.NOT_COMPARABLE);} // singleton
public boolean isEmptyValueMeaningless()
@@ -128,4 +130,40 @@
{
return ByteBufferUtil.bytes(-toLong(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.abs(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((long) Math.exp(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((long) Math.log(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((long) Math.log10(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.clone(input);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DateType.java b/src/java/org/apache/cassandra/db/marshal/DateType.java
index 473cedf..f5f786d 100644
--- a/src/java/org/apache/cassandra/db/marshal/DateType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DateType.java
@@ -31,6 +31,9 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
/**
* This is the old version of TimestampType, but has been replaced as it wasn't comparing pre-epoch timestamps
@@ -43,6 +46,8 @@
public static final DateType instance = new DateType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(new Date(0));
+
DateType() {super(ComparisonType.BYTE_ORDER);} // singleton
public boolean isEmptyValueMeaningless()
@@ -50,6 +55,19 @@
return true;
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ // While BYTE_ORDER would still work for this type, making use of the fixed length is more efficient.
+ return ByteSource.optionalFixedLength(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalFixedLength(accessor, comparableBytes, 8);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -123,4 +141,10 @@
{
return 8;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DecimalType.java b/src/java/org/apache/cassandra/db/marshal/DecimalType.java
index 5740fdc..92d688c 100644
--- a/src/java/org/apache/cassandra/db/marshal/DecimalType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DecimalType.java
@@ -24,6 +24,8 @@
import java.nio.ByteBuffer;
import java.util.Objects;
+import com.google.common.primitives.Ints;
+
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Constants;
import org.apache.cassandra.cql3.Term;
@@ -32,15 +34,31 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+import ch.obermuhlner.math.big.BigDecimalMath;
public class DecimalType extends NumberType<BigDecimal>
{
public static final DecimalType instance = new DecimalType();
+
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(BigDecimal.ZERO);
private static final int MIN_SCALE = 32;
private static final int MIN_SIGNIFICANT_DIGITS = MIN_SCALE;
private static final int MAX_SCALE = 1000;
private static final MathContext MAX_PRECISION = new MathContext(10000);
+ // Constants or escaping values needed to encode/decode variable-length floating point numbers (decimals) in our
+ // custom byte-ordered encoding scheme.
+ private static final int POSITIVE_DECIMAL_HEADER_MASK = 0x80;
+ private static final int NEGATIVE_DECIMAL_HEADER_MASK = 0x00;
+ private static final int DECIMAL_EXPONENT_LENGTH_HEADER_MASK = 0x40;
+ private static final byte DECIMAL_LAST_BYTE = (byte) 0x00;
+ private static final BigInteger HUNDRED = BigInteger.valueOf(100);
+
+ private static final ByteBuffer ZERO_BUFFER = instance.decompose(BigDecimal.ZERO);
+
DecimalType() {super(ComparisonType.CUSTOM);} // singleton
public boolean isEmptyValueMeaningless()
@@ -59,6 +77,196 @@
return compareComposed(left, accessorL, right, accessorR, this);
}
+ /**
+ * Constructs a byte-comparable representation.
+ * This is rather difficult and involves reconstructing the decimal.
+ *
+ * To compare, we need a normalized value, i.e. one with a sign, exponent and (0,1) mantissa. To avoid
+ * loss of precision, both exponent and mantissa need to be base-100. We can't get this directly off the serialized
+ * bytes, as they have base-10 scale and base-256 unscaled part.
+ *
+ * We store:
+ * - sign bit inverted * 0x80 + 0x40 + signed exponent length, where exponent is negated if value is negative
+ * - zero or more exponent bytes (as given by length)
+ * - 0x80 + first pair of decimal digits, negative if value is negative, rounded to -inf
+ * - zero or more 0x80 + pair of decimal digits, always positive
+ * - trailing 0x00
+ * Zero is special-cased as 0x80.
+ *
+ * Because the trailing 00 cannot be produced from a pair of decimal digits (positive or not), no value can be
+ * a prefix of another.
+ *
+ * Encoding examples:
+ * 1.1 as c1 = 0x80 (positive number) + 0x40 + (positive exponent) 0x01 (exp length 1)
+ * 01 = exponent 1 (100^1)
+ * 81 = 0x80 + 01 (0.01)
+ * 8a = 0x80 + 10 (....10) 0.0110e2
+ * 00
+ * -1 as 3f = 0x00 (negative number) + 0x40 - (negative exponent) 0x01 (exp length 1)
+ * ff = exponent -1. negative number, thus 100^1
+ * 7f = 0x80 - 01 (-0.01) -0.01e2
+ * 00
+ * -99.9 as 3f = 0x00 (negative number) + 0x40 - (negative exponent) 0x01 (exp length 1)
+ * ff = exponent -1. negative number, thus 100^1
+ * 1c = 0x80 - 100 (-1.00)
+ * 8a = 0x80 + 10 (+....10) -0.999e2
+ * 00
+ *
+ */
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ BigDecimal value = compose(data, accessor);
+ if (value == null)
+ return null;
+ if (value.compareTo(BigDecimal.ZERO) == 0) // Note: 0.equals(0.0) returns false!
+ return ByteSource.oneByte(POSITIVE_DECIMAL_HEADER_MASK);
+
+ long scale = (((long) value.scale()) - value.precision()) & ~1;
+ boolean negative = value.signum() < 0;
+ // Make a base-100 exponent (this will always fit in an int).
+ int exponent = Math.toIntExact(-scale >> 1);
+ // Flip the exponent sign for negative numbers, so that ones with larger magnitudes are propely treated as smaller.
+ final int modulatedExponent = negative ? -exponent : exponent;
+ // We should never have scale > Integer.MAX_VALUE, as we're always subtracting the non-negative precision of
+ // the encoded BigDecimal, and furthermore we're rounding to negative infinity.
+ assert scale <= Integer.MAX_VALUE;
+ // However, we may end up overflowing on the negative side.
+ if (scale < Integer.MIN_VALUE)
+ {
+ // As scaleByPowerOfTen needs an int scale, do the scaling in two steps.
+ int mv = Integer.MIN_VALUE;
+ value = value.scaleByPowerOfTen(mv);
+ scale -= mv;
+ }
+ final BigDecimal mantissa = value.scaleByPowerOfTen(Ints.checkedCast(scale)).stripTrailingZeros();
+ // We now have a smaller-than-one signed mantissa, and a signed and modulated base-100 exponent.
+ assert mantissa.abs().compareTo(BigDecimal.ONE) < 0;
+
+ return new ByteSource()
+ {
+ // Start with up to 5 bytes for sign + exponent.
+ int exponentBytesLeft = 5;
+ BigDecimal current = mantissa;
+
+ @Override
+ public int next()
+ {
+ if (exponentBytesLeft > 0)
+ {
+ --exponentBytesLeft;
+ if (exponentBytesLeft == 4)
+ {
+ // Skip leading zero bytes in the modulatedExponent.
+ exponentBytesLeft -= Integer.numberOfLeadingZeros(Math.abs(modulatedExponent)) / 8;
+ // Now prepare the leading byte which includes the sign of the number plus the sign and length of the modulatedExponent.
+ int explen = DECIMAL_EXPONENT_LENGTH_HEADER_MASK + (modulatedExponent < 0 ? -exponentBytesLeft : exponentBytesLeft);
+ return explen + (negative ? NEGATIVE_DECIMAL_HEADER_MASK : POSITIVE_DECIMAL_HEADER_MASK);
+ }
+ else
+ return (modulatedExponent >> (exponentBytesLeft * 8)) & 0xFF;
+ }
+ else if (current == null)
+ {
+ return END_OF_STREAM;
+ }
+ else if (current.compareTo(BigDecimal.ZERO) == 0)
+ {
+ current = null;
+ return 0x00;
+ }
+ else
+ {
+ BigDecimal v = current.scaleByPowerOfTen(2);
+ BigDecimal floor = v.setScale(0, RoundingMode.FLOOR);
+ current = v.subtract(floor);
+ return floor.byteValueExact() + 0x80;
+ }
+ }
+ };
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ int headerBits = comparableBytes.next();
+ if (headerBits == POSITIVE_DECIMAL_HEADER_MASK)
+ return accessor.valueOf(ZERO_BUFFER);
+
+ // I. Extract the exponent.
+ // The sign of the decimal, and the sign and the length (in bytes) of the decimal exponent, are all encoded in
+ // the first byte.
+ // Get the sign of the decimal...
+ boolean isNegative = headerBits < POSITIVE_DECIMAL_HEADER_MASK;
+ headerBits -= isNegative ? NEGATIVE_DECIMAL_HEADER_MASK : POSITIVE_DECIMAL_HEADER_MASK;
+ headerBits -= DECIMAL_EXPONENT_LENGTH_HEADER_MASK;
+ // Get the sign and the length of the exponent (the latter is encoded as its negative if the sign of the
+ // exponent is negative)...
+ boolean isExponentNegative = headerBits < 0;
+ headerBits = isExponentNegative ? -headerBits : headerBits;
+ // Now consume the exponent bytes. If the exponent is negative and uses less than 4 bytes, the remaining bytes
+ // should be padded with 1s, in order for the constructed int to contain the correct (negative) exponent value.
+ // So, if the exponent is negative, we can just start with all bits set to 1 (i.e. we can start with -1).
+ int exponent = isExponentNegative ? -1 : 0;
+ for (int i = 0; i < headerBits; ++i)
+ exponent = (exponent << 8) | comparableBytes.next();
+ // The encoded exponent also contains the decimal sign, in order to correctly compare exponents in case of
+ // negative decimals (e.g. x * 10^y > x * 10^z if x < 0 && y < z). After the decimal sign is "removed", what's
+ // left is a base-100 exponent following BigDecimal's convention for the exponent sign.
+ exponent = isNegative ? -exponent : exponent;
+
+ // II. Extract the mantissa as a BigInteger value. It was encoded as a BigDecimal value between 0 and 1, in
+ // order to be used for comparison (after the sign of the decimal and the sign and the value of the exponent),
+ // but when decoding we don't need that property on the transient mantissa value.
+ BigInteger mantissa = BigInteger.ZERO;
+ int curr = comparableBytes.next();
+ while (curr != DECIMAL_LAST_BYTE)
+ {
+ // The mantissa value is constructed by a standard positional notation value calculation.
+ // The value of the next digit is the next most-significant mantissa byte as an unsigned integer,
+ // offset by a predetermined value (in this case, 0x80)...
+ int currModified = curr - 0x80;
+ // ...multiply the current value by the base (in this case, 100)...
+ mantissa = mantissa.multiply(HUNDRED);
+ // ...then add the next digit to the modified current value...
+ mantissa = mantissa.add(BigInteger.valueOf(currModified));
+ // ...and finally, adjust the base-100, BigDecimal format exponent accordingly.
+ --exponent;
+ curr = comparableBytes.next();
+ }
+
+ // III. Construct the final BigDecimal value, by combining the mantissa and the exponent, guarding against
+ // underflow or overflow when exponents are close to their boundary values.
+ long base10NonBigDecimalFormatExp = 2L * exponent;
+ // When expressing a sufficiently big decimal, BigDecimal's internal scale value will be negative with very
+ // big absolute value. To compute the encoded exponent, this internal scale has the number of digits of the
+ // unscaled value subtracted from it, after which it's divided by 2, rounding down to negative infinity
+ // (before accounting for the decimal sign). When decoding, this exponent is converted to a base-10 exponent in
+ // non-BigDecimal format, which means that it can very well overflow Integer.MAX_VALUE.
+ // For example, see how <code>new BigDecimal(BigInteger.TEN, Integer.MIN_VALUE)</code> is encoded and decoded.
+ if (base10NonBigDecimalFormatExp > Integer.MAX_VALUE)
+ {
+ // If the base-10 exponent will result in an overflow, some of its powers of 10 need to be absorbed by the
+ // mantissa. How much exactly? As little as needed, in order to avoid complex BigInteger operations, which
+ // means exactly as much as to have a scale of -Integer.MAX_VALUE.
+ int exponentReduction = (int) (base10NonBigDecimalFormatExp - Integer.MAX_VALUE);
+ mantissa = mantissa.multiply(BigInteger.TEN.pow(exponentReduction));
+ base10NonBigDecimalFormatExp = Integer.MAX_VALUE;
+ }
+ assert base10NonBigDecimalFormatExp >= Integer.MIN_VALUE && base10NonBigDecimalFormatExp <= Integer.MAX_VALUE;
+ // Here we negate the exponent, as we are not using BigDecimal.scaleByPowerOfTen, where a positive number means
+ // "multiplying by a positive power of 10", but to BigDecimal's internal scale representation, where a positive
+ // number means "dividing by a positive power of 10".
+ byte[] mantissaBytes = mantissa.toByteArray();
+ V resultBuf = accessor.allocate(4 + mantissaBytes.length);
+ accessor.putInt(resultBuf, 0, (int) -base10NonBigDecimalFormatExp);
+ accessor.copyByteArrayTo(mantissaBytes, 0, resultBuf, 4, mantissaBytes.length);
+ return resultBuf;
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -186,4 +394,67 @@
{
return decompose(toBigDecimal(input).negate());
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return decompose(toBigDecimal(input).abs());
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return decompose(exp(toBigDecimal(input)));
+ }
+
+ protected BigDecimal exp(BigDecimal input)
+ {
+ int precision = input.precision();
+ precision = Math.max(MIN_SIGNIFICANT_DIGITS, precision);
+ precision = Math.min(MAX_PRECISION.getPrecision(), precision);
+ return BigDecimalMath.exp(input, new MathContext(precision, RoundingMode.HALF_EVEN));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return decompose(log(toBigDecimal(input)));
+ }
+
+ protected BigDecimal log(BigDecimal input)
+ {
+ if (input.compareTo(BigDecimal.ZERO) <= 0) throw new ArithmeticException("Natural log of number zero or less");
+ int precision = input.precision();
+ precision = Math.max(MIN_SIGNIFICANT_DIGITS, precision);
+ precision = Math.min(MAX_PRECISION.getPrecision(), precision);
+ return BigDecimalMath.log(input, new MathContext(precision, RoundingMode.HALF_EVEN));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return decompose(log10(toBigDecimal(input)));
+ }
+
+ protected BigDecimal log10(BigDecimal input)
+ {
+ if (input.compareTo(BigDecimal.ZERO) <= 0) throw new ArithmeticException("Log10 of number zero or less");
+ int precision = input.precision();
+ precision = Math.max(MIN_SIGNIFICANT_DIGITS, precision);
+ precision = Math.min(MAX_PRECISION.getPrecision(), precision);
+ return BigDecimalMath.log10(input, new MathContext(precision, RoundingMode.HALF_EVEN));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return DecimalType.instance.decompose(
+ toBigDecimal(input).setScale(0, RoundingMode.HALF_UP));
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DoubleType.java b/src/java/org/apache/cassandra/db/marshal/DoubleType.java
index 570d420..944bab0 100644
--- a/src/java/org/apache/cassandra/db/marshal/DoubleType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DoubleType.java
@@ -27,11 +27,16 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class DoubleType extends NumberType<Double>
{
public static final DoubleType instance = new DoubleType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(0d);
+
DoubleType() {super(ComparisonType.CUSTOM);} // singleton
public boolean isEmptyValueMeaningless()
@@ -50,6 +55,18 @@
return compareComposed(left, accessorL, right, accessorR, this);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ return ByteSource.optionalSignedFixedLengthFloat(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalSignedFixedLengthFloat(accessor, comparableBytes, 8);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -164,4 +181,40 @@
{
return ByteBufferUtil.bytes(-toDouble(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.abs(toDouble(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.exp(toDouble(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.log(toDouble(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.log10(toDouble(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((double) Math.round(toDouble(input)));
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DurationType.java b/src/java/org/apache/cassandra/db/marshal/DurationType.java
index 2afbfc1..1f4a199 100644
--- a/src/java/org/apache/cassandra/db/marshal/DurationType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DurationType.java
@@ -36,6 +36,8 @@
{
public static final DurationType instance = new DurationType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(Duration.newInstance(0, 0, 0));
+
DurationType()
{
super(ComparisonType.BYTE_ORDER);
@@ -86,4 +88,10 @@
{
return true;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
index 5df3600..e7a2360 100644
--- a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java
@@ -19,9 +19,16 @@
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +41,9 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import static com.google.common.collect.Iterables.any;
@@ -60,7 +70,11 @@
{
private static final Logger logger = LoggerFactory.getLogger(DynamicCompositeType.class);
+ private static final ByteSource[] EMPTY_BYTE_SOURCE_ARRAY = new ByteSource[0];
+ private static final String REVERSED_TYPE = ReversedType.class.getSimpleName();
+
private final Map<Byte, AbstractType<?>> aliases;
+ private final Map<AbstractType<?>, Byte> inverseMapping;
// interning instances
private static final ConcurrentHashMap<Map<Byte, AbstractType<?>>, DynamicCompositeType> instances = new ConcurrentHashMap<>();
@@ -81,6 +95,9 @@
private DynamicCompositeType(Map<Byte, AbstractType<?>> aliases)
{
this.aliases = aliases;
+ this.inverseMapping = new HashMap<>();
+ for (Map.Entry<Byte, AbstractType<?>> en : aliases.entrySet())
+ this.inverseMapping.put(en.getValue(), en.getKey());
}
protected <V> boolean readIsStatic(V value, ValueAccessor<V> accessor)
@@ -197,6 +214,196 @@
}
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+ {
+ List<ByteSource> srcs = new ArrayList<>();
+ int length = accessor.size(data);
+
+ // statics go first
+ boolean isStatic = readIsStatic(data, accessor);
+ int offset = startingOffset(isStatic);
+ srcs.add(isStatic ? null : ByteSource.EMPTY);
+
+ byte lastEoc = 0;
+ int i = 0;
+ while (offset < length)
+ {
+ // Only the end-of-component byte of the last component of this composite can be non-zero, so the
+ // component before can't have a non-zero end-of-component byte.
+ assert lastEoc == 0 : lastEoc;
+
+ AbstractType<?> comp = getComparator(data, accessor, offset);
+ offset += getComparatorSize(i, data, accessor, offset);
+ // The comparable bytes for the component need to ensure comparisons consistent with
+ // AbstractCompositeType.compareCustom(ByteBuffer, ByteBuffer) and
+ // DynamicCompositeType.getComparator(int, ByteBuffer, ByteBuffer):
+ if (version == Version.LEGACY || !(comp instanceof ReversedType))
+ {
+ // ...most often that means just adding the short name of the type, followed by the full name of the type.
+ srcs.add(ByteSource.of(comp.getClass().getSimpleName(), version));
+ srcs.add(ByteSource.of(comp.getClass().getName(), version));
+ }
+ else
+ {
+ // ...however some times the component uses a complex type (currently the only supported complex type
+ // is ReversedType - we can't have elements that are of MapType, CompositeType, TupleType, etc.)...
+ ReversedType<?> reversedComp = (ReversedType<?>) comp;
+ // ...in this case, we need to add the short name of ReversedType before the short name of the base
+ // type, to ensure consistency with DynamicCompositeType.getComparator(int, ByteBuffer, ByteBuffer).
+ srcs.add(ByteSource.of(REVERSED_TYPE, version));
+ srcs.add(ByteSource.of(reversedComp.baseType.getClass().getSimpleName(), version));
+ srcs.add(ByteSource.of(reversedComp.baseType.getClass().getName(), version));
+ }
+ // Only then the payload of the component gets encoded.
+ int componentLength = accessor.getUnsignedShort(data, offset);
+ offset += 2;
+ srcs.add(comp.asComparableBytes(accessor, accessor.slice(data, offset, componentLength), version));
+ offset += componentLength;
+ // The end-of-component byte also takes part in the comparison, and therefore needs to be encoded.
+ lastEoc = accessor.getByte(data, offset);
+ offset += 1;
+ srcs.add(ByteSource.oneByte(version == Version.LEGACY ? lastEoc : lastEoc & 0xFF ^ 0x80));
+ ++i;
+ }
+
+ return ByteSource.withTerminatorMaybeLegacy(version, ByteSource.END_OF_STREAM, srcs.toArray(EMPTY_BYTE_SOURCE_ARRAY));
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, Version version)
+ {
+ // For ByteComparable.Version.LEGACY the terminator byte is ByteSource.END_OF_STREAM. Just like with
+ // CompositeType, this means that in multi-component sequences the terminator may be transformed to a regular
+ // component separator, but unlike CompositeType (where we have the expected number of types/components),
+ // this can make the end of the whole dynamic composite type indistinguishable from the end of a component
+ // somewhere in the middle of the dynamic composite type. Because of that, DynamicCompositeType elements
+ // cannot always be safely decoded using that encoding version.
+ // Even more so than with CompositeType, we just take advantage of the fact that we don't need to decode from
+ // Version.LEGACY, assume that we never do that, and assert it here.
+ assert version != Version.LEGACY;
+
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ // The first byte is the isStatic flag which we don't need but must consume to continue past it.
+ comparableBytes.next();
+
+ List<AbstractType<?>> types = new ArrayList<>();
+ List<V> values = new ArrayList<>();
+ byte lastEoc = 0;
+
+ for (int separator = comparableBytes.next(); separator != ByteSource.TERMINATOR; separator = comparableBytes.next())
+ {
+ // Solely the end-of-component byte of the last component of this composite can be non-zero.
+ assert lastEoc == 0 : lastEoc;
+
+ boolean isReversed = false;
+ // Decode the next type's simple class name that is encoded before its fully qualified class name (in order
+ // for comparisons to work correctly).
+ String simpleClassName = ByteSourceInverse.getString(ByteSourceInverse.nextComponentSource(comparableBytes, separator));
+ if (REVERSED_TYPE.equals(simpleClassName))
+ {
+ // Special-handle if the type is reversed (and decode the actual base type simple class name).
+ isReversed = true;
+ simpleClassName = ByteSourceInverse.getString(ByteSourceInverse.nextComponentSource(comparableBytes));
+ }
+
+ // Decode the type's fully qualified class name and parse the actual type from it.
+ String fullClassName = ByteSourceInverse.getString(ByteSourceInverse.nextComponentSource(comparableBytes));
+ assert fullClassName.endsWith(simpleClassName);
+ if (isReversed)
+ fullClassName = REVERSED_TYPE + '(' + fullClassName + ')';
+ AbstractType<?> type = TypeParser.parse(fullClassName);
+ assert type != null;
+ types.add(type);
+
+ // Decode the payload from this type.
+ V value = type.fromComparableBytes(accessor, ByteSourceInverse.nextComponentSource(comparableBytes), version);
+ values.add(value);
+
+ // Also decode the corresponding end-of-component byte - the last one we decode will be taken into
+ // account when we deserialize the decoded data into an object.
+ lastEoc = ByteSourceInverse.getSignedByte(ByteSourceInverse.nextComponentSource(comparableBytes));
+ }
+ return build(accessor, types, inverseMapping, values, lastEoc);
+ }
+
+ public static ByteBuffer build(List<String> types, List<ByteBuffer> values)
+ {
+ return build(ByteBufferAccessor.instance,
+ Lists.transform(types, TypeParser::parse),
+ Collections.emptyMap(),
+ values,
+ (byte) 0);
+ }
+
+ @VisibleForTesting
+ public static <V> V build(ValueAccessor<V> accessor,
+ List<AbstractType<?>> types,
+ Map<AbstractType<?>, Byte> inverseMapping,
+ List<V> values,
+ byte lastEoc)
+ {
+ assert types.size() == values.size();
+
+ int numComponents = types.size();
+ // Compute the total number of bytes that we'll need to store the types and their payloads.
+ int totalLength = 0;
+ for (int i = 0; i < numComponents; ++i)
+ {
+ AbstractType<?> type = types.get(i);
+ Byte alias = inverseMapping.get(type);
+ int typeNameLength = alias == null ? type.toString().getBytes(StandardCharsets.UTF_8).length : 0;
+ // The type data will be stored by means of the type's fully qualified name, not by aliasing, so:
+ // 1. The type data header should be the fully qualified name length in bytes.
+ // 2. The length should be small enough so that it fits in 15 bits (2 bytes with the first bit zero).
+ assert typeNameLength <= 0x7FFF;
+ int valueLength = accessor.size(values.get(i));
+ // The value length should also expect its first bit to be 0, as the length should be stored as a signed
+ // 2-byte value (short).
+ assert valueLength <= 0x7FFF;
+ totalLength += 2 + typeNameLength + 2 + valueLength + 1;
+ }
+
+ V result = accessor.allocate(totalLength);
+ int offset = 0;
+ for (int i = 0; i < numComponents; ++i)
+ {
+ AbstractType<?> type = types.get(i);
+ Byte alias = inverseMapping.get(type);
+ if (alias == null)
+ {
+ // Write the type data (2-byte length header + the fully qualified type name in UTF-8).
+ byte[] typeNameBytes = type.toString().getBytes(StandardCharsets.UTF_8);
+ accessor.putShort(result,
+ offset,
+ (short) typeNameBytes.length); // this should work fine also if length >= 32768
+ offset += 2;
+ accessor.copyByteArrayTo(typeNameBytes, 0, result, offset, typeNameBytes.length);
+ offset += typeNameBytes.length;
+ }
+ else
+ {
+ accessor.putShort(result, offset, (short) (alias | 0x8000));
+ offset += 2;
+ }
+
+ // Write the type payload data (2-byte length header + the payload).
+ V value = values.get(i);
+ int bytesToCopy = accessor.size(value);
+ accessor.putShort(result, offset, (short) bytesToCopy);
+ offset += 2;
+ accessor.copyTo(value, 0, result, accessor, offset, bytesToCopy);
+ offset += bytesToCopy;
+
+ // Write the end-of-component byte.
+ accessor.putByte(result, offset, i != numComponents - 1 ? (byte) 0 : lastEoc);
+ offset += 1;
+ }
+ return result;
+ }
+
protected ParsedComparator parseComparator(int i, String part)
{
return new DynamicParsedComparator(part);
diff --git a/src/java/org/apache/cassandra/db/marshal/EmptyType.java b/src/java/org/apache/cassandra/db/marshal/EmptyType.java
index 357b6e8..33c4524 100644
--- a/src/java/org/apache/cassandra/db/marshal/EmptyType.java
+++ b/src/java/org/apache/cassandra/db/marshal/EmptyType.java
@@ -33,8 +33,12 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.NoSpamLogger;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SERIALIZATION_EMPTY_TYPE_NONEMPTY_BEHAVIOR;
+
/**
* A type that only accept empty data.
* It is only useful as a value validation type, not as a comparator since column names can't be empty.
@@ -44,13 +48,12 @@
private enum NonEmptyWriteBehavior { FAIL, LOG_DATA_LOSS, SILENT_DATA_LOSS }
private static final Logger logger = LoggerFactory.getLogger(EmptyType.class);
- private static final String KEY_EMPTYTYPE_NONEMPTY_BEHAVIOR = "cassandra.serialization.emptytype.nonempty_behavior";
private static final NoSpamLogger NON_EMPTY_WRITE_LOGGER = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
private static final NonEmptyWriteBehavior NON_EMPTY_WRITE_BEHAVIOR = parseNonEmptyWriteBehavior();
private static NonEmptyWriteBehavior parseNonEmptyWriteBehavior()
{
- String value = System.getProperty(KEY_EMPTYTYPE_NONEMPTY_BEHAVIOR);
+ String value = SERIALIZATION_EMPTY_TYPE_NONEMPTY_BEHAVIOR.getString();
if (value == null)
return NonEmptyWriteBehavior.FAIL;
try
@@ -59,7 +62,7 @@
}
catch (Exception e)
{
- logger.warn("Unable to parse property " + KEY_EMPTYTYPE_NONEMPTY_BEHAVIOR + ", falling back to FAIL", e);
+ logger.warn("Unable to parse property " + SERIALIZATION_EMPTY_TYPE_NONEMPTY_BEHAVIOR.getKey() + ", falling back to FAIL", e);
return NonEmptyWriteBehavior.FAIL;
}
}
@@ -68,6 +71,18 @@
private EmptyType() {super(ComparisonType.CUSTOM);} // singleton
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ return null;
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return accessor.empty();
+ }
+
public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
{
return 0;
@@ -168,4 +183,10 @@
super(message);
}
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return ByteBufferUtil.EMPTY_BYTE_BUFFER;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/FloatType.java b/src/java/org/apache/cassandra/db/marshal/FloatType.java
index 35abee0..74411dc 100644
--- a/src/java/org/apache/cassandra/db/marshal/FloatType.java
+++ b/src/java/org/apache/cassandra/db/marshal/FloatType.java
@@ -27,12 +27,17 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class FloatType extends NumberType<Float>
{
public static final FloatType instance = new FloatType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(0f);
+
FloatType() {super(ComparisonType.CUSTOM);} // singleton
public boolean isEmptyValueMeaningless()
@@ -51,6 +56,18 @@
return compareComposed(left, accessorL, right, accessorR, this);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ return ByteSource.optionalSignedFixedLengthFloat(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalSignedFixedLengthFloat(accessor, comparableBytes, 4);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -159,4 +176,40 @@
{
return ByteBufferUtil.bytes(-toFloat(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.abs(toFloat(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((float) Math.exp(toFloat(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((float) Math.log(toFloat(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((float) Math.log10(toFloat(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((float) Math.round(toFloat(input)));
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/InetAddressType.java b/src/java/org/apache/cassandra/db/marshal/InetAddressType.java
index 6838c83..7e0e58c 100644
--- a/src/java/org/apache/cassandra/db/marshal/InetAddressType.java
+++ b/src/java/org/apache/cassandra/db/marshal/InetAddressType.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.marshal;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import org.apache.cassandra.cql3.CQL3Type;
@@ -33,6 +34,8 @@
{
public static final InetAddressType instance = new InetAddressType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(new InetSocketAddress(0).getAddress());
+
InetAddressType() {super(ComparisonType.BYTE_ORDER);} // singleton
public boolean isEmptyValueMeaningless()
@@ -94,4 +97,10 @@
{
return InetAddressSerializer.instance;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/Int32Type.java b/src/java/org/apache/cassandra/db/marshal/Int32Type.java
index 98f4c83..2d70ac4 100644
--- a/src/java/org/apache/cassandra/db/marshal/Int32Type.java
+++ b/src/java/org/apache/cassandra/db/marshal/Int32Type.java
@@ -28,11 +28,16 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class Int32Type extends NumberType<Integer>
{
public static final Int32Type instance = new Int32Type();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(0);
+
Int32Type()
{
super(ComparisonType.CUSTOM);
@@ -55,6 +60,18 @@
return ValueAccessor.compare(left, accessorL, right, accessorR);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ return ByteSource.optionalSignedFixedLengthNumber(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalSignedFixedLength(accessor, comparableBytes, 4);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -159,4 +176,40 @@
{
return ByteBufferUtil.bytes(-toInt(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.abs(toInt(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((int) Math.exp(toInt(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((int) Math.log(toInt(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((int) Math.log10(toInt(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.clone(input);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/IntegerType.java b/src/java/org/apache/cassandra/db/marshal/IntegerType.java
index 4c913d5..2918953 100644
--- a/src/java/org/apache/cassandra/db/marshal/IntegerType.java
+++ b/src/java/org/apache/cassandra/db/marshal/IntegerType.java
@@ -30,11 +30,25 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public final class IntegerType extends NumberType<BigInteger>
{
public static final IntegerType instance = new IntegerType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(BigInteger.ZERO);
+
+ // Constants or escaping values needed to encode/decode variable-length integers in our custom byte-ordered
+ // encoding scheme.
+ private static final int POSITIVE_VARINT_HEADER = 0x80;
+ private static final int NEGATIVE_VARINT_LENGTH_HEADER = 0x00;
+ private static final int POSITIVE_VARINT_LENGTH_HEADER = 0xFF;
+ private static final byte BIG_INTEGER_NEGATIVE_LEADING_ZERO = (byte) 0xFF;
+ private static final byte BIG_INTEGER_POSITIVE_LEADING_ZERO = (byte) 0x00;
+ public static final int FULL_FORM_THRESHOLD = 7;
+
private static <V> int findMostSignificantByte(V value, ValueAccessor<V> accessor)
{
int len = accessor.size(value) - 1;
@@ -131,6 +145,301 @@
return 0;
}
+ /**
+ * Constructs a byte-comparable representation of the number.
+ *
+ * In the current format we represent it:
+ * directly as varint, if the length is 6 or smaller (the encoding has non-00/FF first byte)
+ * <signbyte><length as unsigned integer - 7><7 or more bytes>, otherwise
+ * where <signbyte> is 00 for negative numbers and FF for positive ones, and the length's bytes are inverted if
+ * the number is negative (so that longer length sorts smaller).
+ *
+ * Because we present the sign separately, we don't need to include 0x00 prefix for positive integers whose first
+ * byte is >= 0x80 or 0xFF prefix for negative integers whose first byte is < 0x80. Note that we do this before
+ * taking the length for the purposes of choosing between varint and full-form encoding.
+ *
+ * The representations are prefix-free, because the choice between varint and full-form encoding is determined by
+ * the first byte where varints are properly ordered between full-form negative and full-form positive, varint
+ * encoding is prefix-free, and full-form representations of different length always have length bytes that differ.
+ *
+ * Examples:
+ * -1 as 7F
+ * 0 as 80
+ * 1 as 81
+ * 127 as C07F
+ * 255 as C0FF
+ * 2^32-1 as F8FFFFFFFF
+ * 2^32 as F900000000
+ * 2^56-1 as FEFFFFFFFFFFFFFF
+ * 2^56 as FF000100000000000000
+ *
+ * See {@link #asComparableBytesLegacy} for description of the legacy format.
+ */
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ final int limit = accessor.size(data);
+ if (limit == 0)
+ return null;
+
+ // skip any leading sign-only byte(s)
+ int p = 0;
+ final byte signbyte = accessor.getByte(data, p);
+ if (signbyte == BIG_INTEGER_NEGATIVE_LEADING_ZERO || signbyte == BIG_INTEGER_POSITIVE_LEADING_ZERO)
+ {
+ while (p + 1 < limit)
+ {
+ if (accessor.getByte(data, ++p) != signbyte)
+ break;
+ }
+ }
+
+ if (version != ByteComparable.Version.LEGACY)
+ return (limit - p < FULL_FORM_THRESHOLD)
+ ? encodeAsVarInt(accessor, data, limit)
+ : asComparableBytesCurrent(accessor, data, p, limit, (signbyte >> 7) & 0xFF);
+ else
+ return asComparableBytesLegacy(accessor, data, p, limit, signbyte);
+ }
+
+ /**
+ * Encode the BigInteger stored in the given buffer as a variable-length signed integer.
+ * The length of the number is given in the limit argument, and must be <= 8.
+ */
+ private <V> ByteSource encodeAsVarInt(ValueAccessor<V> accessor, V data, int limit)
+ {
+ long v;
+ switch (limit)
+ {
+ case 1:
+ v = accessor.getByte(data, 0);
+ break;
+ case 2:
+ v = accessor.getShort(data, 0);
+ break;
+ case 3:
+ v = (accessor.getShort(data, 0) << 8) | (accessor.getByte(data, 2) & 0xFF);
+ break;
+ case 4:
+ v = accessor.getInt(data, 0);
+ break;
+ case 5:
+ v = ((long) accessor.getInt(data, 0) << 8) | (accessor.getByte(data, 4) & 0xFF);
+ break;
+ case 6:
+ v = ((long) accessor.getInt(data, 0) << 16) | (accessor.getShort(data, 4) & 0xFFFF);
+ break;
+ case 7:
+ v = ((long) accessor.getInt(data, 0) << 24) | ((accessor.getShort(data, 4) & 0xFFFF) << 8) | (accessor.getByte(data, 6) & 0xFF);
+ break;
+ case 8:
+ // This is not reachable within the encoding; added for completeness.
+ v = accessor.getLong(data, 0);
+ break;
+ default:
+ throw new AssertionError();
+ }
+ return ByteSource.variableLengthInteger(v);
+ }
+
+ /**
+ * Constructs a full-form byte-comparable representation of the number in the current format.
+ *
+ * This contains:
+ * <signbyte><length as unsigned integer - 7><7 or more bytes>, otherwise
+ * where <signbyte> is 00 for negative numbers and FF for positive ones, and the length's bytes are inverted if
+ * the number is negative (so that longer length sorts smaller).
+ *
+ * Because we present the sign separately, we don't need to include 0x00 prefix for positive integers whose first
+ * byte is >= 0x80 or 0xFF prefix for negative integers whose first byte is < 0x80.
+ *
+ * The representations are prefix-free, because representations of different length always have length bytes that
+ * differ.
+ */
+ private <V> ByteSource asComparableBytesCurrent(ValueAccessor<V> accessor, V data, int startpos, int limit, int signbyte)
+ {
+ // start with sign as a byte, then variable-length-encoded length, then bytes (stripped leading sign)
+ return new ByteSource()
+ {
+ int pos = -2;
+ ByteSource lengthEncoding = new VariableLengthUnsignedInteger(limit - startpos - FULL_FORM_THRESHOLD);
+
+ @Override
+ public int next()
+ {
+ if (pos == -2)
+ {
+ ++pos;
+ return signbyte ^ 0xFF; // 00 for negative/FF for positive (01-FE for direct varint encoding)
+ }
+ else if (pos == -1)
+ {
+ int nextByte = lengthEncoding.next();
+ if (nextByte != END_OF_STREAM)
+ return nextByte ^ signbyte;
+ pos = startpos;
+ }
+
+ if (pos == limit)
+ return END_OF_STREAM;
+
+ return accessor.getByte(data, pos++) & 0xFF;
+ }
+ };
+ }
+
+ /**
+ * Constructs a byte-comparable representation of the number in the legacy format.
+ * We represent it as
+ * <zero or more length_bytes where length = 128> <length_byte> <first_significant_byte> <zero or more bytes>
+ * where a length_byte is:
+ * - 0x80 + (length - 1) for positive numbers (so that longer length sorts bigger)
+ * - 0x7F - (length - 1) for negative numbers (so that longer length sorts smaller)
+ *
+ * Because we include the sign in the length byte:
+ * - unlike fixed-length ints, we don't need to sign-invert the first significant byte,
+ * - unlike BigInteger, we don't need to include 0x00 prefix for positive integers whose first byte is >= 0x80
+ * or 0xFF prefix for negative integers whose first byte is < 0x80.
+ *
+ * The representations are prefix-free, because representations of different length always have length bytes that
+ * differ.
+ *
+ * Examples:
+ * 0 as 8000
+ * 1 as 8001
+ * 127 as 807F
+ * 255 as 80FF
+ * 2^31-1 as 837FFFFFFF
+ * 2^31 as 8380000000
+ * 2^32 as 840100000000
+ */
+ private <V> ByteSource asComparableBytesLegacy(ValueAccessor<V> accessor, V data, int startpos, int limit, int signbyte)
+ {
+ return new ByteSource()
+ {
+ int pos = startpos;
+ int sizeToReport = limit - startpos;
+ boolean sizeReported = false;
+
+ public int next()
+ {
+ if (!sizeReported)
+ {
+ if (sizeToReport >= 128)
+ {
+ sizeToReport -= 128;
+ return signbyte >= 0
+ ? POSITIVE_VARINT_LENGTH_HEADER
+ : NEGATIVE_VARINT_LENGTH_HEADER;
+ }
+ else
+ {
+ sizeReported = true;
+ return signbyte >= 0
+ ? POSITIVE_VARINT_HEADER + (sizeToReport - 1)
+ : POSITIVE_VARINT_HEADER - sizeToReport;
+ }
+ }
+
+ if (pos == limit)
+ return END_OF_STREAM;
+
+ return accessor.getByte(data, pos++) & 0xFF;
+ }
+ };
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ assert version != ByteComparable.Version.LEGACY;
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ // Consume the first byte to determine whether the encoded number is positive and
+ // start iterating through the length header bytes and collecting the number of value bytes.
+ int sign = comparableBytes.peek() ^ 0xFF; // FF if negative, 00 if positive
+ if (sign != 0xFF && sign != 0x00)
+ return extractVarIntBytes(accessor, ByteSourceInverse.getVariableLengthInteger(comparableBytes));
+
+ // consume the sign byte
+ comparableBytes.next();
+
+ // Read the length (inverted if the number is negative)
+ int valueBytes = Math.toIntExact(ByteSourceInverse.getVariableLengthUnsignedIntegerXoring(comparableBytes, sign) + FULL_FORM_THRESHOLD);
+ // Get the bytes.
+ return extractBytes(accessor, comparableBytes, sign, valueBytes);
+ }
+
+ private <V> V extractVarIntBytes(ValueAccessor<V> accessor, long value)
+ {
+ int length = (64 - Long.numberOfLeadingZeros(value ^ (value >> 63)) + 8) / 8; // number of bytes needed: 7 bits -> one byte, 8 bits -> 2 bytes
+ V buf = accessor.allocate(length);
+ switch (length)
+ {
+ case 1:
+ accessor.putByte(buf, 0, (byte) value);
+ break;
+ case 2:
+ accessor.putShort(buf, 0, (short) value);
+ break;
+ case 3:
+ accessor.putShort(buf, 0, (short) (value >> 8));
+ accessor.putByte(buf, 2, (byte) value);
+ break;
+ case 4:
+ accessor.putInt(buf, 0, (int) value);
+ break;
+ case 5:
+ accessor.putInt(buf, 0, (int) (value >> 8));
+ accessor.putByte(buf, 4, (byte) value);
+ break;
+ case 6:
+ accessor.putInt(buf, 0, (int) (value >> 16));
+ accessor.putShort(buf, 4, (short) value);
+ break;
+ case 7:
+ accessor.putInt(buf, 0, (int) (value >> 24));
+ accessor.putShort(buf, 4, (short) (value >> 8));
+ accessor.putByte(buf, 6, (byte) value);
+ break;
+ case 8:
+ // This is not reachable within the encoding; added for completeness.
+ accessor.putLong(buf, 0, value);
+ break;
+ default:
+ throw new AssertionError();
+ }
+ return buf;
+ }
+
+ private <V> V extractBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, int sign, int valueBytes)
+ {
+ int writtenBytes = 0;
+ V buf;
+ // Add "leading zero" if needed (i.e. in case the leading byte of a positive number corresponds to a negative
+ // value, or in case the leading byte of a negative number corresponds to a non-negative value).
+ // Size the array containing all the value bytes accordingly.
+ int curr = comparableBytes.next();
+ if ((curr & 0x80) != (sign & 0x80))
+ {
+ ++valueBytes;
+ buf = accessor.allocate(valueBytes);
+ accessor.putByte(buf, writtenBytes++, (byte) sign);
+ }
+ else
+ buf = accessor.allocate(valueBytes);
+ // Don't forget to add the first consumed value byte after determining whether leading zero should be added
+ // and sizing the value bytes array.
+ accessor.putByte(buf, writtenBytes++, (byte) curr);
+
+ // Consume exactly the number of expected value bytes.
+ while (writtenBytes < valueBytes)
+ accessor.putByte(buf, writtenBytes++, (byte) comparableBytes.next());
+
+ return buf;
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -252,4 +561,54 @@
{
return decompose(toBigInteger(input).negate());
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return decompose(toBigInteger(input).abs());
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ BigInteger bi = toBigInteger(input);
+ BigDecimal bd = new BigDecimal(bi);
+ BigDecimal result = DecimalType.instance.exp(bd);
+ BigInteger out = result.toBigInteger();
+ return IntegerType.instance.decompose(out);
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ BigInteger bi = toBigInteger(input);
+ if (bi.compareTo(BigInteger.ZERO) <= 0) throw new ArithmeticException("Natural log of number zero or less");
+ BigDecimal bd = new BigDecimal(bi);
+ BigDecimal result = DecimalType.instance.log(bd);
+ BigInteger out = result.toBigInteger();
+ return IntegerType.instance.decompose(out);
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ BigInteger bi = toBigInteger(input);
+ if (bi.compareTo(BigInteger.ZERO) <= 0) throw new ArithmeticException("Log10 of number zero or less");
+ BigDecimal bd = new BigDecimal(bi);
+ BigDecimal result = DecimalType.instance.log10(bd);
+ BigInteger out = result.toBigInteger();
+ return IntegerType.instance.decompose(out);
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.clone(input);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/LegacyTimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/LegacyTimeUUIDType.java
index 528a3e8..62280b9 100644
--- a/src/java/org/apache/cassandra/db/marshal/LegacyTimeUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/LegacyTimeUUIDType.java
@@ -17,18 +17,28 @@
*/
package org.apache.cassandra.db.marshal;
+import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.UUIDSerializer;
+import org.apache.cassandra.utils.TimeUUID;
// Fully compatible with UUID, and indeed is interpreted as UUID for UDF
public class LegacyTimeUUIDType extends AbstractTimeUUIDType<UUID>
{
public static final LegacyTimeUUIDType instance = new LegacyTimeUUIDType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(TimeUUID.minAtUnixMillis(0).asUUID());
+
public TypeSerializer<UUID> getSerializer()
{
return UUIDSerializer.instance;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java
index 6dd4161..eac94a4 100644
--- a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java
@@ -26,11 +26,16 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.UUIDSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class LexicalUUIDType extends AbstractType<UUID>
{
public static final LexicalUUIDType instance = new LexicalUUIDType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(UUID.fromString("00000000-0000-0000-0000-000000000000"));
+
LexicalUUIDType()
{
super(ComparisonType.CUSTOM);
@@ -48,6 +53,46 @@
return accessorL.toUUID(left).compareTo(accessorR.toUUID(right));
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ if (data == null || accessor.isEmpty(data))
+ return null;
+
+ // fixed-length (hence prefix-free) representation, but
+ // we have to sign-flip the highest bytes of the two longs
+ return new ByteSource()
+ {
+ int bufpos = 0;
+
+ public int next()
+ {
+ if (bufpos >= accessor.size(data))
+ return END_OF_STREAM;
+ int v = accessor.getByte(data, bufpos) & 0xFF;
+ if (bufpos == 0 || bufpos == 8)
+ v ^= 0x80;
+ ++bufpos;
+ return v;
+ }
+ };
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ // Optional-style encoding of empty values as null sources
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ long hiBits = ByteSourceInverse.getSignedLong(comparableBytes);
+ long loBits = ByteSourceInverse.getSignedLong(comparableBytes);
+
+ // Lexical UUIDs are stored as just two signed longs. The decoding of these longs flips their sign bit back, so
+ // they can directly be used for constructing the original UUID.
+ return UUIDType.makeUuidBytes(accessor, hiBits, loBits);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -88,4 +133,10 @@
{
return 16;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java
index 281f7ee..159680d 100644
--- a/src/java/org/apache/cassandra/db/marshal/ListType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ListType.java
@@ -18,21 +18,25 @@
package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
-import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.Lists;
import org.apache.cassandra.cql3.Term;
-import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.ListSerializer;
import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.JsonUtils;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
public class ListType<T> extends CollectionType<List<T>>
{
@@ -156,14 +160,14 @@
public boolean isCompatibleWithFrozen(CollectionType<?> previous)
{
assert !isMultiCell;
- return this.elements.isCompatibleWith(((ListType) previous).elements);
+ return this.elements.isCompatibleWith(((ListType<?>) previous).elements);
}
@Override
public boolean isValueCompatibleWithFrozen(CollectionType<?> previous)
{
assert !isMultiCell;
- return this.elements.isValueCompatibleWithInternal(((ListType) previous).elements);
+ return this.elements.isValueCompatibleWithInternal(((ListType<?>) previous).elements);
}
public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
@@ -171,29 +175,16 @@
return compareListOrSet(elements, left, accessorL, right, accessorR);
}
- static <VL, VR> int compareListOrSet(AbstractType<?> elementsComparator, VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
{
- // Note that this is only used if the collection is frozen
- if (accessorL.isEmpty(left) || accessorR.isEmpty(right))
- return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left));
+ return asComparableBytesListOrSet(getElementsType(), accessor, data, version);
+ }
- int sizeL = CollectionSerializer.readCollectionSize(left, accessorL, ProtocolVersion.V3);
- int offsetL = CollectionSerializer.sizeOfCollectionSize(sizeL, ProtocolVersion.V3);
- int sizeR = CollectionSerializer.readCollectionSize(right, accessorR, ProtocolVersion.V3);
- int offsetR = TypeSizes.INT_SIZE;
-
- for (int i = 0; i < Math.min(sizeL, sizeR); i++)
- {
- VL v1 = CollectionSerializer.readValue(left, accessorL, offsetL, ProtocolVersion.V3);
- offsetL += CollectionSerializer.sizeOfValue(v1, accessorL, ProtocolVersion.V3);
- VR v2 = CollectionSerializer.readValue(right, accessorR, offsetR, ProtocolVersion.V3);
- offsetR += CollectionSerializer.sizeOfValue(v2, accessorR, ProtocolVersion.V3);
- int cmp = elementsComparator.compare(v1, accessorL, v2, accessorR);
- if (cmp != 0)
- return cmp;
- }
-
- return sizeL == sizeR ? 0 : (sizeL < sizeR ? -1 : 1);
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, Version version)
+ {
+ return fromComparableBytesListOrSet(accessor, comparableBytes, version, getElementsType());
}
@Override
@@ -224,13 +215,13 @@
public Term fromJSONObject(Object parsed) throws MarshalException
{
if (parsed instanceof String)
- parsed = Json.decodeJson((String) parsed);
+ parsed = JsonUtils.decodeJson((String) parsed);
if (!(parsed instanceof List))
throw new MarshalException(String.format(
"Expected a list, but got a %s: %s", parsed.getClass().getSimpleName(), parsed));
- List list = (List) parsed;
+ List<?> list = (List<?>) parsed;
List<Term> terms = new ArrayList<>(list.size());
for (Object element : list)
{
@@ -242,23 +233,6 @@
return new Lists.DelayedValue(terms);
}
- public static String setOrListToJsonString(ByteBuffer buffer, AbstractType elementsType, ProtocolVersion protocolVersion)
- {
- ByteBuffer value = buffer.duplicate();
- StringBuilder sb = new StringBuilder("[");
- int size = CollectionSerializer.readCollectionSize(value, protocolVersion);
- int offset = CollectionSerializer.sizeOfCollectionSize(size, protocolVersion);
- for (int i = 0; i < size; i++)
- {
- if (i > 0)
- sb.append(", ");
- ByteBuffer element = CollectionSerializer.readValue(value, ByteBufferAccessor.instance, offset, protocolVersion);
- offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance, protocolVersion);
- sb.append(elementsType.toJSONString(element, protocolVersion));
- }
- return sb.append("]").toString();
- }
-
public ByteBuffer getSliceFromSerialized(ByteBuffer collection, ByteBuffer from, ByteBuffer to)
{
// We don't support slicing on lists so we don't need that function
@@ -270,4 +244,16 @@
{
return setOrListToJsonString(buffer, elements, protocolVersion);
}
+
+ @Override
+ public void forEach(ByteBuffer input, Consumer<ByteBuffer> action)
+ {
+ serializer.forEach(input, action);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return decompose(Collections.emptyList());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/LongType.java b/src/java/org/apache/cassandra/db/marshal/LongType.java
index ad539f7..c8095fc 100644
--- a/src/java/org/apache/cassandra/db/marshal/LongType.java
+++ b/src/java/org/apache/cassandra/db/marshal/LongType.java
@@ -28,11 +28,16 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class LongType extends NumberType<Long>
{
public static final LongType instance = new LongType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(0L);
+
LongType() {super(ComparisonType.CUSTOM);} // singleton
public boolean isEmptyValueMeaningless()
@@ -57,6 +62,28 @@
return ValueAccessor.compare(left, accessorL, right, accessorR);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+ if (version == ByteComparable.Version.LEGACY)
+ return ByteSource.signedFixedLengthNumber(accessor, data);
+ else
+ return ByteSource.variableLengthInteger(accessor.getLong(data, 0));
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ if (comparableBytes == null)
+ return accessor.empty();
+ if (version == ByteComparable.Version.LEGACY)
+ return ByteSourceInverse.getSignedFixedLength(accessor, comparableBytes, 8);
+ else
+ return accessor.valueOf(ByteSourceInverse.getVariableLengthInteger(comparableBytes));
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -173,4 +200,40 @@
{
return ByteBufferUtil.bytes(-toLong(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes(Math.abs(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((long) Math.exp(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((long) Math.log(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((long) Math.log10(toLong(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.clone(input);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java
index 9473e29..40cd8bf 100644
--- a/src/java/org/apache/cassandra/db/marshal/MapType.java
+++ b/src/java/org/apache/cassandra/db/marshal/MapType.java
@@ -18,19 +18,30 @@
package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
-import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.Maps;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.serializers.CollectionSerializer;
-import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.MapSerializer;
+import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.JsonUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.cassandra.utils.Pair;
public class MapType<K, V> extends CollectionType<Map<K, V>>
@@ -166,7 +177,7 @@
public boolean isCompatibleWithFrozen(CollectionType<?> previous)
{
assert !isMultiCell;
- MapType tprev = (MapType) previous;
+ MapType<?, ?> tprev = (MapType<?, ?>) previous;
return keys.isCompatibleWith(tprev.keys) && values.isCompatibleWith(tprev.values);
}
@@ -174,7 +185,7 @@
public boolean isValueCompatibleWithFrozen(CollectionType<?> previous)
{
assert !isMultiCell;
- MapType tprev = (MapType) previous;
+ MapType<?, ?> tprev = (MapType<?, ?>) previous;
return keys.isCompatibleWith(tprev.keys) && values.isValueCompatibleWith(tprev.values);
}
@@ -189,33 +200,95 @@
return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left));
- ProtocolVersion protocolVersion = ProtocolVersion.V3;
- int sizeL = CollectionSerializer.readCollectionSize(left, accessorL, protocolVersion);
- int sizeR = CollectionSerializer.readCollectionSize(right, accessorR, protocolVersion);
+ int sizeL = CollectionSerializer.readCollectionSize(left, accessorL);
+ int sizeR = CollectionSerializer.readCollectionSize(right, accessorR);
- int offsetL = CollectionSerializer.sizeOfCollectionSize(sizeL, protocolVersion);
- int offsetR = CollectionSerializer.sizeOfCollectionSize(sizeR, protocolVersion);
+ int offsetL = CollectionSerializer.sizeOfCollectionSize();
+ int offsetR = CollectionSerializer.sizeOfCollectionSize();
for (int i = 0; i < Math.min(sizeL, sizeR); i++)
{
- TL k1 = CollectionSerializer.readValue(left, accessorL, offsetL, protocolVersion);
- offsetL += CollectionSerializer.sizeOfValue(k1, accessorL, protocolVersion);
- TR k2 = CollectionSerializer.readValue(right, accessorR, offsetR, protocolVersion);
- offsetR += CollectionSerializer.sizeOfValue(k2, accessorR, protocolVersion);
+ TL k1 = CollectionSerializer.readValue(left, accessorL, offsetL);
+ offsetL += CollectionSerializer.sizeOfValue(k1, accessorL);
+ TR k2 = CollectionSerializer.readValue(right, accessorR, offsetR);
+ offsetR += CollectionSerializer.sizeOfValue(k2, accessorR);
int cmp = keysComparator.compare(k1, accessorL, k2, accessorR);
if (cmp != 0)
return cmp;
- TL v1 = CollectionSerializer.readValue(left, accessorL, offsetL, protocolVersion);
- offsetL += CollectionSerializer.sizeOfValue(v1, accessorL, protocolVersion);
- TR v2 = CollectionSerializer.readValue(right, accessorR, offsetR, protocolVersion);
- offsetR += CollectionSerializer.sizeOfValue(v2, accessorR, protocolVersion);
+ TL v1 = CollectionSerializer.readValue(left, accessorL, offsetL);
+ offsetL += CollectionSerializer.sizeOfValue(v1, accessorL);
+ TR v2 = CollectionSerializer.readValue(right, accessorR, offsetR);
+ offsetR += CollectionSerializer.sizeOfValue(v2, accessorR);
cmp = valuesComparator.compare(v1, accessorL, v2, accessorR);
if (cmp != 0)
return cmp;
}
- return sizeL == sizeR ? 0 : (sizeL < sizeR ? -1 : 1);
+ return Integer.compare(sizeL, sizeR);
+ }
+
+ @Override
+ public <T> ByteSource asComparableBytes(ValueAccessor<T> accessor, T data, Version version)
+ {
+ return asComparableBytesMap(getKeysType(), getValuesType(), accessor, data, version);
+ }
+
+ @Override
+ public <T> T fromComparableBytes(ValueAccessor<T> accessor, ByteSource.Peekable comparableBytes, Version version)
+ {
+ return fromComparableBytesMap(accessor, comparableBytes, version, getKeysType(), getValuesType());
+ }
+
+ static <V> ByteSource asComparableBytesMap(AbstractType<?> keysComparator,
+ AbstractType<?> valuesComparator,
+ ValueAccessor<V> accessor,
+ V data,
+ Version version)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+
+ int offset = 0;
+ int size = CollectionSerializer.readCollectionSize(data, accessor);
+ offset += CollectionSerializer.sizeOfCollectionSize();
+ ByteSource[] srcs = new ByteSource[size * 2];
+ for (int i = 0; i < size; ++i)
+ {
+ V k = CollectionSerializer.readValue(data, accessor, offset);
+ offset += CollectionSerializer.sizeOfValue(k, accessor);
+ srcs[i * 2 + 0] = keysComparator.asComparableBytes(accessor, k, version);
+ V v = CollectionSerializer.readValue(data, accessor, offset);
+ offset += CollectionSerializer.sizeOfValue(v, accessor);
+ srcs[i * 2 + 1] = valuesComparator.asComparableBytes(accessor, v, version);
+ }
+ return ByteSource.withTerminatorMaybeLegacy(version, 0x00, srcs);
+ }
+
+ static <V> V fromComparableBytesMap(ValueAccessor<V> accessor,
+ ByteSource.Peekable comparableBytes,
+ Version version,
+ AbstractType<?> keysComparator,
+ AbstractType<?> valuesComparator)
+ {
+ if (comparableBytes == null)
+ return accessor.empty();
+ assert version != ByteComparable.Version.LEGACY; // legacy translation is not reversible
+
+ List<V> buffers = new ArrayList<>();
+ int separator = comparableBytes.next();
+ while (separator != ByteSource.TERMINATOR)
+ {
+ buffers.add(ByteSourceInverse.nextComponentNull(separator)
+ ? null
+ : keysComparator.fromComparableBytes(accessor, comparableBytes, version));
+ separator = comparableBytes.next();
+ buffers.add(ByteSourceInverse.nextComponentNull(separator)
+ ? null
+ : valuesComparator.fromComparableBytes(accessor, comparableBytes, version));
+ separator = comparableBytes.next();
+ }
+ return CollectionSerializer.pack(buffers, accessor,buffers.size() / 2);
}
@Override
@@ -260,15 +333,15 @@
public Term fromJSONObject(Object parsed) throws MarshalException
{
if (parsed instanceof String)
- parsed = Json.decodeJson((String) parsed);
+ parsed = JsonUtils.decodeJson((String) parsed);
if (!(parsed instanceof Map))
throw new MarshalException(String.format(
"Expected a map, but got a %s: %s", parsed.getClass().getSimpleName(), parsed));
- Map<Object, Object> map = (Map<Object, Object>) parsed;
+ Map<?, ?> map = (Map<?, ?>) parsed;
Map<Term, Term> terms = new HashMap<>(map.size());
- for (Map.Entry<Object, Object> entry : map.entrySet())
+ for (Map.Entry<?, ?> entry : map.entrySet())
{
if (entry.getKey() == null)
throw new MarshalException("Invalid null key in map");
@@ -286,27 +359,39 @@
{
ByteBuffer value = buffer.duplicate();
StringBuilder sb = new StringBuilder("{");
- int size = CollectionSerializer.readCollectionSize(value, protocolVersion);
- int offset = CollectionSerializer.sizeOfCollectionSize(size, protocolVersion);
+ int size = CollectionSerializer.readCollectionSize(value, ByteBufferAccessor.instance);
+ int offset = CollectionSerializer.sizeOfCollectionSize();
for (int i = 0; i < size; i++)
{
if (i > 0)
sb.append(", ");
// map keys must be JSON strings, so convert non-string keys to strings
- ByteBuffer kv = CollectionSerializer.readValue(value, ByteBufferAccessor.instance, offset, protocolVersion);
- offset += CollectionSerializer.sizeOfValue(kv, ByteBufferAccessor.instance, protocolVersion);
+ ByteBuffer kv = CollectionSerializer.readValue(value, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(kv, ByteBufferAccessor.instance);
String key = keys.toJSONString(kv, protocolVersion);
if (key.startsWith("\""))
sb.append(key);
else
- sb.append('"').append(Json.quoteAsJsonString(key)).append('"');
+ sb.append('"').append(JsonUtils.quoteAsJsonString(key)).append('"');
sb.append(": ");
- ByteBuffer vv = CollectionSerializer.readValue(value, ByteBufferAccessor.instance, offset, protocolVersion);
- offset += CollectionSerializer.sizeOfValue(vv, ByteBufferAccessor.instance, protocolVersion);
+ ByteBuffer vv = CollectionSerializer.readValue(value, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(vv, ByteBufferAccessor.instance);
sb.append(values.toJSONString(vv, protocolVersion));
}
return sb.append("}").toString();
}
+
+ @Override
+ public void forEach(ByteBuffer input, Consumer<ByteBuffer> action)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return decompose(Collections.emptyMap());
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/NumberType.java b/src/java/org/apache/cassandra/db/marshal/NumberType.java
index 9e7697f..5b10720 100644
--- a/src/java/org/apache/cassandra/db/marshal/NumberType.java
+++ b/src/java/org/apache/cassandra/db/marshal/NumberType.java
@@ -220,4 +220,53 @@
* @return the negated argument
*/
public abstract ByteBuffer negate(ByteBuffer input);
+
+ /**
+ * Takes the absolute value of the argument.
+ *
+ * @param input the argument to take the absolute value of
+ * @return a ByteBuffer containing the absolute value of the argument. The type of the contents of the ByteBuffer
+ * will match that of the input.
+ */
+ public abstract ByteBuffer abs(ByteBuffer input);
+
+ /**
+ * Raises e to the power of the argument.
+ *
+ * @param input the argument to raise e to
+ * @return a ByteBuffer containg the value of e (the base of natural logarithms) raised to the power of the
+ * argument. The type of the contents of the ByteBuffer will be a double, unless the input is a DecimalType or
+ * IntegerType, in which the ByteBuffer will contain a DecimalType.
+ */
+ public abstract ByteBuffer exp(ByteBuffer input);
+
+ /**
+ * Takes the natural logrithm of the argument.
+ *
+ * @param input the argument to take the natural log (ln) of
+ * @return a ByteBuffer containg the log base e (ln) of the argument. The type of the contents of the ByteBuffer
+ * will be a double, unless the input is a DecimalType or IntegerType, in which the ByteBuffer will contain a
+ * DecimalType.
+ */
+ public abstract ByteBuffer log(ByteBuffer input);
+
+ /**
+ * Takes the log base 10 of the arguement.
+ *
+ * @param input the argument to take the log base ten of
+ * @return a ByteBuffer containg the log base 10 of the argument. The type of the contents of the ByteBuffer
+ * will be a double, unless the input is a DecimalType or IntegerType, in which the ByteBuffer will contain a
+ * DecimalType.
+ */
+ public abstract ByteBuffer log10(ByteBuffer input);
+
+ /**
+ * Rounds the argument to the nearest whole number.
+ *
+ * @param input the argument to round
+ * @return a ByteBuffer containg the rounded argument. If the input is an integral type, the ByteBuffer will contain
+ * a copy of the input. If the input is a float, the ByteBuffer will contain an int32. If the input is a double, the
+ * output will contain a long. If the input is a DecimalType, the output will contain an IntegerType.
+ */
+ public abstract ByteBuffer round(ByteBuffer input);
}
diff --git a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java
index 89241b4..1ad1aea 100644
--- a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java
+++ b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java
@@ -22,11 +22,15 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.FBUtilities;
/** for sorting columns representing row keys in the row ordering as determined by a partitioner.
@@ -94,6 +98,33 @@
}
@Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+ {
+ // Partitioners work with ByteBuffers only.
+ ByteBuffer buf = ByteBufferAccessor.instance.convert(data, accessor);
+ if (version != Version.LEGACY)
+ {
+ // For ByteComparable.Version.OSS50 and above we encode an empty key with a null byte source. This
+ // way we avoid the need to special-handle a sentinel value when we decode the byte source for such a key
+ // (e.g. for ByteComparable.Version.Legacy we use the minimum key bound of the partitioner's minimum token as
+ // a sentinel value, and that results in the need to go twice through the byte source that is being
+ // decoded).
+ return buf.hasRemaining() ? partitioner.decorateKey(buf).asComparableBytes(version) : null;
+ }
+ return PartitionPosition.ForKey.get(buf, partitioner).asComparableBytes(version);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ assert version != Version.LEGACY;
+ if (comparableBytes == null)
+ return accessor.empty();
+ byte[] keyBytes = DecoratedKey.keyFromByteSource(comparableBytes, version, partitioner);
+ return accessor.valueOf(keyBytes);
+ }
+
+ @Override
public void validate(ByteBuffer bytes) throws MarshalException
{
throw new IllegalStateException("You shouldn't be validating this.");
diff --git a/src/java/org/apache/cassandra/db/marshal/ReversedType.java b/src/java/org/apache/cassandra/db/marshal/ReversedType.java
index ceea84a..079836d 100644
--- a/src/java/org/apache/cassandra/db/marshal/ReversedType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ReversedType.java
@@ -28,6 +28,8 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
public class ReversedType<T> extends AbstractType<T>
{
@@ -63,6 +65,32 @@
return baseType.isEmptyValueMeaningless();
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ ByteSource src = baseType.asComparableBytes(accessor, data, version);
+ if (src == null) // Note: this will only compare correctly if used within a sequence
+ return null;
+ // Invert all bytes.
+ // The comparison requirements for the original type ensure that this encoding will compare correctly with
+ // respect to the reversed comparator function (and, specifically, prefixes of escaped byte-ordered types will
+ // compare as larger). Additionally, the weak prefix-freedom requirement ensures this encoding will also be
+ // weakly prefix-free.
+ return () ->
+ {
+ int v = src.next();
+ if (v == ByteSource.END_OF_STREAM)
+ return v;
+ return v ^ 0xFF;
+ };
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return baseType.fromComparableBytes(accessor, ReversedPeekableByteSource.of(comparableBytes), version);
+ }
+
public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
{
return baseType.compare(right, accessorR, left, accessorL);
@@ -156,4 +184,44 @@
{
return getClass().getName() + "(" + baseType + ")";
}
+
+ private static final class ReversedPeekableByteSource extends ByteSource.Peekable
+ {
+ private final ByteSource.Peekable original;
+
+ static ByteSource.Peekable of(ByteSource.Peekable original)
+ {
+ return original != null ? new ReversedPeekableByteSource(original) : null;
+ }
+
+ private ReversedPeekableByteSource(ByteSource.Peekable original)
+ {
+ super(null);
+ this.original = original;
+ }
+
+ @Override
+ public int next()
+ {
+ int v = original.next();
+ if (v != END_OF_STREAM)
+ return v ^ 0xFF;
+ return END_OF_STREAM;
+ }
+
+ @Override
+ public int peek()
+ {
+ int v = original.peek();
+ if (v != END_OF_STREAM)
+ return v ^ 0xFF;
+ return END_OF_STREAM;
+ }
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return baseType.getMaskedValue();
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java
index e5bdada..1bb1ed5 100644
--- a/src/java/org/apache/cassandra/db/marshal/SetType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SetType.java
@@ -20,8 +20,8 @@
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
-import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.Sets;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.db.rows.Cell;
@@ -30,6 +30,9 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.SetSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.JsonUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
public class SetType<T> extends CollectionType<Set<T>>
{
@@ -142,7 +145,7 @@
public boolean isCompatibleWithFrozen(CollectionType<?> previous)
{
assert !isMultiCell;
- return this.elements.isCompatibleWith(((SetType) previous).elements);
+ return this.elements.isCompatibleWith(((SetType<?>) previous).elements);
}
@Override
@@ -154,7 +157,19 @@
public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR)
{
- return ListType.compareListOrSet(elements, left, accessorL, right, accessorR);
+ return compareListOrSet(elements, left, accessorL, right, accessorR);
+ }
+
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ return asComparableBytesListOrSet(getElementsType(), accessor, data, version);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return fromComparableBytesListOrSet(accessor, comparableBytes, version, getElementsType());
}
public SetSerializer<T> getSerializer()
@@ -179,7 +194,7 @@
public List<ByteBuffer> serializedValues(Iterator<Cell<?>> cells)
{
- List<ByteBuffer> bbs = new ArrayList<ByteBuffer>();
+ List<ByteBuffer> bbs = new ArrayList<>();
while (cells.hasNext())
bbs.add(cells.next().path().get(0));
return bbs;
@@ -189,13 +204,13 @@
public Term fromJSONObject(Object parsed) throws MarshalException
{
if (parsed instanceof String)
- parsed = Json.decodeJson((String) parsed);
+ parsed = JsonUtils.decodeJson((String) parsed);
if (!(parsed instanceof List))
throw new MarshalException(String.format(
"Expected a list (representing a set), but got a %s: %s", parsed.getClass().getSimpleName(), parsed));
- List list = (List) parsed;
+ List<?> list = (List<?>) parsed;
Set<Term> terms = new HashSet<>(list.size());
for (Object element : list)
{
@@ -210,6 +225,18 @@
@Override
public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
{
- return ListType.setOrListToJsonString(buffer, elements, protocolVersion);
+ return setOrListToJsonString(buffer, elements, protocolVersion);
+ }
+
+ @Override
+ public void forEach(ByteBuffer input, Consumer<ByteBuffer> action)
+ {
+ serializer.forEach(input, action);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return decompose(Collections.emptySet());
}
}
diff --git a/src/java/org/apache/cassandra/db/marshal/ShortType.java b/src/java/org/apache/cassandra/db/marshal/ShortType.java
index 03dcf5d..24d8e92 100644
--- a/src/java/org/apache/cassandra/db/marshal/ShortType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ShortType.java
@@ -28,11 +28,16 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class ShortType extends NumberType<Short>
{
public static final ShortType instance = new ShortType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose((short) 0);
+
ShortType()
{
super(ComparisonType.CUSTOM);
@@ -46,6 +51,19 @@
return ValueAccessor.compare(left, accessorL, right, accessorR);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ // This type does not allow non-present values, but we do just to avoid future complexity.
+ return ByteSource.optionalSignedFixedLengthNumber(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalSignedFixedLength(accessor, comparableBytes, 2);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -134,4 +152,40 @@
{
return ByteBufferUtil.bytes((short) -toShort(input));
}
+
+ @Override
+ public ByteBuffer abs(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((short) Math.abs(toShort(input)));
+ }
+
+ @Override
+ public ByteBuffer exp(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((short) Math.exp(toShort(input)));
+ }
+
+ @Override
+ public ByteBuffer log(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((short) Math.log(toShort(input)));
+ }
+
+ @Override
+ public ByteBuffer log10(ByteBuffer input)
+ {
+ return ByteBufferUtil.bytes((short) Math.log10(toShort(input)));
+ }
+
+ @Override
+ public ByteBuffer round(ByteBuffer input)
+ {
+ return ByteBufferUtil.clone(input);
+ }
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
index 8f1d677..a474d39 100644
--- a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java
@@ -28,6 +28,10 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
@@ -35,8 +39,24 @@
{
public static final SimpleDateType instance = new SimpleDateType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(SimpleDateSerializer.timeInMillisToDay(0));
+
SimpleDateType() {super(ComparisonType.BYTE_ORDER);} // singleton
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+ {
+ // While BYTE_ORDER would still work for this type, making use of the fixed length is more efficient.
+ // This type does not allow non-present values, but we do just to avoid future complexity.
+ return ByteSource.optionalFixedLength(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalFixedLength(accessor, comparableBytes, 4);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
return ByteBufferUtil.bytes(SimpleDateSerializer.dateStringToDays(source));
@@ -98,4 +118,10 @@
if (!duration.hasDayPrecision())
throw invalidRequest("The duration must have a day precision. Was: %s", duration);
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TimeType.java b/src/java/org/apache/cassandra/db/marshal/TimeType.java
index fd8fac4..d750095 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimeType.java
@@ -28,6 +28,10 @@
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
/**
* Nanosecond resolution time values
@@ -35,6 +39,8 @@
public class TimeType extends TemporalType<Long>
{
public static final TimeType instance = new TimeType();
+
+ private static final ByteBuffer DEFAULT_MASKED_VALUE = instance.decompose(0L);
private TimeType() {super(ComparisonType.BYTE_ORDER);} // singleton
public ByteBuffer fromString(String source) throws MarshalException
@@ -43,6 +49,20 @@
}
@Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+ {
+ // While BYTE_ORDER would still work for this type, making use of the fixed length is more efficient.
+ // This type does not allow non-present values, but we do just to avoid future complexity.
+ return ByteSource.optionalFixedLength(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalFixedLength(accessor, comparableBytes, 8);
+ }
+
+ @Override
public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
{
return this == otherType || otherType == LongType.instance;
@@ -83,4 +103,10 @@
{
return decompose(LocalTime.now(ZoneOffset.UTC).toNanoOfDay());
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return DEFAULT_MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java
index d3b0dec..40e5956 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.db.marshal;
+import java.nio.ByteBuffer;
+
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.utils.TimeUUID;
@@ -25,6 +27,8 @@
{
public static final TimeUUIDType instance = new TimeUUIDType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(TimeUUID.minAtUnixMillis(0));
+
public TypeSerializer<TimeUUID> getSerializer()
{
return TimeUUID.Serializer.instance;
@@ -35,4 +39,10 @@
{
return LegacyTimeUUIDType.instance;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TimestampType.java b/src/java/org/apache/cassandra/db/marshal/TimestampType.java
index ccf1da3..6797676 100644
--- a/src/java/org/apache/cassandra/db/marshal/TimestampType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TimestampType.java
@@ -32,6 +32,9 @@
import org.apache.cassandra.serializers.TimestampSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
@@ -48,6 +51,8 @@
public static final TimestampType instance = new TimestampType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(new Date(0));
+
private TimestampType() {super(ComparisonType.CUSTOM);} // singleton
public boolean isEmptyValueMeaningless()
@@ -60,6 +65,18 @@
return LongType.compareLongs(left, accessorL, right, accessorR);
}
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ return ByteSource.optionalSignedFixedLengthNumber(accessor, data);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return ByteSourceInverse.getOptionalSignedFixedLength(accessor, comparableBytes, 8);
+ }
+
public ByteBuffer fromString(String source) throws MarshalException
{
// Return an empty ByteBuffer for an empty string.
@@ -155,4 +172,10 @@
if (!duration.hasMillisecondPrecision())
throw invalidRequest("The duration must have a millisecond precision. Was: %s", duration);
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/TupleType.java b/src/java/org/apache/cassandra/db/marshal/TupleType.java
index cc08487..79f9219 100644
--- a/src/java/org/apache/cassandra/db/marshal/TupleType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TupleType.java
@@ -30,11 +30,13 @@
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.serializers.*;
import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JsonUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.transform;
@@ -200,47 +202,136 @@
return true;
}
- /**
- * Split a tuple value into its component values.
- */
- public ByteBuffer[] split(ByteBuffer value)
+ @Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
{
- return split(value, size(), this);
+ switch (version)
+ {
+ case LEGACY:
+ return asComparableBytesLegacy(accessor, data);
+ case OSS50:
+ return asComparableBytesNew(accessor, data, version);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private <V> ByteSource asComparableBytesLegacy(ValueAccessor<V> accessor, V data)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+
+ V[] bufs = split(accessor, data); // this may be shorter than types.size -- other srcs remain null in that case
+ ByteSource[] srcs = new ByteSource[types.size()];
+ for (int i = 0; i < bufs.length; ++i)
+ srcs[i] = bufs[i] != null ? types.get(i).asComparableBytes(accessor, bufs[i], ByteComparable.Version.LEGACY) : null;
+
+ // We always have a fixed number of sources, with the trailing ones possibly being nulls.
+ // This can only result in a prefix if the last type in the tuple allows prefixes. Since that type is required
+ // to be weakly prefix-free, so is the tuple.
+ return ByteSource.withTerminatorLegacy(ByteSource.END_OF_STREAM, srcs);
+ }
+
+ private <V> ByteSource asComparableBytesNew(ValueAccessor<V> accessor, V data, ByteComparable.Version version)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+
+ V[] bufs = split(accessor, data);
+ int lengthWithoutTrailingNulls = 0;
+ for (int i = 0; i < bufs.length; ++i)
+ if (bufs[i] != null)
+ lengthWithoutTrailingNulls = i + 1;
+
+ ByteSource[] srcs = new ByteSource[lengthWithoutTrailingNulls];
+ for (int i = 0; i < lengthWithoutTrailingNulls; ++i)
+ srcs[i] = bufs[i] != null ? types.get(i).asComparableBytes(accessor, bufs[i], version) : null;
+
+ // Because we stop early when there are trailing nulls, there needs to be an explicit terminator to make the
+ // type prefix-free.
+ return ByteSource.withTerminator(ByteSource.TERMINATOR, srcs);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ assert version == ByteComparable.Version.OSS50; // Reverse translation is not supported for the legacy version.
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ V[] componentBuffers = accessor.createArray(types.size());
+ for (int i = 0; i < types.size(); ++i)
+ {
+ if (comparableBytes.peek() == ByteSource.TERMINATOR)
+ break; // the rest of the fields remain null
+ AbstractType<?> componentType = types.get(i);
+ ByteSource.Peekable component = ByteSourceInverse.nextComponentSource(comparableBytes);
+ if (component != null)
+ componentBuffers[i] = componentType.fromComparableBytes(accessor, component, version);
+ else
+ componentBuffers[i] = null;
+ }
+ // consume terminator
+ int terminator = comparableBytes.next();
+ assert terminator == ByteSource.TERMINATOR : String.format("Expected TERMINATOR (0x%2x) after %d components",
+ ByteSource.TERMINATOR,
+ types.size());
+ return buildValue(accessor, componentBuffers);
}
/**
* Split a tuple value into its component values.
*/
- public static ByteBuffer[] split(ByteBuffer value, int numberOfElements, TupleType type)
+ public <V> V[] split(ValueAccessor<V> accessor, V value)
{
- ByteBuffer[] components = new ByteBuffer[numberOfElements];
- ByteBuffer input = value.duplicate();
+ return split(accessor, value, size(), this);
+ }
+
+ /**
+ * Split a tuple value into its component values.
+ */
+ public static <V> V[] split(ValueAccessor<V> accessor, V value, int numberOfElements, TupleType type)
+ {
+ V[] components = accessor.createArray(numberOfElements);
+ int length = accessor.size(value);
+ int position = 0;
for (int i = 0; i < numberOfElements; i++)
{
- if (!input.hasRemaining())
+ if (position == length)
return Arrays.copyOfRange(components, 0, i);
- int size = input.getInt();
-
- if (input.remaining() < size)
+ if (position + 4 > length)
throw new MarshalException(String.format("Not enough bytes to read %dth component", i));
+ int size = accessor.getInt(value, position);
+ position += 4;
+
// size < 0 means null value
- components[i] = size < 0 ? null : ByteBufferUtil.readBytes(input, size);
+ if (size >= 0)
+ {
+ if (position + size > length)
+ throw new MarshalException(String.format("Not enough bytes to read %dth component", i));
+
+ components[i] = accessor.slice(value, position, size);
+ position += size;
+ }
+ else
+ components[i] = null;
}
// error out if we got more values in the tuple/UDT than we expected
- if (input.hasRemaining())
+ if (position < length)
{
- throw new InvalidRequestException(String.format(
- "Expected %s %s for %s column, but got more",
- numberOfElements, numberOfElements == 1 ? "value" : "values", type.asCQL3Type()));
+ throw new MarshalException(String.format("Expected %s %s for %s column, but got more",
+ numberOfElements, numberOfElements == 1 ? "value" : "values",
+ type.asCQL3Type()));
}
return components;
}
- public static <V> V buildValue(ValueAccessor<V> accessor, V[] components)
+ @SafeVarargs
+ public static <V> V buildValue(ValueAccessor<V> accessor, V... components)
{
int totalLength = 0;
for (V component : components)
@@ -264,7 +355,7 @@
return result;
}
- public static ByteBuffer buildValue(ByteBuffer[] components)
+ public static ByteBuffer buildValue(ByteBuffer... components)
{
return buildValue(ByteBufferAccessor.instance, components);
}
@@ -333,13 +424,13 @@
public Term fromJSONObject(Object parsed) throws MarshalException
{
if (parsed instanceof String)
- parsed = Json.decodeJson((String) parsed);
+ parsed = JsonUtils.decodeJson((String) parsed);
if (!(parsed instanceof List))
throw new MarshalException(String.format(
"Expected a list representation of a tuple, but got a %s: %s", parsed.getClass().getSimpleName(), parsed));
- List list = (List) parsed;
+ List<?> list = (List<?>) parsed;
if (list.size() > types.size())
throw new MarshalException(String.format("Tuple contains extra items (expected %s): %s", types.size(), parsed));
@@ -375,8 +466,8 @@
if (i > 0)
sb.append(", ");
- ByteBuffer value = CollectionSerializer.readValue(duplicated, ByteBufferAccessor.instance, offset, protocolVersion);
- offset += CollectionSerializer.sizeOfValue(value, ByteBufferAccessor.instance, protocolVersion);
+ ByteBuffer value = CollectionSerializer.readValue(duplicated, ByteBufferAccessor.instance, offset);
+ offset += CollectionSerializer.sizeOfValue(value, ByteBufferAccessor.instance);
if (value == null)
sb.append("null");
else
@@ -459,4 +550,17 @@
{
return getClass().getName() + TypeParser.stringifyTypeParameters(types, true);
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ ByteBuffer[] buffers = new ByteBuffer[types.size()];
+ for (int i = 0; i < types.size(); i++)
+ {
+ AbstractType<?> type = types.get(i);
+ buffers[i] = type.getMaskedValue();
+ }
+
+ return serializer.serialize(buildValue(buffers));
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/UTF8Type.java b/src/java/org/apache/cassandra/db/marshal/UTF8Type.java
index e256070..20de5fc 100644
--- a/src/java/org/apache/cassandra/db/marshal/UTF8Type.java
+++ b/src/java/org/apache/cassandra/db/marshal/UTF8Type.java
@@ -22,7 +22,6 @@
import java.nio.charset.StandardCharsets;
import org.apache.cassandra.cql3.Constants;
-import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Term;
@@ -31,11 +30,14 @@
import org.apache.cassandra.serializers.UTF8Serializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JsonUtils;
public class UTF8Type extends StringType
{
public static final UTF8Type instance = new UTF8Type();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose("****");
+
UTF8Type() {super(ComparisonType.BYTE_ORDER);} // singleton
public ByteBuffer fromString(String source)
@@ -62,7 +64,7 @@
{
try
{
- return '"' + Json.quoteAsJsonString(ByteBufferUtil.string(buffer, StandardCharsets.UTF_8)) + '"';
+ return '"' + JsonUtils.quoteAsJsonString(ByteBufferUtil.string(buffer, StandardCharsets.UTF_8)) + '"';
}
catch (CharacterCodingException exc)
{
@@ -87,4 +89,10 @@
{
return UTF8Serializer.instance;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/UUIDType.java b/src/java/org/apache/cassandra/db/marshal/UUIDType.java
index 55ce59d..0de9041 100644
--- a/src/java/org/apache/cassandra/db/marshal/UUIDType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UUIDType.java
@@ -30,6 +30,9 @@
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.UUIDSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.cassandra.utils.UUIDGen;
/**
@@ -46,6 +49,8 @@
{
public static final UUIDType instance = new UUIDType();
+ private static final ByteBuffer MASKED_VALUE = instance.decompose(UUID.fromString("00000000-0000-0000-0000-000000000000"));
+
UUIDType()
{
super(ComparisonType.CUSTOM);
@@ -96,10 +101,72 @@
return c;
}
+ // Amusingly (or not so much), although UUIDType freely takes time UUIDs (UUIDs with version 1), it compares
+ // them differently than TimeUUIDType. This is evident in the least significant bytes comparison (the code
+ // below for UUIDType), where UUIDType treats them as unsigned bytes, while TimeUUIDType compares the bytes
+ // signed. See CASSANDRA-8730 for details around this discrepancy.
return UnsignedLongs.compare(accessorL.getLong(left, 8), accessorR.getLong(right, 8));
}
@Override
+ public <V> ByteSource asComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version v)
+ {
+ if (accessor.isEmpty(data))
+ return null;
+
+ long msb = accessor.getLong(data, 0);
+ long version = ((msb >>> 12) & 0xf);
+ ByteBuffer swizzled = ByteBuffer.allocate(16);
+
+ if (version == 1)
+ swizzled.putLong(0, TimeUUIDType.reorderTimestampBytes(msb));
+ else
+ swizzled.putLong(0, (version << 60) | ((msb >>> 4) & 0x0FFFFFFFFFFFF000L) | (msb & 0xFFFL));
+
+ swizzled.putLong(8, accessor.getLong(data, 8));
+
+ // fixed-length thus prefix-free
+ return ByteSource.fixedLength(swizzled);
+ }
+
+ @Override
+ public <V> V fromComparableBytes(ValueAccessor<V> accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ // Optional-style encoding of empty values as null sources
+ if (comparableBytes == null)
+ return accessor.empty();
+
+ // The UUID bits are stored as an unsigned fixed-length 128-bit integer.
+ long hiBits = ByteSourceInverse.getUnsignedFixedLengthAsLong(comparableBytes, 8);
+ long loBits = ByteSourceInverse.getUnsignedFixedLengthAsLong(comparableBytes, 8);
+
+ long uuidVersion = hiBits >>> 60 & 0xF;
+ if (uuidVersion == 1)
+ {
+ // If the version bits are set to 1, this is a time-based UUID, and its high bits are significantly more
+ // shuffled than in other UUIDs. Revert the shuffle.
+ hiBits = TimeUUIDType.reorderBackTimestampBytes(hiBits);
+ }
+ else
+ {
+ // For non-time UUIDs, the only thing that's needed is to put the version bits back where they were originally.
+ hiBits = hiBits << 4 & 0xFFFFFFFFFFFF0000L
+ | uuidVersion << 12
+ | hiBits & 0x0000000000000FFFL;
+ }
+
+ return makeUuidBytes(accessor, hiBits, loBits);
+ }
+
+ static <V> V makeUuidBytes(ValueAccessor<V> accessor, long high, long low)
+ {
+ V buffer = accessor.allocate(16);
+ accessor.putLong(buffer, 0, high);
+ accessor.putLong(buffer, 8, low);
+ return buffer;
+ }
+
+ @Override
public boolean isValueCompatibleWithInternal(AbstractType<?> otherType)
{
return otherType instanceof UUIDType || otherType instanceof TimeUUIDType;
@@ -173,4 +240,10 @@
{
return 16;
}
+
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ return MASKED_VALUE;
+ }
}
diff --git a/src/java/org/apache/cassandra/db/marshal/UserType.java b/src/java/org/apache/cassandra/db/marshal/UserType.java
index 64bdcdd..28f654e 100644
--- a/src/java/org/apache/cassandra/db/marshal/UserType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UserType.java
@@ -36,10 +36,12 @@
import org.apache.cassandra.serializers.UserTypeSerializer;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JsonUtils;
import org.apache.cassandra.utils.Pair;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TYPE_UDT_CONFLICT_BEHAVIOR;
import static org.apache.cassandra.cql3.ColumnIdentifier.maybeQuote;
/**
@@ -211,7 +213,7 @@
public Term fromJSONObject(Object parsed) throws MarshalException
{
if (parsed instanceof String)
- parsed = Json.decodeJson((String) parsed);
+ parsed = JsonUtils.decodeJson((String) parsed);
if (!(parsed instanceof Map))
throw new MarshalException(String.format(
@@ -219,7 +221,7 @@
Map<String, Object> map = (Map<String, Object>) parsed;
- Json.handleCaseSensitivity(map);
+ JsonUtils.handleCaseSensitivity(map);
List<Term> terms = new ArrayList<>(types.size());
@@ -258,7 +260,7 @@
@Override
public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion)
{
- ByteBuffer[] buffers = split(buffer);
+ ByteBuffer[] buffers = split(ByteBufferAccessor.instance, buffer);
StringBuilder sb = new StringBuilder("{");
for (int i = 0; i < types.size(); i++)
{
@@ -270,7 +272,7 @@
name = "\"" + name + "\"";
sb.append('"');
- sb.append(Json.quoteAsJsonString(name));
+ sb.append(JsonUtils.quoteAsJsonString(name));
sb.append("\": ");
ByteBuffer valueBuffer = (i >= buffers.length) ? null : buffers[i];
@@ -522,18 +524,16 @@
{
throw new AssertionError(String.format("Duplicate names found in UDT %s.%s for column %s; " +
- "to resolve set -D" + UDT_CONFLICT_BEHAVIOR + "=LOG on startup and remove the type",
- maybeQuote(keyspace), maybeQuote(name), maybeQuote(fieldName)));
+ "to resolve set -D%s=LOG on startup and remove the type",
+ maybeQuote(keyspace), maybeQuote(name), maybeQuote(fieldName), TYPE_UDT_CONFLICT_BEHAVIOR.getKey()));
}
};
- private static final String UDT_CONFLICT_BEHAVIOR = "cassandra.type.udt.conflict_behavior";
-
abstract void onConflict(String keyspace, String name, String fieldName);
static ConflictBehavior get()
{
- String value = System.getProperty(UDT_CONFLICT_BEHAVIOR, REJECT.name());
+ String value = TYPE_UDT_CONFLICT_BEHAVIOR.getString(REJECT.name());
return ConflictBehavior.valueOf(value);
}
}
diff --git a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
index a51836e..0bb6677 100644
--- a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
+++ b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
@@ -68,6 +68,7 @@
Cell<V> cell(ColumnMetadata column, long timestamp, int ttl, int localDeletionTime, V value, CellPath path);
Clustering<V> clustering(V... values);
Clustering<V> clustering();
+ Clustering<V> staticClustering();
ClusteringBound<V> bound(ClusteringPrefix.Kind kind, V... values);
ClusteringBound<V> bound(ClusteringPrefix.Kind kind);
ClusteringBoundary<V> boundary(ClusteringPrefix.Kind kind, V... values);
@@ -105,7 +106,6 @@
{
return boundary(reversed ? INCL_END_EXCL_START_BOUNDARY : EXCL_END_INCL_START_BOUNDARY, boundValues);
}
-
}
/**
* @return the size of the given value
@@ -162,7 +162,7 @@
default void writeWithVIntLength(V value, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(size(value));
+ out.writeUnsignedVInt32(size(value));
write(value, out);
}
@@ -331,6 +331,12 @@
Ballot toBallot(V value);
/**
+ * writes the byte value {@param value} to {@param dst} at offset {@param offset}
+ * @return the number of bytes written to {@param value}
+ */
+ int putByte(V dst, int offset, byte value);
+
+ /**
* writes the short value {@param value} to {@param dst} at offset {@param offset}
* @return the number of bytes written to {@param value}
*/
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
index ae12516..8526dac 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
@@ -46,6 +46,7 @@
import org.apache.cassandra.utils.memory.MemtablePool;
import org.apache.cassandra.utils.memory.NativePool;
import org.apache.cassandra.utils.memory.SlabPool;
+import org.github.jamm.Unmetered;
/**
* A memtable that uses memory tracked and maybe allocated via a MemtableAllocator from a MemtablePool.
@@ -57,17 +58,24 @@
public static final MemtablePool MEMORY_POOL = AbstractAllocatorMemtable.createMemtableAllocatorPool();
+ @Unmetered
protected final Owner owner;
+ @Unmetered // total pool size should not be included in memtable's deep size
protected final MemtableAllocator allocator;
// Record the comparator of the CFS at the creation of the memtable. This
// is only used when a user update the CF comparator, to know if the
// memtable was created with the new or old comparator.
+ @Unmetered
protected final ClusteringComparator initialComparator;
+ // As above, used to determine if the memtable needs to be flushed on schema change.
+ @Unmetered
+ public final Factory initialFactory;
private final long creationNano = Clock.Global.nanoTime();
- private static MemtablePool createMemtableAllocatorPool()
+ @VisibleForTesting
+ static MemtablePool createMemtableAllocatorPool()
{
Config.MemtableAllocationType allocationType = DatabaseDescriptor.getMemtableAllocationType();
long heapLimit = DatabaseDescriptor.getMemtableHeapSpaceInMiB() << 20;
@@ -109,6 +117,7 @@
super(metadataRef, commitLogLowerBound);
this.allocator = MEMORY_POOL.newAllocator(metadataRef.toString());
this.initialComparator = metadata.get().comparator;
+ this.initialFactory = metadata().params.memtable.factory();
this.owner = owner;
scheduleFlush();
}
@@ -125,7 +134,7 @@
{
case SCHEMA_CHANGE:
return initialComparator != metadata().comparator // If the CF comparator has changed, because our partitions reference the old one
- || metadata().params.memtable.factory() != factory(); // If a different type of memtable is requested
+ || !initialFactory.equals(metadata().params.memtable.factory()); // If a different type of memtable is requested
case OWNED_RANGES_CHANGE:
return false; // by default we don't use the local ranges, thus this has no effect
default:
@@ -150,8 +159,6 @@
throw new AssertionError("performSnapshot must be implemented if shouldSwitch(SNAPSHOT) can return false.");
}
- protected abstract Factory factory();
-
public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
{
super.switchOut(writeBarrier, commitLogUpperBound);
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
index ca6dbf6..8e50456 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
@@ -35,6 +35,7 @@
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
+import org.github.jamm.Unmetered;
public abstract class AbstractMemtable implements Memtable
{
@@ -48,6 +49,7 @@
// Note: statsCollector has corresponding statistics to the two above, but starts with an epoch value which is not
// correct for their usage.
+ @Unmetered
protected TableMetadataRef metadata;
public AbstractMemtable(TableMetadataRef metadataRef)
@@ -74,13 +76,6 @@
return currentOperations.get();
}
- /**
- * Returns the minTS if one available, otherwise NO_MIN_TIMESTAMP.
- *
- * EncodingStats uses a synthetic epoch TS at 2015. We don't want to leak that (CASSANDRA-18118) so we return NO_MIN_TIMESTAMP instead.
- *
- * @return The minTS or NO_MIN_TIMESTAMP if none available
- */
public long getMinTimestamp()
{
return minTimestamp.get() != EncodingStats.NO_STATS.minTimestamp ? minTimestamp.get() : NO_MIN_TIMESTAMP;
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java
new file mode 100644
index 0000000..0815701
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractShardedMemtable.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.github.jamm.Unmetered;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+
+public abstract class AbstractShardedMemtable extends AbstractAllocatorMemtable
+{
+ private static final Logger logger = LoggerFactory.getLogger(AbstractShardedMemtable.class);
+
+ public static final String SHARDS_OPTION = "shards";
+ public static final String SHARDED_MEMTABLE_CONFIG_OBJECT_NAME = "org.apache.cassandra.db:type=ShardedMemtableConfig";
+ static
+ {
+ MBeanWrapper.instance.registerMBean(new ShardedMemtableConfig(), SHARDED_MEMTABLE_CONFIG_OBJECT_NAME, MBeanWrapper.OnException.LOG);
+ }
+
+ // default shard count, used when a specific number of shards is not specified in the options
+ private static volatile int defaultShardCount = MEMTABLE_SHARD_COUNT.getInt(FBUtilities.getAvailableProcessors());
+
+ // The boundaries for the keyspace as they were calculated when the memtable is created.
+ // The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
+ // The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same shard
+ // for a given key, even if we race with the StorageService initialization or with topology changes.
+ @Unmetered
+ protected final ShardBoundaries boundaries;
+
+ AbstractShardedMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound,
+ TableMetadataRef metadataRef,
+ Owner owner,
+ Integer shardCountOption)
+ {
+ super(commitLogLowerBound, metadataRef, owner);
+ int shardCount = shardCountOption != null ? shardCountOption : defaultShardCount;
+ this.boundaries = owner.localRangeSplits(shardCount);
+ }
+
+ private static class ShardedMemtableConfig implements ShardedMemtableConfigMXBean
+ {
+ @Override
+ public void setDefaultShardCount(String shardCount)
+ {
+ if ("auto".equalsIgnoreCase(shardCount))
+ {
+ defaultShardCount = FBUtilities.getAvailableProcessors();
+ }
+ else
+ {
+ try
+ {
+ defaultShardCount = Integer.parseInt(shardCount);
+ }
+ catch (NumberFormatException ex)
+ {
+ logger.warn("Unable to parse {} as valid value for shard count", shardCount);
+ return;
+ }
+ }
+ logger.info("Requested setting shard count to {}; set to: {}", shardCount, defaultShardCount);
+ }
+
+ @Override
+ public String getDefaultShardCount()
+ {
+ return Integer.toString(defaultShardCount);
+ }
+ }
+
+ public static int getDefaultShardCount()
+ {
+ return defaultShardCount;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java
index 1a31374..afe7a12 100644
--- a/src/java/org/apache/cassandra/db/memtable/Flushing.java
+++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java
@@ -27,6 +27,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.DiskBoundaries;
@@ -82,7 +83,9 @@
}
catch (Throwable e)
{
- throw Throwables.propagate(abortRunnables(runnables, e));
+ Throwable t = abortRunnables(runnables, e);
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
}
@@ -95,12 +98,12 @@
Directories.DataDirectory flushLocation)
{
Memtable.FlushablePartitionSet<?> flushSet = memtable.getFlushSet(from, to);
- SSTableFormat.Type formatType = SSTableFormat.Type.current();
- long estimatedSize = formatType.info.getWriterFactory().estimateSize(flushSet);
+ SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
+ long estimatedSize = format.getWriterFactory().estimateSize(flushSet);
Descriptor descriptor = flushLocation == null
- ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), formatType)
- : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), formatType);
+ ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), format)
+ : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), format);
SSTableMultiWriter writer = createFlushWriter(cfs,
flushSet,
diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable_API.md b/src/java/org/apache/cassandra/db/memtable/Memtable_API.md
index 8a582e3..70f8f0b 100644
--- a/src/java/org/apache/cassandra/db/memtable/Memtable_API.md
+++ b/src/java/org/apache/cassandra/db/memtable/Memtable_API.md
@@ -45,13 +45,32 @@
class_name: SkipListMemtable
sharded:
class_name: ShardedSkipListMemtable
+ trie:
+ class_name: TrieMemtable
default:
- inherits: sharded
+ inherits: trie
```
Note that the database will only validate the memtable class and its parameters when a configuration needs to be
instantiated for a table.
+## Implementations provided
+
+Cassandra currently comes with three memtable implementations:
+
+- `SkipListMemtable` is the default and matches the memtable format of Cassandra versions up to 4.1. It organizes
+ partitions into a single concurrent skip list.
+- `ShardedSkipListMemtable` splits the partition skip-list into several independent skip-lists each covering a roughly
+ equal part of the token space served by this node. This reduces congestion of the skip-list from concurrent writes and
+ can lead to improved write throughput. Its configuration takes two parameters:
+ - `shards`: the number of shards to split into, defaulting to the number of CPU cores on the machine.
+ - `serialize_writes`: if false (default), each shard may serve multiple writes in parallel; if true, writes to each
+ shard are synchronized.
+- `TrieMemtable` is a novel solution that organizes partitions into an in-memory trie which places the partition
+ indexing structure in a buffer, off-heap if desired, which significantly improves garbage collection efficiency. It
+ also improves the memtable's space efficiency and lookup performance. Its configuration can take a single parameter
+ `shards` as above.
+
## Memtable selection
Once a configuration has been defined, it can be used by specifying it in the `memtable` parameter of a `CREATE TABLE`
diff --git a/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java b/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java
index fb9cc98..864899f 100644
--- a/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java
+++ b/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java
@@ -22,7 +22,6 @@
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.Token;
@@ -83,7 +82,7 @@
if (boundaries.length == 0)
return 0;
- assert (key.getPartitioner() == DatabaseDescriptor.getPartitioner());
+ assert (key.getPartitioner() == boundaries[0].getPartitioner());
return getShardForToken(key.getToken());
}
diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedMemtableConfigMXBean.java b/src/java/org/apache/cassandra/db/memtable/ShardedMemtableConfigMXBean.java
new file mode 100644
index 0000000..60ff10e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/ShardedMemtableConfigMXBean.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+public interface ShardedMemtableConfigMXBean
+{
+ /**
+ * Adjust the shard count for sharded memtables that do not specify it explicitly in the memtable options.
+ * Changes will apply on the next memtable flush.
+ */
+ public void setDefaultShardCount(String numShards);
+
+ /**
+ * Returns the shard count for sharded memtables that do not specify it explicitly in the memtable options.
+ */
+ public String getDefaultShardCount();
+}
diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
index 51cd5f2..036d742 100644
--- a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
@@ -41,6 +41,7 @@
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
import org.apache.cassandra.db.partitions.AtomicBTreePartition;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -51,10 +52,9 @@
import org.apache.cassandra.dht.IncludingExcludingBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.index.transactions.UpdateTransaction;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.Cloner;
import org.apache.cassandra.utils.memory.MemtableAllocator;
@@ -72,46 +72,26 @@
*
* Also see Memtable_API.md.
*/
-public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
+public class ShardedSkipListMemtable extends AbstractShardedMemtable
{
private static final Logger logger = LoggerFactory.getLogger(ShardedSkipListMemtable.class);
- public static final String SHARDS_OPTION = "shards";
public static final String LOCKING_OPTION = "serialize_writes";
- // The boundaries for the keyspace as they were calculated when the memtable is created.
- // The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
- // The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same shard
- // for a given key, even if we race with the StorageService initialization or with topology changes.
- @Unmetered
- final ShardBoundaries boundaries;
-
/**
- * Core-specific memtable regions. All writes must go through the specific core. The data structures used
- * are concurrent-read safe, thus reads can be carried out from any thread.
+ * Sharded memtable sections. Each is responsible for a contiguous range of the token space (between boundaries[i]
+ * and boundaries[i+1]) and is written to by one thread at a time, while reads are carried out concurrently
+ * (including with any write).
*/
final MemtableShard[] shards;
- @VisibleForTesting
- public static final String SHARD_COUNT_PROPERTY = "cassandra.memtable.shard.count";
-
- // default shard count, used when a specific number of shards is not specified in the parameters
- private static final int SHARD_COUNT = Integer.getInteger(SHARD_COUNT_PROPERTY, FBUtilities.getAvailableProcessors());
-
- private final Factory factory;
-
- // only to be used by init(), to setup the very first memtable for the cfs
ShardedSkipListMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound,
TableMetadataRef metadataRef,
Owner owner,
- Integer shardCountOption,
- Factory factory)
+ Integer shardCountOption)
{
- super(commitLogLowerBound, metadataRef, owner);
- int shardCount = shardCountOption != null ? shardCountOption : SHARD_COUNT;
- this.boundaries = owner.localRangeSplits(shardCount);
+ super(commitLogLowerBound, metadataRef, owner, shardCountOption);
this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef);
- this.factory = factory;
}
private static MemtableShard[] generatePartitionShards(int splits,
@@ -128,17 +108,11 @@
public boolean isClean()
{
for (MemtableShard shard : shards)
- if (!shard.isEmpty())
+ if (!shard.isClean())
return false;
return true;
}
- @Override
- protected Memtable.Factory factory()
- {
- return factory;
- }
-
/**
* Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
* OpOrdering.
@@ -396,14 +370,14 @@
}
}
- long[] pair = previous.addAllWithSizeDelta(update, cloner, opGroup, indexer);
+ BTreePartitionUpdater updater = previous.addAll(update, cloner, opGroup, indexer);
updateMin(minTimestamp, update.stats().minTimestamp);
updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
- liveDataSize.addAndGet(initialSize + pair[0]);
+ liveDataSize.addAndGet(initialSize + updater.dataSize);
columnsCollector.update(update.columns());
statsCollector.update(update.stats());
currentOperations.addAndGet(update.operationCount());
- return pair[1];
+ return updater.colUpdateTimeDelta;
}
private Map<PartitionPosition, AtomicBTreePartition> getPartitionsSubMap(PartitionPosition left,
@@ -432,7 +406,7 @@
}
}
- public boolean isEmpty()
+ public boolean isClean()
{
return partitions.isEmpty();
}
@@ -500,9 +474,9 @@
static class Locking extends ShardedSkipListMemtable
{
- Locking(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption, Factory factory)
+ Locking(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption)
{
- super(commitLogLowerBound, metadataRef, owner, shardCountOption, factory);
+ super(commitLogLowerBound, metadataRef, owner, shardCountOption);
}
/**
@@ -547,8 +521,8 @@
Owner owner)
{
return isLocking
- ? new Locking(commitLogLowerBound, metadataRef, owner, shardCount, this)
- : new ShardedSkipListMemtable(commitLogLowerBound, metadataRef, owner, shardCount, this);
+ ? new Locking(commitLogLowerBound, metadataRef, owner, shardCount)
+ : new ShardedSkipListMemtable(commitLogLowerBound, metadataRef, owner, shardCount);
}
public boolean equals(Object o)
diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
index 17c0cce..e6fecec 100644
--- a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.db.memtable;
+import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentNavigableMap;
@@ -36,9 +37,10 @@
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.partitions.AbstractBTreePartition;
import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
import org.apache.cassandra.db.partitions.AtomicBTreePartition;
+import org.apache.cassandra.db.partitions.BTreePartitionData;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -49,14 +51,14 @@
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.index.transactions.UpdateTransaction;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.Cloner;
import org.apache.cassandra.utils.memory.MemtableAllocator;
+import org.apache.cassandra.utils.memory.NativeAllocator;
import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_COMPUTE_STEPS;
import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE;
@@ -90,12 +92,6 @@
}
@Override
- protected Factory factory()
- {
- return FACTORY;
- }
-
- @Override
public boolean isClean()
{
return partitions.isEmpty();
@@ -131,14 +127,14 @@
}
}
- long[] pair = previous.addAllWithSizeDelta(update, cloner, opGroup, indexer);
+ BTreePartitionUpdater updater = previous.addAll(update, cloner, opGroup, indexer);
updateMin(minTimestamp, update.stats().minTimestamp);
updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
- liveDataSize.addAndGet(initialSize + pair[0]);
+ liveDataSize.addAndGet(initialSize + updater.dataSize);
columnsCollector.update(update.columns());
statsCollector.update(update.stats());
currentOperations.addAndGet(update.operationCount());
- return pair[1];
+ return updater.colUpdateTimeDelta;
}
@Override
@@ -227,13 +223,18 @@
Cloner cloner = allocator.cloner(group);
ConcurrentNavigableMap<PartitionPosition, Object> partitions = new ConcurrentSkipListMap<>();
final Object val = new Object();
+ final int testBufferSize = 8;
for (int i = 0 ; i < count ; i++)
- partitions.put(cloner.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER)), val);
- double avgSize = ObjectSizes.measureDeep(partitions) / (double) count;
+ partitions.put(cloner.clone(new BufferDecoratedKey(new LongToken(i), ByteBuffer.allocate(testBufferSize))), val);
+ double avgSize = ObjectSizes.measureDeepOmitShared(partitions) / (double) count;
rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize));
- rowOverhead -= ObjectSizes.measureDeep(new LongToken(0));
+ rowOverhead -= new LongToken(0).getHeapSize();
rowOverhead += AtomicBTreePartition.EMPTY_SIZE;
- rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE;
+ rowOverhead += BTreePartitionData.UNSHARED_HEAP_SIZE;
+ if (!(allocator instanceof NativeAllocator))
+ rowOverhead -= testBufferSize; // measureDeepOmitShared includes the given number of bytes even for
+ // off-heap buffers, but not for direct memory.
+ // Decorated key overhead with byte buffer (if needed) is included
allocator.setDiscarding();
allocator.setDiscarded();
return rowOverhead;
diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
new file mode 100644
index 0000000..e8d5978
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
@@ -0,0 +1,729 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.memtable;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.BTreePartitionData;
+import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.metrics.TrieMemtableMetricsView;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.EnsureOnHeap;
+import org.apache.cassandra.utils.memory.MemtableAllocator;
+import org.github.jamm.Unmetered;
+
+/**
+ * Trie memtable implementation. Improves memory usage, garbage collection efficiency and lookup performance.
+ * The implementation is described in detail in the paper:
+ * https://www.vldb.org/pvldb/vol15/p3359-lambov.pdf
+ *
+ * The configuration takes a single parameter:
+ * - shards: the number of shards to split into, defaulting to the number of CPU cores.
+ *
+ * Also see Memtable_API.md.
+ */
+public class TrieMemtable extends AbstractShardedMemtable
+{
+ private static final Logger logger = LoggerFactory.getLogger(TrieMemtable.class);
+
+ /** Buffer type to use for memtable tries (on- vs off-heap) */
+ public static final BufferType BUFFER_TYPE;
+
+ static
+ {
+ switch (DatabaseDescriptor.getMemtableAllocationType())
+ {
+ case unslabbed_heap_buffers:
+ case heap_buffers:
+ BUFFER_TYPE = BufferType.ON_HEAP;
+ break;
+ case offheap_buffers:
+ case offheap_objects:
+ BUFFER_TYPE = BufferType.OFF_HEAP;
+ break;
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /** If keys is below this length, we will use a recursive procedure for inserting data in the memtable trie. */
+ @VisibleForTesting
+ public static final int MAX_RECURSIVE_KEY_LENGTH = 128;
+
+ /** The byte-ordering conversion version to use for memtables. */
+ public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
+
+ // Set to true when the memtable requests a switch (e.g. for trie size limit being reached) to ensure only one
+ // thread calls cfs.switchMemtableIfCurrent.
+ private final AtomicBoolean switchRequested = new AtomicBoolean(false);
+
+ /**
+ * Sharded memtable sections. Each is responsible for a contiguous range of the token space (between boundaries[i]
+ * and boundaries[i+1]) and is written to by one thread at a time, while reads are carried out concurrently
+ * (including with any write).
+ */
+ private final MemtableShard[] shards;
+
+ /**
+ * A merged view of the memtable map. Used for partition range queries and flush.
+ * For efficiency we serve single partition requests off the shard which offers more direct InMemoryTrie methods.
+ */
+ private final Trie<BTreePartitionData> mergedTrie;
+
+ @Unmetered
+ private final TrieMemtableMetricsView metrics;
+
+ TrieMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption)
+ {
+ super(commitLogLowerBound, metadataRef, owner, shardCountOption);
+ this.metrics = new TrieMemtableMetricsView(metadataRef.keyspace, metadataRef.name);
+ this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef, metrics);
+ this.mergedTrie = makeMergedTrie(shards);
+ }
+
+ private static MemtableShard[] generatePartitionShards(int splits,
+ MemtableAllocator allocator,
+ TableMetadataRef metadata,
+ TrieMemtableMetricsView metrics)
+ {
+ MemtableShard[] partitionMapContainer = new MemtableShard[splits];
+ for (int i = 0; i < splits; i++)
+ partitionMapContainer[i] = new MemtableShard(metadata, allocator, metrics);
+
+ return partitionMapContainer;
+ }
+
+ private static Trie<BTreePartitionData> makeMergedTrie(MemtableShard[] shards)
+ {
+ List<Trie<BTreePartitionData>> tries = new ArrayList<>(shards.length);
+ for (MemtableShard shard : shards)
+ tries.add(shard.data);
+ return Trie.mergeDistinct(tries);
+ }
+
+ @Override
+ public boolean isClean()
+ {
+ for (MemtableShard shard : shards)
+ if (!shard.isClean())
+ return false;
+ return true;
+ }
+
+ @Override
+ public void discard()
+ {
+ super.discard();
+ // metrics here are not thread safe, but I think we can live with that
+ metrics.lastFlushShardDataSizes.reset();
+ for (MemtableShard shard : shards)
+ {
+ metrics.lastFlushShardDataSizes.update(shard.liveDataSize());
+ }
+ // the buffer release is a longer-running process, do it in a separate loop to not make the metrics update wait
+ for (MemtableShard shard : shards)
+ {
+ shard.data.discardBuffers();
+ }
+ }
+
+ /**
+ * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
+ * OpOrdering.
+ *
+ * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
+ */
+ @Override
+ public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+ {
+ try
+ {
+ DecoratedKey key = update.partitionKey();
+ MemtableShard shard = shards[boundaries.getShardForKey(key)];
+ long colUpdateTimeDelta = shard.put(key, update, indexer, opGroup);
+
+ if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true))
+ {
+ logger.info("Scheduling flush due to trie size limit reached.");
+ owner.signalFlushRequired(this, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT);
+ }
+
+ return colUpdateTimeDelta;
+ }
+ catch (InMemoryTrie.SpaceExhaustedException e)
+ {
+ // This should never happen as {@link InMemoryTrie#reachedAllocatedSizeThreshold} should become
+ // true and trigger a memtable switch long before this limit is reached.
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Technically we should scatter gather on all the core threads because the size in following calls are not
+ * using volatile variables, but for metrics purpose this should be good enough.
+ */
+ @Override
+ public long getLiveDataSize()
+ {
+ long total = 0L;
+ for (MemtableShard shard : shards)
+ total += shard.liveDataSize();
+ return total;
+ }
+
+ @Override
+ public long operationCount()
+ {
+ long total = 0L;
+ for (MemtableShard shard : shards)
+ total += shard.currentOperations();
+ return total;
+ }
+
+ @Override
+ public long partitionCount()
+ {
+ int total = 0;
+ for (MemtableShard shard : shards)
+ total += shard.size();
+ return total;
+ }
+
+ /**
+ * Returns the minTS if one available, otherwise NO_MIN_TIMESTAMP.
+ *
+ * EncodingStats uses a synthetic epoch TS at 2015. We don't want to leak that (CASSANDRA-18118) so we return NO_MIN_TIMESTAMP instead.
+ *
+ * @return The minTS or NO_MIN_TIMESTAMP if none available
+ */
+ @Override
+ public long getMinTimestamp()
+ {
+ long min = Long.MAX_VALUE;
+ for (MemtableShard shard : shards)
+ min = Long.min(min, shard.minTimestamp());
+ return min != EncodingStats.NO_STATS.minTimestamp ? min : NO_MIN_TIMESTAMP;
+ }
+
+ @Override
+ public int getMinLocalDeletionTime()
+ {
+ int min = Integer.MAX_VALUE;
+ for (MemtableShard shard : shards)
+ min = Integer.min(min, shard.minLocalDeletionTime());
+ return min;
+ }
+
+ @Override
+ RegularAndStaticColumns columns()
+ {
+ for (MemtableShard shard : shards)
+ columnsCollector.update(shard.columnsCollector);
+ return columnsCollector.get();
+ }
+
+ @Override
+ EncodingStats encodingStats()
+ {
+ for (MemtableShard shard : shards)
+ statsCollector.update(shard.statsCollector.get());
+ return statsCollector.get();
+ }
+
+ @Override
+ public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter,
+ final DataRange dataRange,
+ SSTableReadsListener readsListener)
+ {
+ AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
+
+ PartitionPosition left = keyRange.left;
+ PartitionPosition right = keyRange.right;
+ if (left.isMinimum())
+ left = null;
+ if (right.isMinimum())
+ right = null;
+
+ boolean isBound = keyRange instanceof Bounds;
+ boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
+ boolean includeStop = isBound || keyRange instanceof Range;
+
+ Trie<BTreePartitionData> subMap = mergedTrie.subtrie(left, includeStart, right, includeStop);
+
+ return new MemtableUnfilteredPartitionIterator(metadata(),
+ allocator.ensureOnHeap(),
+ subMap,
+ columnFilter,
+ dataRange);
+ // readsListener is ignored as it only accepts sstable signals
+ }
+
+ private Partition getPartition(DecoratedKey key)
+ {
+ int shardIndex = boundaries.getShardForKey(key);
+ BTreePartitionData data = shards[shardIndex].data.get(key);
+ if (data != null)
+ return createPartition(metadata(), allocator.ensureOnHeap(), key, data);
+ else
+ return null;
+ }
+
+ @Override
+ public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
+ {
+ Partition p = getPartition(key);
+ if (p == null)
+ return null;
+ else
+ return p.unfilteredIterator(selectedColumns, slices, reversed);
+ }
+
+ @Override
+ public UnfilteredRowIterator rowIterator(DecoratedKey key)
+ {
+ Partition p = getPartition(key);
+ return p != null ? p.unfilteredIterator() : null;
+ }
+
+ private static MemtablePartition createPartition(TableMetadata metadata, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
+ {
+ return new MemtablePartition(metadata, ensureOnHeap, key, data);
+ }
+
+ private static MemtablePartition getPartitionFromTrieEntry(TableMetadata metadata, EnsureOnHeap ensureOnHeap, Map.Entry<ByteComparable, BTreePartitionData> en)
+ {
+ DecoratedKey key = BufferDecoratedKey.fromByteComparable(en.getKey(),
+ BYTE_COMPARABLE_VERSION,
+ metadata.partitioner);
+ return createPartition(metadata, ensureOnHeap, key, en.getValue());
+ }
+
+
+ @Override
+ public FlushablePartitionSet<MemtablePartition> getFlushSet(PartitionPosition from, PartitionPosition to)
+ {
+ Trie<BTreePartitionData> toFlush = mergedTrie.subtrie(from, true, to, false);
+ long keySize = 0;
+ int keyCount = 0;
+
+ for (Iterator<Map.Entry<ByteComparable, BTreePartitionData>> it = toFlush.entryIterator(); it.hasNext(); )
+ {
+ Map.Entry<ByteComparable, BTreePartitionData> en = it.next();
+ byte[] keyBytes = DecoratedKey.keyFromByteSource(ByteSource.peekable(en.getKey().asComparableBytes(BYTE_COMPARABLE_VERSION)),
+ BYTE_COMPARABLE_VERSION,
+ metadata().partitioner);
+ keySize += keyBytes.length;
+ keyCount++;
+ }
+ long partitionKeySize = keySize;
+ int partitionCount = keyCount;
+
+ return new AbstractFlushablePartitionSet<MemtablePartition>()
+ {
+ public Memtable memtable()
+ {
+ return TrieMemtable.this;
+ }
+
+ public PartitionPosition from()
+ {
+ return from;
+ }
+
+ public PartitionPosition to()
+ {
+ return to;
+ }
+
+ public long partitionCount()
+ {
+ return partitionCount;
+ }
+
+ public Iterator<MemtablePartition> iterator()
+ {
+ return Iterators.transform(toFlush.entryIterator(),
+ // During flushing we are certain the memtable will remain at least until
+ // the flush completes. No copying to heap is necessary.
+ entry -> getPartitionFromTrieEntry(metadata(), EnsureOnHeap.NOOP, entry));
+ }
+
+ public long partitionKeysSize()
+ {
+ return partitionKeySize;
+ }
+ };
+ }
+
+ static class MemtableShard
+ {
+ // The following fields are volatile as we have to make sure that when we
+ // collect results from all sub-ranges, the thread accessing the value
+ // is guaranteed to see the changes to the values.
+
+ // The smallest timestamp for all partitions stored in this shard
+ private volatile long minTimestamp = Long.MAX_VALUE;
+
+ private volatile int minLocalDeletionTime = Integer.MAX_VALUE;
+
+ private volatile long liveDataSize = 0;
+
+ private volatile long currentOperations = 0;
+
+ @Unmetered
+ private final ReentrantLock writeLock = new ReentrantLock();
+
+ // Content map for the given shard. This is implemented as a memtable trie which uses the prefix-free
+ // byte-comparable ByteSource representations of the keys to address the partitions.
+ //
+ // This map is used in a single-producer, multi-consumer fashion: only one thread will insert items but
+ // several threads may read from it and iterate over it. Iterators (especially partition range iterators)
+ // may operate for a long period of time and thus iterators should not throw ConcurrentModificationExceptions
+ // if the underlying map is modified during iteration, they should provide a weakly consistent view of the map
+ // instead.
+ //
+ // Also, this data is backed by memtable memory, when accessing it callers must specify if it can be accessed
+ // unsafely, meaning that the memtable will not be discarded as long as the data is used, or whether the data
+ // should be copied on heap for off-heap allocators.
+ @VisibleForTesting
+ final InMemoryTrie<BTreePartitionData> data;
+
+ private final ColumnsCollector columnsCollector;
+
+ private final StatsCollector statsCollector;
+
+ @Unmetered // total pool size should not be included in memtable's deep size
+ private final MemtableAllocator allocator;
+
+ @Unmetered
+ private final TrieMemtableMetricsView metrics;
+
+ @VisibleForTesting
+ MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator, TrieMemtableMetricsView metrics)
+ {
+ this.data = new InMemoryTrie<>(BUFFER_TYPE);
+ this.columnsCollector = new AbstractMemtable.ColumnsCollector(metadata.get().regularAndStaticColumns());
+ this.statsCollector = new AbstractMemtable.StatsCollector();
+ this.allocator = allocator;
+ this.metrics = metrics;
+ }
+
+ public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws InMemoryTrie.SpaceExhaustedException
+ {
+ BTreePartitionUpdater updater = new BTreePartitionUpdater(allocator, allocator.cloner(opGroup), opGroup, indexer);
+ boolean locked = writeLock.tryLock();
+ if (locked)
+ {
+ metrics.uncontendedPuts.inc();
+ }
+ else
+ {
+ metrics.contendedPuts.inc();
+ long lockStartTime = Clock.Global.nanoTime();
+ writeLock.lock();
+ metrics.contentionTime.addNano(Clock.Global.nanoTime() - lockStartTime);
+ }
+ try
+ {
+ try
+ {
+ long onHeap = data.sizeOnHeap();
+ long offHeap = data.sizeOffHeap();
+ // Use the fast recursive put if we know the key is small enough to not cause a stack overflow.
+ data.putSingleton(key,
+ update,
+ updater::mergePartitions,
+ key.getKeyLength() < MAX_RECURSIVE_KEY_LENGTH);
+ allocator.offHeap().adjust(data.sizeOffHeap() - offHeap, opGroup);
+ allocator.onHeap().adjust(data.sizeOnHeap() - onHeap, opGroup);
+ }
+ finally
+ {
+ minTimestamp = Math.min(minTimestamp, update.stats().minTimestamp);
+ minLocalDeletionTime = Math.min(minLocalDeletionTime, update.stats().minLocalDeletionTime);
+ liveDataSize += updater.dataSize;
+ currentOperations += update.operationCount();
+
+ columnsCollector.update(update.columns());
+ statsCollector.update(update.stats());
+ }
+ }
+ finally
+ {
+ writeLock.unlock();
+ }
+ return updater.colUpdateTimeDelta;
+ }
+
+ public boolean isClean()
+ {
+ return data.isEmpty();
+ }
+
+ public int size()
+ {
+ return data.valuesCount();
+ }
+
+ long minTimestamp()
+ {
+ return minTimestamp;
+ }
+
+ long liveDataSize()
+ {
+ return liveDataSize;
+ }
+
+ long currentOperations()
+ {
+ return currentOperations;
+ }
+
+ int minLocalDeletionTime()
+ {
+ return minLocalDeletionTime;
+ }
+ }
+
+ static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
+ {
+ private final TableMetadata metadata;
+ private final EnsureOnHeap ensureOnHeap;
+ private final Iterator<Map.Entry<ByteComparable, BTreePartitionData>> iter;
+ private final ColumnFilter columnFilter;
+ private final DataRange dataRange;
+
+ public MemtableUnfilteredPartitionIterator(TableMetadata metadata,
+ EnsureOnHeap ensureOnHeap,
+ Trie<BTreePartitionData> source,
+ ColumnFilter columnFilter,
+ DataRange dataRange)
+ {
+ this.metadata = metadata;
+ this.ensureOnHeap = ensureOnHeap;
+ this.iter = source.entryIterator();
+ this.columnFilter = columnFilter;
+ this.dataRange = dataRange;
+ }
+
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ public boolean hasNext()
+ {
+ return iter.hasNext();
+ }
+
+ public UnfilteredRowIterator next()
+ {
+ Partition partition = getPartitionFromTrieEntry(metadata(), ensureOnHeap, iter.next());
+ DecoratedKey key = partition.partitionKey();
+ ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
+
+ return filter.getUnfilteredRowIterator(columnFilter, partition);
+ }
+ }
+
+ static class MemtablePartition extends ImmutableBTreePartition
+ {
+
+ private final EnsureOnHeap ensureOnHeap;
+
+ private MemtablePartition(TableMetadata table, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
+ {
+ super(table, key, data);
+ this.ensureOnHeap = ensureOnHeap;
+ }
+
+ @Override
+ protected boolean canHaveShadowedData()
+ {
+ // The BtreePartitionData we store in the memtable are build iteratively by BTreePartitionData.add(), which
+ // doesn't make sure there isn't shadowed data, so we'll need to eliminate any.
+ return true;
+ }
+
+
+ @Override
+ public DeletionInfo deletionInfo()
+ {
+ return ensureOnHeap.applyToDeletionInfo(super.deletionInfo());
+ }
+
+ @Override
+ public Row staticRow()
+ {
+ return ensureOnHeap.applyToStatic(super.staticRow());
+ }
+
+ @Override
+ public DecoratedKey partitionKey()
+ {
+ return ensureOnHeap.applyToPartitionKey(super.partitionKey());
+ }
+
+ @Override
+ public Row getRow(Clustering<?> clustering)
+ {
+ return ensureOnHeap.applyToRow(super.getRow(clustering));
+ }
+
+ @Override
+ public Row lastRow()
+ {
+ return ensureOnHeap.applyToRow(super.lastRow());
+ }
+
+ @Override
+ public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
+ {
+ return unfilteredIterator(holder(), selection, slices, reversed);
+ }
+
+ @Override
+ public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
+ {
+ return ensureOnHeap.applyToPartition(super.unfilteredIterator(selection, clusteringsInQueryOrder, reversed));
+ }
+
+ @Override
+ public UnfilteredRowIterator unfilteredIterator()
+ {
+ return unfilteredIterator(ColumnFilter.selection(super.columns()), Slices.ALL, false);
+ }
+
+ @Override
+ public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
+ {
+ return ensureOnHeap.applyToPartition(super.unfilteredIterator(current, selection, slices, reversed));
+ }
+
+ @Override
+ public Iterator<Row> iterator()
+ {
+ return ensureOnHeap.applyToPartition(super.iterator());
+ }
+ }
+
+ public static Factory factory(Map<String, String> optionsCopy)
+ {
+ String shardsString = optionsCopy.remove(SHARDS_OPTION);
+ Integer shardCount = shardsString != null ? Integer.parseInt(shardsString) : null;
+ return new Factory(shardCount);
+ }
+
+ static class Factory implements Memtable.Factory
+ {
+ final Integer shardCount;
+
+ Factory(Integer shardCount)
+ {
+ this.shardCount = shardCount;
+ }
+
+ public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound,
+ TableMetadataRef metadaRef,
+ Owner owner)
+ {
+ return new TrieMemtable(commitLogLowerBound, metadaRef, owner, shardCount);
+ }
+
+ @Override
+ public TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef)
+ {
+ TrieMemtableMetricsView metrics = new TrieMemtableMetricsView(metadataRef.keyspace, metadataRef.name);
+ return metrics::release;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ Factory factory = (Factory) o;
+ return Objects.equals(shardCount, factory.shardCount);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(shardCount);
+ }
+ }
+
+ @VisibleForTesting
+ public long unusedReservedMemory()
+ {
+ long size = 0;
+ for (MemtableShard shard : shards)
+ size += shard.data.unusedReservedMemory();
+ return size;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java b/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java
index d681e4b..2435699 100644
--- a/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java
+++ b/src/java/org/apache/cassandra/db/monitoring/MonitoringTask.java
@@ -33,11 +33,12 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.utils.NoSpamLogger;
-import static java.lang.System.getProperty;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MONITORING_MAX_OPERATIONS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MONITORING_REPORT_INTERVAL_MS;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQueue;
@@ -48,20 +49,20 @@
*/
class MonitoringTask
{
- private static final String LINE_SEPARATOR = getProperty("line.separator");
+ private static final String LINE_SEPARATOR = CassandraRelevantProperties.LINE_SEPARATOR.getString();
private static final Logger logger = LoggerFactory.getLogger(MonitoringTask.class);
private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 5L, TimeUnit.MINUTES);
/**
* Defines the interval for reporting any operations that have timed out.
*/
- private static final int REPORT_INTERVAL_MS = Math.max(0, Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "monitoring_report_interval_ms", "5000")));
+ private static final int REPORT_INTERVAL_MS = Math.max(0, MONITORING_REPORT_INTERVAL_MS.getInt());
/**
* Defines the maximum number of unique timed out queries that will be reported in the logs.
* Use a negative number to remove any limit.
*/
- private static final int MAX_OPERATIONS = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "monitoring_max_operations", "50"));
+ private static final int MAX_OPERATIONS = MONITORING_MAX_OPERATIONS.getInt();
@VisibleForTesting
static MonitoringTask instance = make(REPORT_INTERVAL_MS, MAX_OPERATIONS);
diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
index 5926ced..3327237 100644
--- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java
@@ -22,14 +22,12 @@
import java.util.Iterator;
import java.util.NavigableSet;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
@@ -37,12 +35,9 @@
public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
{
- protected static final Holder EMPTY = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
- public static final long HOLDER_UNSHARED_HEAP_SIZE = ObjectSizes.measure(EMPTY);
-
protected final DecoratedKey partitionKey;
- protected abstract Holder holder();
+ protected abstract BTreePartitionData holder();
protected abstract boolean canHaveShadowedData();
protected AbstractBTreePartition(DecoratedKey partitionKey)
@@ -50,31 +45,6 @@
this.partitionKey = partitionKey;
}
- @VisibleForTesting
- public static final class Holder
- {
- public final RegularAndStaticColumns columns;
- public final DeletionInfo deletionInfo;
- // the btree of rows
- public final Object[] tree;
- public final Row staticRow;
- public final EncodingStats stats;
-
- Holder(RegularAndStaticColumns columns, Object[] tree, DeletionInfo deletionInfo, Row staticRow, EncodingStats stats)
- {
- this.columns = columns;
- this.tree = tree;
- this.deletionInfo = deletionInfo;
- this.staticRow = staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow;
- this.stats = stats;
- }
-
- protected Holder withColumns(RegularAndStaticColumns columns)
- {
- return new Holder(columns, this.tree, this.deletionInfo, this.staticRow, this.stats);
- }
- }
-
public DeletionInfo deletionInfo()
{
return holder().deletionInfo;
@@ -87,13 +57,13 @@
public boolean isEmpty()
{
- Holder holder = holder();
+ BTreePartitionData holder = holder();
return holder.deletionInfo.isLive() && BTree.isEmpty(holder.tree) && holder.staticRow.isEmpty();
}
public boolean hasRows()
{
- Holder holder = holder();
+ BTreePartitionData holder = holder();
return !BTree.isEmpty(holder.tree);
}
@@ -122,7 +92,7 @@
public Row getRow(Clustering<?> clustering)
{
ColumnFilter columns = ColumnFilter.selection(columns());
- Holder holder = holder();
+ BTreePartitionData holder = holder();
if (clustering == Clustering.STATIC_CLUSTERING)
{
@@ -152,7 +122,7 @@
return row.filter(columns, activeDeletion, true, metadata());
}
- private Row staticRow(Holder current, ColumnFilter columns, boolean setActiveDeletionToRow)
+ private Row staticRow(BTreePartitionData current, ColumnFilter columns, boolean setActiveDeletionToRow)
{
DeletionTime partitionDeletion = current.deletionInfo.getPartitionDeletion();
if (columns.fetchedColumns().statics.isEmpty() || (current.staticRow.isEmpty() && partitionDeletion.isLive()))
@@ -185,7 +155,7 @@
return unfilteredIterator(holder(), selection, slices, reversed);
}
- public UnfilteredRowIterator unfilteredIterator(Holder current, ColumnFilter selection, Slices slices, boolean reversed)
+ public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
{
Row staticRow = staticRow(current, selection, false);
if (slices.size() == 0)
@@ -199,7 +169,7 @@
: new SlicesIterator(selection, slices, reversed, current, staticRow);
}
- private UnfilteredRowIterator sliceIterator(ColumnFilter selection, Slice slice, boolean reversed, Holder current, Row staticRow)
+ private UnfilteredRowIterator sliceIterator(ColumnFilter selection, Slice slice, boolean reversed, BTreePartitionData current, Row staticRow)
{
ClusteringBound<?> start = slice.start().isBottom() ? null : slice.start();
ClusteringBound<?> end = slice.end().isTop() ? null : slice.end();
@@ -209,7 +179,7 @@
}
private RowAndDeletionMergeIterator merge(Iterator<Row> rowIter, Iterator<RangeTombstone> deleteIter,
- ColumnFilter selection, boolean reversed, Holder current, Row staticRow)
+ ColumnFilter selection, boolean reversed, BTreePartitionData current, Row staticRow)
{
return new RowAndDeletionMergeIterator(metadata(), partitionKey(), current.deletionInfo.getPartitionDeletion(),
selection, staticRow, reversed, current.stats,
@@ -219,10 +189,10 @@
private abstract class AbstractIterator extends AbstractUnfilteredRowIterator
{
- final Holder current;
+ final BTreePartitionData current;
final ColumnFilter selection;
- private AbstractIterator(Holder current, Row staticRow, ColumnFilter selection, boolean isReversed)
+ private AbstractIterator(BTreePartitionData current, Row staticRow, ColumnFilter selection, boolean isReversed)
{
super(AbstractBTreePartition.this.metadata(),
AbstractBTreePartition.this.partitionKey(),
@@ -245,7 +215,7 @@
private int idx;
private Iterator<Unfiltered> currentSlice;
- private SlicesIterator(ColumnFilter selection, Slices slices, boolean isReversed, Holder current, Row staticRow)
+ private SlicesIterator(ColumnFilter selection, Slices slices, boolean isReversed, BTreePartitionData current, Row staticRow)
{
super(current, staticRow, selection, isReversed);
this.slices = slices;
@@ -283,7 +253,7 @@
private ClusteringsIterator(ColumnFilter selection,
NavigableSet<Clustering<?>> clusteringsInQueryOrder,
boolean isReversed,
- Holder current,
+ BTreePartitionData current,
Row staticRow)
{
super(current, staticRow, selection, isReversed);
@@ -326,12 +296,12 @@
}
}
- protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity)
+ protected static BTreePartitionData build(UnfilteredRowIterator iterator, int initialRowCapacity)
{
return build(iterator, initialRowCapacity, true);
}
- protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered)
+ protected static BTreePartitionData build(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered)
{
TableMetadata metadata = iterator.metadata();
RegularAndStaticColumns columns = iterator.columns();
@@ -353,12 +323,12 @@
if (reversed)
builder.reverse();
- return new Holder(columns, builder.build(), deletionBuilder.build(), iterator.staticRow(), iterator.stats());
+ return new BTreePartitionData(columns, builder.build(), deletionBuilder.build(), iterator.staticRow(), iterator.stats());
}
// Note that when building with a RowIterator, deletion will generally be LIVE, but we allow to pass it nonetheless because PartitionUpdate
// passes a MutableDeletionInfo that it mutates later.
- protected static Holder build(RowIterator rows, DeletionInfo deletion, boolean buildEncodingStats)
+ protected static BTreePartitionData build(RowIterator rows, DeletionInfo deletion, boolean buildEncodingStats)
{
RegularAndStaticColumns columns = rows.columns();
boolean reversed = rows.isReverseOrder();
@@ -375,7 +345,7 @@
Row staticRow = rows.staticRow();
EncodingStats stats = buildEncodingStats ? EncodingStats.Collector.collect(staticRow, BTree.iterator(tree), deletion)
: EncodingStats.NO_STATS;
- return new Holder(columns, tree, deletion, staticRow, stats);
+ return new BTreePartitionData(columns, tree, deletion, staticRow, stats);
}
}
@@ -420,7 +390,7 @@
return false;
PartitionUpdate that = (PartitionUpdate) obj;
- Holder a = this.holder(), b = that.holder();
+ BTreePartitionData a = this.holder(), b = that.holder();
return partitionKey.equals(that.partitionKey)
&& metadata().id.equals(that.metadata().id)
&& a.deletionInfo.equals(b.deletionInfo)
@@ -446,16 +416,4 @@
return BTree.findByIndex(tree, BTree.size(tree) - 1);
}
-
- @VisibleForTesting
- public static Holder unsafeGetEmptyHolder()
- {
- return EMPTY;
- }
-
- @VisibleForTesting
- public static Holder unsafeConstructHolder(RegularAndStaticColumns columns, Object[] tree, DeletionInfo deletionInfo, Row staticRow, EncodingStats stats)
- {
- return new Holder(columns, tree, deletionInfo, staticRow, stats);
- }
}
diff --git a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
index 986e707..c9035be 100644
--- a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
@@ -18,28 +18,24 @@
package org.apache.cassandra.db.partitions;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.List;
-import java.util.NavigableSet;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.index.transactions.UpdateTransaction;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.btree.BTree;
-import org.apache.cassandra.utils.btree.UpdateFunction;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.Cloner;
-import org.apache.cassandra.utils.memory.HeapCloner;
import org.apache.cassandra.utils.memory.MemtableAllocator;
-import com.google.common.annotations.VisibleForTesting;
+import org.github.jamm.Unmetered;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@@ -71,7 +67,7 @@
// CLOCK_GRANULARITY = 1^9ns >> CLOCK_SHIFT == 132us == (1/7.63)ms
private static final AtomicIntegerFieldUpdater<AtomicBTreePartition> wasteTrackerUpdater = AtomicIntegerFieldUpdater.newUpdater(AtomicBTreePartition.class, "wasteTracker");
- private static final AtomicReferenceFieldUpdater<AtomicBTreePartition, Holder> refUpdater = AtomicReferenceFieldUpdater.newUpdater(AtomicBTreePartition.class, Holder.class, "ref");
+ private static final AtomicReferenceFieldUpdater<AtomicBTreePartition, BTreePartitionData> refUpdater = AtomicReferenceFieldUpdater.newUpdater(AtomicBTreePartition.class, BTreePartitionData.class, "ref");
/**
* (clock + allocation) granularity are combined to give us an acceptable (waste) allocation rate that is defined by
@@ -83,9 +79,12 @@
*/
private volatile int wasteTracker = TRACKER_NEVER_WASTED;
+ @Unmetered
private final MemtableAllocator allocator;
- private volatile Holder ref;
+ private volatile BTreePartitionData ref;
+
+ @Unmetered
private final TableMetadataRef metadata;
public AtomicBTreePartition(TableMetadataRef metadata, DecoratedKey partitionKey, MemtableAllocator allocator)
@@ -94,10 +93,10 @@
super(partitionKey);
this.metadata = metadata;
this.allocator = allocator;
- this.ref = EMPTY;
+ this.ref = BTreePartitionData.EMPTY;
}
- protected Holder holder()
+ protected BTreePartitionData holder()
{
return ref;
}
@@ -112,92 +111,78 @@
return true;
}
- private long[] addAllWithSizeDeltaInternal(RowUpdater updater, PartitionUpdate update, UpdateTransaction indexer)
- {
- Holder current = ref;
- updater.reset();
-
- if (!update.deletionInfo().getPartitionDeletion().isLive())
- indexer.onPartitionDeletion(update.deletionInfo().getPartitionDeletion());
-
- if (update.deletionInfo().hasRanges())
- update.deletionInfo().rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
-
- DeletionInfo deletionInfo;
- if (update.deletionInfo().mayModify(current.deletionInfo))
- {
- if (updater.inputDeletionInfoCopy == null)
- updater.inputDeletionInfoCopy = update.deletionInfo().clone(HeapCloner.instance);
-
- deletionInfo = current.deletionInfo.mutableCopy().add(updater.inputDeletionInfoCopy);
- updater.onAllocatedOnHeap(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize());
- }
- else
- {
- deletionInfo = current.deletionInfo;
- }
-
- RegularAndStaticColumns columns = update.columns().mergeTo(current.columns);
- updater.onAllocatedOnHeap(columns.unsharedHeapSize() - current.columns.unsharedHeapSize());
- Row newStatic = update.staticRow();
- Row staticRow = newStatic.isEmpty()
- ? current.staticRow
- : (current.staticRow.isEmpty() ? updater.insert(newStatic) : updater.merge(current.staticRow, newStatic));
- Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, updater);
- EncodingStats newStats = current.stats.mergeWith(update.stats());
- updater.onAllocatedOnHeap(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize());
-
- if (tree != null && refUpdater.compareAndSet(this, current, new Holder(columns, tree, deletionInfo, staticRow, newStats)))
- {
- updater.finish();
- return new long[]{ updater.dataSize, updater.colUpdateTimeDelta };
- }
- else
- {
- return null;
- }
- }
/**
* Adds a given update to this in-memtable partition.
*
* @return an array containing first the difference in size seen after merging the updates, and second the minimum
- * time detla between updates.
+ * time delta between updates.
*/
- public long[] addAllWithSizeDelta(final PartitionUpdate update,
- Cloner cloner,
- OpOrder.Group writeOp,
- UpdateTransaction indexer)
+ public BTreePartitionUpdater addAll(final PartitionUpdate update, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
{
- RowUpdater updater = new RowUpdater(allocator, cloner, writeOp, indexer);
- try
- {
- boolean shouldLock = shouldLock(writeOp);
- indexer.start();
+ return new Updater(allocator, cloner, writeOp, indexer).addAll(update);
+ }
- while (true)
+ @VisibleForTesting
+ public void unsafeSetHolder(BTreePartitionData holder)
+ {
+ ref = holder;
+ }
+
+ @VisibleForTesting
+ public BTreePartitionData unsafeGetHolder()
+ {
+ return ref;
+ }
+
+ class Updater extends BTreePartitionUpdater
+ {
+ BTreePartitionData current;
+
+ public Updater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
+ {
+ super(allocator, cloner, writeOp, indexer);
+ }
+
+ Updater addAll(final PartitionUpdate update)
+ {
+ try
{
- if (shouldLock)
+ boolean shouldLock = shouldLock(writeOp);
+ indexer.start();
+
+ while (true)
{
- synchronized (this)
+ if (shouldLock)
{
- long[] result = addAllWithSizeDeltaInternal(updater, update, indexer);
- if (result != null)
- return result;
+ synchronized (this)
+ {
+ if (tryUpdateData(update))
+ return this;
+ }
+ }
+ else
+ {
+ if (tryUpdateData(update))
+ return this;
+
+ shouldLock = shouldLock(heapSize, writeOp);
}
}
- else
- {
- long[] result = addAllWithSizeDeltaInternal(updater, update, indexer);
- if (result != null)
- return result;
-
- shouldLock = shouldLock(updater.heapSize, writeOp);
- }
+ }
+ finally
+ {
+ indexer.commit();
+ reportAllocatedMemory();
}
}
- finally
+
+ private boolean tryUpdateData(PartitionUpdate update)
{
- indexer.commit();
+ current = ref;
+ this.dataSize = 0;
+ this.heapSize = 0;
+ BTreePartitionData result = makeMergedPartition(current, update);
+ return refUpdater.compareAndSet(AtomicBTreePartition.this, current, result);
}
}
@@ -232,25 +217,7 @@
}
@Override
- public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
- {
- return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator(selection, slices, reversed));
- }
-
- @Override
- public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
- {
- return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator(selection, clusteringsInQueryOrder, reversed));
- }
-
- @Override
- public UnfilteredRowIterator unfilteredIterator()
- {
- return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator());
- }
-
- @Override
- public UnfilteredRowIterator unfilteredIterator(Holder current, ColumnFilter selection, Slices slices, boolean reversed)
+ public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
{
return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator(current, selection, slices, reversed));
}
@@ -334,118 +301,4 @@
return wasteTracker + 1;
return wasteTracker;
}
-
- @VisibleForTesting
- public void unsafeSetHolder(Holder holder)
- {
- ref = holder;
- }
-
- @VisibleForTesting
- public Holder unsafeGetHolder()
- {
- return ref;
- }
-
- // the function we provide to the btree utilities to perform any column replacements
- private static final class RowUpdater implements UpdateFunction<Row, Row>, ColumnData.PostReconciliationFunction
- {
- final MemtableAllocator allocator;
- final OpOrder.Group writeOp;
- final UpdateTransaction indexer;
- final Cloner cloner;
- long dataSize;
- long heapSize;
- long colUpdateTimeDelta = Long.MAX_VALUE;
- List<Row> inserted; // TODO: replace with walk of aborted BTree
-
- DeletionInfo inputDeletionInfoCopy = null;
-
- private RowUpdater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
- {
- this.allocator = allocator;
- this.writeOp = writeOp;
- this.indexer = indexer;
- this.cloner = cloner;
- }
-
- @Override
- public Row insert(Row insert)
- {
- Row data = insert.clone(cloner);
- indexer.onInserted(insert);
-
- this.dataSize += data.dataSize();
- this.heapSize += data.unsharedHeapSizeExcludingData();
- if (inserted == null)
- inserted = new ArrayList<>();
- inserted.add(data);
- return data;
- }
-
- public Row merge(Row existing, Row update)
- {
- Row reconciled = Rows.merge(existing, update, this);
- indexer.onUpdated(existing, reconciled);
-
- if (inserted == null)
- inserted = new ArrayList<>();
- inserted.add(reconciled);
-
- return reconciled;
- }
-
- public Row retain(Row existing)
- {
- return existing;
- }
-
- protected void reset()
- {
- this.dataSize = 0;
- this.heapSize = 0;
- if (inserted != null)
- inserted.clear();
- }
-
- public Cell<?> merge(Cell<?> previous, Cell<?> insert)
- {
- if (insert == previous)
- return insert;
- long timeDelta = Math.abs(insert.timestamp() - previous.timestamp());
- if (timeDelta < colUpdateTimeDelta)
- colUpdateTimeDelta = timeDelta;
- if (cloner != null)
- insert = cloner.clone(insert);
- dataSize += insert.dataSize() - previous.dataSize();
- heapSize += insert.unsharedHeapSizeExcludingData() - previous.unsharedHeapSizeExcludingData();
- return insert;
- }
-
- public ColumnData insert(ColumnData insert)
- {
- if (cloner != null)
- insert = insert.clone(cloner);
- dataSize += insert.dataSize();
- heapSize += insert.unsharedHeapSizeExcludingData();
- return insert;
- }
-
- @Override
- public void delete(ColumnData existing)
- {
- dataSize -= existing.dataSize();
- heapSize -= existing.unsharedHeapSizeExcludingData();
- }
-
- public void onAllocatedOnHeap(long heapSize)
- {
- this.heapSize += heapSize;
- }
-
- protected void finish()
- {
- allocator.onHeap().adjust(heapSize, writeOp);
- }
- }
}
diff --git a/src/java/org/apache/cassandra/db/partitions/BTreePartitionData.java b/src/java/org/apache/cassandra/db/partitions/BTreePartitionData.java
new file mode 100644
index 0000000..aefcd1d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/BTreePartitionData.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import java.util.Arrays;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.btree.BTree;
+
+/**
+ * Holder of the content of a partition, see {@link AbstractBTreePartition}.
+ * When updating a partition one holder is swapped for another atomically.
+ */
+public final class BTreePartitionData
+{
+ public static final BTreePartitionData EMPTY = new BTreePartitionData(RegularAndStaticColumns.NONE,
+ BTree.empty(),
+ DeletionInfo.LIVE,
+ Rows.EMPTY_STATIC_ROW,
+ EncodingStats.NO_STATS);
+ public static final long UNSHARED_HEAP_SIZE = ObjectSizes.measure(EMPTY);
+
+
+ final RegularAndStaticColumns columns;
+ final DeletionInfo deletionInfo;
+ // the btree of rows
+ final Object[] tree;
+ final Row staticRow;
+ public final EncodingStats stats;
+
+ BTreePartitionData(RegularAndStaticColumns columns,
+ Object[] tree,
+ DeletionInfo deletionInfo,
+ Row staticRow,
+ EncodingStats stats)
+ {
+ this.columns = columns;
+ this.tree = tree;
+ this.deletionInfo = deletionInfo;
+ this.staticRow = staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow;
+ this.stats = stats;
+ }
+
+ BTreePartitionData withColumns(RegularAndStaticColumns columns)
+ {
+ return new BTreePartitionData(columns, this.tree, this.deletionInfo, this.staticRow, this.stats);
+ }
+
+ @VisibleForTesting
+ public static BTreePartitionData unsafeGetEmpty()
+ {
+ return EMPTY;
+ }
+
+ @VisibleForTesting
+ public static BTreePartitionData unsafeConstruct(RegularAndStaticColumns columns,
+ Object[] tree,
+ DeletionInfo deletionInfo,
+ Row staticRow,
+ EncodingStats stats)
+ {
+ return new BTreePartitionData(columns, tree, deletionInfo, staticRow, stats);
+ }
+
+ @VisibleForTesting
+ public static void unsafeInvalidate(AtomicBTreePartition partition)
+ {
+ BTreePartitionData holder = partition.unsafeGetHolder();
+ if (!BTree.isEmpty(holder.tree))
+ {
+ partition.unsafeSetHolder(unsafeConstruct(holder.columns,
+ Arrays.copyOf(holder.tree, holder.tree.length),
+ holder.deletionInfo,
+ holder.staticRow,
+ holder.stats));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java b/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java
new file mode 100644
index 0000000..0230192
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/partitions/BTreePartitionUpdater.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.partitions;
+
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.utils.btree.BTree;
+import org.apache.cassandra.utils.btree.UpdateFunction;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.Cloner;
+import org.apache.cassandra.utils.memory.HeapCloner;
+import org.apache.cassandra.utils.memory.MemtableAllocator;
+
+/**
+ * the function we provide to the trie and btree utilities to perform any row and column replacements
+ */
+public class BTreePartitionUpdater implements UpdateFunction<Row, Row>, ColumnData.PostReconciliationFunction
+{
+ final MemtableAllocator allocator;
+ final OpOrder.Group writeOp;
+ final Cloner cloner;
+ final UpdateTransaction indexer;
+ public long dataSize;
+ long heapSize;
+ public long colUpdateTimeDelta = Long.MAX_VALUE;
+
+ public BTreePartitionUpdater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
+ {
+ this.allocator = allocator;
+ this.cloner = cloner;
+ this.writeOp = writeOp;
+ this.indexer = indexer;
+ this.heapSize = 0;
+ this.dataSize = 0;
+ }
+
+ public BTreePartitionData mergePartitions(BTreePartitionData current, final PartitionUpdate update)
+ {
+ if (current == null)
+ {
+ current = BTreePartitionData.EMPTY;
+ onAllocatedOnHeap(BTreePartitionData.UNSHARED_HEAP_SIZE);
+ }
+
+ try
+ {
+ indexer.start();
+
+ return makeMergedPartition(current, update);
+ }
+ finally
+ {
+ indexer.commit();
+ reportAllocatedMemory();
+ }
+ }
+
+ protected BTreePartitionData makeMergedPartition(BTreePartitionData current, PartitionUpdate update)
+ {
+ DeletionInfo newDeletionInfo = merge(current.deletionInfo, update.deletionInfo());
+
+ RegularAndStaticColumns columns = current.columns;
+ RegularAndStaticColumns newColumns = update.columns().mergeTo(columns);
+ onAllocatedOnHeap(newColumns.unsharedHeapSize() - columns.unsharedHeapSize());
+ Row newStatic = mergeStatic(current.staticRow, update.staticRow());
+
+ Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, this);
+ EncodingStats newStats = current.stats.mergeWith(update.stats());
+ onAllocatedOnHeap(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize());
+
+ return new BTreePartitionData(newColumns, tree, newDeletionInfo, newStatic, newStats);
+ }
+
+ private Row mergeStatic(Row current, Row update)
+ {
+ if (update.isEmpty())
+ return current;
+ if (current.isEmpty())
+ return insert(update);
+
+ return merge(current, update);
+ }
+
+ private DeletionInfo merge(DeletionInfo existing, DeletionInfo update)
+ {
+ if (update.isLive() || !update.mayModify(existing))
+ return existing;
+
+ if (!update.getPartitionDeletion().isLive())
+ indexer.onPartitionDeletion(update.getPartitionDeletion());
+
+ if (update.hasRanges())
+ update.rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
+
+ // Like for rows, we have to clone the update in case internal buffers (when it has range tombstones) reference
+ // memory we shouldn't hold into. But we don't ever store this off-heap currently so we just default to the
+ // HeapAllocator (rather than using 'allocator').
+ DeletionInfo newInfo = existing.mutableCopy().add(update.clone(HeapCloner.instance));
+ onAllocatedOnHeap(newInfo.unsharedHeapSize() - existing.unsharedHeapSize());
+ return newInfo;
+ }
+
+ @Override
+ public Row insert(Row insert)
+ {
+ Row data = insert.clone(cloner);
+ indexer.onInserted(insert);
+
+ dataSize += data.dataSize();
+ heapSize += data.unsharedHeapSizeExcludingData();
+ return data;
+ }
+
+ public Row merge(Row existing, Row update)
+ {
+ Row reconciled = Rows.merge(existing, update, this);
+ indexer.onUpdated(existing, reconciled);
+
+ return reconciled;
+ }
+
+ public Cell<?> merge(Cell<?> previous, Cell<?> insert)
+ {
+ if (insert == previous)
+ return insert;
+
+ long timeDelta = Math.abs(insert.timestamp() - previous.timestamp());
+ if (timeDelta < colUpdateTimeDelta)
+ colUpdateTimeDelta = timeDelta;
+ if (cloner != null)
+ insert = cloner.clone(insert);
+ dataSize += insert.dataSize() - previous.dataSize();
+ heapSize += insert.unsharedHeapSizeExcludingData() - previous.unsharedHeapSizeExcludingData();
+ return insert;
+ }
+
+ public ColumnData insert(ColumnData insert)
+ {
+ if (cloner != null)
+ insert = insert.clone(cloner);
+ dataSize += insert.dataSize();
+ heapSize += insert.unsharedHeapSizeExcludingData();
+ return insert;
+ }
+
+ @Override
+ public void delete(ColumnData existing)
+ {
+ dataSize -= existing.dataSize();
+ heapSize -= existing.unsharedHeapSizeExcludingData();
+ }
+
+ public void onAllocatedOnHeap(long heapSize)
+ {
+ this.heapSize += heapSize;
+ }
+
+ public void reportAllocatedMemory()
+ {
+ allocator.onHeap().adjust(heapSize, writeOp);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java
index 2183a98..f09f75a 100644
--- a/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/CachedBTreePartition.java
@@ -40,7 +40,7 @@
private CachedBTreePartition(TableMetadata metadata,
DecoratedKey partitionKey,
- Holder holder,
+ BTreePartitionData holder,
int createdAtInSec,
int cachedLiveRows,
int rowsWithNonExpiringCells)
@@ -80,7 +80,7 @@
*/
public static CachedBTreePartition create(UnfilteredRowIterator iterator, int initialRowCapacity, int nowInSec)
{
- Holder holder = ImmutableBTreePartition.build(iterator, initialRowCapacity);
+ BTreePartitionData holder = ImmutableBTreePartition.build(iterator, initialRowCapacity);
int cachedLiveRows = 0;
int rowsWithNonExpiringCells = 0;
@@ -180,7 +180,7 @@
UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, DeserializationHelper.Flag.LOCAL);
assert !header.isReversed && header.rowEstimate >= 0;
- Holder holder;
+ BTreePartitionData holder;
try (UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, DeserializationHelper.Flag.LOCAL, header))
{
holder = ImmutableBTreePartition.build(partition, header.rowEstimate);
diff --git a/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java
index 5139d40..6617255 100644
--- a/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java
+++ b/src/java/org/apache/cassandra/db/partitions/ImmutableBTreePartition.java
@@ -27,7 +27,7 @@
public class ImmutableBTreePartition extends AbstractBTreePartition
{
- protected final Holder holder;
+ protected final BTreePartitionData holder;
protected final TableMetadata metadata;
public ImmutableBTreePartition(TableMetadata metadata,
@@ -40,12 +40,12 @@
{
super(partitionKey);
this.metadata = metadata;
- this.holder = new Holder(columns, tree, deletionInfo, staticRow, stats);
+ this.holder = new BTreePartitionData(columns, tree, deletionInfo, staticRow, stats);
}
protected ImmutableBTreePartition(TableMetadata metadata,
DecoratedKey partitionKey,
- Holder holder)
+ BTreePartitionData holder)
{
super(partitionKey);
this.metadata = metadata;
@@ -119,7 +119,7 @@
return metadata;
}
- protected Holder holder()
+ protected BTreePartitionData holder()
{
return holder;
}
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java b/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java
index 6d37640..7c3ba15 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java
@@ -22,9 +22,10 @@
public interface PartitionStatisticsCollector
{
- public void update(LivenessInfo info);
- public void update(DeletionTime deletionTime);
- public void update(Cell<?> cell);
- public void updateColumnSetPerRow(long columnSetInRow);
- public void updateHasLegacyCounterShards(boolean hasLegacyCounterShards);
-}
+ void update(LivenessInfo info);
+ void updatePartitionDeletion(DeletionTime dt);
+ void update(DeletionTime deletionTime);
+ void update(Cell<?> cell);
+ void updateColumnSetPerRow(long columnSetInRow);
+ void updateHasLegacyCounterShards(boolean hasLegacyCounterShards);
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
index 6019d56..7f2ca7d 100644
--- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
+++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
@@ -36,7 +36,10 @@
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.index.IndexRegistry;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
@@ -67,7 +70,7 @@
public static final PartitionUpdateSerializer serializer = new PartitionUpdateSerializer();
- private final Holder holder;
+ private final BTreePartitionData holder;
private final DeletionInfo deletionInfo;
private final TableMetadata metadata;
@@ -75,7 +78,7 @@
private PartitionUpdate(TableMetadata metadata,
DecoratedKey key,
- Holder holder,
+ BTreePartitionData holder,
MutableDeletionInfo deletionInfo,
boolean canHaveShadowedData)
{
@@ -97,7 +100,7 @@
public static PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey key)
{
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
- Holder holder = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+ BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
}
@@ -114,7 +117,7 @@
public static PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, int nowInSec)
{
MutableDeletionInfo deletionInfo = new MutableDeletionInfo(timestamp, nowInSec);
- Holder holder = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
+ BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
}
@@ -131,7 +134,7 @@
public static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row, Row staticRow)
{
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
- Holder holder = new Holder(
+ BTreePartitionData holder = new BTreePartitionData(
new RegularAndStaticColumns(
staticRow == null ? Columns.NONE : Columns.from(staticRow),
row == null ? Columns.NONE : Columns.from(row)
@@ -187,7 +190,7 @@
public static PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
{
iterator = UnfilteredRowIterators.withOnlyQueriedData(iterator, filter);
- Holder holder = build(iterator, 16);
+ BTreePartitionData holder = build(iterator, 16);
MutableDeletionInfo deletionInfo = (MutableDeletionInfo) holder.deletionInfo;
return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
}
@@ -208,7 +211,7 @@
{
iterator = RowIterators.withOnlyQueriedData(iterator, filter);
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
- Holder holder = build(iterator, deletionInfo, true);
+ BTreePartitionData holder = build(iterator, deletionInfo, true);
return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
}
@@ -373,7 +376,7 @@
return holder.columns;
}
- protected Holder holder()
+ protected BTreePartitionData holder()
{
return holder;
}
@@ -463,6 +466,62 @@
return marks;
}
+ /**
+ *
+ * @return the estimated number of rows affected by this mutation
+ */
+ public int affectedRowCount()
+ {
+ // If there is a partition-level deletion, we intend to delete at least one row.
+ if (!partitionLevelDeletion().isLive())
+ return 1;
+
+ int count = 0;
+
+ // Each range delete should correspond to at least one intended row deletion.
+ if (deletionInfo().hasRanges())
+ count += deletionInfo().rangeCount();
+
+ count += rowCount();
+
+ if (!staticRow().isEmpty())
+ count++;
+
+ return count;
+ }
+
+ /**
+ *
+ * @return the estimated total number of columns that either have live data or are covered by a delete
+ */
+ public int affectedColumnCount()
+ {
+ // If there is a partition-level deletion, we intend to delete at least the columns of one row.
+ if (!partitionLevelDeletion().isLive())
+ return metadata().regularAndStaticColumns().size();
+
+ int count = 0;
+
+ // Each range delete should correspond to at least one intended row deletion, and with it, its regular columns.
+ if (deletionInfo().hasRanges())
+ count += deletionInfo().rangeCount() * metadata().regularColumns().size();
+
+ for (Row row : this)
+ {
+ if (row.deletion().isLive())
+ // If the row is live, this will include simple tombstones as well as cells w/ actual data.
+ count += row.columnCount();
+ else
+ // We have a row deletion, so account for the columns that might be deleted.
+ count += metadata().regularColumns().size();
+ }
+
+ if (!staticRow().isEmpty())
+ count += staticRow().columnCount();
+
+ return count;
+ }
+
private static void addMarksForRow(Row row, List<CounterMark> marks)
{
for (Cell<?> cell : row.cells())
@@ -494,7 +553,7 @@
@VisibleForTesting
public static PartitionUpdate unsafeConstruct(TableMetadata metadata,
DecoratedKey key,
- Holder holder,
+ BTreePartitionData holder,
MutableDeletionInfo deletionInfo,
boolean canHaveShadowedData)
{
@@ -690,7 +749,7 @@
MutableDeletionInfo deletionInfo = deletionBuilder.build();
return new PartitionUpdate(metadata,
header.key,
- new Holder(header.sHeader.columns(), rows, deletionInfo, header.staticRow, header.sHeader.stats()),
+ new BTreePartitionData(header.sHeader.columns(), rows, deletionInfo, header.staticRow, header.sHeader.stats()),
deletionInfo,
false);
}
@@ -702,7 +761,7 @@
if (position >= in.limit())
throw new EOFException();
// DecoratedKey key = metadata.decorateKey(ByteBufferUtil.readWithVIntLength(in));
- int keyLength = (int) VIntCoding.getUnsignedVInt(in, position);
+ int keyLength = VIntCoding.getUnsignedVInt32(in, position);
position += keyLength + VIntCoding.computeUnsignedVIntSize(keyLength);
if (position >= in.limit())
throw new EOFException();
@@ -800,7 +859,7 @@
RegularAndStaticColumns columns,
int initialRowCapacity,
boolean canHaveShadowedData,
- Holder holder)
+ BTreePartitionData holder)
{
this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, holder.staticRow, holder.deletionInfo, holder.tree);
}
@@ -909,11 +968,11 @@
isBuilt = true;
return new PartitionUpdate(metadata,
partitionKey(),
- new Holder(columns,
- merged,
- deletionInfo,
- staticRow,
- newStats),
+ new BTreePartitionData(columns,
+ merged,
+ deletionInfo,
+ staticRow,
+ newStats),
deletionInfo,
canHaveShadowedData);
}
diff --git a/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java b/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java
index 09f3ae3..5d97fd3 100644
--- a/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java
+++ b/src/java/org/apache/cassandra/db/partitions/PurgeFunction.java
@@ -31,13 +31,15 @@
private final boolean enforceStrictLiveness;
private boolean isReverseOrder;
+ private boolean ignoreGcGraceSeconds;
+
public PurgeFunction(int nowInSec, int gcBefore, int oldestUnrepairedTombstone, boolean onlyPurgeRepairedTombstones,
boolean enforceStrictLiveness)
{
this.nowInSec = nowInSec;
this.purger = (timestamp, localDeletionTime) ->
!(onlyPurgeRepairedTombstones && localDeletionTime >= oldestUnrepairedTombstone)
- && localDeletionTime < gcBefore
+ && (localDeletionTime < gcBefore || ignoreGcGraceSeconds)
&& getPurgeEvaluator().test(timestamp);
this.enforceStrictLiveness = enforceStrictLiveness;
}
@@ -59,6 +61,13 @@
{
}
+ // Called at the beginning of each new partition
+ // Return true if the current partitionKey ignores the gc_grace_seconds during compaction.
+ protected boolean shouldIgnoreGcGrace()
+ {
+ return false;
+ }
+
protected void setReverseOrder(boolean isReverseOrder)
{
this.isReverseOrder = isReverseOrder;
@@ -70,6 +79,8 @@
{
onNewPartition(partition.partitionKey());
+ ignoreGcGraceSeconds = shouldIgnoreGcGrace();
+
setReverseOrder(partition.isReverseOrder());
UnfilteredRowIterator purged = Transformation.apply(partition, this);
if (purged.isEmpty())
diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
index a051ee1..02fdde0 100644
--- a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
+++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
@@ -386,4 +386,4 @@
};
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
index af9888a..55138e8 100644
--- a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
+++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
@@ -35,6 +35,7 @@
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.concurrent.FutureTask;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -67,8 +68,8 @@
public class PendingAntiCompaction
{
private static final Logger logger = LoggerFactory.getLogger(PendingAntiCompaction.class);
- private static final int ACQUIRE_SLEEP_MS = Integer.getInteger("cassandra.acquire_sleep_ms", 1000);
- private static final int ACQUIRE_RETRY_SECONDS = Integer.getInteger("cassandra.acquire_retry_seconds", 60);
+ private static final int ACQUIRE_SLEEP_MS = CassandraRelevantProperties.ACQUIRE_SLEEP_MS.getInt();
+ private static final int ACQUIRE_RETRY_SECONDS = CassandraRelevantProperties.ACQUIRE_RETRY_SECONDS.getInt();
public static class AcquireResult
{
@@ -214,6 +215,11 @@
return null;
}
+ protected AcquireResult acquireSSTables()
+ {
+ return cfs.runWithCompactionsDisabled(this::acquireTuple, predicate, OperationType.ANTICOMPACTION, false, false, false);
+ }
+
public AcquireResult call()
{
logger.debug("acquiring sstables for pending anti compaction on session {}", sessionID);
@@ -231,7 +237,7 @@
{
// Note that anticompactions are not disabled when running this. This is safe since runWithCompactionsDisabled
// is synchronized - acquireTuple and predicate can only be run by a single thread (for the given cfs).
- return cfs.runWithCompactionsDisabled(this::acquireTuple, predicate, false, false, false);
+ return acquireSSTables();
}
catch (SSTableAcquisitionException e)
{
diff --git a/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java b/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java
new file mode 100644
index 0000000..ed6e39a
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import java.util.Objects;
+
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class ArtificialBoundMarker extends RangeTombstoneBoundMarker
+{
+ public ArtificialBoundMarker(ClusteringBound<?> bound)
+ {
+ super(bound, DeletionTime.LIVE);
+ assert bound.isArtificial();
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ if (this == other)
+ return true;
+
+ if (!(other instanceof ArtificialBoundMarker))
+ return false;
+
+ ArtificialBoundMarker that = (ArtificialBoundMarker) other;
+ return Objects.equals(bound, that.bound);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(bound);
+ }
+
+ @Override
+ public String toString(TableMetadata metadata)
+ {
+ return String.format("LowerBoundMarker %s", bound.toString(metadata));
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/rows/EncodingStats.java b/src/java/org/apache/cassandra/db/rows/EncodingStats.java
index 37dd34e..a29ce8f 100644
--- a/src/java/org/apache/cassandra/db/rows/EncodingStats.java
+++ b/src/java/org/apache/cassandra/db/rows/EncodingStats.java
@@ -67,7 +67,7 @@
// We should use this sparingly obviously
public static final EncodingStats NO_STATS = new EncodingStats(TIMESTAMP_EPOCH, DELETION_TIME_EPOCH, TTL_EPOCH);
- public static long HEAP_SIZE = ObjectSizes.measure(NO_STATS);
+ public static final long HEAP_SIZE = ObjectSizes.measure(NO_STATS);
public static final Serializer serializer = new Serializer();
@@ -215,6 +215,12 @@
updateLocalDeletionTime(deletionTime.localDeletionTime());
}
+ @Override
+ public void updatePartitionDeletion(DeletionTime dt)
+ {
+ update(dt);
+ }
+
public void updateTimestamp(long timestamp)
{
isTimestampSet = true;
@@ -266,8 +272,8 @@
public void serialize(EncodingStats stats, DataOutputPlus out) throws IOException
{
out.writeUnsignedVInt(stats.minTimestamp - TIMESTAMP_EPOCH);
- out.writeUnsignedVInt(stats.minLocalDeletionTime - DELETION_TIME_EPOCH);
- out.writeUnsignedVInt(stats.minTTL - TTL_EPOCH);
+ out.writeUnsignedVInt32(stats.minLocalDeletionTime - DELETION_TIME_EPOCH);
+ out.writeUnsignedVInt32(stats.minTTL - TTL_EPOCH);
}
public int serializedSize(EncodingStats stats)
@@ -280,9 +286,9 @@
public EncodingStats deserialize(DataInputPlus in) throws IOException
{
long minTimestamp = in.readUnsignedVInt() + TIMESTAMP_EPOCH;
- int minLocalDeletionTime = (int)in.readUnsignedVInt() + DELETION_TIME_EPOCH;
- int minTTL = (int)in.readUnsignedVInt() + TTL_EPOCH;
+ int minLocalDeletionTime = in.readUnsignedVInt32() + DELETION_TIME_EPOCH;
+ int minTTL = in.readUnsignedVInt32() + TTL_EPOCH;
return new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL);
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java
index d8bd36f..8a8b229 100644
--- a/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java
@@ -17,17 +17,17 @@
*/
package org.apache.cassandra.db.rows;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.db.*;
-
/**
- * Abstract class to create UnfilteredRowIterator that lazily initialize themselves.
- *
+ * Abstract class to create UnfilteredRowIterator that lazily initializes itself.
+ * <p>
* This is used during partition range queries when we know the partition key but want
- * to defer the initialization of the rest of the UnfilteredRowIterator until we need those informations.
- * See {@link org.apache.cassandra.io.sstable.format.big.BigTableScanner.KeyScanningIterator} for instance.
+ * to defer the initialization of the rest of the UnfilteredRowIterator until we need it.
*/
public abstract class LazilyInitializedUnfilteredRowIterator extends AbstractIterator<Unfiltered> implements UnfilteredRowIterator
{
@@ -48,11 +48,6 @@
iterator = initializeIterator();
}
- public boolean initialized()
- {
- return iterator != null;
- }
-
public TableMetadata metadata()
{
maybeInit();
@@ -103,6 +98,14 @@
public void close()
{
if (iterator != null)
+ {
iterator.close();
+ iterator = null;
+ }
+ }
+
+ public boolean isOpen()
+ {
+ return iterator != null;
}
}
diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java
index bf8c051..9e8276a 100644
--- a/src/java/org/apache/cassandra/db/rows/Row.java
+++ b/src/java/org/apache/cassandra/db/rows/Row.java
@@ -150,10 +150,10 @@
public ComplexColumnData getComplexColumnData(ColumnMetadata c);
/**
- * The data for a regular or complex column.
+ * Returns the {@link ColumnData} for the specified column.
*
- * @param c the column for which to return the complex data.
- * @return the data for {@code c} or {@code null} if the row has no data for this column.
+ * @param c the column for which to fetch the data.
+ * @return the data for the column or {@code null} if the row has no data for this column.
*/
public ColumnData getColumnData(ColumnMetadata c);
diff --git a/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java b/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java
index cbbac64..2474084 100644
--- a/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java
@@ -21,13 +21,13 @@
import java.util.Collections;
import java.util.Iterator;
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.CloseableIterator;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* A utility class to split the given {@link#UnfilteredRowIterator} into smaller chunks each
* having at most {@link #throttle} + 1 unfiltereds.
@@ -83,7 +83,7 @@
return throttledItr = origin;
}
- throttledItr = new WrappingUnfilteredRowIterator(origin)
+ throttledItr = new WrappingUnfilteredRowIterator()
{
private int count = 0;
private boolean isFirst = throttledItr == null;
@@ -97,9 +97,15 @@
private RangeTombstoneMarker closeMarker = null;
@Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return origin;
+ }
+
+ @Override
public boolean hasNext()
{
- return (withinLimit() && wrapped.hasNext()) || closeMarker != null;
+ return (withinLimit() && origin.hasNext()) || closeMarker != null;
}
@Override
@@ -119,7 +125,7 @@
if (overflowed.hasNext())
next = overflowed.next();
else
- next = wrapped.next();
+ next = origin.next();
recordNext(next);
return next;
}
@@ -132,8 +138,8 @@
// when reach throttle with a remaining openMarker, we need to create corresponding closeMarker.
if (count == throttle && openMarker != null)
{
- assert wrapped.hasNext();
- closeOpenMarker(wrapped.next());
+ assert origin.hasNext();
+ closeOpenMarker(origin.next());
}
}
@@ -191,13 +197,13 @@
@Override
public DeletionTime partitionLevelDeletion()
{
- return isFirst ? wrapped.partitionLevelDeletion() : DeletionTime.LIVE;
+ return isFirst ? origin.partitionLevelDeletion() : DeletionTime.LIVE;
}
@Override
public Row staticRow()
{
- return isFirst ? wrapped.staticRow() : Rows.EMPTY_STATIC_ROW;
+ return isFirst ? origin.staticRow() : Rows.EMPTY_STATIC_ROW;
}
@Override
@@ -258,4 +264,4 @@
}
};
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java
index 11541ee..1be3d54 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java
@@ -17,15 +17,21 @@
*/
package org.apache.cassandra.db.rows;
-import java.io.IOException;
import java.io.IOError;
+import java.io.IOException;
import java.nio.BufferOverflowException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.io.sstable.format.big.BigFormatPartitionWriter;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableMetadata;
@@ -60,7 +66,7 @@
*
* Please note that the format described above is the on-wire format. On-disk, the format is basically the
* same, but the header is written once per sstable, not once per-partition. Further, the actual row and
- * range tombstones are not written using this class, but rather by {@link ColumnIndex}.
+ * range tombstones are not written using this class, but rather by {@link BigFormatPartitionWriter}.
*/
public class UnfilteredRowIteratorSerializer
{
@@ -140,7 +146,7 @@
UnfilteredSerializer.serializer.serialize(staticRow, helper, out, version);
if (rowEstimate >= 0)
- out.writeUnsignedVInt(rowEstimate);
+ out.writeUnsignedVInt32(rowEstimate);
while (iterator.hasNext())
UnfilteredSerializer.serializer.serialize(iterator.next(), helper, out, version);
@@ -211,7 +217,7 @@
if (hasStatic)
staticRow = UnfilteredSerializer.serializer.deserializeStaticRow(in, header, new DeserializationHelper(metadata, version, flag));
- int rowEstimate = hasRowEstimate ? (int)in.readUnsignedVInt() : -1;
+ int rowEstimate = hasRowEstimate ? in.readUnsignedVInt32() : -1;
return new Header(header, key, isReversed, false, partitionDeletion, staticRow, rowEstimate);
}
@@ -280,4 +286,4 @@
sHeader, key, isReversed, isEmpty, partitionDeletion, staticRow, rowEstimate);
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
index 4d1d71b..fba163f 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
@@ -20,21 +20,27 @@
*/
package org.apache.cassandra.db.rows;
-import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.Comparator;
-import java.util.List;
+import java.util.Optional;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.Clusterable;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.transform.RTBoundValidator;
-import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.IteratorWithLowerBound;
/**
@@ -48,10 +54,11 @@
public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilteredRowIterator implements IteratorWithLowerBound<Unfiltered>
{
private final SSTableReader sstable;
- private final ClusteringIndexFilter filter;
+ private final Slices slices;
+ private final boolean isReverseOrder;
private final ColumnFilter selectedColumns;
private final SSTableReadsListener listener;
- private ClusteringBound<?> lowerBound;
+ private Optional<Unfiltered> lowerBoundMarker;
private boolean firstItemRetrieved;
public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey,
@@ -60,25 +67,44 @@
ColumnFilter selectedColumns,
SSTableReadsListener listener)
{
+ this(partitionKey, sstable, filter.getSlices(sstable.metadata()), filter.isReversed(), selectedColumns, listener);
+ }
+
+ @VisibleForTesting
+ public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey,
+ SSTableReader sstable,
+ Slices slices,
+ boolean isReverseOrder,
+ ColumnFilter selectedColumns,
+ SSTableReadsListener listener)
+ {
super(partitionKey);
this.sstable = sstable;
- this.filter = filter;
+ this.slices = slices;
+ this.isReverseOrder = isReverseOrder;
this.selectedColumns = selectedColumns;
this.listener = listener;
- this.lowerBound = null;
this.firstItemRetrieved = false;
}
public Unfiltered lowerBound()
{
- if (lowerBound != null)
- return makeBound(lowerBound);
+ if (lowerBoundMarker != null)
+ return lowerBoundMarker.orElse(null);
- // The partition index lower bound is more accurate than the sstable metadata lower bound but it is only
- // present if the iterator has already been initialized, which we only do when there are tombstones since in
- // this case we cannot use the sstable metadata clustering values
- ClusteringBound<?> ret = getPartitionIndexLowerBound();
- return ret != null ? makeBound(ret) : makeBound(getMetadataLowerBound());
+ // lower bound from cache may be more accurate as it stores information about clusterings range for that exact
+ // row, so we try it first (without initializing iterator)
+ ClusteringBound<?> lowerBound = maybeGetLowerBoundFromKeyCache();
+ if (lowerBound == null)
+ // If we couldn't get the lower bound from cache, we try with metadata
+ lowerBound = maybeGetLowerBoundFromMetadata();
+
+ if (lowerBound != null)
+ lowerBoundMarker = Optional.of(makeBound(lowerBound));
+ else
+ lowerBoundMarker = Optional.empty();
+
+ return lowerBoundMarker.orElse(null);
}
private Unfiltered makeBound(ClusteringBound<?> bound)
@@ -86,21 +112,15 @@
if (bound == null)
return null;
- if (lowerBound != bound)
- lowerBound = bound;
-
- return new RangeTombstoneBoundMarker(lowerBound, DeletionTime.LIVE);
+ return new ArtificialBoundMarker(bound);
}
@Override
protected UnfilteredRowIterator initializeIterator()
{
@SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
- UnfilteredRowIterator iter = RTBoundValidator.validate(
- sstable.rowIterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener),
- RTBoundValidator.Stage.SSTABLE,
- false
- );
+ UnfilteredRowIterator iter = RTBoundValidator.validate(sstable.rowIterator(partitionKey(), slices, selectedColumns, isReverseOrder, listener),
+ RTBoundValidator.Stage.SSTABLE, false);
return iter;
}
@@ -113,19 +133,20 @@
// Check that the lower bound is not bigger than the first item retrieved
firstItemRetrieved = true;
+ Unfiltered lowerBound = lowerBound();
if (lowerBound != null && ret != null)
- assert comparator().compare(lowerBound, ret.clustering()) <= 0
- : String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s",
- lowerBound.toString(metadata()),
- ret.toString(metadata()),
- sstable.getFilename());
+ assert comparator().compare(lowerBound.clustering(), ret.clustering()) <= 0
+ : String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s",
+ lowerBound.clustering().toString(metadata()),
+ ret.toString(metadata()),
+ sstable.getFilename());
return ret;
}
private Comparator<Clusterable> comparator()
{
- return filter.isReversed() ? metadata().comparator.reversed() : metadata().comparator;
+ return isReverseOrder ? metadata().comparator.reversed() : metadata().comparator;
}
@Override
@@ -137,7 +158,7 @@
@Override
public boolean isReverseOrder()
{
- return filter.isReversed();
+ return isReverseOrder;
}
@Override
@@ -155,7 +176,7 @@
@Override
public DeletionTime partitionLevelDeletion()
{
- if (!sstable.mayHaveTombstones())
+ if (!sstable.getSSTableMetadata().hasPartitionLevelDeletions)
return DeletionTime.LIVE;
return super.partitionLevelDeletion();
@@ -170,93 +191,66 @@
return super.staticRow();
}
- private static <V> ClusteringBound<V> createInclusiveOpen(boolean isReversed, ClusteringPrefix<V> from)
- {
- return from.accessor().factory().inclusiveOpen(isReversed, from.getRawValues());
- }
-
/**
* @return the lower bound stored on the index entry for this partition, if available.
*/
- private ClusteringBound<?> getPartitionIndexLowerBound()
+ private ClusteringBound<?> maybeGetLowerBoundFromKeyCache()
{
- // NOTE: CASSANDRA-11206 removed the lookup against the key-cache as the IndexInfo objects are no longer
- // in memory for not heap backed IndexInfo objects (so, these are on disk).
- // CASSANDRA-11369 is there to fix this afterwards.
+ if (sstable instanceof KeyCacheSupport<?>)
+ return ((KeyCacheSupport<?>) sstable).getLowerBoundPrefixFromCache(partitionKey(), isReverseOrder);
- // Creating the iterator ensures that rowIndexEntry is loaded if available (partitions bigger than
- // DatabaseDescriptor.column_index_size)
- if (!canUseMetadataLowerBound())
- maybeInit();
-
- RowIndexEntry rowIndexEntry = sstable.getCachedPosition(partitionKey(), false);
- if (rowIndexEntry == null || !rowIndexEntry.indexOnHeap())
- return null;
-
- try (RowIndexEntry.IndexInfoRetriever onHeapRetriever = rowIndexEntry.openWithIndex(null))
- {
- IndexInfo column = onHeapRetriever.columnsIndex(filter.isReversed() ? rowIndexEntry.columnsIndexCount() - 1 : 0);
- ClusteringPrefix<?> lowerBoundPrefix = filter.isReversed() ? column.lastName : column.firstName;
- assert lowerBoundPrefix.getRawValues().length <= metadata().comparator.size() :
- String.format("Unexpected number of clustering values %d, expected %d or fewer for %s",
- lowerBoundPrefix.getRawValues().length,
- metadata().comparator.size(),
- sstable.getFilename());
- return createInclusiveOpen(filter.isReversed(), lowerBoundPrefix);
- }
- catch (IOException e)
- {
- throw new RuntimeException("should never occur", e);
- }
+ return null;
}
/**
* Whether we can use the clustering values in the stats of the sstable to build the lower bound.
- * <p>
- * Currently, the clustering values of the stats file records for each clustering component the min and max
- * value seen, null excluded. In other words, having a non-null value for a component in those min/max clustering
- * values does _not_ guarantee that there isn't an unfiltered in the sstable whose clustering has either no value for
- * that component (it's a prefix) or a null value.
- * <p>
- * This is problematic as this means we can't in general build a lower bound from those values since the "min"
- * values doesn't actually guarantee minimality.
- * <p>
- * However, we can use those values if we can guarantee that no clustering in the sstable 1) is a true prefix and
- * 2) uses null values. Nat having true prefixes means having no range tombstone markers since rows use
- * {@link Clustering} which is always "full" (all components are always present). As for null values, we happen to
- * only allow those in compact tables (for backward compatibility), so we can simply exclude those tables.
- * <p>
- * Note that the information we currently have at our disposal make this condition less precise that it could be.
- * In particular, {@link SSTableReader#mayHaveTombstones} could return {@code true} (making us not use the stats)
- * because of cell tombstone or even expiring cells even if the sstable has no range tombstone markers, even though
- * it's really only markers we want to exclude here (more precisely, as said above, we want to exclude anything
- * whose clustering is not "full", but that's only markers). It wouldn't be very hard to collect whether a sstable
- * has any range tombstone marker however so it's a possible improvement.
*/
private boolean canUseMetadataLowerBound()
{
- // Side-note: pre-2.1 sstable stat file had clustering value arrays whose size may not match the comparator size
- // and that would break getMetadataLowerBound. We don't support upgrade from 2.0 to 3.0 directly however so it's
- // not a true concern. Besides, !sstable.mayHaveTombstones already ensure this is a 3.0 sstable anyway.
- return !sstable.mayHaveTombstones() && !sstable.metadata().isCompactTable();
+ if (sstable.metadata().isCompactTable())
+ return false;
+
+ Slices requestedSlices = slices;
+
+ if (requestedSlices.isEmpty())
+ return true;
+
+ // Simply exclude the cases where lower bound would not be used anyway, that is, the start of covered range of
+ // clusterings in sstable is lower than the requested slice. In such case, we need to access that sstable's
+ // iterator anyway so there is no need to use a lower bound optimization extra complexity.
+ if (!isReverseOrder())
+ {
+ return !requestedSlices.hasLowerBound() ||
+ metadata().comparator.compare(requestedSlices.start(), sstable.getSSTableMetadata().coveredClustering.start()) < 0;
+ }
+ else
+ {
+ return !requestedSlices.hasUpperBound() ||
+ metadata().comparator.compare(requestedSlices.end(), sstable.getSSTableMetadata().coveredClustering.end()) > 0;
+ }
}
/**
* @return a global lower bound made from the clustering values stored in the sstable metadata, note that
* this currently does not correctly compare tombstone bounds, especially ranges.
*/
- private ClusteringBound<?> getMetadataLowerBound()
+ private ClusteringBound<?> maybeGetLowerBoundFromMetadata()
{
if (!canUseMetadataLowerBound())
return null;
final StatsMetadata m = sstable.getSSTableMetadata();
- List<ByteBuffer> vals = filter.isReversed() ? m.maxClusteringValues : m.minClusteringValues;
- assert vals.size() <= metadata().comparator.size() :
- String.format("Unexpected number of clustering values %d, expected %d or fewer for %s",
- vals.size(),
- metadata().comparator.size(),
- sstable.getFilename());
- return ByteBufferAccessor.instance.factory().inclusiveOpen(filter.isReversed(), vals.toArray(new ByteBuffer[vals.size()]));
+ ClusteringBound<?> bound = m.coveredClustering.open(isReverseOrder);
+ assertBoundSize(bound, sstable);
+ return bound.artificialLowerBound(isReverseOrder);
}
-}
+
+ public static void assertBoundSize(ClusteringPrefix<?> lowerBound, SSTable sstable)
+ {
+ assert lowerBound.size() <= sstable.metadata().comparator.size() :
+ String.format("Unexpected number of clustering values %d, expected %d or fewer for %s",
+ lowerBound.size(),
+ sstable.metadata().comparator.size(),
+ sstable.getFilename());
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
index 2eb5d8f..95cf080 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
@@ -17,12 +17,17 @@
*/
package org.apache.cassandra.db.rows;
-import java.util.*;
+import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.transform.FilteredRows;
import org.apache.cassandra.db.transform.MoreRows;
@@ -263,16 +268,22 @@
/**
* Returns an iterator that concatenate the specified atom with the iterator.
*/
- public static UnfilteredRowIterator concat(final Unfiltered first, final UnfilteredRowIterator rest)
+ public static UnfilteredRowIterator concat(final Unfiltered first, final UnfilteredRowIterator wrapped)
{
- return new WrappingUnfilteredRowIterator(rest)
+ return new WrappingUnfilteredRowIterator()
{
private boolean hasReturnedFirst;
@Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return wrapped;
+ }
+
+ @Override
public boolean hasNext()
{
- return hasReturnedFirst ? super.hasNext() : true;
+ return hasReturnedFirst ? wrapped.hasNext() : true;
}
@Override
@@ -283,7 +294,7 @@
hasReturnedFirst = true;
return first;
}
- return super.next();
+ return wrapped.next();
}
};
}
@@ -604,4 +615,4 @@
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
index 1c0dcd4..d528a70 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
@@ -20,14 +20,14 @@
import java.io.IOException;
import net.nicoulaj.compilecommand.annotations.Inline;
-import org.apache.cassandra.db.marshal.ByteArrayAccessor;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
import org.apache.cassandra.db.rows.Row.Deletion;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.WrappedException;
@@ -196,7 +196,7 @@
// We write the size of the previous unfiltered to make reverse queries more efficient (and simpler).
// This is currently not used however and using it is tbd.
out.writeUnsignedVInt(previousUnfilteredSize);
- out.write(dob.getData(), 0, dob.getLength());
+ out.write(dob.unsafeGetBufferAndFlip());
}
}
else
@@ -270,7 +270,7 @@
if (hasComplexDeletion)
header.writeDeletionTime(data.complexDeletion(), out);
- out.writeUnsignedVInt(data.cellsCount());
+ out.writeUnsignedVInt32(data.cellsCount());
for (Cell<?> cell : data)
Cell.serializer.serialize(cell, column, out, rowLiveness, header);
}
@@ -662,7 +662,7 @@
builder.addComplexDeletion(column, complexDeletion);
}
- int count = (int) in.readUnsignedVInt();
+ int count = in.readUnsignedVInt32();
while (--count >= 0)
{
Cell<byte[]> cell = Cell.serializer.deserialize(in, rowLiveness, column, header, helper, ByteArrayAccessor.instance);
@@ -680,7 +680,7 @@
public void skipRowBody(DataInputPlus in) throws IOException
{
- int rowSize = (int)in.readUnsignedVInt();
+ int rowSize = in.readUnsignedVInt32();
in.skipBytesFully(rowSize);
}
@@ -695,7 +695,7 @@
public void skipMarkerBody(DataInputPlus in) throws IOException
{
- int markerSize = (int)in.readUnsignedVInt();
+ int markerSize = in.readUnsignedVInt32();
in.skipBytesFully(markerSize);
}
@@ -705,7 +705,7 @@
if (hasComplexDeletion)
header.skipDeletionTime(in);
- int count = (int) in.readUnsignedVInt();
+ int count = in.readUnsignedVInt32();
while (--count >= 0)
Cell.serializer.skip(in, column, header);
}
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java
index b984522..62c22ed 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java
@@ -23,7 +23,7 @@
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
/**
* Common data access interface for sstables and memtables.
diff --git a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java
index 02e1cec..e387136 100644
--- a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java
+++ b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java
@@ -17,76 +17,72 @@
*/
package org.apache.cassandra.db.rows;
-import com.google.common.collect.UnmodifiableIterator;
-
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
/**
* Abstract class to make writing unfiltered iterators that wrap another iterator
* easier. By default, the wrapping iterator simply delegate every call to
- * the wrapped iterator so concrete implementations will have to override
- * some of the methods.
+ * the wrapped iterator so concrete implementations will have to override some methods.
* <p>
* Note that if most of what you want to do is modifying/filtering the returned
- * {@code Unfiltered}, {@link org.apache.cassandra.db.transform.Transformation#merge(UnfilteredRowIterator,Transformation)} can be a simpler option.
+ * {@code Unfiltered}, {@link org.apache.cassandra.db.transform.Transformation#apply(UnfilteredRowIterator, Transformation)}
+ * can be a simpler option.
*/
-public abstract class WrappingUnfilteredRowIterator extends UnmodifiableIterator<Unfiltered> implements UnfilteredRowIterator
+public interface WrappingUnfilteredRowIterator extends UnfilteredRowIterator
{
- protected final UnfilteredRowIterator wrapped;
+ UnfilteredRowIterator wrapped();
- protected WrappingUnfilteredRowIterator(UnfilteredRowIterator wrapped)
+ default TableMetadata metadata()
{
- this.wrapped = wrapped;
+ return wrapped().metadata();
}
- public TableMetadata metadata()
+ default RegularAndStaticColumns columns()
{
- return wrapped.metadata();
+ return wrapped().columns();
}
- public RegularAndStaticColumns columns()
+ default boolean isReverseOrder()
{
- return wrapped.columns();
+ return wrapped().isReverseOrder();
}
- public boolean isReverseOrder()
+ default DecoratedKey partitionKey()
{
- return wrapped.isReverseOrder();
+ return wrapped().partitionKey();
}
- public DecoratedKey partitionKey()
+ default DeletionTime partitionLevelDeletion()
{
- return wrapped.partitionKey();
+ return wrapped().partitionLevelDeletion();
}
- public DeletionTime partitionLevelDeletion()
+ default Row staticRow()
{
- return wrapped.partitionLevelDeletion();
+ return wrapped().staticRow();
}
- public Row staticRow()
+ default EncodingStats stats()
{
- return wrapped.staticRow();
+ return wrapped().stats();
}
- public EncodingStats stats()
+ default boolean hasNext()
{
- return wrapped.stats();
+ return wrapped().hasNext();
}
- public boolean hasNext()
+ default Unfiltered next()
{
- return wrapped.hasNext();
+ return wrapped().next();
}
- public Unfiltered next()
+ default void close()
{
- return wrapped.next();
+ wrapped().close();
}
-
- public void close()
- {
- wrapped.close();
- }
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java
index 005a9aa..5a6e710 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java
@@ -76,7 +76,7 @@
{
TrackedDataInputPlus in = new TrackedDataInputPlus(cis);
deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
- writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
+ writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format);
String filename = writer.getFilename();
String sectionName = filename + '-' + fileSeqNum;
int sectionIdx = 0;
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
index 41fd9b1..806a74a 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java
@@ -27,12 +27,12 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.streaming.ProgressInfo;
-import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.utils.FBUtilities;
/**
@@ -71,7 +71,7 @@
int sectionIdx = 0;
// stream each of the required sections of the file
- String filename = sstable.descriptor.filenameFor(Component.DATA);
+ String filename = sstable.descriptor.fileFor(Components.DATA).toString();
for (Section section : sections)
{
// length of the section to stream
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
index 515c85d..9c5e048 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
@@ -25,17 +25,20 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IOOptions;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.big.BigTableZeroCopyWriter;
+import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.SequentialWriterOption;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamReceiver;
@@ -60,9 +63,6 @@
public CassandraEntireSSTableStreamReader(StreamMessageHeader messageHeader, CassandraStreamHeader streamHeader, StreamSession session)
{
- if (streamHeader.format != SSTableFormat.Type.BIG)
- throw new AssertionError("Unsupported SSTable format " + streamHeader.format);
-
if (session.getPendingRepair() != null)
{
// we should only ever be streaming pending repair sstables if the session has a pending repair id
@@ -84,7 +84,7 @@
*/
@SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed
@Override
- public SSTableMultiWriter read(DataInputPlus in) throws Throwable
+ public SSTableMultiWriter read(DataInputPlus in) throws IOException
{
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
if (cfs == null)
@@ -103,7 +103,7 @@
prettyPrintMemory(totalSize),
cfs.metadata());
- BigTableZeroCopyWriter writer = null;
+ SSTableZeroCopyWriter writer = null;
try
{
@@ -122,7 +122,7 @@
prettyPrintMemory(totalSize));
writer.writeComponent(component.type, in, length);
- session.progress(writer.descriptor.filenameFor(component), ProgressInfo.Direction.IN, length, length, length);
+ session.progress(writer.descriptor.fileFor(component).toString(), ProgressInfo.Direction.IN, length, length, length);
bytesRead += length;
logger.debug("[Stream #{}] Finished receiving {} component from {}, componentSize = {}, readBytes = {}, totalSize = {}",
@@ -145,7 +145,11 @@
{
logger.error("[Stream {}] Error while reading sstable from stream for table = {}", session.planId(), cfs.metadata(), e);
if (writer != null)
- e = writer.abort(e);
+ {
+ Throwable e2 = writer.abort(null);
+ if (e2 != null)
+ e.addSuppressed(e2);
+ }
throw e;
}
}
@@ -165,7 +169,7 @@
}
@SuppressWarnings("resource")
- protected BigTableZeroCopyWriter createWriter(ColumnFamilyStore cfs, long totalSize, Collection<Component> components) throws IOException
+ protected SSTableZeroCopyWriter createWriter(ColumnFamilyStore cfs, long totalSize, Collection<Component> components) throws IOException
{
File dataDir = getDataDir(cfs, totalSize);
@@ -174,10 +178,26 @@
LifecycleNewTracker lifecycleNewTracker = CassandraStreamReceiver.fromReceiver(session.getAggregator(tableId)).createLifecycleNewTracker();
- Descriptor desc = cfs.newSSTableDescriptor(dataDir, header.version, header.format);
+ Descriptor desc = cfs.newSSTableDescriptor(dataDir, header.version);
- logger.debug("[Table #{}] {} Components to write: {}", cfs.metadata(), desc.filenameFor(Component.DATA), components);
+ IOOptions ioOptions = new IOOptions(DatabaseDescriptor.getDiskOptimizationStrategy(),
+ DatabaseDescriptor.getDiskAccessMode(),
+ DatabaseDescriptor.getIndexAccessMode(),
+ DatabaseDescriptor.getDiskOptimizationEstimatePercentile(),
+ SequentialWriterOption.newBuilder()
+ .trickleFsync(false)
+ .bufferSize(2 << 20)
+ .bufferType(BufferType.OFF_HEAP)
+ .build(),
+ DatabaseDescriptor.getFlushCompression());
- return new BigTableZeroCopyWriter(desc, cfs.metadata, lifecycleNewTracker, components);
+ logger.debug("[Table #{}] {} Components to write: {}", cfs.metadata(), desc, components);
+ return desc.getFormat()
+ .getWriterFactory()
+ .builder(desc)
+ .setComponents(components)
+ .setTableMetadataRef(cfs.metadata)
+ .setIOOptions(ioOptions)
+ .createZeroCopyWriter(lifecycleNewTracker, cfs);
}
}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java
index 3d679a5..54fec49 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java
@@ -27,9 +27,9 @@
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.streaming.ProgressInfo;
-import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import static org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
@@ -93,7 +93,7 @@
long bytesWritten = out.writeFileToChannel(channel, limiter);
progress += bytesWritten;
- session.progress(sstable.descriptor.filenameFor(component), ProgressInfo.Direction.OUT, bytesWritten, bytesWritten, length);
+ session.progress(sstable.descriptor.fileFor(component).toString(), ProgressInfo.Direction.OUT, bytesWritten, bytesWritten, length);
logger.debug("[Stream #{}] Finished streaming {}.{} gen {} component {} to {}, xfered = {}, length = {}, totalSize = {}",
session.planId(),
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
index 367c304..88ecff8 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java
@@ -31,9 +31,9 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.OutgoingStream;
-import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Ref;
@@ -76,16 +76,13 @@
boolean shouldStreamEntireSSTable,
ComponentManifest manifest)
{
- boolean keepSSTableLevel = operation == StreamOperation.BOOTSTRAP || operation == StreamOperation.REBUILD;
-
CompressionInfo compressionInfo = sstable.compression
? CompressionInfo.newLazyInstance(sstable.getCompressionMetadata(), sections)
: null;
return CassandraStreamHeader.builder()
- .withSSTableFormat(sstable.descriptor.formatType)
.withSSTableVersion(sstable.descriptor.version)
- .withSSTableLevel(keepSSTableLevel ? sstable.getSSTableLevel() : 0)
+ .withSSTableLevel(operation.keepSSTableLevel() ? sstable.getSSTableLevel() : 0)
.withEstimatedKeys(estimatedKeys)
.withSections(sections)
.withCompressionInfo(compressionInfo)
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java
index c9e10cf..2b1223b 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java
@@ -26,6 +26,7 @@
import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SerializationHeader;
@@ -47,8 +48,6 @@
/** SSTable version */
public final Version version;
- /** SSTable format **/
- public final SSTableFormat.Type format;
public final long estimatedKeys;
public final List<SSTableReader.PartitionPositionBounds> sections;
public final CompressionInfo compressionInfo;
@@ -68,7 +67,6 @@
private CassandraStreamHeader(Builder builder)
{
version = builder.version;
- format = builder.format;
estimatedKeys = builder.estimatedKeys;
sections = builder.sections;
compressionInfo = builder.compressionInfo;
@@ -124,7 +122,6 @@
sstableLevel == that.sstableLevel &&
isEntireSSTable == that.isEntireSSTable &&
Objects.equals(version, that.version) &&
- format == that.format &&
Objects.equals(sections, that.sections) &&
Objects.equals(compressionInfo, that.compressionInfo) &&
Objects.equals(serializationHeader, that.serializationHeader) &&
@@ -136,7 +133,7 @@
@Override
public int hashCode()
{
- return Objects.hash(version, format, estimatedKeys, sections, compressionInfo, sstableLevel, serializationHeader, componentManifest,
+ return Objects.hash(version, estimatedKeys, sections, compressionInfo, sstableLevel, serializationHeader, componentManifest,
isEntireSSTable, firstKey, tableId);
}
@@ -145,7 +142,7 @@
{
return "CassandraStreamHeader{" +
"version=" + version +
- ", format=" + format +
+ ", format=" + version.format.name() +
", estimatedKeys=" + estimatedKeys +
", sections=" + sections +
", sstableLevel=" + sstableLevel +
@@ -163,7 +160,7 @@
public void serialize(CassandraStreamHeader header, DataOutputPlus out, int version) throws IOException
{
out.writeUTF(header.version.toString());
- out.writeUTF(header.format.name);
+ out.writeUTF(header.version.format.name());
out.writeLong(header.estimatedKeys);
out.writeInt(header.sections.size());
@@ -182,7 +179,7 @@
if (header.isEntireSSTable)
{
- ComponentManifest.serializer.serialize(header.componentManifest, out, version);
+ ComponentManifest.serializers.get(header.version.format.name()).serialize(header.componentManifest, out, version);
ByteBufferUtil.writeWithVIntLength(header.firstKey.getKey(), out);
}
}
@@ -201,8 +198,11 @@
@VisibleForTesting
public CassandraStreamHeader deserialize(DataInputPlus in, int version, Function<TableId, IPartitioner> partitionerMapper) throws IOException
{
- Version sstableVersion = SSTableFormat.Type.current().info.getVersion(in.readUTF());
- SSTableFormat.Type format = SSTableFormat.Type.validate(in.readUTF());
+ String sstableVersionString = in.readUTF();
+ String formatName = in.readUTF();
+ SSTableFormat<?, ?> format = Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(formatName),
+ String.format("Unknown SSTable format '%s'", formatName));
+ Version sstableVersion = format.getVersion(sstableVersionString);
long estimatedKeys = in.readLong();
int count = in.readInt();
@@ -221,7 +221,7 @@
if (isEntireSSTable)
{
- manifest = ComponentManifest.serializer.deserialize(in, version);
+ manifest = ComponentManifest.serializers.get(format.name()).deserialize(in, version);
ByteBuffer keyBuf = ByteBufferUtil.readWithVIntLength(in);
IPartitioner partitioner = partitionerMapper.apply(tableId);
if (partitioner == null)
@@ -229,8 +229,7 @@
firstKey = partitioner.decorateKey(keyBuf);
}
- return builder().withSSTableFormat(format)
- .withSSTableVersion(sstableVersion)
+ return builder().withSSTableVersion(sstableVersion)
.withSSTableLevel(sstableLevel)
.withEstimatedKeys(estimatedKeys)
.withSections(sections)
@@ -247,7 +246,7 @@
{
long size = 0;
size += TypeSizes.sizeof(header.version.toString());
- size += TypeSizes.sizeof(header.format.name);
+ size += TypeSizes.sizeof(header.version.format.name());
size += TypeSizes.sizeof(header.estimatedKeys);
size += TypeSizes.sizeof(header.sections.size());
@@ -267,7 +266,7 @@
if (header.isEntireSSTable)
{
- size += ComponentManifest.serializer.serializedSize(header.componentManifest, version);
+ size += ComponentManifest.serializers.get(header.version.format.name()).serializedSize(header.componentManifest, version);
size += ByteBufferUtil.serializedSizeWithVIntLength(header.firstKey.getKey());
}
return size;
@@ -277,7 +276,6 @@
public static final class Builder
{
private Version version;
- private SSTableFormat.Type format;
private long estimatedKeys;
private List<SSTableReader.PartitionPositionBounds> sections;
private CompressionInfo compressionInfo;
@@ -288,12 +286,6 @@
private DecoratedKey firstKey;
private TableId tableId;
- public Builder withSSTableFormat(SSTableFormat.Type format)
- {
- this.format = format;
- return this;
- }
-
public Builder withSSTableVersion(Version version)
{
this.version = version;
@@ -357,7 +349,6 @@
public CassandraStreamHeader build()
{
checkNotNull(version);
- checkNotNull(format);
checkNotNull(sections);
checkNotNull(serializationHeader);
checkNotNull(tableId);
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
index 46cf253..8ca7ac5 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
@@ -30,6 +30,7 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.service.ActiveRepairService;
@@ -105,7 +106,10 @@
}
else
{
- predicate = s -> s.isPendingRepair() && s.getSSTableMetadata().pendingRepair.equals(pendingRepair);
+ predicate = s -> {
+ StatsMetadata sstableMetadata = s.getSSTableMetadata();
+ return sstableMetadata.pendingRepair != ActiveRepairService.NO_PENDING_REPAIR && sstableMetadata.pendingRepair.equals(pendingRepair);
+ };
}
for (Range<PartitionPosition> keyRange : keyRanges)
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
index 04268f0..2325828 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
@@ -39,9 +39,9 @@
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.exceptions.UnknownColumnException;
+import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.SSTableSimpleIterator;
-import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
@@ -73,7 +73,6 @@
protected final Version inputVersion;
protected final long repairedAt;
protected final TimeUUID pendingRepair;
- protected final SSTableFormat.Type format;
protected final int sstableLevel;
protected final SerializationHeader.Component header;
protected final int fileSeqNum;
@@ -93,7 +92,6 @@
this.inputVersion = streamHeader.version;
this.repairedAt = header.repairedAt;
this.pendingRepair = header.pendingRepair;
- this.format = streamHeader.format;
this.sstableLevel = streamHeader.sstableLevel;
this.header = streamHeader.serializationHeader;
this.fileSeqNum = header.sequenceNumber;
@@ -125,7 +123,7 @@
{
TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream);
deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
- writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
+ writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format);
String sequenceName = writer.getFilename() + '-' + fileSeqNum;
long lastBytesRead = 0;
while (in.getBytesRead() < totalSize)
@@ -157,7 +155,7 @@
return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader
}
@SuppressWarnings("resource")
- protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat.Type format) throws IOException
+ protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat<?, ?> format) throws IOException
{
Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
if (localDir == null)
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
index 48de8b5..518d537 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
@@ -25,18 +25,16 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
-
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.io.sstable.SSTable;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.ThrottledUnfilteredIterator;
@@ -45,6 +43,7 @@
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.streaming.IncomingStream;
@@ -54,11 +53,13 @@
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Refs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH;
+
public class CassandraStreamReceiver implements StreamReceiver
{
private static final Logger logger = LoggerFactory.getLogger(CassandraStreamReceiver.class);
- private static final int MAX_ROWS_PER_BATCH = Integer.getInteger("cassandra.repair.mutation_repair_rows_per_batch", 100);
+ private static final int MAX_ROWS_PER_BATCH = REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH.getInt();
private final ColumnFamilyStore cfs;
private final StreamSession session;
@@ -172,23 +173,31 @@
return cfs.metadata().params.cdc;
}
+ // returns true iif it is a cdc table and cdc on repair is enabled.
+ private boolean cdcRequiresWriteCommitLog(ColumnFamilyStore cfs)
+ {
+ return DatabaseDescriptor.isCDCOnRepairEnabled() && hasCDC(cfs);
+ }
+
/*
* We have a special path for views and for CDC.
*
* For views, since the view requires cleaning up any pre-existing state, we must put all partitions
* through the same write path as normal mutations. This also ensures any 2is are also updated.
*
- * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they
- * can be archived by the CDC process on discard.
+ * For CDC-enabled tables and write path for CDC is enabled, we want to ensure that the mutations are
+ * run through the CommitLog, so they can be archived by the CDC process on discard.
*/
private boolean requiresWritePath(ColumnFamilyStore cfs)
{
- return hasCDC(cfs) || cfs.streamToMemtable() || (session.streamOperation().requiresViewBuild() && hasViews(cfs));
+ return cdcRequiresWriteCommitLog(cfs)
+ || cfs.streamToMemtable()
+ || (session.streamOperation().requiresViewBuild() && hasViews(cfs));
}
private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReader> readers)
{
- boolean hasCdc = hasCDC(cfs);
+ boolean writeCDCCommitLog = cdcRequiresWriteCommitLog(cfs);
ColumnFilter filter = ColumnFilter.all(cfs.metadata());
for (SSTableReader reader : readers)
{
@@ -206,7 +215,7 @@
// If the CFS has CDC, however, these updates need to be written to the CommitLog
// so they get archived into the cdc_raw folder
ks.apply(new Mutation(PartitionUpdate.fromIterator(throttledPartitions.next(), filter)),
- hasCdc,
+ writeCDCCommitLog,
true,
false);
}
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
index 9d9ea3c..3e98c7d 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java
@@ -21,23 +21,21 @@
import java.nio.ByteBuffer;
import java.util.Collection;
-import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import org.apache.cassandra.io.compress.BufferType;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.ChannelProxy;
-import org.apache.cassandra.io.util.DataIntegrityMetadata;
import org.apache.cassandra.io.util.DataIntegrityMetadata.ChecksumValidator;
import org.apache.cassandra.streaming.ProgressInfo;
-import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamingDataOutputPlus;
import org.apache.cassandra.streaming.async.StreamCompressionSerializer;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.memory.BufferPools;
@@ -84,9 +82,7 @@
sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
try(ChannelProxy proxy = sstable.getDataChannel().newChannel();
- ChecksumValidator validator = new File(sstable.descriptor.filenameFor(Component.CRC)).exists()
- ? DataIntegrityMetadata.checksumValidator(sstable.descriptor)
- : null)
+ ChecksumValidator validator = sstable.maybeGetChecksumValidator())
{
int bufferSize = validator == null ? DEFAULT_CHUNK_SIZE: validator.chunkSize;
@@ -94,7 +90,7 @@
long progress = 0L;
// stream each of the required sections of the file
- String filename = sstable.descriptor.filenameFor(Component.DATA);
+ String filename = sstable.descriptor.fileFor(Components.DATA).toString();
for (SSTableReader.PartitionPositionBounds section : sections)
{
long start = validator == null ? section.lowerPosition : validator.chunkStart(section.lowerPosition);
diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentContext.java b/src/java/org/apache/cassandra/db/streaming/ComponentContext.java
index c8c08aa..164dd6b 100644
--- a/src/java/org/apache/cassandra/db/streaming/ComponentContext.java
+++ b/src/java/org/apache/cassandra/db/streaming/ComponentContext.java
@@ -18,32 +18,23 @@
package org.apache.cassandra.db.streaming;
-import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Map;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.util.FileUtils;
-
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Mutable SSTable components and their hardlinks to avoid concurrent sstable component modification
- * during entire-sstable-streaming.
- */
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
public class ComponentContext implements AutoCloseable
{
private static final Logger logger = LoggerFactory.getLogger(ComponentContext.class);
- private static final Set<Component> MUTABLE_COMPONENTS = ImmutableSet.of(Component.STATS, Component.SUMMARY);
-
private final Map<Component, File> hardLinks;
private final ComponentManifest manifest;
@@ -57,13 +48,13 @@
{
Map<Component, File> hardLinks = new HashMap<>(1);
- for (Component component : MUTABLE_COMPONENTS)
+ for (Component component : descriptor.getFormat().mutableComponents())
{
- File file = new File(descriptor.filenameFor(component));
+ File file = descriptor.fileFor(component);
if (!file.exists())
continue;
- File hardlink = new File(descriptor.tmpFilenameForStreaming(component));
+ File hardlink = descriptor.tmpFileForStreaming(component);
FileUtils.createHardLink(file, hardlink);
hardLinks.put(component, hardlink);
}
@@ -81,9 +72,9 @@
*/
public FileChannel channel(Descriptor descriptor, Component component, long size) throws IOException
{
- String toTransfer = hardLinks.containsKey(component) ? hardLinks.get(component).path() : descriptor.filenameFor(component);
+ File toTransfer = hardLinks.containsKey(component) ? hardLinks.get(component) : descriptor.fileFor(component);
@SuppressWarnings("resource") // file channel will be closed by Caller
- FileChannel channel = new File(toTransfer).newReadChannel();
+ FileChannel channel = toTransfer.newReadChannel();
assert size == channel.size() : String.format("Entire sstable streaming expects %s file size to be %s but got %s.",
component, size, channel.size());
diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java
index b77b594..5e3cc0c 100644
--- a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java
+++ b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java
@@ -18,30 +18,32 @@
package org.apache.cassandra.db.streaming;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-
-import java.io.IOException;
-import java.util.*;
+import org.apache.cassandra.io.util.File;
/**
* SSTable components and their sizes to be tranfered via entire-sstable-streaming
*/
-import org.apache.cassandra.io.util.File;
-
public final class ComponentManifest implements Iterable<Component>
{
- private static final List<Component> STREAM_COMPONENTS = ImmutableList.of(Component.DATA, Component.PRIMARY_INDEX, Component.STATS,
- Component.COMPRESSION_INFO, Component.FILTER, Component.SUMMARY,
- Component.DIGEST, Component.CRC);
-
private final LinkedHashMap<Component, Long> components;
public ComponentManifest(Map<Component, Long> components)
@@ -52,11 +54,11 @@
@VisibleForTesting
public static ComponentManifest create(Descriptor descriptor)
{
- LinkedHashMap<Component, Long> components = new LinkedHashMap<>(STREAM_COMPONENTS.size());
+ LinkedHashMap<Component, Long> components = new LinkedHashMap<>(descriptor.getFormat().streamingComponents().size());
- for (Component component : STREAM_COMPONENTS)
+ for (Component component : descriptor.getFormat().streamingComponents())
{
- File file = new File(descriptor.filenameFor(component));
+ File file = descriptor.fileFor(component);
if (!file.exists())
continue;
@@ -114,45 +116,58 @@
'}';
}
- public static final IVersionedSerializer<ComponentManifest> serializer = new IVersionedSerializer<ComponentManifest>()
+ public static final Map<String, IVersionedSerializer<ComponentManifest>> serializers;
+
+ static
{
- public void serialize(ComponentManifest manifest, DataOutputPlus out, int version) throws IOException
+ ImmutableMap.Builder<String, IVersionedSerializer<ComponentManifest>> b = ImmutableMap.builder();
+ for (SSTableFormat<?, ?> format : DatabaseDescriptor.getSSTableFormats().values())
{
- out.writeUnsignedVInt(manifest.components.size());
- for (Map.Entry<Component, Long> entry : manifest.components.entrySet())
+ IVersionedSerializer<ComponentManifest> serializer = new IVersionedSerializer<ComponentManifest>()
{
- out.writeUTF(entry.getKey().name);
- out.writeUnsignedVInt(entry.getValue());
- }
+
+ public void serialize(ComponentManifest manifest, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUnsignedVInt32(manifest.components.size());
+ for (Map.Entry<Component, Long> entry : manifest.components.entrySet())
+ {
+ out.writeUTF(entry.getKey().name);
+ out.writeUnsignedVInt(entry.getValue());
+ }
+ }
+
+ public ComponentManifest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ int size = in.readUnsignedVInt32();
+
+ LinkedHashMap<Component, Long> components = new LinkedHashMap<>(size);
+
+ for (int i = 0; i < size; i++)
+ {
+ Component component = Component.parse(in.readUTF(), format);
+ long length = in.readUnsignedVInt();
+ components.put(component, length);
+ }
+
+ return new ComponentManifest(components);
+ }
+
+ public long serializedSize(ComponentManifest manifest, int version)
+ {
+ long size = TypeSizes.sizeofUnsignedVInt(manifest.components.size());
+ for (Map.Entry<Component, Long> entry : manifest.components.entrySet())
+ {
+ size += TypeSizes.sizeof(entry.getKey().name);
+ size += TypeSizes.sizeofUnsignedVInt(entry.getValue());
+ }
+ return size;
+ }
+ };
+
+ b.put(format.name(), serializer);
}
-
- public ComponentManifest deserialize(DataInputPlus in, int version) throws IOException
- {
- int size = (int) in.readUnsignedVInt();
-
- LinkedHashMap<Component, Long> components = new LinkedHashMap<>(size);
-
- for (int i = 0; i < size; i++)
- {
- Component component = Component.parse(in.readUTF());
- long length = in.readUnsignedVInt();
- components.put(component, length);
- }
-
- return new ComponentManifest(components);
- }
-
- public long serializedSize(ComponentManifest manifest, int version)
- {
- long size = TypeSizes.sizeofUnsignedVInt(manifest.components.size());
- for (Map.Entry<Component, Long> entry : manifest.components.entrySet())
- {
- size += TypeSizes.sizeof(entry.getKey().name);
- size += TypeSizes.sizeofUnsignedVInt(entry.getValue());
- }
- return size;
- }
- };
+ serializers = b.build();
+ }
@Override
public Iterator<Component> iterator()
diff --git a/src/java/org/apache/cassandra/db/transform/BasePartitions.java b/src/java/org/apache/cassandra/db/transform/BasePartitions.java
index 464ae6f..79e8952 100644
--- a/src/java/org/apache/cassandra/db/transform/BasePartitions.java
+++ b/src/java/org/apache/cassandra/db/transform/BasePartitions.java
@@ -20,8 +20,6 @@
*/
package org.apache.cassandra.db.transform;
-import java.util.Collections;
-
import org.apache.cassandra.db.partitions.BasePartitionIterator;
import org.apache.cassandra.db.rows.BaseRowIterator;
import org.apache.cassandra.utils.Throwables;
@@ -112,7 +110,7 @@
catch (Throwable t)
{
if (next != null)
- Throwables.close(t, Collections.singleton(next));
+ Throwables.close(t, next);
throw t;
}
}
diff --git a/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java b/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java
index eb37f4b..e197ce2 100644
--- a/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java
+++ b/src/java/org/apache/cassandra/db/transform/RTBoundValidator.java
@@ -17,11 +17,12 @@
*/
package org.apache.cassandra.db.transform;
+import java.util.Objects;
+
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.RangeTombstoneMarker;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.schema.TableMetadata;
/**
* A validating transformation that sanity-checks the sequence of RT bounds and boundaries in every partition.
@@ -51,29 +52,27 @@
public static UnfilteredRowIterator validate(UnfilteredRowIterator partition, Stage stage, boolean enforceIsClosed)
{
- return Transformation.apply(partition, new RowsTransformation(stage, partition.metadata(), partition.isReverseOrder(), enforceIsClosed));
+ return Transformation.apply(partition, new RowsTransformation(stage, partition, enforceIsClosed));
}
@Override
public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
{
- return Transformation.apply(partition, new RowsTransformation(stage, partition.metadata(), partition.isReverseOrder(), enforceIsClosed));
+ return Transformation.apply(partition, new RowsTransformation(stage, partition, enforceIsClosed));
}
private final static class RowsTransformation extends Transformation
{
private final Stage stage;
- private final TableMetadata metadata;
- private final boolean isReverseOrder;
private final boolean enforceIsClosed;
+ private final UnfilteredRowIterator partition;
private DeletionTime openMarkerDeletionTime;
- private RowsTransformation(Stage stage, TableMetadata metadata, boolean isReverseOrder, boolean enforceIsClosed)
+ private RowsTransformation(Stage stage, UnfilteredRowIterator partition, boolean enforceIsClosed)
{
this.stage = stage;
- this.metadata = metadata;
- this.isReverseOrder = isReverseOrder;
+ this.partition = partition;
this.enforceIsClosed = enforceIsClosed;
}
@@ -83,25 +82,25 @@
if (null == openMarkerDeletionTime)
{
// there is no open RT in the stream - we are expecting a *_START_BOUND
- if (marker.isClose(isReverseOrder))
- throw ise("unexpected end bound or boundary " + marker.toString(metadata));
+ if (marker.isClose(partition.isReverseOrder()))
+ throw ise("unexpected end bound or boundary " + marker.toString(partition.metadata()));
}
else
{
// there is an open RT in the stream - we are expecting a *_BOUNDARY or an *_END_BOUND
- if (!marker.isClose(isReverseOrder))
- throw ise("start bound followed by another start bound " + marker.toString(metadata));
+ if (!marker.isClose(partition.isReverseOrder()))
+ throw ise("start bound followed by another start bound " + marker.toString(partition.metadata()));
// deletion times of open/close markers must match
- DeletionTime deletionTime = marker.closeDeletionTime(isReverseOrder);
+ DeletionTime deletionTime = marker.closeDeletionTime(partition.isReverseOrder());
if (!deletionTime.equals(openMarkerDeletionTime))
- throw ise("open marker and close marker have different deletion times");
+ throw ise("open marker and close marker have different deletion times, close=" + deletionTime);
openMarkerDeletionTime = null;
}
- if (marker.isOpen(isReverseOrder))
- openMarkerDeletionTime = marker.openDeletionTime(isReverseOrder);
+ if (marker.isOpen(partition.isReverseOrder()))
+ openMarkerDeletionTime = marker.openDeletionTime(partition.isReverseOrder());
return marker;
}
@@ -115,9 +114,17 @@
private IllegalStateException ise(String why)
{
- String message =
- String.format("%s UnfilteredRowIterator for %s has an illegal RT bounds sequence: %s", stage, metadata, why);
- throw new IllegalStateException(message);
+ throw new IllegalStateException(message(why));
+ }
+
+ private String message(String why)
+ {
+ return String.format("%s UnfilteredRowIterator for %s (key: %s omdt: [%s]) has an illegal RT bounds sequence: %s",
+ stage,
+ partition.metadata(),
+ partition.metadata().partitionKeyType.getString(partition.partitionKey().getKey()),
+ Objects.toString(openMarkerDeletionTime, "not present"),
+ why);
}
}
}
diff --git a/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java b/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java
new file mode 100644
index 0000000..0336910
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/CollectionMergeTrie.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * A merged view of multiple tries.
+ *
+ * This is accomplished by walking the cursors in parallel; the merged cursor takes the position and features of the
+ * smallest and advances with it; when multiple cursors are equal, all of them are advanced. The ordered view of the
+ * cursors is maintained using a custom binary min-heap, built for efficiently reforming the heap when the top elements
+ * are advanced (see {@link CollectionMergeCursor}).
+ *
+ * Crucial for the efficiency of this is the fact that when they are advanced like this, we can compare cursors'
+ * positions by their depth descending and then incomingTransition ascending.
+ *
+ * See Trie.md for further details.
+ */
+class CollectionMergeTrie<T> extends Trie<T>
+{
+ private final CollectionMergeResolver<T> resolver; // only called on more than one input
+ protected final Collection<? extends Trie<T>> inputs;
+
+ CollectionMergeTrie(Collection<? extends Trie<T>> inputs, CollectionMergeResolver<T> resolver)
+ {
+ this.resolver = resolver;
+ this.inputs = inputs;
+ }
+
+ @Override
+ protected Cursor<T> cursor()
+ {
+ return new CollectionMergeCursor<>(resolver, inputs);
+ }
+
+ /**
+ * Compare the positions of two cursors. One is before the other when
+ * - its depth is greater, or
+ * - its depth is equal, and the incoming transition is smaller.
+ */
+ static <T> boolean greaterCursor(Cursor<T> c1, Cursor<T> c2)
+ {
+ int c1depth = c1.depth();
+ int c2depth = c2.depth();
+ if (c1depth != c2depth)
+ return c1depth < c2depth;
+ return c1.incomingTransition() > c2.incomingTransition();
+ }
+
+ static <T> boolean equalCursor(Cursor<T> c1, Cursor<T> c2)
+ {
+ return c1.depth() == c2.depth() && c1.incomingTransition() == c2.incomingTransition();
+ }
+
+ /*
+ * The merge cursor is a variation of the idea of a merge iterator with one key observation: because we advance
+ * the source iterators together, we can compare them just by depth and incoming transition.
+ *
+ * The most straightforward way to implement merging of iterators is to use a {@code PriorityQueue},
+ * {@code poll} it to find the next item to consume, then {@code add} the iterator back after advancing.
+ * This is not very efficient as {@code poll} and {@code add} in all cases require at least
+ * {@code log(size)} comparisons and swaps (usually more than {@code 2*log(size)}) per consumed item, even
+ * if the input is suitable for fast iteration.
+ *
+ * The implementation below makes use of the fact that replacing the top element in a binary heap can be
+ * done much more efficiently than separately removing it and placing it back, especially in the cases where
+ * the top iterator is to be used again very soon (e.g. when there are large sections of the output where
+ * only a limited number of input iterators overlap, which is normally the case in many practically useful
+ * situations, e.g. levelled compaction).
+ *
+ * The implementation builds and maintains a binary heap of sources (stored in an array), where we do not
+ * add items after the initial construction. Instead we advance the smallest element (which is at the top
+ * of the heap) and push it down to find its place for its new position. Should this source be exhausted,
+ * we swap it with the last source in the heap and proceed by pushing that down in the heap.
+ *
+ * In the case where we have multiple sources with matching positions, the merging algorithm
+ * must be able to merge all equal values. To achieve this {@code content} walks the heap to
+ * find all equal cursors without advancing them, and separately {@code advance} advances
+ * all equal sources and restores the heap structure.
+ *
+ * The latter is done equivalently to the process of initial construction of a min-heap using back-to-front
+ * heapification as done in the classic heapsort algorithm. It only needs to heapify subheaps whose top item
+ * is advanced (i.e. one whose position matches the current), and we can do that recursively from
+ * bottom to top. Should a source be exhausted when advancing, it can be thrown away by swapping in the last
+ * source in the heap (note: we must be careful to advance that source too if required).
+ *
+ * To make it easier to advance efficienty in single-sourced branches of tries, we extract the current smallest
+ * cursor (the head) separately, and start any advance with comparing that to the heap's first. When the smallest
+ * cursor remains the same (e.g. in branches coming from a single source) this makes it possible to advance with
+ * just one comparison instead of two at the expense of increasing the number by one in the general case.
+ *
+ * Note: This is a simplification of the MergeIterator code from CASSANDRA-8915, without the leading ordered
+ * section and equalParent flag since comparisons of cursor positions are cheap.
+ */
+ static class CollectionMergeCursor<T> implements Cursor<T>
+ {
+ private final CollectionMergeResolver<T> resolver;
+
+ /**
+ * The smallest cursor, tracked separately to improve performance in single-source sections of the trie.
+ */
+ private Cursor<T> head;
+
+ /**
+ * Binary heap of the remaining cursors. The smallest element is at position 0.
+ * Every element i is smaller than or equal to its two children, i.e.
+ * heap[i] <= heap[i*2 + 1] && heap[i] <= heap[i*2 + 2]
+ */
+ private final Cursor<T>[] heap;
+
+ /**
+ * A list used to collect contents during content() calls.
+ */
+ private final List<T> contents;
+
+ public CollectionMergeCursor(CollectionMergeResolver<T> resolver, Collection<? extends Trie<T>> inputs)
+ {
+ this.resolver = resolver;
+ int count = inputs.size();
+ // Get cursors for all inputs. Put one of them in head and the rest in the heap.
+ heap = new Cursor[count - 1];
+ contents = new ArrayList<>(count);
+ int i = -1;
+ for (Trie<T> trie : inputs)
+ {
+ Cursor<T> cursor = trie.cursor();
+ assert cursor.depth() == 0;
+ if (i >= 0)
+ heap[i] = cursor;
+ else
+ head = cursor;
+ ++i;
+ }
+ // The cursors are all currently positioned on the root and thus in valid heap order.
+ }
+
+ /**
+ * Interface for internal operations that can be applied to the equal top elements of the heap.
+ */
+ interface HeapOp<T>
+ {
+ void apply(CollectionMergeCursor<T> self, Cursor<T> cursor, int index);
+ }
+
+ /**
+ * Apply a non-interfering operation, i.e. one that does not change the cursor state, to all inputs in the heap
+ * that are on equal position to the head.
+ * For interfering operations like advancing the cursors, use {@link #advanceEqualAndRestoreHeap(AdvancingHeapOp)}.
+ */
+ private void applyToEqualOnHeap(HeapOp<T> action)
+ {
+ applyToEqualElementsInHeap(action, 0);
+ }
+
+ /**
+ * Interface for internal advancing operations that can be applied to the heap cursors. This interface provides
+ * the code to restore the heap structure after advancing the cursors.
+ */
+ interface AdvancingHeapOp<T> extends HeapOp<T>
+ {
+ void apply(Cursor<T> cursor);
+
+ default void apply(CollectionMergeCursor<T> self, Cursor<T> cursor, int index)
+ {
+ // Apply the operation, which should advance the position of the element.
+ apply(cursor);
+
+ // This method is called on the back path of the recursion. At this point the heaps at both children are
+ // advanced and well-formed.
+ // Place current node in its proper position.
+ self.heapifyDown(cursor, index);
+ // The heap rooted at index is now advanced and well-formed.
+ }
+ }
+
+
+ /**
+ * Advance the state of all inputs in the heap that are on equal position as the head and restore the heap
+ * invariant.
+ */
+ private void advanceEqualAndRestoreHeap(AdvancingHeapOp<T> action)
+ {
+ applyToEqualElementsInHeap(action, 0);
+ }
+
+ /**
+ * Apply an operation to all elements on the heap that are equal to the head. Descends recursively in the heap
+ * structure to all equal children and applies the operation on the way back.
+ *
+ * This operation can be something that does not change the cursor state (see {@link #content}) or an operation
+ * that advances the cursor to a new state, wrapped in a {@link AdvancingHeapOp} ({@link #advance} or
+ * {@link #skipChildren}). The latter interface takes care of pushing elements down in the heap after advancing
+ * and restores the subheap state on return from each level of the recursion.
+ */
+ private void applyToEqualElementsInHeap(HeapOp<T> action, int index)
+ {
+ if (index >= heap.length)
+ return;
+ Cursor<T> item = heap[index];
+ if (!equalCursor(item, head))
+ return;
+
+ // If the children are at the same position, they also need advancing and their subheap
+ // invariant to be restored.
+ applyToEqualElementsInHeap(action, index * 2 + 1);
+ applyToEqualElementsInHeap(action, index * 2 + 2);
+
+ // Apply the action. This is done on the reverse direction to give the action a chance to form proper
+ // subheaps and combine them on processing the parent.
+ action.apply(this, item, index);
+ }
+
+ /**
+ * Push the given state down in the heap from the given index until it finds its proper place among
+ * the subheap rooted at that position.
+ */
+ private void heapifyDown(Cursor<T> item, int index)
+ {
+ while (true)
+ {
+ int next = index * 2 + 1;
+ if (next >= heap.length)
+ break;
+ // Select the smaller of the two children to push down to.
+ if (next + 1 < heap.length && greaterCursor(heap[next], heap[next + 1]))
+ ++next;
+ // If the child is greater or equal, the invariant has been restored.
+ if (!greaterCursor(item, heap[next]))
+ break;
+ heap[index] = heap[next];
+ index = next;
+ }
+ heap[index] = item;
+ }
+
+ /**
+ * Check if the head is greater than the top element in the heap, and if so, swap them and push down the new
+ * top until its proper place.
+ * @param headDepth the depth of the head cursor (as returned by e.g. advance).
+ * @return the new head element's depth
+ */
+ private int maybeSwapHead(int headDepth)
+ {
+ int heap0Depth = heap[0].depth();
+ if (headDepth > heap0Depth ||
+ (headDepth == heap0Depth && head.incomingTransition() <= heap[0].incomingTransition()))
+ return headDepth; // head is still smallest
+
+ // otherwise we need to swap heap and heap[0]
+ Cursor<T> newHeap0 = head;
+ head = heap[0];
+ heapifyDown(newHeap0, 0);
+ return heap0Depth;
+ }
+
+ @Override
+ public int advance()
+ {
+ advanceEqualAndRestoreHeap(Cursor::advance);
+ return maybeSwapHead(head.advance());
+ }
+
+ @Override
+ public int advanceMultiple(TransitionsReceiver receiver)
+ {
+ // If the current position is present in just one cursor, we can safely descend multiple levels within
+ // its branch as no one of the other tries has content for it.
+ if (equalCursor(heap[0], head))
+ return advance(); // More than one source at current position, do single-step advance.
+
+ // If there are no children, i.e. the cursor ascends, we have to check if it's become larger than some
+ // other candidate.
+ return maybeSwapHead(head.advanceMultiple(receiver));
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ advanceEqualAndRestoreHeap(Cursor::skipChildren);
+ return maybeSwapHead(head.skipChildren());
+ }
+
+ @Override
+ public int depth()
+ {
+ return head.depth();
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return head.incomingTransition();
+ }
+
+ @Override
+ public T content()
+ {
+ applyToEqualOnHeap(CollectionMergeCursor::collectContent);
+ collectContent(head, -1);
+
+ T toReturn;
+ switch (contents.size())
+ {
+ case 0:
+ toReturn = null;
+ break;
+ case 1:
+ toReturn = contents.get(0);
+ break;
+ default:
+ toReturn = resolver.resolve(contents);
+ break;
+ }
+ contents.clear();
+ return toReturn;
+ }
+
+ private void collectContent(Cursor<T> item, int index)
+ {
+ T itemContent = item.content();
+ if (itemContent != null)
+ contents.add(itemContent);
+ }
+ }
+
+ /**
+ * Special instance for sources that are guaranteed distinct. The main difference is that we can form unordered
+ * value list by concatenating sources.
+ */
+ static class Distinct<T> extends CollectionMergeTrie<T>
+ {
+ Distinct(Collection<? extends Trie<T>> inputs)
+ {
+ super(inputs, throwingResolver());
+ }
+
+ @Override
+ public Iterable<T> valuesUnordered()
+ {
+ return Iterables.concat(Iterables.transform(inputs, Trie::valuesUnordered));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java b/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java
new file mode 100644
index 0000000..88f5987
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java
@@ -0,0 +1,920 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.function.Function;
+
+import org.agrona.concurrent.UnsafeBuffer;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * In-memory trie built for fast modification and reads executing concurrently with writes from a single mutator thread.
+ *
+ * This class provides the read-only functionality, expanded in {@link InMemoryTrie} to writes.
+ */
+public class InMemoryReadTrie<T> extends Trie<T>
+{
+ /*
+ TRIE FORMAT AND NODE TYPES
+
+ The memtable trie uses five different types of nodes:
+ - "leaf" nodes, which have content and no children;
+ - single-transition "chain" nodes, which have exactly one child; while each node is a single transition, they are
+ called "chain" because multiple such transition are packed in a block.
+ - "sparse" nodes which have between two and six children;
+ - "split" nodes for anything above six children;
+ - "prefix" nodes that augment one of the other types (except leaf) with content.
+
+ The data for all nodes except leaf ones is stored in a contiguous 'node buffer' and laid out in blocks of 32 bytes.
+ A block only contains data for a single type of node, but there is no direct correspondence between block and node
+ in that:
+ - a single block can contain multiple "chain" nodes.
+ - a sparse node occupies exactly one block.
+ - a split node occupies a variable number of blocks.
+ - a prefix node can be placed in the same block as the node it augments, or in a separate block.
+
+ Nodes are referenced in that buffer by an integer position/pointer, the 'node pointer'. Note that node pointers are
+ not pointing at the beginning of blocks, and we call 'pointer offset' the offset of the node pointer to the block it
+ points into. The value of a 'node pointer' is used to decide what kind of node is pointed:
+
+ - If the pointer is negative, we have a leaf node. Since a leaf has no children, we need no data outside of its
+ content to represent it, and that content is stored in a 'content list', not in the nodes buffer. The content
+ of a particular leaf node is located at the ~pointer position in the content list (~ instead of - so that -1 can
+ correspond to position 0).
+
+ - If the 'pointer offset' is smaller than 28, we have a chain node with one transition. The transition character is
+ the byte at the position pointed in the 'node buffer', and the child is pointed by:
+ - the integer value at offset 28 of the block pointed if the 'pointer offset' is 27
+ - pointer + 1 (which is guaranteed to have offset smaller than 28, i.e. to be a chain node), otherwise
+ In other words, a chain block contains a sequence of characters that leads to the child whose address is at
+ offset 28. It may have between 1 and 28 characters depending on the pointer with which the block is entered.
+
+ - If the 'pointer offset' is 30, we have a sparse node. The data of a sparse node occupies a full block and is laid
+ out as:
+ - six pointers to children at offsets 0 to 24
+ - six transition characters at offsets 24 to 30
+ - an order word stored in the two bytes at offset 30
+ To enable in-place addition of children, the pointers and transition characters are not stored ordered.
+ Instead, we use an order encoding in the last 2 bytes of the node. The encoding is a base-6 number which
+ describes the order of the transitions (least significant digit being the smallest).
+ The node must have at least two transitions and the transition at position 0 is never the biggest (we can
+ enforce this by choosing for position 0 the smaller of the two transitions a sparse node starts with). This
+ allows iteration over the order word (which divides said word by 6 each step) to finish when the result becomes 0.
+
+ - If the 'pointer offset' is 28, the node is a split one. Split nodes are dense, meaning that there is a direct
+ mapping between a transition character and the address of the associated pointer, and new children can easily be
+ added in place.
+ Split nodes occupy multiple blocks, and a child is located by traversing 3 layers of pointers:
+ - the first pointer is within the top-level block (the one pointed by the pointer) and points to a "mid" block.
+ The top-level block has 4 such pointers to "mid" block, located between offset 16 and 32.
+ - the 2nd pointer is within the "mid" block and points to a "tail" block. A "mid" block has 8 such pointers
+ occupying the whole block.
+ - the 3rd pointer is with the "tail" block and is the actual child pointer. Like "mid" block, there are 8 such
+ pointers (so we finally address 4 * 8 * 8 = 256 children).
+ To find a child, we thus need to know the index of the pointer to follow within the top-level block, the index
+ of the one in the "mid" block and the index in the "tail" block. For that, we split the transition byte in a
+ sequence of 2-3-3 bits:
+ - the first 2 bits are the index in the top-level block;
+ - the next 3 bits, the index in the "mid" block;
+ - and the last 3 bits the index in the "tail" block.
+ This layout allows the node to use the smaller fixed-size blocks (instead of 256*4 bytes for the whole character
+ space) and also leaves some room in the head block (the 16 first bytes) for additional information (which we can
+ use to store prefix nodes containing things like deletion times).
+ One split node may need up to 1 + 4 + 4*8 blocks (1184 bytes) to store all its children.
+
+ - If the pointer offset is 31, we have a prefix node. These are two types:
+ -- Embedded prefix nodes occupy the free bytes in a chain or split node. The byte at offset 4 has the offset
+ within the 32-byte block for the augmented node.
+ -- Full prefix nodes have 0xFF at offset 4 and a pointer at 28, pointing to the augmented node.
+ Both types contain an index for content at offset 0. The augmented node cannot be a leaf or NONE -- in the former
+ case the leaf itself contains the content index, in the latter we use a leaf instead.
+ The term "node" when applied to these is a bit of a misnomer as they are not presented as separate nodes during
+ traversals. Instead, they augment a node, changing only its content. Internally we create a Node object for the
+ augmented node and wrap a PrefixNode around it, which changes the `content()` method and routes all other
+ calls to the augmented node's methods.
+
+ When building a trie we first allocate the content, then create a chain node leading to it. While we only have
+ single transitions leading to a chain node, we can expand that node (attaching a character and using pointer - 1)
+ instead of creating a new one. When a chain node already has a child and needs a new one added we change the type
+ (i.e. create a new node and remap the parent) to sparse with two children. When a six-child sparse node needs a new
+ child, we switch to split.
+
+ Blocks currently are not reused, because we do not yet have a mechanism to tell when readers are done with blocks
+ they are referencing. This currently causes a very low overhead (because we change data in place with the only
+ exception of nodes needing to change type) and is planned to be addressed later.
+
+ For further descriptions and examples of the mechanics of the trie, see InMemoryTrie.md.
+ */
+
+ static final int BLOCK_SIZE = 32;
+
+ // Biggest block offset that can contain a pointer.
+ static final int LAST_POINTER_OFFSET = BLOCK_SIZE - 4;
+
+ /*
+ Block offsets used to identify node types (by comparing them to the node 'pointer offset').
+ */
+
+ // split node (dense, 2-3-3 transitions), laid out as 4 pointers to "mid" block, with has 8 pointers to "tail" block,
+ // which has 8 pointers to children
+ static final int SPLIT_OFFSET = BLOCK_SIZE - 4;
+ // sparse node, unordered list of up to 6 transition, laid out as 6 transition pointers followed by 6 transition
+ // bytes. The last two bytes contain an ordering of the transitions (in base-6) which is used for iteration. On
+ // update the pointer is set last, i.e. during reads the node may show that a transition exists and list a character
+ // for it, but pointer may still be null.
+ static final int SPARSE_OFFSET = BLOCK_SIZE - 2;
+ // min and max offset for a chain node. A block of chain node is laid out as a pointer at LAST_POINTER_OFFSET,
+ // preceded by characters that lead to it. Thus a full chain block contains BLOCK_SIZE-4 transitions/chain nodes.
+ static final int CHAIN_MIN_OFFSET = 0;
+ static final int CHAIN_MAX_OFFSET = BLOCK_SIZE - 5;
+ // Prefix node, an intermediate node augmenting its child node with content.
+ static final int PREFIX_OFFSET = BLOCK_SIZE - 1;
+
+ /*
+ Offsets and values for navigating in a block for particular node type. Those offsets are 'from the node pointer'
+ (not the block start) and can be thus negative since node pointers points towards the end of blocks.
+ */
+
+ // Limit for the starting cell / sublevel (2 bits -> 4 pointers).
+ static final int SPLIT_START_LEVEL_LIMIT = 4;
+ // Limit for the other sublevels (3 bits -> 8 pointers).
+ static final int SPLIT_OTHER_LEVEL_LIMIT = 8;
+ // Bitshift between levels.
+ static final int SPLIT_LEVEL_SHIFT = 3;
+
+ static final int SPARSE_CHILD_COUNT = 6;
+ // Offset to the first child pointer of a spare node (laid out from the start of the block)
+ static final int SPARSE_CHILDREN_OFFSET = 0 - SPARSE_OFFSET;
+ // Offset to the first transition byte of a sparse node (laid out after the child pointers)
+ static final int SPARSE_BYTES_OFFSET = SPARSE_CHILD_COUNT * 4 - SPARSE_OFFSET;
+ // Offset to the order word of a sparse node (laid out after the children (pointer + transition byte))
+ static final int SPARSE_ORDER_OFFSET = SPARSE_CHILD_COUNT * 5 - SPARSE_OFFSET; // 0
+
+ // Offset of the flag byte in a prefix node. In shared blocks, this contains the offset of the next node.
+ static final int PREFIX_FLAGS_OFFSET = 4 - PREFIX_OFFSET;
+ // Offset of the content id
+ static final int PREFIX_CONTENT_OFFSET = 0 - PREFIX_OFFSET;
+ // Offset of the next pointer in a non-shared prefix node
+ static final int PREFIX_POINTER_OFFSET = LAST_POINTER_OFFSET - PREFIX_OFFSET;
+
+ /**
+ * Value used as null for node pointers.
+ * No node can use this address (we enforce this by not allowing chain nodes to grow to position 0).
+ * Do not change this as the code relies there being a NONE placed in all bytes of the block that are not set.
+ */
+ static final int NONE = 0;
+
+ volatile int root;
+
+ /*
+ EXPANDABLE DATA STORAGE
+
+ The tries will need more and more space in buffers and content lists as they grow. Instead of using ArrayList-like
+ reallocation with copying, which may be prohibitively expensive for large buffers, we use a sequence of
+ buffers/content arrays that double in size on every expansion.
+
+ For a given address x the index of the buffer can be found with the following calculation:
+ index_of_most_significant_set_bit(x / min_size + 1)
+ (relying on sum (2^i) for i in [0, n-1] == 2^n - 1) which can be performed quickly on modern hardware.
+
+ Finding the offset within the buffer is then
+ x + min - (min << buffer_index)
+
+ The allocated space starts 256 bytes for the buffer and 16 entries for the content list.
+
+ Note that a buffer is not allowed to split 32-byte blocks (code assumes same buffer can be used for all bytes
+ inside the block).
+ */
+
+ static final int BUF_START_SHIFT = 8;
+ static final int BUF_START_SIZE = 1 << BUF_START_SHIFT;
+
+ static final int CONTENTS_START_SHIFT = 4;
+ static final int CONTENTS_START_SIZE = 1 << CONTENTS_START_SHIFT;
+
+ static
+ {
+ assert BUF_START_SIZE % BLOCK_SIZE == 0 : "Initial buffer size must fit a full block.";
+ }
+
+ final UnsafeBuffer[] buffers;
+ final AtomicReferenceArray<T>[] contentArrays;
+
+ InMemoryReadTrie(UnsafeBuffer[] buffers, AtomicReferenceArray<T>[] contentArrays, int root)
+ {
+ this.buffers = buffers;
+ this.contentArrays = contentArrays;
+ this.root = root;
+ }
+
+ /*
+ Buffer, content list and block management
+ */
+ int getChunkIdx(int pos, int minChunkShift, int minChunkSize)
+ {
+ return 31 - minChunkShift - Integer.numberOfLeadingZeros(pos + minChunkSize);
+ }
+
+ int inChunkPointer(int pos, int chunkIndex, int minChunkSize)
+ {
+ return pos + minChunkSize - (minChunkSize << chunkIndex);
+ }
+
+ UnsafeBuffer getChunk(int pos)
+ {
+ int leadBit = getChunkIdx(pos, BUF_START_SHIFT, BUF_START_SIZE);
+ return buffers[leadBit];
+ }
+
+ int inChunkPointer(int pos)
+ {
+ int leadBit = getChunkIdx(pos, BUF_START_SHIFT, BUF_START_SIZE);
+ return inChunkPointer(pos, leadBit, BUF_START_SIZE);
+ }
+
+
+ /**
+ * Pointer offset for a node pointer.
+ */
+ int offset(int pos)
+ {
+ return pos & (BLOCK_SIZE - 1);
+ }
+
+ final int getUnsignedByte(int pos)
+ {
+ return getChunk(pos).getByte(inChunkPointer(pos)) & 0xFF;
+ }
+
+ final int getUnsignedShort(int pos)
+ {
+ return getChunk(pos).getShort(inChunkPointer(pos)) & 0xFFFF;
+ }
+
+ final int getInt(int pos)
+ {
+ return getChunk(pos).getInt(inChunkPointer(pos));
+ }
+
+ T getContent(int index)
+ {
+ int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+ int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+ AtomicReferenceArray<T> array = contentArrays[leadBit];
+ return array.get(ofs);
+ }
+
+ /*
+ Reading node content
+ */
+
+ boolean isNull(int node)
+ {
+ return node == NONE;
+ }
+
+ boolean isLeaf(int node)
+ {
+ return node < NONE;
+ }
+
+ boolean isNullOrLeaf(int node)
+ {
+ return node <= NONE;
+ }
+
+ /**
+ * Returns the number of transitions in a chain block entered with the given pointer.
+ */
+ private int chainBlockLength(int node)
+ {
+ return LAST_POINTER_OFFSET - offset(node);
+ }
+
+ /**
+ * Get a node's child for the given transition character
+ */
+ int getChild(int node, int trans)
+ {
+ if (isNullOrLeaf(node))
+ return NONE;
+
+ node = followContentTransition(node);
+
+ switch (offset(node))
+ {
+ case SPARSE_OFFSET:
+ return getSparseChild(node, trans);
+ case SPLIT_OFFSET:
+ return getSplitChild(node, trans);
+ case CHAIN_MAX_OFFSET:
+ if (trans != getUnsignedByte(node))
+ return NONE;
+ return getInt(node + 1);
+ default:
+ if (trans != getUnsignedByte(node))
+ return NONE;
+ return node + 1;
+ }
+ }
+
+ protected int followContentTransition(int node)
+ {
+ if (isNullOrLeaf(node))
+ return NONE;
+
+ if (offset(node) == PREFIX_OFFSET)
+ {
+ int b = getUnsignedByte(node + PREFIX_FLAGS_OFFSET);
+ if (b < BLOCK_SIZE)
+ node = node - PREFIX_OFFSET + b;
+ else
+ node = getInt(node + PREFIX_POINTER_OFFSET);
+
+ assert node >= 0 && offset(node) != PREFIX_OFFSET;
+ }
+ return node;
+ }
+
+ /**
+ * Advance as long as the cell pointed to by the given pointer will let you.
+ * <p>
+ * This is the same as getChild(node, first), except for chain nodes where it would walk the fill chain as long as
+ * the input source matches.
+ */
+ int advance(int node, int first, ByteSource rest)
+ {
+ if (isNullOrLeaf(node))
+ return NONE;
+
+ node = followContentTransition(node);
+
+ switch (offset(node))
+ {
+ case SPARSE_OFFSET:
+ return getSparseChild(node, first);
+ case SPLIT_OFFSET:
+ return getSplitChild(node, first);
+ default:
+ // Check the first byte matches the expected
+ if (getUnsignedByte(node++) != first)
+ return NONE;
+ // Check the rest of the bytes provided by the chain node
+ for (int length = chainBlockLength(node); length > 0; --length)
+ {
+ first = rest.next();
+ if (getUnsignedByte(node++) != first)
+ return NONE;
+ }
+ // All bytes matched, node is now positioned on the child pointer. Follow it.
+ return getInt(node);
+ }
+ }
+
+ /**
+ * Get the child for the given transition character, knowing that the node is sparse
+ */
+ int getSparseChild(int node, int trans)
+ {
+ for (int i = 0; i < SPARSE_CHILD_COUNT; ++i)
+ {
+ if (getUnsignedByte(node + SPARSE_BYTES_OFFSET + i) == trans)
+ {
+ int child = getInt(node + SPARSE_CHILDREN_OFFSET + i * 4);
+
+ // we can't trust the transition character read above, because it may have been fetched before a
+ // concurrent update happened, and the update may have managed to modify the pointer by now.
+ // However, if we read it now that we have accessed the volatile pointer, it must have the correct
+ // value as it is set before the pointer.
+ if (child != NONE && getUnsignedByte(node + SPARSE_BYTES_OFFSET + i) == trans)
+ return child;
+ }
+ }
+ return NONE;
+ }
+
+ /**
+ * Given a transition, returns the corresponding index (within the node block) of the pointer to the mid block of
+ * a split node.
+ */
+ int splitNodeMidIndex(int trans)
+ {
+ // first 2 bits of the 2-3-3 split
+ return (trans >> 6) & 0x3;
+ }
+
+ /**
+ * Given a transition, returns the corresponding index (within the mid block) of the pointer to the tail block of
+ * a split node.
+ */
+ int splitNodeTailIndex(int trans)
+ {
+ // second 3 bits of the 2-3-3 split
+ return (trans >> 3) & 0x7;
+ }
+
+ /**
+ * Given a transition, returns the corresponding index (within the tail block) of the pointer to the child of
+ * a split node.
+ */
+ int splitNodeChildIndex(int trans)
+ {
+ // third 3 bits of the 2-3-3 split
+ return trans & 0x7;
+ }
+
+ /**
+ * Get the child for the given transition character, knowing that the node is split
+ */
+ int getSplitChild(int node, int trans)
+ {
+ int mid = getSplitBlockPointer(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+ if (isNull(mid))
+ return NONE;
+
+ int tail = getSplitBlockPointer(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ if (isNull(tail))
+ return NONE;
+ return getSplitBlockPointer(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ }
+
+ /**
+ * Get the content for a given node
+ */
+ T getNodeContent(int node)
+ {
+ if (isLeaf(node))
+ return getContent(~node);
+
+ if (offset(node) != PREFIX_OFFSET)
+ return null;
+
+ int index = getInt(node + PREFIX_CONTENT_OFFSET);
+ return (index >= 0)
+ ? getContent(index)
+ : null;
+ }
+
+ int splitBlockPointerAddress(int node, int childIndex, int subLevelLimit)
+ {
+ return node - SPLIT_OFFSET + (8 - subLevelLimit + childIndex) * 4;
+ }
+
+ int getSplitBlockPointer(int node, int childIndex, int subLevelLimit)
+ {
+ return getInt(splitBlockPointerAddress(node, childIndex, subLevelLimit));
+ }
+
+ /**
+ * Backtracking state for a cursor.
+ *
+ * To avoid allocations and pointer-chasing, the backtracking data is stored in a simple int array with
+ * BACKTRACK_INTS_PER_ENTRY ints for each level.
+ */
+ private static class CursorBacktrackingState
+ {
+ static final int BACKTRACK_INTS_PER_ENTRY = 3;
+ static final int BACKTRACK_INITIAL_SIZE = 16;
+ private int[] backtrack = new int[BACKTRACK_INITIAL_SIZE * BACKTRACK_INTS_PER_ENTRY];
+ int backtrackDepth = 0;
+
+ void addBacktrack(int node, int data, int depth)
+ {
+ if (backtrackDepth * BACKTRACK_INTS_PER_ENTRY >= backtrack.length)
+ backtrack = Arrays.copyOf(backtrack, backtrack.length * 2);
+ backtrack[backtrackDepth * BACKTRACK_INTS_PER_ENTRY + 0] = node;
+ backtrack[backtrackDepth * BACKTRACK_INTS_PER_ENTRY + 1] = data;
+ backtrack[backtrackDepth * BACKTRACK_INTS_PER_ENTRY + 2] = depth;
+ ++backtrackDepth;
+ }
+
+ int node(int backtrackDepth)
+ {
+ return backtrack[backtrackDepth * BACKTRACK_INTS_PER_ENTRY + 0];
+ }
+
+ int data(int backtrackDepth)
+ {
+ return backtrack[backtrackDepth * BACKTRACK_INTS_PER_ENTRY + 1];
+ }
+
+ int depth(int backtrackDepth)
+ {
+ return backtrack[backtrackDepth * BACKTRACK_INTS_PER_ENTRY + 2];
+ }
+ }
+
+ /*
+ * Cursor implementation.
+ *
+ * InMemoryTrie cursors maintain their backtracking state in CursorBacktrackingState where they store
+ * information about the node to backtrack to and the transitions still left to take or attempt.
+ *
+ * This information is different for the different types of node:
+ * - for leaf and chain no backtracking is saved (because we know there are no further transitions)
+ * - for sparse we store the remainder of the order word
+ * - for split we store one entry per sub-level of the 2-3-3 split
+ *
+ * When the cursor is asked to advance it first checks the current node for children, and if there aren't any
+ * (i.e. it is positioned on a leaf node), it goes one level up the backtracking chain, where we are guaranteed to
+ * have a remaining child to advance to. When there's nothing to backtrack to, the trie is exhausted.
+ */
+ class MemtableCursor extends CursorBacktrackingState implements Cursor<T>
+ {
+ private int currentNode;
+ private int incomingTransition;
+ private T content;
+ private int depth = -1;
+
+ MemtableCursor()
+ {
+ descendInto(root, -1);
+ }
+
+ @Override
+ public int advance()
+ {
+ if (isNullOrLeaf(currentNode))
+ return backtrack();
+ else
+ return advanceToFirstChild(currentNode);
+ }
+
+ @Override
+ public int advanceMultiple(TransitionsReceiver receiver)
+ {
+ int node = currentNode;
+ if (!isChainNode(node))
+ return advance();
+
+ // Jump directly to the chain's child.
+ UnsafeBuffer chunk = getChunk(node);
+ int inChunkNode = inChunkPointer(node);
+ int bytesJumped = chainBlockLength(node) - 1; // leave the last byte for incomingTransition
+ if (receiver != null && bytesJumped > 0)
+ receiver.addPathBytes(chunk, inChunkNode, bytesJumped);
+ depth += bytesJumped; // descendInto will add one
+ inChunkNode += bytesJumped;
+
+ // inChunkNode is now positioned on the last byte of the chain.
+ // Consume it to be the new state's incomingTransition.
+ int transition = chunk.getByte(inChunkNode++) & 0xFF;
+ // inChunkNode is now positioned on the child pointer.
+ int child = chunk.getInt(inChunkNode);
+ return descendInto(child, transition);
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ return backtrack();
+ }
+
+ @Override
+ public int depth()
+ {
+ return depth;
+ }
+
+ @Override
+ public T content()
+ {
+ return content;
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return incomingTransition;
+ }
+
+ private int backtrack()
+ {
+ if (--backtrackDepth < 0)
+ return depth = -1;
+
+ depth = depth(backtrackDepth);
+ return advanceToNextChild(node(backtrackDepth), data(backtrackDepth));
+ }
+
+ private int advanceToFirstChild(int node)
+ {
+ assert (!isNullOrLeaf(node));
+
+ switch (offset(node))
+ {
+ case SPLIT_OFFSET:
+ return descendInSplitSublevel(node, SPLIT_START_LEVEL_LIMIT, 0, SPLIT_LEVEL_SHIFT * 2);
+ case SPARSE_OFFSET:
+ return nextValidSparseTransition(node, getUnsignedShort(node + SPARSE_ORDER_OFFSET));
+ default:
+ return getChainTransition(node);
+ }
+ }
+
+ private int advanceToNextChild(int node, int data)
+ {
+ assert (!isNullOrLeaf(node));
+
+ switch (offset(node))
+ {
+ case SPLIT_OFFSET:
+ return nextValidSplitTransition(node, data);
+ case SPARSE_OFFSET:
+ return nextValidSparseTransition(node, data);
+ default:
+ throw new AssertionError("Unexpected node type in backtrack state.");
+ }
+ }
+
+ /**
+ * Descend into the sub-levels of a split node. Advances to the first child and creates backtracking entries
+ * for the following ones. We use the bits of trans (lowest non-zero ones) to identify which sub-level an
+ * entry refers to.
+ *
+ * @param node The node or block id, must have offset SPLIT_OFFSET.
+ * @param limit The transition limit for the current sub-level (4 for the start, 8 for the others).
+ * @param collected The transition bits collected from the parent chain (e.g. 0x40 after following 1 on the top
+ * sub-level).
+ * @param shift This level's bit shift (6 for start, 3 for mid and 0 for tail).
+ * @return the depth reached after descending.
+ */
+ private int descendInSplitSublevel(int node, int limit, int collected, int shift)
+ {
+ while (true)
+ {
+ assert offset(node) == SPLIT_OFFSET;
+ int childIndex;
+ int child = NONE;
+ // find the first non-null child
+ for (childIndex = 0; childIndex < limit; ++childIndex)
+ {
+ child = getSplitBlockPointer(node, childIndex, limit);
+ if (!isNull(child))
+ break;
+ }
+ // there must be at least one child
+ assert childIndex < limit && child != NONE;
+
+ // look for any more valid transitions and add backtracking if found
+ maybeAddSplitBacktrack(node, childIndex, limit, collected, shift);
+
+ // add the bits just found
+ collected |= childIndex << shift;
+ // descend to next sub-level or child
+ if (shift == 0)
+ return descendInto(child, collected);
+ // continue with next sublevel; same as
+ // return descendInSplitSublevel(child + SPLIT_OFFSET, 8, collected, shift - 3)
+ node = child;
+ limit = SPLIT_OTHER_LEVEL_LIMIT;
+ shift -= SPLIT_LEVEL_SHIFT;
+ }
+ }
+
+ /**
+ * Backtrack to a split sub-level. The level is identified by the lowest non-0 bits in trans.
+ */
+ private int nextValidSplitTransition(int node, int trans)
+ {
+ assert trans >= 0 && trans <= 0xFF;
+ int childIndex = splitNodeChildIndex(trans);
+ if (childIndex > 0)
+ {
+ maybeAddSplitBacktrack(node,
+ childIndex,
+ SPLIT_OTHER_LEVEL_LIMIT,
+ trans & -(1 << (SPLIT_LEVEL_SHIFT * 1)),
+ SPLIT_LEVEL_SHIFT * 0);
+ int child = getSplitBlockPointer(node, childIndex, SPLIT_OTHER_LEVEL_LIMIT);
+ return descendInto(child, trans);
+ }
+ int tailIndex = splitNodeTailIndex(trans);
+ if (tailIndex > 0)
+ {
+ maybeAddSplitBacktrack(node,
+ tailIndex,
+ SPLIT_OTHER_LEVEL_LIMIT,
+ trans & -(1 << (SPLIT_LEVEL_SHIFT * 2)),
+ SPLIT_LEVEL_SHIFT * 1);
+ int tail = getSplitBlockPointer(node, tailIndex, SPLIT_OTHER_LEVEL_LIMIT);
+ return descendInSplitSublevel(tail,
+ SPLIT_OTHER_LEVEL_LIMIT,
+ trans,
+ SPLIT_LEVEL_SHIFT * 0);
+ }
+ int midIndex = splitNodeMidIndex(trans);
+ assert midIndex > 0;
+ maybeAddSplitBacktrack(node,
+ midIndex,
+ SPLIT_START_LEVEL_LIMIT,
+ 0,
+ SPLIT_LEVEL_SHIFT * 2);
+ int mid = getSplitBlockPointer(node, midIndex, SPLIT_START_LEVEL_LIMIT);
+ return descendInSplitSublevel(mid,
+ SPLIT_OTHER_LEVEL_LIMIT,
+ trans,
+ SPLIT_LEVEL_SHIFT * 1);
+ }
+
+ /**
+ * Look for any further non-null transitions on this sub-level and, if found, add a backtracking entry.
+ */
+ private void maybeAddSplitBacktrack(int node, int startAfter, int limit, int collected, int shift)
+ {
+ int nextChildIndex;
+ for (nextChildIndex = startAfter + 1; nextChildIndex < limit; ++nextChildIndex)
+ {
+ if (!isNull(getSplitBlockPointer(node, nextChildIndex, limit)))
+ break;
+ }
+ if (nextChildIndex < limit)
+ addBacktrack(node, collected | (nextChildIndex << shift), depth);
+ }
+
+ private int nextValidSparseTransition(int node, int data)
+ {
+ UnsafeBuffer chunk = getChunk(node);
+ int inChunkNode = inChunkPointer(node);
+
+ // Peel off the next index.
+ int index = data % SPARSE_CHILD_COUNT;
+ data = data / SPARSE_CHILD_COUNT;
+
+ // If there are remaining transitions, add backtracking entry.
+ if (data > 0)
+ addBacktrack(node, data, depth);
+
+ // Follow the transition.
+ int child = chunk.getInt(inChunkNode + SPARSE_CHILDREN_OFFSET + index * 4);
+ int transition = chunk.getByte(inChunkNode + SPARSE_BYTES_OFFSET + index) & 0xFF;
+ return descendInto(child, transition);
+ }
+
+ private int getChainTransition(int node)
+ {
+ // No backtracking needed.
+ UnsafeBuffer chunk = getChunk(node);
+ int inChunkNode = inChunkPointer(node);
+ int transition = chunk.getByte(inChunkNode) & 0xFF;
+ int next = node + 1;
+ if (offset(next) <= CHAIN_MAX_OFFSET)
+ return descendIntoChain(next, transition);
+ else
+ return descendInto(chunk.getInt(inChunkNode + 1), transition);
+ }
+
+ private int descendInto(int child, int transition)
+ {
+ ++depth;
+ incomingTransition = transition;
+ content = getNodeContent(child);
+ currentNode = followContentTransition(child);
+ return depth;
+ }
+
+ private int descendIntoChain(int child, int transition)
+ {
+ ++depth;
+ incomingTransition = transition;
+ content = null;
+ currentNode = child;
+ return depth;
+ }
+ }
+
+ private boolean isChainNode(int node)
+ {
+ return !isNullOrLeaf(node) && offset(node) <= CHAIN_MAX_OFFSET;
+ }
+
+ public MemtableCursor cursor()
+ {
+ return new MemtableCursor();
+ }
+
+ /*
+ Direct read methods
+ */
+
+ /**
+ * Get the content mapped by the specified key.
+ * Fast implementation using integer node addresses.
+ */
+ public T get(ByteComparable path)
+ {
+ int n = root;
+ ByteSource source = path.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ while (!isNull(n))
+ {
+ int c = source.next();
+ if (c == ByteSource.END_OF_STREAM)
+ return getNodeContent(n);
+
+ n = advance(n, c, source);
+ }
+
+ return null;
+ }
+
+ public boolean isEmpty()
+ {
+ return isNull(root);
+ }
+
+ /**
+ * Override of dump to provide more detailed printout that includes the type of each node in the trie.
+ * We do this via a wrapping cursor that returns a content string for the type of node for every node we return.
+ */
+ @Override
+ public String dump(Function<T, String> contentToString)
+ {
+ MemtableCursor source = cursor();
+ class TypedNodesCursor implements Cursor<String>
+ {
+ @Override
+ public int advance()
+ {
+ return source.advance();
+ }
+
+
+ @Override
+ public int advanceMultiple(TransitionsReceiver receiver)
+ {
+ return source.advanceMultiple(receiver);
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ return source.skipChildren();
+ }
+
+ @Override
+ public int depth()
+ {
+ return source.depth();
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return source.incomingTransition();
+ }
+
+ @Override
+ public String content()
+ {
+ String type = null;
+ int node = source.currentNode;
+ if (!isNullOrLeaf(node))
+ {
+ switch (offset(node))
+ {
+ case SPARSE_OFFSET:
+ type = "[SPARSE]";
+ break;
+ case SPLIT_OFFSET:
+ type = "[SPLIT]";
+ break;
+ case PREFIX_OFFSET:
+ throw new AssertionError("Unexpected prefix as cursor currentNode.");
+ default:
+ type = "[CHAIN]";
+ break;
+ }
+ }
+ T content = source.content();
+ if (content != null)
+ {
+ if (type != null)
+ return contentToString.apply(content) + " -> " + type;
+ else
+ return contentToString.apply(content);
+ }
+ else
+ return type;
+ }
+ }
+ return process(new TrieDumper<>(Function.identity()), new TypedNodesCursor());
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java
new file mode 100644
index 0000000..19f28c3
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.java
@@ -0,0 +1,1028 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.agrona.concurrent.UnsafeBuffer;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.github.jamm.MemoryLayoutSpecification;
+
+/**
+ * In-memory trie built for fast modification and reads executing concurrently with writes from a single mutator thread.
+ *
+ * This class can currently only provide atomicity (i.e. reads seeing either the content before a write, or the
+ * content after it; any read seeing the write enforcing any subsequent (i.e. started after it completed) reads to
+ * also see it) for singleton writes (i.e. calls to {@link #putRecursive}, {@link #putSingleton} or {@link #apply}
+ * with a singleton trie as argument).
+ *
+ * Because it uses 32-bit pointers in byte buffers, this trie has a fixed size limit of 2GB.
+ */
+public class InMemoryTrie<T> extends InMemoryReadTrie<T>
+{
+ // See the trie format description in InMemoryReadTrie.
+
+ /**
+ * Trie size limit. This is not enforced, but users must check from time to time that it is not exceeded (using
+ * {@link #reachedAllocatedSizeThreshold()}) and start switching to a new trie if it is.
+ * This must be done to avoid tries growing beyond their hard 2GB size limit (due to the 32-bit pointers).
+ */
+ @VisibleForTesting
+ static final int ALLOCATED_SIZE_THRESHOLD;
+ static
+ {
+ // Default threshold + 10% == 2 GB. This should give the owner enough time to react to the
+ // {@link #reachedAllocatedSizeThreshold()} signal and switch this trie out before it fills up.
+ int limitInMB = CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE.getInt(2048 * 10 / 11);
+ if (limitInMB < 1 || limitInMB > 2047)
+ throw new AssertionError(CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE.getKey() +
+ " must be within 1 and 2047");
+ ALLOCATED_SIZE_THRESHOLD = 1024 * 1024 * limitInMB;
+ }
+
+ private int allocatedPos = 0;
+ private int contentCount = 0;
+
+ private final BufferType bufferType; // on or off heap
+
+ // constants for space calculations
+ private static final long EMPTY_SIZE_ON_HEAP;
+ private static final long EMPTY_SIZE_OFF_HEAP;
+ private static final long REFERENCE_ARRAY_ON_HEAP_SIZE = ObjectSizes.measureDeep(new AtomicReferenceArray<>(0));
+
+ static
+ {
+ InMemoryTrie<Object> empty = new InMemoryTrie<>(BufferType.ON_HEAP);
+ EMPTY_SIZE_ON_HEAP = ObjectSizes.measureDeep(empty);
+ empty = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ EMPTY_SIZE_OFF_HEAP = ObjectSizes.measureDeep(empty);
+ }
+
+ public InMemoryTrie(BufferType bufferType)
+ {
+ super(new UnsafeBuffer[31 - BUF_START_SHIFT], // last one is 1G for a total of ~2G bytes
+ new AtomicReferenceArray[29 - CONTENTS_START_SHIFT], // takes at least 4 bytes to write pointer to one content -> 4 times smaller than buffers
+ NONE);
+ this.bufferType = bufferType;
+ }
+
+ // Buffer, content list and block management
+
+ /**
+ * Because we use buffers and 32-bit pointers, the trie cannot grow over 2GB of size. This exception is thrown if
+ * a trie operation needs it to grow over that limit.
+ *
+ * To avoid this problem, users should query {@link #reachedAllocatedSizeThreshold} from time to time. If the call
+ * returns true, they should switch to a new trie (e.g. by flushing a memtable) as soon as possible. The threshold
+ * is configurable, and is set by default to 10% under the 2GB limit to give ample time for the switch to happen.
+ */
+ public static class SpaceExhaustedException extends Exception
+ {
+ public SpaceExhaustedException()
+ {
+ super("The hard 2GB limit on trie size has been exceeded");
+ }
+ }
+
+ final void putInt(int pos, int value)
+ {
+ getChunk(pos).putInt(inChunkPointer(pos), value);
+ }
+
+ final void putIntVolatile(int pos, int value)
+ {
+ getChunk(pos).putIntVolatile(inChunkPointer(pos), value);
+ }
+
+ final void putShort(int pos, short value)
+ {
+ getChunk(pos).putShort(inChunkPointer(pos), value);
+ }
+
+ final void putShortVolatile(int pos, short value)
+ {
+ getChunk(pos).putShort(inChunkPointer(pos), value);
+ }
+
+ final void putByte(int pos, byte value)
+ {
+ getChunk(pos).putByte(inChunkPointer(pos), value);
+ }
+
+
+ private int allocateBlock() throws SpaceExhaustedException
+ {
+ // Note: If this method is modified, please run InMemoryTrieTest.testOver1GSize to verify it acts correctly
+ // close to the 2G limit.
+ int v = allocatedPos;
+ if (inChunkPointer(v) == 0)
+ {
+ int leadBit = getChunkIdx(v, BUF_START_SHIFT, BUF_START_SIZE);
+ if (leadBit + BUF_START_SHIFT == 31)
+ throw new SpaceExhaustedException();
+
+ ByteBuffer newBuffer = bufferType.allocate(BUF_START_SIZE << leadBit);
+ buffers[leadBit] = new UnsafeBuffer(newBuffer);
+ // Note: Since we are not moving existing data to a new buffer, we are okay with no happens-before enforcing
+ // writes. Any reader that sees a pointer in the new buffer may only do so after reading the volatile write
+ // that attached the new path.
+ }
+
+ allocatedPos += BLOCK_SIZE;
+ return v;
+ }
+
+ private int addContent(T value)
+ {
+ int index = contentCount++;
+ int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+ int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+ AtomicReferenceArray<T> array = contentArrays[leadBit];
+ if (array == null)
+ {
+ assert ofs == 0 : "Error in content arrays configuration.";
+ contentArrays[leadBit] = array = new AtomicReferenceArray<>(CONTENTS_START_SIZE << leadBit);
+ }
+ array.lazySet(ofs, value); // no need for a volatile set here; at this point the item is not referenced
+ // by any node in the trie, and a volatile set will be made to reference it.
+ return index;
+ }
+
+ private void setContent(int index, T value)
+ {
+ int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+ int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+ AtomicReferenceArray<T> array = contentArrays[leadBit];
+ array.set(ofs, value);
+ }
+
+ public void discardBuffers()
+ {
+ if (bufferType == BufferType.ON_HEAP)
+ return; // no cleaning needed
+
+ for (UnsafeBuffer b : buffers)
+ {
+ if (b != null)
+ FileUtils.clean(b.byteBuffer());
+ }
+ }
+
+ // Write methods
+
+ // Write visibility model: writes are not volatile, with the exception of the final write before a call returns
+ // the same value that was present before (e.g. content was updated in-place / existing node got a new child or had
+ // a child pointer updated); if the whole path including the root node changed, the root itself gets a volatile
+ // write.
+ // This final write is the point where any new cells created during the write become visible for readers for the
+ // first time, and such readers must pass through reading that pointer, which forces a happens-before relationship
+ // that extends to all values written by this thread before it.
+
+ /**
+ * Attach a child to the given non-content node. This may be an update for an existing branch, or a new child for
+ * the node. An update _is_ required (i.e. this is only called when the newChild pointer is not the same as the
+ * existing value).
+ */
+ private int attachChild(int node, int trans, int newChild) throws SpaceExhaustedException
+ {
+ assert !isLeaf(node) : "attachChild cannot be used on content nodes.";
+
+ switch (offset(node))
+ {
+ case PREFIX_OFFSET:
+ assert false : "attachChild cannot be used on content nodes.";
+ case SPARSE_OFFSET:
+ return attachChildToSparse(node, trans, newChild);
+ case SPLIT_OFFSET:
+ attachChildToSplit(node, trans, newChild);
+ return node;
+ case LAST_POINTER_OFFSET - 1:
+ // If this is the last character in a Chain block, we can modify the child in-place
+ if (trans == getUnsignedByte(node))
+ {
+ putIntVolatile(node + 1, newChild);
+ return node;
+ }
+ // else pass through
+ default:
+ return attachChildToChain(node, trans, newChild);
+ }
+ }
+
+ /**
+ * Attach a child to the given split node. This may be an update for an existing branch, or a new child for the node.
+ */
+ private void attachChildToSplit(int node, int trans, int newChild) throws SpaceExhaustedException
+ {
+ int midPos = splitBlockPointerAddress(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+ int mid = getInt(midPos);
+ if (isNull(mid))
+ {
+ mid = createEmptySplitNode();
+ int tailPos = splitBlockPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ int tail = createEmptySplitNode();
+ int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ putInt(childPos, newChild);
+ putInt(tailPos, tail);
+ putIntVolatile(midPos, mid);
+ return;
+ }
+
+ int tailPos = splitBlockPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ int tail = getInt(tailPos);
+ if (isNull(tail))
+ {
+ tail = createEmptySplitNode();
+ int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ putInt(childPos, newChild);
+ putIntVolatile(tailPos, tail);
+ return;
+ }
+
+ int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ putIntVolatile(childPos, newChild);
+ }
+
+ /**
+ * Attach a child to the given sparse node. This may be an update for an existing branch, or a new child for the node.
+ */
+ private int attachChildToSparse(int node, int trans, int newChild) throws SpaceExhaustedException
+ {
+ int index;
+ int smallerCount = 0;
+ // first check if this is an update and modify in-place if so
+ for (index = 0; index < SPARSE_CHILD_COUNT; ++index)
+ {
+ if (isNull(getInt(node + SPARSE_CHILDREN_OFFSET + index * 4)))
+ break;
+ final int existing = getUnsignedByte(node + SPARSE_BYTES_OFFSET + index);
+ if (existing == trans)
+ {
+ putIntVolatile(node + SPARSE_CHILDREN_OFFSET + index * 4, newChild);
+ return node;
+ }
+ else if (existing < trans)
+ ++smallerCount;
+ }
+ int childCount = index;
+
+ if (childCount == SPARSE_CHILD_COUNT)
+ {
+ // Node is full. Switch to split
+ int split = createEmptySplitNode();
+ for (int i = 0; i < SPARSE_CHILD_COUNT; ++i)
+ {
+ int t = getUnsignedByte(node + SPARSE_BYTES_OFFSET + i);
+ int p = getInt(node + SPARSE_CHILDREN_OFFSET + i * 4);
+ attachChildToSplitNonVolatile(split, t, p);
+ }
+ attachChildToSplitNonVolatile(split, trans, newChild);
+ return split;
+ }
+
+ // Add a new transition. They are not kept in order, so append it at the first free position.
+ putByte(node + SPARSE_BYTES_OFFSET + childCount, (byte) trans);
+
+ // Update order word.
+ int order = getUnsignedShort(node + SPARSE_ORDER_OFFSET);
+ int newOrder = insertInOrderWord(order, childCount, smallerCount);
+
+ // Sparse nodes have two access modes: via the order word, when listing transitions, or directly to characters
+ // and addresses.
+ // To support the former, we volatile write to the order word last, and everything is correctly set up.
+ // The latter does not touch the order word. To support that too, we volatile write the address, as the reader
+ // can't determine if the position is in use based on the character byte alone (00 is also a valid transition).
+ // Note that this means that reader must check the transition byte AFTER the address, to ensure they get the
+ // correct value (see getSparseChild).
+
+ // setting child enables reads to start seeing the new branch
+ putIntVolatile(node + SPARSE_CHILDREN_OFFSET + childCount * 4, newChild);
+
+ // some readers will decide whether to check the pointer based on the order word
+ // write that volatile to make sure they see the new change too
+ putShortVolatile(node + SPARSE_ORDER_OFFSET, (short) newOrder);
+ return node;
+ }
+
+ /**
+ * Insert the given newIndex in the base-6 encoded order word in the correct position with respect to the ordering.
+ *
+ * E.g.
+ * - insertOrderWord(120, 3, 0) must return 1203 (decimal 48*6 + 3)
+ * - insertOrderWord(120, 3, 1, ptr) must return 1230 (decimal 8*36 + 3*6 + 0)
+ * - insertOrderWord(120, 3, 2, ptr) must return 1320 (decimal 1*216 + 3*36 + 12)
+ * - insertOrderWord(120, 3, 3, ptr) must return 3120 (decimal 3*216 + 48)
+ */
+ private static int insertInOrderWord(int order, int newIndex, int smallerCount)
+ {
+ int r = 1;
+ for (int i = 0; i < smallerCount; ++i)
+ r *= 6;
+ int head = order / r;
+ int tail = order % r;
+ // insert newIndex after the ones we have passed (order % r) and before the remaining (order / r)
+ return tail + (head * 6 + newIndex) * r;
+ }
+
+ /**
+ * Non-volatile version of attachChildToSplit. Used when the split node is not reachable yet (during the conversion
+ * from sparse).
+ */
+ private void attachChildToSplitNonVolatile(int node, int trans, int newChild) throws SpaceExhaustedException
+ {
+ assert offset(node) == SPLIT_OFFSET : "Invalid split node in trie";
+ int midPos = splitBlockPointerAddress(node, splitNodeMidIndex(trans), SPLIT_START_LEVEL_LIMIT);
+ int mid = getInt(midPos);
+ if (isNull(mid))
+ {
+ mid = createEmptySplitNode();
+ putInt(midPos, mid);
+ }
+
+ assert offset(mid) == SPLIT_OFFSET : "Invalid split node in trie";
+ int tailPos = splitBlockPointerAddress(mid, splitNodeTailIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ int tail = getInt(tailPos);
+ if (isNull(tail))
+ {
+ tail = createEmptySplitNode();
+ putInt(tailPos, tail);
+ }
+
+ assert offset(tail) == SPLIT_OFFSET : "Invalid split node in trie";
+ int childPos = splitBlockPointerAddress(tail, splitNodeChildIndex(trans), SPLIT_OTHER_LEVEL_LIMIT);
+ putInt(childPos, newChild);
+ }
+
+ /**
+ * Attach a child to the given chain node. This may be an update for an existing branch with different target
+ * address, or a second child for the node.
+ * This method always copies the node -- with the exception of updates that change the child of the last node in a
+ * chain block with matching transition byte (which this method is not used for, see attachChild), modifications to
+ * chain nodes cannot be done in place, either because we introduce a new transition byte and have to convert from
+ * the single-transition chain type to sparse, or because we have to remap the child from the implicit node + 1 to
+ * something else.
+ */
+ private int attachChildToChain(int node, int transitionByte, int newChild) throws SpaceExhaustedException
+ {
+ int existingByte = getUnsignedByte(node);
+ if (transitionByte == existingByte)
+ {
+ // This will only be called if new child is different from old, and the update is not on the final child
+ // where we can change it in place (see attachChild). We must always create something new.
+ // If the child is a chain, we can expand it (since it's a different value, its branch must be new and
+ // nothing can already reside in the rest of the block).
+ return expandOrCreateChainNode(transitionByte, newChild);
+ }
+
+ // The new transition is different, so we no longer have only one transition. Change type.
+ int existingChild = node + 1;
+ if (offset(existingChild) == LAST_POINTER_OFFSET)
+ {
+ existingChild = getInt(existingChild);
+ }
+ return createSparseNode(existingByte, existingChild, transitionByte, newChild);
+ }
+
+ private boolean isExpandableChain(int newChild)
+ {
+ int newOffset = offset(newChild);
+ return newChild > 0 && newChild - 1 > NONE && newOffset > CHAIN_MIN_OFFSET && newOffset <= CHAIN_MAX_OFFSET;
+ }
+
+ /**
+ * Create a sparse node with two children.
+ */
+ private int createSparseNode(int byte1, int child1, int byte2, int child2) throws SpaceExhaustedException
+ {
+ assert byte1 != byte2 : "Attempted to create a sparse node with two of the same transition";
+ if (byte1 > byte2)
+ {
+ // swap them so the smaller is byte1, i.e. there's always something bigger than child 0 so 0 never is
+ // at the end of the order
+ int t = byte1; byte1 = byte2; byte2 = t;
+ t = child1; child1 = child2; child2 = t;
+ }
+
+ int node = allocateBlock() + SPARSE_OFFSET;
+ putByte(node + SPARSE_BYTES_OFFSET + 0, (byte) byte1);
+ putByte(node + SPARSE_BYTES_OFFSET + 1, (byte) byte2);
+ putInt(node + SPARSE_CHILDREN_OFFSET + 0 * 4, child1);
+ putInt(node + SPARSE_CHILDREN_OFFSET + 1 * 4, child2);
+ putShort(node + SPARSE_ORDER_OFFSET, (short) (1 * 6 + 0));
+ // Note: this does not need a volatile write as it is a new node, returning a new pointer, which needs to be
+ // put in an existing node or the root. That action ends in a happens-before enforcing write.
+ return node;
+ }
+
+ /**
+ * Creates a chain node with the single provided transition (pointing to the provided child).
+ * Note that to avoid creating inefficient tries with under-utilized chain nodes, this should only be called from
+ * {@link #expandOrCreateChainNode} and other call-sites should call {@link #expandOrCreateChainNode}.
+ */
+ private int createNewChainNode(int transitionByte, int newChild) throws SpaceExhaustedException
+ {
+ int newNode = allocateBlock() + LAST_POINTER_OFFSET - 1;
+ putByte(newNode, (byte) transitionByte);
+ putInt(newNode + 1, newChild);
+ // Note: this does not need a volatile write as it is a new node, returning a new pointer, which needs to be
+ // put in an existing node or the root. That action ends in a happens-before enforcing write.
+ return newNode;
+ }
+
+ /** Like {@link #createNewChainNode}, but if the new child is already a chain node and has room, expand
+ * it instead of creating a brand new node. */
+ private int expandOrCreateChainNode(int transitionByte, int newChild) throws SpaceExhaustedException
+ {
+ if (isExpandableChain(newChild))
+ {
+ // attach as a new character in child node
+ int newNode = newChild - 1;
+ putByte(newNode, (byte) transitionByte);
+ return newNode;
+ }
+
+ return createNewChainNode(transitionByte, newChild);
+ }
+
+ private int createEmptySplitNode() throws SpaceExhaustedException
+ {
+ return allocateBlock() + SPLIT_OFFSET;
+ }
+
+ private int createPrefixNode(int contentIndex, int child, boolean isSafeChain) throws SpaceExhaustedException
+ {
+ assert !isNullOrLeaf(child) : "Prefix node cannot reference a childless node.";
+
+ int offset = offset(child);
+ int node;
+ if (offset == SPLIT_OFFSET || isSafeChain && offset > (PREFIX_FLAGS_OFFSET + PREFIX_OFFSET) && offset <= CHAIN_MAX_OFFSET)
+ {
+ // We can do an embedded prefix node
+ // Note: for chain nodes we have a risk that the node continues beyond the current point, in which case
+ // creating the embedded node may overwrite information that is still needed by concurrent readers or the
+ // mutation process itself.
+ node = (child & -BLOCK_SIZE) | PREFIX_OFFSET;
+ putByte(node + PREFIX_FLAGS_OFFSET, (byte) offset);
+ }
+ else
+ {
+ // Full prefix node
+ node = allocateBlock() + PREFIX_OFFSET;
+ putByte(node + PREFIX_FLAGS_OFFSET, (byte) 0xFF);
+ putInt(node + PREFIX_POINTER_OFFSET, child);
+ }
+
+ putInt(node + PREFIX_CONTENT_OFFSET, contentIndex);
+ return node;
+ }
+
+ private int updatePrefixNodeChild(int node, int child) throws SpaceExhaustedException
+ {
+ assert offset(node) == PREFIX_OFFSET : "updatePrefix called on non-prefix node";
+ assert !isNullOrLeaf(child) : "Prefix node cannot reference a childless node.";
+
+ // We can only update in-place if we have a full prefix node
+ if (!isEmbeddedPrefixNode(node))
+ {
+ // This attaches the child branch and makes it reachable -- the write must be volatile.
+ putIntVolatile(node + PREFIX_POINTER_OFFSET, child);
+ return node;
+ }
+ else
+ {
+ int contentIndex = getInt(node + PREFIX_CONTENT_OFFSET);
+ return createPrefixNode(contentIndex, child, true);
+ }
+ }
+
+ private boolean isEmbeddedPrefixNode(int node)
+ {
+ return getUnsignedByte(node + PREFIX_FLAGS_OFFSET) < BLOCK_SIZE;
+ }
+
+ /**
+ * Copy the content from an existing node, if it has any, to a newly-prepared update for its child.
+ *
+ * @param existingPreContentNode pointer to the existing node before skipping over content nodes, i.e. this is
+ * either the same as existingPostContentNode or a pointer to a prefix or leaf node
+ * whose child is existingPostContentNode
+ * @param existingPostContentNode pointer to the existing node being updated, after any content nodes have been
+ * skipped and before any modification have been applied; always a non-content node
+ * @param updatedPostContentNode is the updated node, i.e. the node to which all relevant modifications have been
+ * applied; if the modifications were applied in-place, this will be the same as
+ * existingPostContentNode, otherwise a completely different pointer; always a non-
+ * content node
+ * @return a node which has the children of updatedPostContentNode combined with the content of
+ * existingPreContentNode
+ */
+ private int preserveContent(int existingPreContentNode,
+ int existingPostContentNode,
+ int updatedPostContentNode) throws SpaceExhaustedException
+ {
+ if (existingPreContentNode == existingPostContentNode)
+ return updatedPostContentNode; // no content to preserve
+
+ if (existingPostContentNode == updatedPostContentNode)
+ return existingPreContentNode; // child didn't change, no update necessary
+
+ // else we have existing prefix node, and we need to reference a new child
+ if (isLeaf(existingPreContentNode))
+ {
+ return createPrefixNode(~existingPreContentNode, updatedPostContentNode, true);
+ }
+
+ assert offset(existingPreContentNode) == PREFIX_OFFSET : "Unexpected content in non-prefix and non-leaf node.";
+ return updatePrefixNodeChild(existingPreContentNode, updatedPostContentNode);
+ }
+
+ final ApplyState applyState = new ApplyState();
+
+ /**
+ * Represents the state for an {@link #apply} operation. Contains a stack of all nodes we descended through
+ * and used to update the nodes with any new data during ascent.
+ *
+ * To make this as efficient and GC-friendly as possible, we use an integer array (instead of is an object stack)
+ * and we reuse the same object. The latter is safe because memtable tries cannot be mutated in parallel by multiple
+ * writers.
+ */
+ class ApplyState
+ {
+ int[] data = new int[16 * 5];
+ int currentDepth = -1;
+
+ void reset()
+ {
+ currentDepth = -1;
+ }
+
+ /**
+ * Pointer to the existing node before skipping over content nodes, i.e. this is either the same as
+ * existingPostContentNode or a pointer to a prefix or leaf node whose child is existingPostContentNode.
+ */
+ int existingPreContentNode()
+ {
+ return data[currentDepth * 5 + 0];
+ }
+ void setExistingPreContentNode(int value)
+ {
+ data[currentDepth * 5 + 0] = value;
+ }
+
+ /**
+ * Pointer to the existing node being updated, after any content nodes have been skipped and before any
+ * modification have been applied. Always a non-content node.
+ */
+ int existingPostContentNode()
+ {
+ return data[currentDepth * 5 + 1];
+ }
+ void setExistingPostContentNode(int value)
+ {
+ data[currentDepth * 5 + 1] = value;
+ }
+
+ /**
+ * The updated node, i.e. the node to which the relevant modifications are being applied. This will change as
+ * children are processed and attached to the node. After all children have been processed, this will contain
+ * the fully updated node (i.e. the union of existingPostContentNode and mutationNode) without any content,
+ * which will be processed separately and, if necessary, attached ahead of this. If the modifications were
+ * applied in-place, this will be the same as existingPostContentNode, otherwise a completely different
+ * pointer. Always a non-content node.
+ */
+ int updatedPostContentNode()
+ {
+ return data[currentDepth * 5 + 2];
+ }
+ void setUpdatedPostContentNode(int value)
+ {
+ data[currentDepth * 5 + 2] = value;
+ }
+
+ /**
+ * The transition we took on the way down.
+ */
+ int transition()
+ {
+ return data[currentDepth * 5 + 3];
+ }
+ void setTransition(int transition)
+ {
+ data[currentDepth * 5 + 3] = transition;
+ }
+
+ /**
+ * The compiled content index. Needed because we can only access a cursor's content on the way down but we can't
+ * attach it until we ascend from the node.
+ */
+ int contentIndex()
+ {
+ return data[currentDepth * 5 + 4];
+ }
+ void setContentIndex(int value)
+ {
+ data[currentDepth * 5 + 4] = value;
+ }
+
+ /**
+ * Descend to a child node. Prepares a new entry in the stack for the node.
+ */
+ <U> void descend(int transition, U mutationContent, final UpsertTransformer<T, U> transformer)
+ {
+ int existingPreContentNode;
+ if (currentDepth < 0)
+ existingPreContentNode = root;
+ else
+ {
+ setTransition(transition);
+ existingPreContentNode = isNull(existingPostContentNode())
+ ? NONE
+ : getChild(existingPostContentNode(), transition);
+ }
+
+ ++currentDepth;
+ if (currentDepth * 5 >= data.length)
+ data = Arrays.copyOf(data, currentDepth * 5 * 2);
+ setExistingPreContentNode(existingPreContentNode);
+
+ int existingContentIndex = -1;
+ int existingPostContentNode;
+ if (isLeaf(existingPreContentNode))
+ {
+ existingContentIndex = ~existingPreContentNode;
+ existingPostContentNode = NONE;
+ }
+ else if (offset(existingPreContentNode) == PREFIX_OFFSET)
+ {
+ existingContentIndex = getInt(existingPreContentNode + PREFIX_CONTENT_OFFSET);
+ existingPostContentNode = followContentTransition(existingPreContentNode);
+ }
+ else
+ existingPostContentNode = existingPreContentNode;
+ setExistingPostContentNode(existingPostContentNode);
+ setUpdatedPostContentNode(existingPostContentNode);
+
+ int contentIndex = updateContentIndex(mutationContent, existingContentIndex, transformer);
+ setContentIndex(contentIndex);
+ }
+
+ /**
+ * Combine existing and new content.
+ */
+ private <U> int updateContentIndex(U mutationContent, int existingContentIndex, final UpsertTransformer<T, U> transformer)
+ {
+ if (mutationContent != null)
+ {
+ if (existingContentIndex != -1)
+ {
+ final T existingContent = getContent(existingContentIndex);
+ T combinedContent = transformer.apply(existingContent, mutationContent);
+ assert (combinedContent != null) : "Transformer cannot be used to remove content.";
+ setContent(existingContentIndex, combinedContent);
+ return existingContentIndex;
+ }
+ else
+ {
+ T combinedContent = transformer.apply(null, mutationContent);
+ assert (combinedContent != null) : "Transformer cannot be used to remove content.";
+ return addContent(combinedContent);
+ }
+ }
+ else
+ return existingContentIndex;
+ }
+
+ /**
+ * Attach a child to the current node.
+ */
+ private void attachChild(int transition, int child) throws SpaceExhaustedException
+ {
+ int updatedPostContentNode = updatedPostContentNode();
+ if (isNull(updatedPostContentNode))
+ setUpdatedPostContentNode(expandOrCreateChainNode(transition, child));
+ else
+ setUpdatedPostContentNode(InMemoryTrie.this.attachChild(updatedPostContentNode,
+ transition,
+ child));
+ }
+
+ /**
+ * Apply the collected content to a node. Converts NONE to a leaf node, and adds or updates a prefix for all
+ * others.
+ */
+ private int applyContent() throws SpaceExhaustedException
+ {
+ int contentIndex = contentIndex();
+ int updatedPostContentNode = updatedPostContentNode();
+ if (contentIndex == -1)
+ return updatedPostContentNode;
+
+ if (isNull(updatedPostContentNode))
+ return ~contentIndex;
+
+ int existingPreContentNode = existingPreContentNode();
+ int existingPostContentNode = existingPostContentNode();
+
+ // We can't update in-place if there was no preexisting prefix, or if the prefix was embedded and the target
+ // node must change.
+ if (existingPreContentNode == existingPostContentNode ||
+ isNull(existingPostContentNode) ||
+ isEmbeddedPrefixNode(existingPreContentNode) && updatedPostContentNode != existingPostContentNode)
+ return createPrefixNode(contentIndex, updatedPostContentNode, isNull(existingPostContentNode));
+
+ // Otherwise modify in place
+ if (updatedPostContentNode != existingPostContentNode) // to use volatile write but also ensure we don't corrupt embedded nodes
+ putIntVolatile(existingPreContentNode + PREFIX_POINTER_OFFSET, updatedPostContentNode);
+ assert contentIndex == getInt(existingPreContentNode + PREFIX_CONTENT_OFFSET) : "Unexpected change of content index.";
+ return existingPreContentNode;
+ }
+
+ /**
+ * After a node's children are processed, this is called to ascend from it. This means applying the collected
+ * content to the compiled updatedPostContentNode and creating a mapping in the parent to it (or updating if
+ * one already exists).
+ * Returns true if still have work to do, false if the operation is completed.
+ */
+ private boolean attachAndMoveToParentState() throws SpaceExhaustedException
+ {
+ int updatedPreContentNode = applyContent();
+ int existingPreContentNode = existingPreContentNode();
+ --currentDepth;
+ if (currentDepth == -1)
+ {
+ assert root == existingPreContentNode : "Unexpected change to root. Concurrent trie modification?";
+ if (updatedPreContentNode != existingPreContentNode)
+ {
+ // Only write to root if they are different (value doesn't change, but
+ // we don't want to invalidate the value in other cores' caches unnecessarily).
+ root = updatedPreContentNode;
+ }
+ return false;
+ }
+ if (updatedPreContentNode != existingPreContentNode)
+ attachChild(transition(), updatedPreContentNode);
+ return true;
+ }
+ }
+
+ /**
+ * Somewhat similar to {@link MergeResolver}, this encapsulates logic to be applied whenever new content is being
+ * upserted into a {@link InMemoryTrie}. Unlike {@link MergeResolver}, {@link UpsertTransformer} will be applied no
+ * matter if there's pre-existing content for that trie key/path or not.
+ *
+ * @param <T> The content type for this {@link InMemoryTrie}.
+ * @param <U> The type of the new content being applied to this {@link InMemoryTrie}.
+ */
+ public interface UpsertTransformer<T, U>
+ {
+ /**
+ * Called when there's content in the updating trie.
+ *
+ * @param existing Existing content for this key, or null if there isn't any.
+ * @param update The update, always non-null.
+ * @return The combined value to use. Cannot be null.
+ */
+ T apply(T existing, U update);
+ }
+
+ /**
+ * Modify this trie to apply the mutation given in the form of a trie. Any content in the mutation will be resolved
+ * with the given function before being placed in this trie (even if there's no pre-existing content in this trie).
+ * @param mutation the mutation to be applied, given in the form of a trie. Note that its content can be of type
+ * different than the element type for this memtable trie.
+ * @param transformer a function applied to the potentially pre-existing value for the given key, and the new
+ * value. Applied even if there's no pre-existing value in the memtable trie.
+ */
+ public <U> void apply(Trie<U> mutation, final UpsertTransformer<T, U> transformer) throws SpaceExhaustedException
+ {
+ Cursor<U> mutationCursor = mutation.cursor();
+ assert mutationCursor.depth() == 0 : "Unexpected non-fresh cursor.";
+ ApplyState state = applyState;
+ state.reset();
+ state.descend(-1, mutationCursor.content(), transformer);
+ assert state.currentDepth == 0 : "Unexpected change to applyState. Concurrent trie modification?";
+
+ while (true)
+ {
+ int depth = mutationCursor.advance();
+ while (state.currentDepth >= depth)
+ {
+ // There are no more children. Ascend to the parent state to continue walk.
+ if (!state.attachAndMoveToParentState())
+ {
+ assert depth == -1 : "Unexpected change to applyState. Concurrent trie modification?";
+ return;
+ }
+ }
+
+ // We have a transition, get child to descend into
+ state.descend(mutationCursor.incomingTransition(), mutationCursor.content(), transformer);
+ assert state.currentDepth == depth : "Unexpected change to applyState. Concurrent trie modification?";
+ }
+ }
+
+ /**
+ * Map-like put method, using the apply machinery above which cannot run into stack overflow. When the correct
+ * position in the trie has been reached, the value will be resolved with the given function before being placed in
+ * the trie (even if there's no pre-existing content in this trie).
+ * @param key the trie path/key for the given value.
+ * @param value the value being put in the memtable trie. Note that it can be of type different than the element
+ * type for this memtable trie. It's up to the {@code transformer} to return the final value that will stay in
+ * the memtable trie.
+ * @param transformer a function applied to the potentially pre-existing value for the given key, and the new
+ * value (of a potentially different type), returning the final value that will stay in the memtable trie. Applied
+ * even if there's no pre-existing value in the memtable trie.
+ */
+ public <R> void putSingleton(ByteComparable key,
+ R value,
+ UpsertTransformer<T, ? super R> transformer) throws SpaceExhaustedException
+ {
+ apply(Trie.singleton(key, value), transformer);
+ }
+
+ /**
+ * A version of putSingleton which uses recursive put if the last argument is true.
+ */
+ public <R> void putSingleton(ByteComparable key,
+ R value,
+ UpsertTransformer<T, ? super R> transformer,
+ boolean useRecursive) throws SpaceExhaustedException
+ {
+ if (useRecursive)
+ putRecursive(key, value, transformer);
+ else
+ putSingleton(key, value, transformer);
+ }
+
+ /**
+ * Map-like put method, using a fast recursive implementation through the key bytes. May run into stack overflow if
+ * the trie becomes too deep. When the correct position in the trie has been reached, the value will be resolved
+ * with the given function before being placed in the trie (even if there's no pre-existing content in this trie).
+ * @param key the trie path/key for the given value.
+ * @param value the value being put in the memtable trie. Note that it can be of type different than the element
+ * type for this memtable trie. It's up to the {@code transformer} to return the final value that will stay in
+ * the memtable trie.
+ * @param transformer a function applied to the potentially pre-existing value for the given key, and the new
+ * value (of a potentially different type), returning the final value that will stay in the memtable trie. Applied
+ * even if there's no pre-existing value in the memtable trie.
+ */
+ public <R> void putRecursive(ByteComparable key, R value, final UpsertTransformer<T, R> transformer) throws SpaceExhaustedException
+ {
+ int newRoot = putRecursive(root, key.asComparableBytes(BYTE_COMPARABLE_VERSION), value, transformer);
+ if (newRoot != root)
+ root = newRoot;
+ }
+
+ private <R> int putRecursive(int node, ByteSource key, R value, final UpsertTransformer<T, R> transformer) throws SpaceExhaustedException
+ {
+ int transition = key.next();
+ if (transition == ByteSource.END_OF_STREAM)
+ return applyContent(node, value, transformer);
+
+ int child = getChild(node, transition);
+
+ int newChild = putRecursive(child, key, value, transformer);
+ if (newChild == child)
+ return node;
+
+ int skippedContent = followContentTransition(node);
+ int attachedChild = !isNull(skippedContent)
+ ? attachChild(skippedContent, transition, newChild) // Single path, no copying required
+ : expandOrCreateChainNode(transition, newChild);
+
+ return preserveContent(node, skippedContent, attachedChild);
+ }
+
+ private <R> int applyContent(int node, R value, UpsertTransformer<T, R> transformer) throws SpaceExhaustedException
+ {
+ if (isNull(node))
+ return ~addContent(transformer.apply(null, value));
+
+ if (isLeaf(node))
+ {
+ int contentIndex = ~node;
+ setContent(contentIndex, transformer.apply(getContent(contentIndex), value));
+ return node;
+ }
+
+ if (offset(node) == PREFIX_OFFSET)
+ {
+ int contentIndex = getInt(node + PREFIX_CONTENT_OFFSET);
+ setContent(contentIndex, transformer.apply(getContent(contentIndex), value));
+ return node;
+ }
+ else
+ return createPrefixNode(addContent(transformer.apply(null, value)), node, false);
+ }
+
+ /**
+ * Returns true if the allocation threshold has been reached. To be called by the the writing thread (ideally, just
+ * after the write completes). When this returns true, the user should switch to a new trie as soon as feasible.
+ *
+ * The trie expects up to 10% growth above this threshold. Any growth beyond that may be done inefficiently, and
+ * the trie will fail altogether when the size grows beyond 2G - 256 bytes.
+ */
+ public boolean reachedAllocatedSizeThreshold()
+ {
+ return allocatedPos >= ALLOCATED_SIZE_THRESHOLD;
+ }
+
+ /**
+ * For tests only! Advance the allocation pointer (and allocate space) by this much to test behaviour close to
+ * full.
+ */
+ @VisibleForTesting
+ int advanceAllocatedPos(int wantedPos) throws SpaceExhaustedException
+ {
+ while (allocatedPos < wantedPos)
+ allocateBlock();
+ return allocatedPos;
+ }
+
+ /** Returns the off heap size of the memtable trie itself, not counting any space taken by referenced content. */
+ public long sizeOffHeap()
+ {
+ return bufferType == BufferType.ON_HEAP ? 0 : allocatedPos;
+ }
+
+ /** Returns the on heap size of the memtable trie itself, not counting any space taken by referenced content. */
+ public long sizeOnHeap()
+ {
+ return contentCount * MemoryLayoutSpecification.SPEC.getReferenceSize() +
+ REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(contentCount, CONTENTS_START_SHIFT, CONTENTS_START_SIZE) +
+ (bufferType == BufferType.ON_HEAP ? allocatedPos + EMPTY_SIZE_ON_HEAP : EMPTY_SIZE_OFF_HEAP) +
+ REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(allocatedPos, BUF_START_SHIFT, BUF_START_SIZE);
+ }
+
+ @Override
+ public Iterable<T> valuesUnordered()
+ {
+ return () -> new Iterator<T>()
+ {
+ int idx = 0;
+
+ public boolean hasNext()
+ {
+ return idx < contentCount;
+ }
+
+ public T next()
+ {
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ return getContent(idx++);
+ }
+ };
+ }
+
+ public int valuesCount()
+ {
+ return contentCount;
+ }
+
+ public long unusedReservedMemory()
+ {
+ int bufferOverhead = 0;
+ if (bufferType == BufferType.ON_HEAP)
+ {
+ int pos = this.allocatedPos;
+ UnsafeBuffer buffer = getChunk(pos);
+ if (buffer != null)
+ bufferOverhead = buffer.capacity() - inChunkPointer(pos);
+ }
+
+ int index = contentCount;
+ int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
+ int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
+ AtomicReferenceArray<T> contentArray = contentArrays[leadBit];
+ int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs) * MemoryLayoutSpecification.SPEC.getReferenceSize();
+
+ return bufferOverhead + contentOverhead;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md
new file mode 100644
index 0000000..09c1408
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md
@@ -0,0 +1,753 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# InMemoryTrie Design
+
+The `InMemoryTrie` is one of the main components of the trie infrastructure, a mutable in-memory trie built for fast
+modification and reads executing concurrently with writes from a single mutator thread.
+
+The main features of its implementation are:
+- full support of the `Trie` interface
+- using nodes of several different types for efficiency
+- support for content on any node, including intermediate (prefix)
+- support for writes from a single mutator thread concurrent with multiple readers
+- maximum trie size of 2GB
+
+
+## Memory layout
+
+One of the main design drivers of the memtable trie is the desire to avoid on-heap storage and Java object management.
+The trie thus implements its own memory management for the structure of the trie (content is, at this time, still given
+as Java objects in a content array). The structure resides in one `UnsafeBuffer` (which can be on or off heap as
+desired) and is broken up in 32-byte "cells" (also called "blocks" in the code), which are the unit of allocation,
+update and reuse.
+
+Like all tries, `InMemoryTrie` is built from nodes and has a root pointer. The nodes reside in cells, but there is no
+1:1 correspondence between nodes and cells - some node types pack multiple in one cell, while other types require
+multiple cells.
+
+### Pointers and node types
+
+A "pointer" is an integer that points to a node in the trie buffer. A pointer specifies the location of the node
+(its starting cell), but also defines the type of node in its 5 lowest-order bits (i.e. the offset within the cell).
+If a pointer has a negative value, it refers to a value in the content array, and implies a leaf node with the specified
+content. Additionally, the special pointer value `NONE` (0) is used to specify "no child". We use 32-bit integers as
+pointers, therefore the size of the trie structure is limited to a little less than 2GB.
+
+For example, the pointer `0x0109E` specifies a node residing in the cell at bytes `0x01080`-`0x0109F` in the buffer
+(specified by the pointers' 27 leading bits), where the node type is `Sparse` (specified by `0x1E` in the last 5 bits).
+
+The pointer `0xFFFFFFF0` specifies a leaf node (being negative), where the content's index is `0xF` (obtained by
+negating all bits of the pointer).
+
+To save space and reduce pointer chasing, we use several different types of nodes that address different common patterns
+in a trie. It is common for a trie to have one or a couple of top levels which have many children, and where it is
+important to make decisions with as few if-then-else branches as possible (served by the `Split` type), another one or
+two levels of nodes with a small number of children, where it is most important to save space as the number of these
+nodes is high (served by the `Sparse` type), and a lot of sequences of single-child nodes containing the trailing bytes
+of the key or of some common key prefix (served by the `Chain` type). Most of the payload/content of the trie resides
+at the leaves, where it makes sense to avoid taking any space for a node (the `Leaf` type), but we must also allow the
+possibility for values to be present in intermediate nodes — because this is rare, we support it with a special
+`Prefix` type instead of reserving a space for payload in all other node types.
+
+The Split-Sparse-Chain-Leaf/Prefix pattern may repeat several times. For example, we could have these four layers for
+the partition key with some metadata associated with the partition, then for the first component of the clustering key,
+then for the second component etc.
+
+The sections below specify the layout of each supported node type.
+
+#### Leaf nodes
+
+Leaf nodes do not have a corresponding cell in the buffer. Instead, they reference a value (i.e. a POJO in the
+`InMemoryTrie`'s content type) in the content array. The index of the value is specified by `~pointer` (unlike `-x`,
+`~x` allows one to also encode 0 in a negative number).
+
+Leaf nodes have no children, and return the specified value for `content()`.
+
+Example: -1 is a leaf cell with content `contentArray[0]`.
+
+#### `Chain` nodes - single path, multiple transitions in one cell
+
+Chain nodes are one-child nodes. Multiple chain nodes, forming a chain of transitions to one target, can reside in a
+single cell. Chain nodes are identified by the lowest 5 bits of a pointer being between `0x00` and `0x1B`. In addition
+to the type of node, in this case the bits also define the length of the chain — the difference between
+`0x1C` and the pointer offset specifies the number of characters in the chain.
+
+The simplest chain node has one transition leading to one child and is laid out like this:
+
+offset|content|example
+---|---|---
+00 - 1A|unused|
+1B |character|41 A
+1C - 1F|child pointer|FFFFFFFF
+
+where the pointer points to the `1B` line in the cell.
+
+Example: The cell `xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxx41 FFFFFFFF` at bytes `0x120`-`0x13F` and
+pointer `0x13B` point to a node with one child with transition `0x41` `A` to a leaf node with content `contentArray[0]`.
+
+Another chain cell, which points to this one, can be added in the same cell by placing a character at offset `1A`. This
+new node is effectively laid out as
+
+offset|content|example
+---|---|---
+00 - 19|unused|
+1A |character|48 H
+1B - 1F|unused|
+
+where the pointer points to line `1A`. This node has one transition, and the child pointer is implicit as the node's
+pointer plus one.
+
+This can continue until all the bytes in the "unused" area are filled.
+
+Example: The cell `xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xx434841 FFFFFFFF` at bytes `0x120`-`0x13F` and
+pointer `0x139` point to a node with one child with transition `0x43` `C` to a node with one child with transition
+`0x48` `H` to a node with one child with transition `0x41` `A` to a leaf node with content `contentArray[0]`.
+
+offset|content|example
+---|---|---
+00 - 18|unused|
+19 |character|43 C
+1A |character|48 H
+1B |character|41 A
+1C - 1F|child pointer|FFFFFFFF
+
+
+In this example `0x13A` and `0x13B` are also valid pointers to the respective chains and could be referenced from other
+nodes (an example will be given below). In any case, the byte pointed directly by the node pointer contains the
+transition byte. The child pointer is either `pointer + 1` (if the lowest 5 pointer bits are less than `0x1B`), or the
+integer stored at `pointer + 1` (if the pointer's last 5 bits are `0x1B`).
+
+
+
+Note: offset `0x00` also specifies a chain node, but the pointer 0 is a special case and care must be taken to ensure no
+28-byte chain node is placed in the cell at bytes `0x00`-`0x1F`.
+
+#### `Sparse` nodes - between 2 and 6 children in one cell
+
+Sparse nodes are used when a node has at least two children, and all pointers and transition characters can fit in one
+cell, which limits the maximum number of children to 6. Their layout is:
+
+offset|content|
+---|---|
+00 - 03|child pointer 0|
+04 - 07|child pointer 1|
+08 - 0B|child pointer 2|
+0C - 0F|child pointer 3|
+10 - 13|child pointer 4|
+14 - 17|child pointer 5|
+18 |character 0|
+19 |character 1|
+1A |character 2|
+1B |character 3|
+1C |character 4|
+1D |character 5|
+1E - 1F|order word|
+
+where the pointer points to the line `1E` (i.e. the type identifier for a sparse node is `0x1E`).
+
+It is important to note that the pointers and characters are not in order. This is done so that an update to a sparse
+node where a new child is inserted can be done while the previous state of the node is still valid and readable for
+any concurrent readers. Instead, new children are appended, and the order is maintained in the "order word". This word
+is a number whose digits specify the order of the children's transition characters (where higher-order digits specify
+bigger characters) encoded, to be able to fit into a 16-bit word, in base 6. Its number of digits also specifies the
+number of children of the node.
+
+To explain this better, we will give an example of the evolution of a sparse node. Suppose we had the `0x139` node from
+the previous section, and some update needs to attach a second child to that, e.g. with the character `A` and child
+`0x238`.
+
+
+
+To do this, the mutating thread will have to convert the chain node into a sparse by allocating a new cell
+(e.g. `0x240`-`0x25F`) and filling in the sparse node `00000238 0000013A 00000000 00000000 00000000 00000000 41430000
+00000006` with pointer `0x25E`:
+
+offset|content|example
+---|---|---
+00 - 03|child pointer 0| 00000238
+04 - 07|child pointer 1| 0000013A
+08 - 17|unused|
+18 |character 0| 41 A
+19 |character 1| 43 C
+1A - 1D|unused|
+1E - 1F|order word, always 10| 0006 = 10 (base 6)
+
+This is the smallest kind of sparse node, with just two children. Two-children sparse nodes always
+put their two children in order (we can do this as this does not happen in response to an addition of a new child to
+an existing sparse node, but this is constructed directly) and thus their order word is always 10 (if they were
+not in order, the order word would have to be 01, which would be misinterpreted as the single-digit 1).
+
+This node has two (the number of digits in the order word) children. The first child is at the position specified by the
+least significant digit of the order word, 0. The second child is specified by the second least significant digit, 1.
+
+Suppose we then need to add a new child, using character `0x35` `5` and child `0x33B`. The node will change to `00000238
+0000013A 0000033B 00000000 00000000 00000000 41433500 00000026` and the pointer to it stays the same.
+
+offset|content|example
+---|---|---
+00 - 03|child pointer 0| 00000238
+04 - 07|child pointer 1| 0000013A
+08 - 0B|child pointer 2| 0000033B
+0C - 17|unused|
+18 |character 0| 41 A
+19 |character 1| 43 C
+1A |character 2| 35 5
+1B - 1D|unused|
+1E - 1F|order word| 0026 = 102 (base 6)
+
+This node has three (the number of digits in the order word) children. The first child is at the position specified by
+the least significant digit of the order word, 2. The second child is specified by the second least significant digit,
+0, and the last child is specified by the leading digit, 1.
+
+Note that because of the ordering of the two children in the smallest sparse node, the digit 0 is always preceded by a
+more-significant 1 in the order word in base 6. Therefore the leading digit of the order word can never be 0 and thus we
+cannot miscount the number of children.
+
+The addition of children can continue until we have 6, for example `00000238 0000013A 0000033B 0000035C 0000037A
+0000041B 41433542 50338129` (pointer `0x25E`) for
+
+offset|content|example
+---|---|---
+00 - 03|child pointer 0| 00000238
+04 - 07|child pointer 1| 0000013A
+08 - 0B|child pointer 2| 0000033B
+0C - 0F|child pointer 3| 0000035C
+10 - 13|child pointer 4| 0000037A
+14 - 17|child pointer 5| 0000041B
+18 |character 0| 41 A
+19 |character 1| 43 C
+1A |character 2| 35 5
+1B |character 3| 42 B
+1C |character 4| 50 P
+1D |character 5| 33 3
+1E - 1F|order word| 8129 = 413025 (base 6)
+
+Beyond 6 children, a node needs to be converted to split.
+
+#### `Split` nodes - up to 256 children in multiple cells
+
+Split nodes are used to handle the nodes with a large number of children. We can only allocate cells of 32 bytes, thus
+we have to distribute the child transitions among cells in some way that is efficient for reading and updating. The
+method we chose is to construct a "mini-trie" with 2-3-3 bit transitions.
+
+A split node is identified by the `0x1C` offset. The starting cell of a split node has this layout:
+
+offset|content|
+---|---|
+00 - 0F|unused|
+10 - 13|mid-cell for leading 00|
+14 - 17|mid-cell for leading 01|
+18 - 1B|mid-cell for leading 10|
+1C - 1F|mid-cell for leading 11|
+
+(pointers to this node point to the `1C` line) and where each mid-cell contains:
+
+offset|content|
+---|---|
+00 - 03|end-cell for middle 000|
+04 - 07|end-cell for middle 001|
+08 - 0B|end-cell for middle 010|
+0C - 0F|end-cell for middle 011|
+10 - 13|end-cell for middle 100|
+14 - 17|end-cell for middle 101|
+18 - 1B|end-cell for middle 110|
+1C - 1F|end-cell for middle 111|
+
+and end-cell:
+
+offset|content|
+---|---|
+00 - 03|pointer to child for ending 000|
+04 - 07|pointer to child for ending 001|
+08 - 0B|pointer to child for ending 010|
+0C - 0F|pointer to child for ending 011|
+10 - 13|pointer to child for ending 100|
+14 - 17|pointer to child for ending 101|
+18 - 1B|pointer to child for ending 110|
+1C - 1F|pointer to child for ending 111|
+
+In any of the cell or pointer positions we can have `NONE`, meaning that such a child (or block of children) does not
+exist. At minimum, a split node occupies 3 cells (one leading, one mid and one end), and at maximum —
+`1 + 4 + 4*8 = 37` cells i.e. `1184` bytes. If we could allocate contiguous arrays, a full split node would use `1024`
+bytes, thus this splitting can add ~15% overhead. However, real data often has additional structure that this can make
+use of to avoid creating some of the blocks, e.g. if the trie encodes US-ASCII or UTF-encoded strings where some
+character ranges are not allowed at all, and others are prevalent. Another benefit is that to change a transition while
+preserving the previous state of the node for concurrent readers we have to only copy three blocks and not the entire
+range of children (applications of this will be given later).
+
+As an example, suppose we need to add a `0x51` `Q` transition to `0x455` to the 6-children sparse node from the previous
+section. This will generate the following structure:
+
+Leading cell (e.g. `0x500`-`0x51F` with pointer `0x51C`)
+
+offset|content|example
+---|---|---
+00 - 0F|unused|
+10 - 13|mid-cell for leading 00|0000053C
+14 - 17|mid-cell for leading 01|0000057C
+18 - 1B|mid-cell for leading 10|00000000 NONE
+1C - 1F|mid-cell for leading 11|00000000 NONE
+
+Mid cell `00` at `0x520`-`0x53F`:
+
+offset|content|example
+---|---|---
+00 - 03|end-cell for middle 000|00000000 NONE
+04 - 07|end-cell for middle 001|00000000 NONE
+08 - 0B|end-cell for middle 010|00000000 NONE
+0C - 0F|end-cell for middle 011|00000000 NONE
+10 - 13|end-cell for middle 100|00000000 NONE
+14 - 17|end-cell for middle 101|00000000 NONE
+18 - 1B|end-cell for middle 110|0000055C
+1C - 1F|end-cell for middle 111|00000000 NONE
+
+End cell `00 110` at `0x540`-`0x55F`:
+
+offset|content|example
+---|---|---
+00 - 03|pointer to child for ending 000|00000000 NONE
+04 - 07|pointer to child for ending 001|00000000 NONE
+08 - 0B|pointer to child for ending 010|00000000 NONE
+0C - 0F|pointer to child for ending 011|0000041B
+10 - 13|pointer to child for ending 100|00000000 NONE
+14 - 17|pointer to child for ending 101|0000033B
+18 - 1B|pointer to child for ending 110|00000000 NONE
+1C - 1F|pointer to child for ending 111|00000000 NONE
+
+Mid cell `01` at `0x560`-`0x57F`:
+
+offset|content|example
+---|---|---
+00 - 03|end-cell for middle 000|0000059C
+04 - 07|end-cell for middle 001|00000000 NONE
+08 - 0B|end-cell for middle 010|000005BC
+0C - 0F|end-cell for middle 011|00000000 NONE
+10 - 13|end-cell for middle 100|00000000 NONE
+14 - 17|end-cell for middle 101|00000000 NONE
+18 - 1B|end-cell for middle 110|00000000 NONE
+1C - 1F|end-cell for middle 111|00000000 NONE
+
+End cell `01 000` at `0x580`-`0x59F`:
+
+offset|content|example
+---|---|---
+00 - 03|pointer to child for ending 000|00000000 NONE
+04 - 07|pointer to child for ending 001|00000238
+08 - 0B|pointer to child for ending 010|0000035C
+0C - 0F|pointer to child for ending 011|0000013A
+10 - 13|pointer to child for ending 100|00000000 NONE
+14 - 17|pointer to child for ending 101|00000000 NONE
+18 - 1B|pointer to child for ending 110|00000000 NONE
+1C - 1F|pointer to child for ending 111|00000000 NONE
+
+End cell `01 010` at `0x5A0`-`0x5BF`:
+
+offset|content|example
+---|---|---
+00 - 03|pointer to child for ending 000|0000037A
+04 - 07|pointer to child for ending 001|00000455
+08 - 0B|pointer to child for ending 010|00000000 NONE
+0C - 0F|pointer to child for ending 011|00000000 NONE
+10 - 13|pointer to child for ending 100|00000000 NONE
+14 - 17|pointer to child for ending 101|00000000 NONE
+18 - 1B|pointer to child for ending 110|00000000 NONE
+1C - 1F|pointer to child for ending 111|00000000 NONE
+
+To find a child in this structure, we follow the transitions along the bits of the mini-trie. For example, for `0x42`
+`B` = `0b01000010` we start at `0x51C`, take the `01` pointer to `0x57C`, then the `000` pointer to `0x59C` and finally
+the `010` index to retrieve the node pointer `0x35C`. Note that the intermediate cells (dashed in the diagram) are not
+reachable with pointers, they only make sense as substructure of the split node.
+
+
+
+#### Content `Prefix`
+
+Prefix nodes are not nodes in themselves, but they add information to the node they lead to. Specifically, they
+encode an index in the content array, and a pointer to the node to which this content is attached. In anything other
+than the content, they are equivalent to the linked node — i.e. a prefix node pointer has the same children as
+the node it links to (another way to see this is as a content-carrying node is one that has an _ε_ transition to the
+linked node and no other features except added content). We do not allow more than one prefix to a node (i.e. prefix
+can't point to another prefix), and the child of a prefix node cannot be a leaf.
+
+There are two types of prefixes:
+- standalone, which has a full 32-bit pointer to the linked node
+- embedded, which occupies unused space in `Chain` or `Split` nodes and specifies the 5-bit offset within the same cell
+of the linked node
+
+Standalone prefixes have this layout:
+
+offset|content|example
+---|---|---
+00 - 03|content index|00000001
+04|standalone flag, 0xFF|FF
+05 - 1B|unused|
+1C - 1F|linked node pointer|0000025E
+
+and pointer offset `0x1F`. The sample values above will be the ones used to link a prefix node to our `Sparse`
+example, where a prefix cannot be embedded as all the bytes of the cell are in use.
+
+If we want to attach the same prefix to the `Split` example, we will place this
+
+offset|content|example
+---|---|---
+00 - 03|content index|00000001
+04|embedded offset within cell|1C
+05 - 1F|unused|
+
+_inside_ the leading split cell, with pointer `0x1F`. Since this is an embedded node, the augmented one resides within
+the same cell, and thus we need only 5 bits to encode the pointer (the other 27 are the same as the prefix's).
+The combined content of the cell at `0x500-0x51F` will then be `00000001 1C000000 00000000 00000000 00000520 00000560
+00000000 00000000`:
+
+offset|content|example
+---|---|---
+00 - 03|content index|00000001
+04|embedded offset within cell|1C
+05 - 0F|unused|
+10 - 13|mid-cell for leading 00|00000520
+14 - 17|mid-cell for leading 01|00000560
+18 - 1B|mid-cell for leading 10|00000000 NONE
+1C - 1F|mid-cell for leading 11|00000000 NONE
+
+Both `0x51C` and `0x51F` are valid pointers in this cell. The former refers to the plain split node, the latter to its
+content-augmented version. The only difference between the two is the result of a call to `content()`.
+
+
+
+
+## Reading a trie
+
+`InMemoryTrie` is mainly meant to be used as an implementation of `Trie`. As such, the main method of retrieval of
+information is via some selection (i.e. intersection) of a subtrie followed by a walk over the content in this
+subtrie. Straightforward methods for direct retrieval of data by key are also provided, but they are mainly for testing.
+
+The methods for iterating over and transforming tries are provided by the `Trie` interface and are built on the cursor
+interface implemented by `InMemoryTrie` (see `Trie.md` for a description of cursors).
+
+
+
+(Edges in black show the trie's structure, and the ones in <span style="color:lightblue">light blue</span> the path the cursor walk takes.)
+
+### Cursors over `InMemoryTrie`
+
+`InMemoryTrie` implements cursors using arrays of integers to store the backtracking state (as the simplest
+possible structure that can be easily walked and garbage collected). No backtracking state is added for `Chain` or
+`Leaf` nodes and any prefix. For `Sparse` we store the node address, depth and the remainder of the sparse order word.
+That is, we read the sparse order word on entry, peel off the next index to descend and store the remainder. When we
+backtrack to the node we peel off another index -- if the remainder becomes 0, there are no further children and the
+backtracking entry can be removed.
+
+For `Split` nodes we store one entry per split node cell. This means:
+- one entry for the head cell with address, depth and next child bits `0bHH000000` where HH is between 1 and 3
+- one entry for the mid cell with address, depth and next child bits `0bHHMMM000` where MMM is between 1 and 7
+- one entry for the tail cell with address, depth and next child bits `0bHHMMMTTT` where TTT is between 1 and 7
+
+On backtracking we recognize the sublevel by the position of the lowest non-zero bit triple. For example, if the last
+three are not `0b000`, this is a tail cell, we can advance in it and use the HHMMM bits to form the transition byte.
+
+This substructure is a little more efficient than storing only one entry for the split node (the head-to-mid and
+mid-to-tail links do not need to be followed for every new child) and also allows us to easily get the precise next
+child and remove the backtracking entry when a cell has no further children.
+
+`InMemoryTrie` cursors also implement `advanceMultiple`, which jumps over intermediate nodes in `Chain` blocks:
+
+
+
+## Mutation
+
+Mutation of `InMemoryTrie` must be done by one thread only (for performance reasons we don't enforce it, the user must
+make sure that's the case), but writes may be concurrent with multiple reads over the data that is being mutated. The
+trie is built to support this by making sure that any modification of a node is safe for any reader that is operating
+concurrently.
+
+The main method for mutating a `InMemoryTrie` is `apply`, which merges the structure of another `Trie` in.
+`InMemoryTrie` also provides simpler recursive method of modification, `putRecursive`, which creates a single
+`key -> value` mapping in the trie. We will describe the mutation process starting with a `putRecursive` example.
+
+### Adding a new key -> value mapping using `putRecursive`
+
+Suppose we want to insert the value `traverse` into the trie described in the previous paragraph. The recursive
+insertion process walks the trie to find corresponding existing nodes for the ones in the path to be inserted.
+When it has to leave the existing trie, because it has no entries for the path, the process continues using `NONE` as
+the trie node.
+
+
+
+When it reaches the end of the path, it needs to attach the value. Unless this is a prefix of an existing entry, the
+matching trie node will either be `NONE` or a leaf node. Here it's `NONE`, so we create a item in the
+content array, `contentArray[3]`, put the value in it, and thus form the leaf node `~3` (`0xFFFFFFFC`). The recursive
+process returns this to the previous step.
+
+The previous step must attach a child with the transition `e` to the node `NONE`. Since this is a new node, we do this
+by creating a new `Chain` node at address `0x0BB` mapping `e` to `~3` and return that. For the node above, we again
+need to attach a child to `NONE`, but this time the child is a `Chain` node, so we can do this by expanding it, i.e.
+writing the new character at the address just before the child pointer, and returning that address (note that the
+child chain node is newly created, so we can't be overwriting any existing data there). We can do this several more
+times.
+
+
+
+(<span style="color:lightblue">Light blue</span> specifies the descent path, <span style="color:pink">pink</span>
+the values returned, <span style="color:blue">blue</span> stands for newly-created nodes and links, and
+<span style="color:lightgray">light gray</span> for obsoleted nodes and links.)
+
+In the next step we must attach the child `0x0B8` with transition `v` to the existing `Chain` node `0x018`. This is a
+different transition from the one that node already has, so the change cannot be accommodated by a node of type `Chain`,
+thus we need to copy this into a new `Sparse` node `0x0DE` with two children, the existing `c -> 0x019` and the new
+`v -> 0x0B8` and return `0x0DE` to the parent step.
+
+The parent step must then change its existing pointer for the character `a` from `0x018` to `0x0DE` which it can do in
+place by writing the new value in its pointer cell for `a`. This is the attachment point for the newly created
+substructure, i.e. before this, the new nodes were not reachable, and now become reachable; before this, the node
+`0x018 ` was reachable, and now becomes unreachable. The attachment is done by a volatile write, to enforce a
+happens-before relationship that makes sure that all the new substructure (all written by this thread) is fully readable
+by all readers who pass through the new pointer (which is the only way they can reach it). The same happens-before also
+ensures that any new readers cannot reach the obsoleted nodes (there may be existing reader threads that are already in
+them).
+
+It can then return its address `0x07E` unchanged up, and no changes need to be done in any of the remaining steps. The
+process finishes in a new value for `root`, which in this case remains unchanged.
+
+
+
+The process created a few new nodes (in blue), and made one obsolete (in grey). What concurrent readers can see depends
+on where they are at the time the attachment point write is done. Forward traversals, if they are in the path below
+`0x07E`, will continue working with the obsoleted data and will not see any of the new changes. If they are above
+`0x07E`, they will see the updated content. If they are _at_ the `0x07E` node, they may see either, depending on the
+time they read the pointer for `a`. Reverse traversals that happen to be in the region to the right of the new nodes
+_will_ see the updated content, as they will read the pointer after it has been updated.
+
+In any case, the obsolete paths remain correct and usable for any thread that has already reached them, and the new
+paths are correct and usable from the moment they become reachable.
+
+Note that if we perform multiple mutations in sequence, and a reader happens to be stalled between them (in iteration
+order), such reader may see only the mutation that is ahead of it _in iteration order_, which is not necessarily the
+mutation that happened first. For the example above, if we also inserted `trespass`, a reader thread that was paused
+at `0x018` in a forward traversal and wakes up after both insertions have completed will see `trespass`, but _will not_
+see `traverse` even though it was inserted earlier.
+
+### In-place modifications
+
+When the backtracking process returns with a new mapping, there are several cases when we can apply a change in place
+(creating an attachment point for the new path). We will explain these in detail, as it is important to understand what
+exactly happens from concurrent readers' point of view in all of them.
+
+Note that if a modification cannot be done in place, we copy the content to a new node. The copied node is always
+unreachable and there will always be an attachment point that makes it reachable somewhere in the parent chain.
+
+#### Changing the child pointer of the last `Chain` node in a chain
+
+This happens when the existing transition matches the transition of the new character, but the pointer is different,
+and only applies to `Chain` nodes whose offset is `0x1B`. In this case the child pointer is written at offset `0x1C`,
+and we can put in the new value by performing a volatile write.
+
+For example, updating `N -> 0x39C` is accomplished by making the volatile write:
+
+offset|content|before|after
+---|---|---|---
+00-1A|irrelevant||
+1B|character|N|N
+1C-1F|pointer|0000031E|_**0000039C**_
+
+(Here and below normal writes are in bold and volatile writes in bold italic.)
+
+Readers have to read the pointer to reach the child (old or new), so this achieves the happens-before guarantees we
+seek. Readers either see the old value (where none of the branch's data has been modified in any way), or the new value
+(where the happens-before guarantees all writes creating the attached substructure are fully visible).
+
+Note that if the node is not the last in the chain, the pointer is implicit and we cannot change it. Thus we have
+to copy, i.e. create a new node, which in this case will also be a `Chain` node, because there is nothing else in the
+original node that needs to be preserved (the only existing transition is replaced by the update).
+
+#### Changing the child pointer of a `Sparse` or `Split` node
+
+Similarly to above, in this case the transition matches an existing one, and thus we already have a 4-byte location
+where the pointer to the old child is written, and we can update it by doing a volatile write.
+
+For example, updating `C -> 0x51E` in a sparse node can be:
+
+offset|content|before|after
+---|---|---|---
+00 - 03|child pointer 0| 00000238|00000238
+04 - 07|child pointer 1| 0000013A|_**0000051E**_
+08 - 0B|child pointer 2| 0000033B|0000033B
+0C - 17|unused|
+18 |character 0| 41 A|41 A
+19 |character 1| 43 C|43 C
+1A |character 2| 35 5|35 5
+1B - 1D|unused|
+1E - 1F|order word| 0026 = 102 (base 6)
+
+
+#### Adding a new child to `Split`
+
+If we already have the substructure that leads to the pointer for the new transition (i.e. a mid- and end-cell for the
+transition's first 2-3 bits already exists), the situation is as above, where the existing pointer is `NONE`, and we can
+simply perform a volatile write.
+
+If an end-cell mapping does not exist, we allocate a new cleared cell (so that all pointers are `NONE`), write the new
+pointer at its position using a non-volatile write, and then create a mapping to this end-cell in the mid cell by
+volatile writing its pointer over the `NONE` in the correct offset. Similarly, if there's no mid-cell either, we create
+empty end-cell and mid-cell, write pointer in end-cell and mapping in mid-cell non-volatile, and write the mapping in
+the leading cell volatile.
+
+In any of these cases, readers have to pass through the volatile update to reach any of the new content.
+
+For example, to add `x -> 0x71A` (`x` is `0x78` or `0b01111000`) to the split node example needs a new end cell for
+`01 111` (for example at `0x720-0x73F`) (these writes can be non-volatile):
+
+offset|content|before|after
+---|---|---|---
+00 - 03|pointer to child for ending 000|n/a|**0000071A**
+04 - 07|pointer to child for ending 001|n/a|**00000000** NONE
+08 - 0B|pointer to child for ending 010|n/a|**00000000** NONE
+0C - 0F|pointer to child for ending 011|n/a|**00000000** NONE
+10 - 13|pointer to child for ending 100|n/a|**00000000** NONE
+14 - 17|pointer to child for ending 101|n/a|**00000000** NONE
+18 - 1B|pointer to child for ending 110|n/a|**00000000** NONE
+1C - 1F|pointer to child for ending 111|n/a|**00000000** NONE
+
+and this volatile write to the mid cell `0x520`:
+
+offset|content|before|after
+---|---|---|---
+00 - 03|end-cell for middle 000|00000000 NONE|00000000 NONE
+04 - 07|end-cell for middle 001|00000000 NONE|00000000 NONE
+08 - 0B|end-cell for middle 010|00000000 NONE|00000000 NONE
+0C - 0F|end-cell for middle 011|00000000 NONE|00000000 NONE
+10 - 13|end-cell for middle 100|00000000 NONE|00000000 NONE
+14 - 17|end-cell for middle 101|00000000 NONE|00000000 NONE
+18 - 1B|end-cell for middle 110|0000055C|0000055C
+1C - 1F|end-cell for middle 111|00000000 NONE|_**0000073C**_
+
+The start cell, and the other mid and end cells remain unchanged.
+
+#### Adding a new child to `Sparse` with 5 or fewer existing children
+
+The need to maintain a correct view for concurrent readers without blocking is the reason why we cannot keep the
+children in a `Sparse` cell ordered (if we insert ordered, we open ourselves to readers possibly seeing the same pointer
+or child twice, or even going back in the iteration order). Instead we always add new characters and pointers at the
+next free position and then update the order word to include it. More precisely:
+- we find the smallest index `i < 6` for which the pointer is `NONE`
+- we write the transition character at position `i`
+- we write the pointer at position `i` over `NONE` volatile
+- we compile a new order word by inserting `i` after all indexes with greater transition and before all indexes with
+ smaller in the base-6 representation (e.g. to insert `j` in sparse node that has `a@4 f@0 g@2 k@1 q@3` we change the
+ order word `31204` to `315204`) and write it volatile
+
+This ensures that any reader that iterates over children (i.e. one that needs the order word) will have to pass through
+the volatile order word update and will see the correct character and pointer values. Readers who have read the order
+word at some earlier time will not include the new pointer or character in the iteration.
+
+Readers that directly select the child for a given transition must read the pointer for each index _before_ reading the
+character to ensure they can see the properly updated value (otherwise they could match e.g. a `00` transition to the
+new branch because the real character was not written when they read the byte, but the pointer was when they got to it)
+and stop searching when they find a `NONE` pointer.
+
+For example, adding `x -> 0x71A` to the sparse example above is done by:
+
+offset|content|before|after
+---|---|---|---
+00 - 03|child pointer 0| 00000238|00000238
+04 - 07|child pointer 1| 0000051E|0000051E
+08 - 0B|child pointer 2| 0000033B|0000033B
+0C - 0F|child pointer 3|any|_**0000071A**_
+10 - 17|unused|NONE|NONE
+18 |character 0| 41 A|41 A
+19 |character 1| 43 C|43 C
+1A |character 2| 35 5|35 5
+1B |character 3| any |**78** x
+1C - 1D|unused|00 00|00 00
+1E - 1F|order word|0026 = 102 (base 6)|_**02AE**_ = 3102 (base 6)
+
+where we first write the character, then volatile write the pointer, and finally the order word.
+
+#### Changing the root pointer
+
+If an update propagates with copying modifications all the way to the root, we must update the root pointer. The latter
+is a volatile variable, so this also enforces the happens-before relationship we need.
+
+### Merging a branch using `apply`
+
+This is a generalization of the mutation procedure above, which applies to more complex branches, where each node may
+potentially need multiple updates to attach more than one child. The process proceeds following the nodes that a cursor
+of the mutation trie produces and we maintain full (i.e. for all nodes in the parent chain) backtracking information in
+an array of integers that contains
+- `existingNode`, the corresponding pointer in the memtable trie,
+- `updatedNode`, the current corresponding pointer in the memtable trie, which may be different from the above if the
+ mutation node is branching and one or more of its children have been already added,
+- `transition`, the incoming edge we took to reach the node.
+
+When we descend, we follow the transitions in the memtable trie corresponding to the ones from an iteration over the
+structure of the mutation trie to obtain the `existingNode` pointers, and initialize `updatedNode` to the same. When the
+iteration processes a child, we apply the update to the node, which may happen in place, or may require copying
+— in the latter case `updatedNode` will change to the new value. Note that if `updatedNode` was different from
+the original `existingNode`, it was pointing to an unreachable copied node which will remain unreachable as we will only
+attach the newer version.
+
+After all modifications coming as the result of application of child branches have been applied, we have an
+`updatedNode` that reflects all. As we ascend we apply that new value to the parent's `updatedNode`.
+
+For example (adding a trie containing "traverse, truck" to the "tractor, tree, trie" one):
+
+
+
+In this diagram `existingNode`s are the ones reached through the <span style="color:lightblue">light blue</span> arrows during the descent phase (e.g.
+`0x018` for the `ApplyState` at `tra`, or `NONE` for `tru`), and `updatedNode`s are the ones ascent (<span style="color:pink">pink</span> arrows)
+returns with (e.g . `0x0DE` and `0x0FA` for the respective states).
+
+During this process, readers can see any modifications made in place (each in-place modification is an attachment point
+which makes part of the new nodes reachable). The update mechanism above makes sure both that the state before the
+update is preserved, and that the state after the update is fully visible for readers that can reach it, but it does not
+guarantee that the mutation is seen atomically by the readers if it contains multiple separate branches.
+It is possible for a reader to see only a part of the update, for example:
+- a reading thread racing with the mutator can iterate over `traverse` but finish iterating before the mutator
+manages to attach `truck`;
+- a reading thread that iterated to `tree` (while `traverse` was not yet attached) and paused, will see `truck` if the
+mutating thread applies the update during the pause.
+
+### Handling prefix nodes
+
+The descriptions above were given without prefix nodes. Handling prefixes is just a little complication over the update
+process where we must augment `updatedNode` with any applicable content before applying the change to the parent.
+To do this we expand the state tracked to:
+- `existingPreContentNode` which points to the existing node including any prefix,
+- `existingPostContentNode` which is obtained by skipping over the prefix (for simplicity we also treat leaf
+ nodes like a prefix with no child) and is the base for all child updates (i.e. it takes the role of
+ `existingNode` in the descriptions above),
+- `updatedPostContentNode` which is the node as changed/copied after children modifications are applied,
+- `contentIndex` which is the index in the content array for the result of merging existing and newly introduced
+ content, (Note: The mutation content is only readable when the cursor enters the node, and we can only attach it when
+ we ascend from it.)
+- `transition` remains as before.
+
+and we then attach `contentIndex` to compile an `updatedPreContentNode` which the parent is made to link to. This will
+be equal to `updatedPostContentNode` if no content applies, i.e. `contentIndex == -1`.
+
+("Pre-" and "post-" refer to descent/iteration order, not to construction order; e.g. `updatedPreContentNode` is
+constructed after `updatedPostContentNode` but links above it in the trie.)
+
+As an example, consider the process of adding `trees` to our sample trie:
+
+
+
+When descending at `tree` we set `existingPreContentNode = ~1`, `existingPostContentNode = NONE` and `contentIndex = 1`.
+Ascending back to add the child `~3`, we add a child to `NONE` and get `updatedPostContentNode = 0x0BB`. To then apply
+the existing content, we create the embedded prefix node `updatedPreContentNode = 0x0BF` with `contentIndex = 1` and
+pass that on to the recursion.
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a1.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a1.svg
new file mode 100644
index 0000000..4237fb5
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.a1.svg
@@ -0,0 +1,599 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+```plantuml
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"; color = "lightgrey"; fontcolor = lightgray]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tra [label = " a"; color = "lightgrey"; fontcolor = lightgray]
+ tra -> trac [label = " c"; color = "lightgrey"; fontcolor = lightgray]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+ node [color = "blue"; fontcolor="blue"]
+
+ start -> root
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trav [label = " v"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+
+ tra2 [label = "Sparse\n0x0DE"]
+ trav [label = "Chain\n0x0B8"]
+ trave [label = "0x0B9"]
+ traver [label = "0x0BA"]
+ travers [label = "0x0BB"]
+ traverse [label = "contentArray[3]"]
+
+ tr -> tru [label = " u"]
+ tru -> truc [label = " c"]
+ truc -> truck [label = " k"]
+
+ tru [label = "Chain\n0x0FA"]
+ truc [label = "0x0FB"]
+ truck [label = "contentArray[4]"]
+ }
+
+ {rank=same tra -> tra2 -> tre -> tri -> tru [style=invis]}
+ {rank=same trac -> trav -> tree -> trie -> truc [style=invis]}
+
+ {
+ edge [color = "blue"]
+ tr -> tra2 [label = " a"]
+ tra2 -> trac [label = " c"]
+ tra2 -> trav [label = " v"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+ tr -> tru [label = " u"]
+ tru -> truc [label = " c"]
+ truc -> truck [label = " k"]
+ }
+
+ subgraph back {
+ edge [color = "pink"; fontcolor="red"; arrowhead="vee"; constrain="false"]
+
+ traverse -> travers [label = " ~3"]
+ travers -> traver [label = "0x0BB"]
+ traver -> trave [label = "0x0BA"]
+ trave -> trav [label = "0x0B9"]
+ trav -> tra2 [label = "0x0B8"]
+ tra2 -> tr [label = "0x0DE"]
+ tr -> t [label = "0x07E"]
+ t -> root [label = "0x09B"]
+ root -> start [label = "0x09A"]
+
+ truck -> truc [label = "~4"]
+ truc -> tru [label = "0x0FB"]
+ tru -> tr [label = "0x0FA"]
+ }
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="757pt" height="846pt"
+ viewBox="0.00 0.00 757.47 845.73" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 841.7251)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-841.7251 753.4738,-841.7251 753.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="357.2369" cy="-808.3095" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="357.2369" y="-812.5095" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="357.2369" y="-795.7095" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="490.2369" cy="-808.3095" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="490.2369" y="-804.1095" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- root->start -->
+ <g id="edge51" class="edge">
+ <title>root->start</title>
+ <path fill="none" stroke="#ffc0cb" d="M390.7984,-808.3095C404.3037,-808.3095 420.2339,-808.3095 435.3398,-808.3095"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="445.6261,-808.3095 435.6262,-812.8096 440.6261,-808.3095 435.6261,-808.3096 435.6261,-808.3096 435.6261,-808.3096 440.6261,-808.3095 435.6261,-803.8096 445.6261,-808.3095 445.6261,-808.3095"/>
+ <text text-anchor="middle" x="418.2509" y="-815.5095" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09A</text>
+ </g>
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="357.2369" cy="-708.0939" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="357.2369" y="-703.8939" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M337.9564,-784.2612C330.7393,-772.3485 325.2768,-757.6462 329.8463,-744.0939 331.1636,-740.1869 333.0926,-736.3684 335.3269,-732.7663"/>
+ <polygon fill="#000000" stroke="#000000" points="338.3844,-734.5068 341.3357,-724.3313 332.683,-730.4454 338.3844,-734.5068"/>
+ <text text-anchor="middle" x="333.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge14" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M357.2369,-778.7835C357.2369,-765.2984 357.2369,-749.4062 357.2369,-736.1092"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="357.2369,-726.0972 361.737,-736.0971 357.2369,-731.0972 357.237,-736.0972 357.237,-736.0972 357.237,-736.0972 357.2369,-731.0972 352.737,-736.0972 357.2369,-726.0972 357.2369,-726.0972"/>
+ <text text-anchor="middle" x="360.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge13" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M446.4042,-805.6333C434.2938,-805.1818 421.1479,-804.9584 409.0184,-805.3095 406.3264,-805.3874 403.5568,-805.4881 400.7643,-805.6051"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="390.6504,-806.089 400.4239,-801.1161 395.6447,-805.85 400.639,-805.611 400.639,-805.611 400.639,-805.611 395.6447,-805.85 400.8541,-810.1058 390.6504,-806.089 390.6504,-806.089"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="74.2369" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="74.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge6" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M74.0297,-177.2006C73.8934,-165.0949 73.7122,-149.0076 73.5575,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="77.0529,-134.8319 73.4404,-124.872 70.0533,-134.9108 77.0529,-134.8319"/>
+ <text text-anchor="middle" x="78.3172" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="78.2369" cy="-284.4" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="78.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge5" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M77.4081,-266.0006C76.8628,-253.8949 76.1381,-237.8076 75.5192,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="78.9975,-223.5044 75.0509,-213.672 72.0046,-223.8194 78.9975,-223.5044"/>
+ <text text-anchor="middle" x="82.4869" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="94.2369" cy="-384.6156" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="94.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge4" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M91.3022,-366.2345C88.9115,-351.2603 85.4955,-329.8643 82.756,-312.7055"/>
+ <polygon fill="#000000" stroke="#000000" points="86.1858,-311.9877 81.1529,-302.6646 79.2734,-313.0913 86.1858,-311.9877"/>
+ <text text-anchor="middle" x="89.9322" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- trav -->
+ <g id="node14" class="node">
+ <title>trav</title>
+ <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="202.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#0000ff">Chain</text>
+ <text text-anchor="middle" x="202.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B8</text>
+ </g>
+ <!-- trac->trav -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#d3d3d3" cx="133.2369" cy="-496.2469" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="133.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Chain</text>
+ <text text-anchor="middle" x="133.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge8" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#d3d3d3" d="M114.5188,-470.6739C110.3016,-463.8622 106.2666,-456.3098 103.5223,-448.8313 99.3758,-437.5318 97.0731,-424.4776 95.7973,-413.1021"/>
+ <polygon fill="#d3d3d3" stroke="#d3d3d3" points="99.2538,-412.4638 94.8622,-402.822 92.2826,-413.098 99.2538,-412.4638"/>
+ <text text-anchor="middle" x="109.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> c</text>
+ </g>
+ <!-- tra->trav -->
+ <g id="edge17" class="edge">
+ <title>tra->trav</title>
+ <path fill="none" stroke="#add8e6" d="M130.7527,-466.6197C131.0399,-455.1889 132.9734,-442.3975 138.7369,-432.0313 144.2651,-422.0881 152.8469,-413.6214 161.933,-406.7148"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="170.346,-400.8129 164.7439,-410.2398 166.2527,-403.6844 162.1595,-406.5559 162.1595,-406.5559 162.1595,-406.5559 166.2527,-403.6844 159.5751,-402.872 170.346,-400.8129 170.346,-400.8129"/>
+ <text text-anchor="middle" x="144.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> v</text>
+ </g>
+ <!-- tra2 -->
+ <g id="node19" class="node">
+ <title>tra2</title>
+ <ellipse fill="none" stroke="#0000ff" cx="244.2369" cy="-496.2469" rx="39.2145" ry="29.3315"/>
+ <text text-anchor="middle" x="244.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Sparse</text>
+ <text text-anchor="middle" x="244.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0DE</text>
+ </g>
+ <!-- tra->tra2 -->
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="348.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="348.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="530.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="530.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="357.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="357.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="357.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M354.8487,-466.6249C353.5073,-449.9873 351.8422,-429.334 350.5186,-412.9163"/>
+ <polygon fill="#000000" stroke="#000000" points="353.9973,-412.5103 349.7049,-402.8239 347.0199,-413.0729 353.9973,-412.5103"/>
+ <text text-anchor="middle" x="359.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="468.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="468.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="468.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tre->tri -->
+ <!-- truc -->
+ <g id="node21" class="node">
+ <title>truc</title>
+ <ellipse fill="none" stroke="#0000ff" cx="676.2369" cy="-384.6156" rx="37.1616" ry="18"/>
+ <text text-anchor="middle" x="676.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0FB</text>
+ </g>
+ <!-- trie->truc -->
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M483.2454,-469.2241C492.9216,-451.8019 505.4604,-429.2258 515.1517,-411.7765"/>
+ <polygon fill="#000000" stroke="#000000" points="518.4109,-413.1168 520.2066,-402.6752 512.2914,-409.718 518.4109,-413.1168"/>
+ <text text-anchor="middle" x="507.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tru -->
+ <g id="node20" class="node">
+ <title>tru</title>
+ <ellipse fill="none" stroke="#0000ff" cx="580.2369" cy="-496.2469" rx="38.626" ry="29.3315"/>
+ <text text-anchor="middle" x="580.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Chain</text>
+ <text text-anchor="middle" x="580.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0FA</text>
+ </g>
+ <!-- tri->tru -->
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="357.2369" cy="-607.8782" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="357.2369" y="-612.0782" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="357.2369" y="-595.2782" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge7" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#d3d3d3" d="M321.8872,-597.2153C295.7144,-588.7379 259.543,-575.7815 229.5223,-560.4626 208.2971,-549.6318 186.0738,-535.0548 168.3323,-522.5268"/>
+ <polygon fill="#d3d3d3" stroke="#d3d3d3" points="170.1754,-519.5418 160.0068,-516.5667 166.1007,-525.2336 170.1754,-519.5418"/>
+ <text text-anchor="middle" x="235.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> a</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge16" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M319.8208,-603.7035C283.8354,-598.2919 229.0725,-586.2816 188.5223,-560.4626 176.3594,-552.7182 165.4058,-541.4906 156.5184,-530.5822"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="150.1392,-522.3658 159.8263,-527.5049 153.2055,-526.3152 156.2719,-530.2646 156.2719,-530.2646 156.2719,-530.2646 153.2055,-526.3152 152.7174,-533.0243 150.1392,-522.3658 150.1392,-522.3658"/>
+ <text text-anchor="middle" x="194.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M357.2369,-578.2562C357.2369,-565.3881 357.2369,-550.1179 357.2369,-536.2631"/>
+ <polygon fill="#000000" stroke="#000000" points="360.737,-536.0074 357.2369,-526.0074 353.737,-536.0074 360.737,-536.0074"/>
+ <text text-anchor="middle" x="362.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M380.5061,-584.4766C397.0825,-567.806 419.5291,-545.2318 437.5674,-527.0908"/>
+ <polygon fill="#000000" stroke="#000000" points="440.4292,-529.1767 444.9983,-519.6177 435.4654,-524.241 440.4292,-529.1767"/>
+ <text text-anchor="middle" x="423.9322" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- tr->t -->
+ <g id="edge49" class="edge">
+ <title>tr->t</title>
+ <path fill="none" stroke="#ffc0cb" d="M355.1399,-637.287C354.605,-648.2159 354.2816,-660.7205 354.6843,-672.0939 354.7703,-674.5242 354.889,-677.0451 355.0283,-679.5733"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="355.6875,-689.7931 350.5531,-680.1035 355.3656,-684.8034 355.0438,-679.8138 355.0438,-679.8138 355.0438,-679.8138 355.3656,-684.8034 359.5344,-679.5241 355.6875,-689.7931 355.6875,-689.7931"/>
+ <text text-anchor="middle" x="373.5132" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x07E</text>
+ </g>
+ <!-- tr->tra2 -->
+ <g id="edge33" class="edge">
+ <title>tr->tra2</title>
+ <path fill="none" stroke="#0000ff" d="M333.9323,-584.2913C321.8427,-572.1071 306.8029,-557.03 293.2369,-543.6626 287.6045,-538.1127 281.5754,-532.2309 275.74,-526.5682"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="278.1153,-523.9962 268.4966,-519.5541 273.2458,-529.0249 278.1153,-523.9962"/>
+ <text text-anchor="middle" x="314.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tru -->
+ <g id="edge22" class="edge">
+ <title>tr->tru</title>
+ <path fill="none" stroke="#add8e6" d="M389.1874,-591.8842C428.4573,-572.2261 495.1969,-538.817 538.4895,-517.1452"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="547.6205,-512.5743 540.6927,-521.0747 543.1494,-514.8125 538.6783,-517.0507 538.6783,-517.0507 538.6783,-517.0507 543.1494,-514.8125 536.664,-513.0267 547.6205,-512.5743 547.6205,-512.5743"/>
+ <text text-anchor="middle" x="486.4869" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> u</text>
+ </g>
+ <!-- tr->tru -->
+ <g id="edge40" class="edge">
+ <title>tr->tru</title>
+ <path fill="none" stroke="#0000ff" d="M393.3967,-599.0169C422.2538,-591.1383 463.1788,-578.126 496.2369,-560.4626 514.2689,-550.8278 532.5102,-537.4219 547.3655,-525.3499"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="549.7845,-527.8908 555.2473,-518.8128 545.3158,-522.5028 549.7845,-527.8908"/>
+ <text text-anchor="middle" x="525.4869" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> u</text>
+ </g>
+ <!-- t->root -->
+ <g id="edge50" class="edge">
+ <title>t->root</title>
+ <path fill="none" stroke="#ffc0cb" d="M362.0886,-726.2786C363.3802,-731.9426 364.5942,-738.2388 365.2369,-744.0939 366.1247,-752.1817 365.87,-760.8327 365.048,-769.081"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="363.7469,-779.2407 360.5537,-768.7501 364.3821,-774.2812 365.0173,-769.3217 365.0173,-769.3217 365.0173,-769.3217 364.3821,-774.2812 369.4808,-769.8934 363.7469,-779.2407 363.7469,-779.2407"/>
+ <text text-anchor="middle" x="384.9052" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09B</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M378.7729,-693.2308C385.3244,-687.4682 391.7179,-680.2751 395.2369,-672.0939 398.1872,-665.2348 397.7115,-662.3385 395.2369,-655.2939 393.2659,-649.6828 390.3202,-644.2436 386.9247,-639.1727"/>
+ <polygon fill="#000000" stroke="#000000" points="389.6469,-636.9677 380.8782,-631.0211 384.0247,-641.138 389.6469,-636.9677"/>
+ <text text-anchor="middle" x="402.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge15" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M340.324,-692.0138C335.3995,-686.2543 330.6543,-679.3722 328.0763,-672.0939 324.4827,-661.9481 326.7954,-651.1983 331.382,-641.4556"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="336.3576,-632.4758 335.4471,-643.4038 333.9342,-636.8493 331.5109,-641.2228 331.5109,-641.2228 331.5109,-641.2228 333.9342,-636.8493 327.5747,-639.0418 336.3576,-632.4758 336.3576,-632.4758"/>
+ <text text-anchor="middle" x="332.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trav->tree -->
+ <!-- trave -->
+ <g id="node15" class="node">
+ <title>trave</title>
+ <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-284.4" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="202.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B9</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge18" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#add8e6" d="M181.4692,-359.4476C176.9833,-352.6281 172.8815,-344.9707 170.5223,-337.2 168.3531,-330.0554 167.8843,-327.3851 170.5223,-320.4 172.1555,-316.0753 174.5712,-311.9338 177.3533,-308.0992"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="183.8676,-300.2431 180.9485,-310.8134 180.6761,-304.0921 177.4845,-307.941 177.4845,-307.941 177.4845,-307.941 180.6761,-304.0921 174.0205,-305.0686 183.8676,-300.2431 183.8676,-300.2431"/>
+ <text text-anchor="middle" x="176.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge36" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#0000ff" d="M202.2369,-355.0897C202.2369,-341.6046 202.2369,-325.7123 202.2369,-312.4153"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-312.4033 202.2369,-302.4033 198.737,-312.4034 205.737,-312.4033"/>
+ <text text-anchor="middle" x="207.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- trav->tra2 -->
+ <g id="edge47" class="edge">
+ <title>trav->tra2</title>
+ <path fill="none" stroke="#ffc0cb" d="M213.0163,-412.9388C215.4002,-419.2228 217.909,-425.8514 220.2369,-432.0313 223.4375,-440.5282 226.8619,-449.6636 230.0924,-458.3025"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="233.6352,-467.7845 225.9197,-459.992 231.8851,-463.1007 230.1351,-458.417 230.1351,-458.417 230.1351,-458.417 231.8851,-463.1007 234.3505,-456.8419 233.6352,-467.7845 233.6352,-467.7845"/>
+ <text text-anchor="middle" x="244.9052" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B8</text>
+ </g>
+ <!-- trave->trav -->
+ <g id="edge46" class="edge">
+ <title>trave->trav</title>
+ <path fill="none" stroke="#ffc0cb" d="M208.9024,-302.4722C210.6782,-308.1317 212.3487,-314.4535 213.2369,-320.4 214.4724,-328.6713 214.0766,-337.5213 212.8888,-345.926"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="211.1036,-355.888 208.4382,-345.2509 211.9856,-350.9664 212.8676,-346.0448 212.8676,-346.0448 212.8676,-346.0448 211.9856,-350.9664 217.297,-346.8386 211.1036,-355.888 211.1036,-355.888"/>
+ <text text-anchor="middle" x="232.9052" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B9</text>
+ </g>
+ <!-- traver -->
+ <g id="node16" class="node">
+ <title>traver</title>
+ <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-195.6" rx="38.8671" ry="18"/>
+ <text text-anchor="middle" x="202.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BA</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge19" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#add8e6" d="M185.324,-268.32C180.3995,-262.5605 175.6543,-255.6784 173.0763,-248.4 170.5834,-241.3618 170.5834,-238.6382 173.0763,-231.6 174.4962,-227.5912 176.5735,-223.7026 178.9762,-220.0556"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="185.0164,-212.0415 182.5911,-222.7358 182.007,-216.0344 178.9975,-220.0273 178.9975,-220.0273 178.9975,-220.0273 182.007,-216.0344 175.4039,-217.3188 185.0164,-212.0415 185.0164,-212.0415"/>
+ <text text-anchor="middle" x="177.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge37" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#0000ff" d="M202.2369,-266.0006C202.2369,-253.8949 202.2369,-237.8076 202.2369,-224.0674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-223.672 202.2369,-213.672 198.737,-223.6721 205.737,-223.672"/>
+ <text text-anchor="middle" x="206.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- traver->trave -->
+ <g id="edge45" class="edge">
+ <title>traver->trave</title>
+ <path fill="none" stroke="#ffc0cb" d="M207.6937,-213.7509C209.1467,-219.4136 210.5127,-225.7174 211.2369,-231.6 212.1492,-239.0107 212.1492,-240.9893 211.2369,-248.4 210.9201,-250.9736 210.4804,-253.6279 209.9643,-256.2738"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="207.6937,-266.2491 205.5255,-255.4997 208.8035,-261.3738 209.9132,-256.4985 209.9132,-256.4985 209.9132,-256.4985 208.8035,-261.3738 214.301,-257.4973 207.6937,-266.2491 207.6937,-266.2491"/>
+ <text text-anchor="middle" x="232.4585" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BA</text>
+ </g>
+ <!-- travers -->
+ <g id="node17" class="node">
+ <title>travers</title>
+ <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-106.8" rx="38.305" ry="18"/>
+ <text text-anchor="middle" x="203.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BB</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge20" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#add8e6" d="M183.9854,-179.6658C174.6965,-169.5397 166.3137,-155.9116 171.2923,-142.8 172.9385,-138.4645 175.3723,-134.3169 178.1749,-130.4793"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="184.7367,-122.6213 181.7812,-133.1814 181.5319,-126.4592 178.3271,-130.2971 178.3271,-130.2971 178.3271,-130.2971 181.5319,-126.4592 174.873,-127.4128 184.7367,-122.6213 184.7367,-122.6213"/>
+ <text text-anchor="middle" x="176.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge38" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#0000ff" d="M202.4441,-177.2006C202.5804,-165.0949 202.7616,-149.0076 202.9163,-135.2674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="206.4205,-134.9108 203.0334,-124.872 199.4209,-134.8319 206.4205,-134.9108"/>
+ <text text-anchor="middle" x="207.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> s</text>
+ </g>
+ <!-- travers->traver -->
+ <g id="edge44" class="edge">
+ <title>travers->traver</title>
+ <path fill="none" stroke="#ffc0cb" d="M209.0666,-124.8376C211.6638,-135.0186 213.8193,-147.9908 212.2369,-159.6 211.8717,-162.2793 211.3602,-165.0407 210.7599,-167.7863"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="208.2983,-177.4866 206.3963,-166.6869 209.5282,-172.6402 210.758,-167.7938 210.758,-167.7938 210.758,-167.7938 209.5282,-172.6402 215.1198,-168.9007 208.2983,-177.4866 208.2983,-177.4866"/>
+ <text text-anchor="middle" x="233.0735" y="-147" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BB</text>
+ </g>
+ <!-- traverse -->
+ <g id="node18" class="node">
+ <title>traverse</title>
+ <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="203.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[3]</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge21" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#add8e6" d="M184.8676,-90.9569C179.5135,-85.2132 174.3474,-78.2805 171.5223,-70.8 168.8843,-63.8149 168.8843,-60.9851 171.5223,-54 172.8668,-50.4399 174.7415,-47.004 176.9144,-43.7612"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="183.203,-35.676 180.6155,-46.3323 180.1332,-39.6227 177.0634,-43.5695 177.0634,-43.5695 177.0634,-43.5695 180.1332,-39.6227 173.5114,-40.8067 183.203,-35.676 183.203,-35.676"/>
+ <text text-anchor="middle" x="177.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge39" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#0000ff" d="M203.2369,-88.4006C203.2369,-76.2949 203.2369,-60.2076 203.2369,-46.4674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="206.737,-46.072 203.2369,-36.072 199.737,-46.0721 206.737,-46.072"/>
+ <text text-anchor="middle" x="208.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- traverse->travers -->
+ <g id="edge43" class="edge">
+ <title>traverse->travers</title>
+ <path fill="none" stroke="#ffc0cb" d="M209.9024,-36.0722C211.6782,-41.7317 213.3487,-48.0535 214.2369,-54 215.3399,-61.3847 215.3399,-63.4153 214.2369,-70.8 213.8344,-73.4945 213.2713,-76.2661 212.6106,-79.0181"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="209.9024,-88.7278 208.2546,-77.8864 211.2458,-83.9116 212.5891,-79.0954 212.5891,-79.0954 212.5891,-79.0954 211.2458,-83.9116 216.9237,-80.3044 209.9024,-88.7278 209.9024,-88.7278"/>
+ <text text-anchor="middle" x="224.2732" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#ff0000"> ~3</text>
+ </g>
+ <!-- tra2->trac -->
+ <g id="edge34" class="edge">
+ <title>tra2->trac</title>
+ <path fill="none" stroke="#0000ff" d="M216.023,-475.2499C189.2207,-455.3034 149.1456,-425.4792 122.3699,-405.5525"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="124.2894,-402.6181 114.1775,-399.4556 120.1102,-408.2337 124.2894,-402.6181"/>
+ <text text-anchor="middle" x="183.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- tra2->tre -->
+ <!-- tra2->tr -->
+ <g id="edge48" class="edge">
+ <title>tra2->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M240.4944,-525.6305C240.4789,-537.419 242.498,-550.5344 249.5777,-560.4626 263.7679,-580.3622 288.2176,-592.0186 310.2338,-598.7899"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="320.1326,-601.5694 309.2883,-603.1984 315.3187,-600.2177 310.5049,-598.866 310.5049,-598.866 310.5049,-598.866 315.3187,-600.2177 311.7214,-594.5335 320.1326,-601.5694 320.1326,-601.5694"/>
+ <text text-anchor="middle" x="270.0665" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0DE</text>
+ </g>
+ <!-- tra2->trav -->
+ <g id="edge35" class="edge">
+ <title>tra2->trav</title>
+ <path fill="none" stroke="#0000ff" d="M222.659,-471.5823C217.635,-464.6387 212.8503,-456.7959 209.7369,-448.8313 206.7154,-441.1019 204.8136,-432.4806 203.6368,-424.1275"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="207.1115,-423.7045 202.5344,-414.1494 200.1538,-424.4733 207.1115,-423.7045"/>
+ <text text-anchor="middle" x="215.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> v</text>
+ </g>
+ <!-- tru->tr -->
+ <g id="edge54" class="edge">
+ <title>tru->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M566.5842,-524.088C558.9936,-536.8985 548.4041,-551.2608 535.2369,-560.4626 496.6598,-587.4217 443.613,-599.0546 405.481,-604.073"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="395.1457,-605.3203 404.5344,-599.6545 400.1097,-604.7212 405.0736,-604.1221 405.0736,-604.1221 405.0736,-604.1221 400.1097,-604.7212 405.6128,-608.5897 395.1457,-605.3203 395.1457,-605.3203"/>
+ <text text-anchor="middle" x="571.6815" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0FA</text>
+ </g>
+ <!-- tru->truc -->
+ <g id="edge23" class="edge">
+ <title>tru->truc</title>
+ <path fill="none" stroke="#add8e6" d="M584.906,-467.0117C587.8872,-455.247 592.7225,-442.1094 600.5223,-432.0313 610.5674,-419.0519 625.2451,-408.7011 639.0092,-401.0235"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="647.8644,-396.3635 641.1106,-405.0028 643.4397,-398.692 639.0149,-401.0205 639.0149,-401.0205 639.0149,-401.0205 643.4397,-398.692 636.9193,-397.0383 647.8644,-396.3635 647.8644,-396.3635"/>
+ <text text-anchor="middle" x="605.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> c</text>
+ </g>
+ <!-- tru->truc -->
+ <g id="edge41" class="edge">
+ <title>tru->truc</title>
+ <path fill="none" stroke="#0000ff" d="M601.5424,-471.4723C617.5177,-452.8958 639.2774,-427.5931 655.1898,-409.0897"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="657.9656,-411.2298 661.8323,-401.3657 652.6583,-406.6656 657.9656,-411.2298"/>
+ <text text-anchor="middle" x="638.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- truc->tru -->
+ <g id="edge53" class="edge">
+ <title>truc->tru</title>
+ <path fill="none" stroke="#ffc0cb" d="M670.9731,-402.6462C666.3263,-416.3625 658.5213,-435.1464 647.2369,-448.8313 639.6641,-458.015 629.8999,-466.2538 620.1806,-473.1865"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="611.7071,-478.9336 617.4572,-469.5962 615.8452,-476.127 619.9832,-473.3204 619.9832,-473.3204 619.9832,-473.3204 615.8452,-476.127 622.5091,-477.0446 611.7071,-478.9336 611.7071,-478.9336"/>
+ <text text-anchor="middle" x="676.2965" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0FB</text>
+ </g>
+ <!-- truck -->
+ <g id="node22" class="node">
+ <title>truck</title>
+ <ellipse fill="none" stroke="#0000ff" cx="676.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="676.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[4]</text>
+ </g>
+ <!-- truc->truck -->
+ <g id="edge24" class="edge">
+ <title>truc->truck</title>
+ <path fill="none" stroke="#add8e6" d="M661.1247,-367.9416C654.3993,-359.4018 647.222,-348.4489 643.7369,-337.2 641.5272,-330.0678 641.0565,-327.369 643.7369,-320.4 645.1585,-316.7039 647.1547,-313.1507 649.4663,-309.8129"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="655.7148,-302.0002 652.9831,-312.6204 652.5918,-305.9049 649.4689,-309.8097 649.4689,-309.8097 649.4689,-309.8097 652.5918,-305.9049 645.9546,-306.999 655.7148,-302.0002 655.7148,-302.0002"/>
+ <text text-anchor="middle" x="648.4869" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> k</text>
+ </g>
+ <!-- truc->truck -->
+ <g id="edge42" class="edge">
+ <title>truc->truck</title>
+ <path fill="none" stroke="#0000ff" d="M676.2369,-366.2345C676.2369,-351.2603 676.2369,-329.8643 676.2369,-312.7055"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="679.737,-312.6645 676.2369,-302.6646 672.737,-312.6646 679.737,-312.6645"/>
+ <text text-anchor="middle" x="681.4869" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> k</text>
+ </g>
+ <!-- truck->truc -->
+ <g id="edge52" class="edge">
+ <title>truck->truc</title>
+ <path fill="none" stroke="#ffc0cb" d="M683.5061,-302.4275C685.4432,-308.0852 687.266,-314.4172 688.2369,-320.4 690.1842,-332.3997 688.3971,-345.6086 685.6614,-356.9103"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="682.9623,-366.6639 681.2924,-355.8259 684.2958,-361.845 685.6294,-357.0261 685.6294,-357.0261 685.6294,-357.0261 684.2958,-361.845 689.9664,-358.2263 682.9623,-366.6639 682.9623,-366.6639"/>
+ <text text-anchor="middle" x="695.5232" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">~4</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g1.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g1.svg
new file mode 100644
index 0000000..e43b324
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g1.svg
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="154pt" height="310pt"
+ viewBox="0.00 0.00 154.47 310.40" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 306.4)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-306.4 150.4738,-306.4 150.4738,4 -4,4"/>
+ <!-- 0x13B -->
+ <g id="node1" class="node">
+ <title>0x13B</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x13B</text>
+ </g>
+ <!-- contentArray[0] -->
+ <g id="node2" class="node">
+ <title>contentArray[0]</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- 0x13B->contentArray[0] -->
+ <g id="edge1" class="edge">
+ <title>0x13B->contentArray[0]</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-88.4006C73.2369,-76.2949 73.2369,-60.2076 73.2369,-46.4674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-46.072 73.2369,-36.072 69.737,-46.0721 76.737,-46.072"/>
+ <text text-anchor="middle" x="81.7902" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000">  A</text>
+ </g>
+ <!-- 0x13A -->
+ <g id="node3" class="node">
+ <title>0x13A</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x13A</text>
+ </g>
+ <!-- 0x13A->0x13B -->
+ <g id="edge2" class="edge">
+ <title>0x13A->0x13B</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-177.2006C73.2369,-165.0949 73.2369,-149.0076 73.2369,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-134.872 73.2369,-124.872 69.737,-134.8721 76.737,-134.872"/>
+ <text text-anchor="middle" x="81.4052" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000">  H</text>
+ </g>
+ <!-- 0x139 -->
+ <g id="node4" class="node">
+ <title>0x139</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x139</text>
+ </g>
+ <!-- 0x139->0x13A -->
+ <g id="edge3" class="edge">
+ <title>0x139->0x13A</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-266.0006C73.2369,-253.8949 73.2369,-237.8076 73.2369,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-223.672 73.2369,-213.672 69.737,-223.6721 76.737,-223.672"/>
+ <text text-anchor="middle" x="81.4052" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  C</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g2.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g2.svg
new file mode 100644
index 0000000..a5c7eed
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g2.svg
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+strict digraph G {
+ "0x139" [color=grey,fontcolor=grey];
+
+ "0x13B" -> "contentArray[0]" [label=" A"];
+ "0x13A" -> "0x13B" [label=" H"];
+ "0x139" -> "0x13A" [label=" C",color="grey",fontcolor="grey"];
+
+ "0x25E" -> "0x238" [label=" A"];
+ "0x25E" -> "0x13A" [label=" C"];
+// "0x25E" -> "0x33B" [label=" 5"];
+// "0x25E" -> "0x35C" [label=" B"];
+// "0x25E" -> "0x37A" [label=" P"];
+// "0x25E" -> "0x41B" [label=" 3"];
+}
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="207pt" height="310pt"
+ viewBox="0.00 0.00 207.49 310.40" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 306.4)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-306.4 203.4867,-306.4 203.4867,4 -4,4"/>
+ <!-- 0x139 -->
+ <g id="node1" class="node">
+ <title>0x139</title>
+ <ellipse fill="none" stroke="#c0c0c0" cx="73.2369" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#c0c0c0">0x139</text>
+ </g>
+ <!-- 0x13A -->
+ <g id="node4" class="node">
+ <title>0x13A</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x13A</text>
+ </g>
+ <!-- 0x139->0x13A -->
+ <g id="edge3" class="edge">
+ <title>0x139->0x13A</title>
+ <path fill="none" stroke="#c0c0c0" d="M73.2369,-266.0006C73.2369,-253.8949 73.2369,-237.8076 73.2369,-224.0674"/>
+ <polygon fill="#c0c0c0" stroke="#c0c0c0" points="76.737,-223.672 73.2369,-213.672 69.737,-223.6721 76.737,-223.672"/>
+ <text text-anchor="middle" x="81.4052" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#c0c0c0">  C</text>
+ </g>
+ <!-- 0x13B -->
+ <g id="node2" class="node">
+ <title>0x13B</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x13B</text>
+ </g>
+ <!-- contentArray[0] -->
+ <g id="node3" class="node">
+ <title>contentArray[0]</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- 0x13B->contentArray[0] -->
+ <g id="edge1" class="edge">
+ <title>0x13B->contentArray[0]</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-88.4006C73.2369,-76.2949 73.2369,-60.2076 73.2369,-46.4674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-46.072 73.2369,-36.072 69.737,-46.0721 76.737,-46.072"/>
+ <text text-anchor="middle" x="81.7902" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000">  A</text>
+ </g>
+ <!-- 0x13A->0x13B -->
+ <g id="edge2" class="edge">
+ <title>0x13A->0x13B</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-177.2006C73.2369,-165.0949 73.2369,-149.0076 73.2369,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-134.872 73.2369,-124.872 69.737,-134.8721 76.737,-134.872"/>
+ <text text-anchor="middle" x="81.4052" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000">  H</text>
+ </g>
+ <!-- 0x25E -->
+ <g id="node5" class="node">
+ <title>0x25E</title>
+ <ellipse fill="none" stroke="#000000" cx="163.2369" cy="-284.4" rx="36.5014" ry="18"/>
+ <text text-anchor="middle" x="163.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x25E</text>
+ </g>
+ <!-- 0x25E->0x13A -->
+ <g id="edge5" class="edge">
+ <title>0x25E->0x13A</title>
+ <path fill="none" stroke="#000000" d="M146.7295,-268.1127C132.7742,-254.3435 112.6083,-234.4465 96.9293,-218.9765"/>
+ <polygon fill="#000000" stroke="#000000" points="99.2162,-216.3161 89.6396,-211.784 94.2998,-221.299 99.2162,-216.3161"/>
+ <text text-anchor="middle" x="135.4052" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  C</text>
+ </g>
+ <!-- 0x238 -->
+ <g id="node6" class="node">
+ <title>0x238</title>
+ <ellipse fill="none" stroke="#000000" cx="164.2369" cy="-195.6" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="164.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x238</text>
+ </g>
+ <!-- 0x25E->0x238 -->
+ <g id="edge4" class="edge">
+ <title>0x25E->0x238</title>
+ <path fill="none" stroke="#000000" d="M163.4441,-266.0006C163.5804,-253.8949 163.7616,-237.8076 163.9163,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="167.4205,-223.7108 164.0334,-213.672 160.4209,-223.6319 167.4205,-223.7108"/>
+ <text text-anchor="middle" x="171.7902" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  A</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g3.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g3.svg
new file mode 100644
index 0000000..b0326c9
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g3.svg
@@ -0,0 +1,253 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# http://www.graphviz.org/content/cluster
+
+strict digraph G {
+ "0x51C";
+
+ subgraph cluster_51C {
+ label= "Split node 0x51C";
+ color=grey;
+ node [style=dashed] "0x53C", "0x55C", "0x57C", "0x59C", "0x5BC";
+
+ "0x51C" -> "0x53C" [label=" 00"];
+ "0x51C" -> "0x57C" [label=" 01"];
+ "0x53C" -> "0x55C" [label=" 110"];
+ "0x57C" -> "0x59C" [label=" 000"];
+ "0x57C" -> "0x5BC" [label=" 010"];
+ }
+
+ "0x55C" -> "0x41B" [label=" 011"];
+ "0x55C" -> "0x33B" [label=" 101"];
+ "0x59C" -> "0x238" [label=" 001"];
+ "0x59C" -> "0x35C" [label=" 010"];
+ "0x59C" -> "0x13A" [label=" 011"];
+ "0x5BC" -> "0x37A" [label=" 000"];
+ "0x5BC" -> "0x455" [label=" 001"];
+
+ "0x13B" -> "contentArray[0]" [label=" A"];
+ "0x13A" -> "0x13B" [label=" H"];
+}
+
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="628pt" height="605pt"
+ viewBox="0.00 0.00 627.97 604.80" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 600.8)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-600.8 623.9695,-600.8 623.9695,4 -4,4"/>
+ <g id="clust1" class="cluster">
+ <title>cluster_51C</title>
+ <polygon fill="none" stroke="#c0c0c0" points="177.7906,-258.4 177.7906,-588.8 444.7906,-588.8 444.7906,-258.4 177.7906,-258.4"/>
+ <text text-anchor="middle" x="311.2906" y="-572.2" font-family="Times,serif" font-size="14.00" fill="#000000">Split node 0x51C</text>
+ </g>
+ <!-- 0x51C -->
+ <g id="node1" class="node">
+ <title>0x51C</title>
+ <ellipse fill="none" stroke="#000000" cx="264.7906" cy="-538" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="264.7906" y="-533.8" font-family="Times,serif" font-size="14.00" fill="#000000">0x51C</text>
+ </g>
+ <!-- 0x53C -->
+ <g id="node2" class="node">
+ <title>0x53C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="220.7906" cy="-411.2" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="220.7906" y="-407" font-family="Times,serif" font-size="14.00" fill="#000000">0x53C</text>
+ </g>
+ <!-- 0x51C->0x53C -->
+ <g id="edge1" class="edge">
+ <title>0x51C->0x53C</title>
+ <path fill="none" stroke="#000000" d="M258.5798,-520.1016C251.2095,-498.8616 238.8421,-463.2211 230.2726,-438.5254"/>
+ <polygon fill="#000000" stroke="#000000" points="233.5547,-437.3073 226.9698,-429.0074 226.9416,-439.6022 233.5547,-437.3073"/>
+ <text text-anchor="middle" x="256.2906" y="-470.4" font-family="Times,serif" font-size="14.00" fill="#000000">  00</text>
+ </g>
+ <!-- 0x57C -->
+ <g id="node4" class="node">
+ <title>0x57C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="308.7906" cy="-411.2" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="308.7906" y="-407" font-family="Times,serif" font-size="14.00" fill="#000000">0x57C</text>
+ </g>
+ <!-- 0x51C->0x57C -->
+ <g id="edge2" class="edge">
+ <title>0x51C->0x57C</title>
+ <path fill="none" stroke="#000000" d="M271.0014,-520.1016C278.3718,-498.8616 290.7391,-463.2211 299.3086,-438.5254"/>
+ <polygon fill="#000000" stroke="#000000" points="302.6397,-439.6022 302.6114,-429.0074 296.0265,-437.3073 302.6397,-439.6022"/>
+ <text text-anchor="middle" x="300.2906" y="-470.4" font-family="Times,serif" font-size="14.00" fill="#000000">  01</text>
+ </g>
+ <!-- 0x55C -->
+ <g id="node3" class="node">
+ <title>0x55C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="220.7906" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="220.7906" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x55C</text>
+ </g>
+ <!-- 0x53C->0x55C -->
+ <g id="edge3" class="edge">
+ <title>0x53C->0x55C</title>
+ <path fill="none" stroke="#000000" d="M220.7906,-393.0327C220.7906,-372.0352 220.7906,-337.2261 220.7906,-312.679"/>
+ <polygon fill="#000000" stroke="#000000" points="224.2907,-312.5336 220.7906,-302.5336 217.2907,-312.5337 224.2907,-312.5336"/>
+ <text text-anchor="middle" x="234.7906" y="-343.6" font-family="Times,serif" font-size="14.00" fill="#000000">  110</text>
+ </g>
+ <!-- 0x41B -->
+ <g id="node7" class="node">
+ <title>0x41B</title>
+ <ellipse fill="none" stroke="#000000" cx="36.7906" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="36.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x41B</text>
+ </g>
+ <!-- 0x55C->0x41B -->
+ <g id="edge6" class="edge">
+ <title>0x55C->0x41B</title>
+ <path fill="none" stroke="#000000" d="M192.599,-273.4087C175.6606,-266.6053 153.7802,-257.4635 134.7906,-248.4 112.7572,-237.8837 88.6053,-224.889 69.791,-214.4226"/>
+ <polygon fill="#000000" stroke="#000000" points="71.4518,-211.3412 61.0164,-209.5097 68.0319,-217.449 71.4518,-211.3412"/>
+ <text text-anchor="middle" x="148.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  011</text>
+ </g>
+ <!-- 0x33B -->
+ <g id="node8" class="node">
+ <title>0x33B</title>
+ <ellipse fill="none" stroke="#000000" cx="128.7906" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="128.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x33B</text>
+ </g>
+ <!-- 0x55C->0x33B -->
+ <g id="edge7" class="edge">
+ <title>0x55C->0x33B</title>
+ <path fill="none" stroke="#000000" d="M203.9164,-268.1127C189.651,-254.3435 169.037,-234.4465 153.0095,-218.9765"/>
+ <polygon fill="#000000" stroke="#000000" points="155.1837,-216.2106 145.5579,-211.784 150.3223,-221.2472 155.1837,-216.2106"/>
+ <text text-anchor="middle" x="197.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  101</text>
+ </g>
+ <!-- 0x59C -->
+ <g id="node5" class="node">
+ <title>0x59C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="308.7906" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="308.7906" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x59C</text>
+ </g>
+ <!-- 0x57C->0x59C -->
+ <g id="edge4" class="edge">
+ <title>0x57C->0x59C</title>
+ <path fill="none" stroke="#000000" d="M308.7906,-393.0327C308.7906,-372.0352 308.7906,-337.2261 308.7906,-312.679"/>
+ <polygon fill="#000000" stroke="#000000" points="312.2907,-312.5336 308.7906,-302.5336 305.2907,-312.5337 312.2907,-312.5336"/>
+ <text text-anchor="middle" x="322.7906" y="-343.6" font-family="Times,serif" font-size="14.00" fill="#000000">  000</text>
+ </g>
+ <!-- 0x5BC -->
+ <g id="node6" class="node">
+ <title>0x5BC</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="399.7906" cy="-284.4" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="399.7906" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x5BC</text>
+ </g>
+ <!-- 0x57C->0x5BC -->
+ <g id="edge5" class="edge">
+ <title>0x57C->0x5BC</title>
+ <path fill="none" stroke="#000000" d="M321.0633,-394.0992C336.5995,-372.451 363.5002,-334.9674 381.471,-309.9267"/>
+ <polygon fill="#000000" stroke="#000000" points="384.4347,-311.7999 387.4218,-301.6348 378.7476,-307.7185 384.4347,-311.7999"/>
+ <text text-anchor="middle" x="373.7906" y="-343.6" font-family="Times,serif" font-size="14.00" fill="#000000">  010</text>
+ </g>
+ <!-- 0x238 -->
+ <g id="node9" class="node">
+ <title>0x238</title>
+ <ellipse fill="none" stroke="#000000" cx="218.7906" cy="-195.6" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="218.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x238</text>
+ </g>
+ <!-- 0x59C->0x238 -->
+ <g id="edge8" class="edge">
+ <title>0x59C->0x238</title>
+ <path fill="none" stroke="#000000" d="M292.2832,-268.1127C278.3279,-254.3435 258.162,-234.4465 242.483,-218.9765"/>
+ <polygon fill="#000000" stroke="#000000" points="244.77,-216.3161 235.1934,-211.784 239.8535,-221.299 244.77,-216.3161"/>
+ <text text-anchor="middle" x="286.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  001</text>
+ </g>
+ <!-- 0x35C -->
+ <g id="node10" class="node">
+ <title>0x35C</title>
+ <ellipse fill="none" stroke="#000000" cx="308.7906" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="308.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x35C</text>
+ </g>
+ <!-- 0x59C->0x35C -->
+ <g id="edge9" class="edge">
+ <title>0x59C->0x35C</title>
+ <path fill="none" stroke="#000000" d="M308.7906,-266.0006C308.7906,-253.8949 308.7906,-237.8076 308.7906,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="312.2907,-223.672 308.7906,-213.672 305.2907,-223.6721 312.2907,-223.672"/>
+ <text text-anchor="middle" x="322.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  010</text>
+ </g>
+ <!-- 0x13A -->
+ <g id="node11" class="node">
+ <title>0x13A</title>
+ <ellipse fill="none" stroke="#000000" cx="400.7906" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="400.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x13A</text>
+ </g>
+ <!-- 0x59C->0x13A -->
+ <g id="edge10" class="edge">
+ <title>0x59C->0x13A</title>
+ <path fill="none" stroke="#000000" d="M325.6648,-268.1127C339.9303,-254.3435 360.5443,-234.4465 376.5717,-218.9765"/>
+ <polygon fill="#000000" stroke="#000000" points="379.259,-221.2472 384.0234,-211.784 374.3976,-216.2106 379.259,-221.2472"/>
+ <text text-anchor="middle" x="377.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  011</text>
+ </g>
+ <!-- 0x37A -->
+ <g id="node12" class="node">
+ <title>0x37A</title>
+ <ellipse fill="none" stroke="#000000" cx="493.7906" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="493.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x37A</text>
+ </g>
+ <!-- 0x5BC->0x37A -->
+ <g id="edge11" class="edge">
+ <title>0x5BC->0x37A</title>
+ <path fill="none" stroke="#000000" d="M417.0317,-268.1127C431.6072,-254.3435 452.6693,-234.4465 469.0452,-218.9765"/>
+ <polygon fill="#000000" stroke="#000000" points="471.7931,-221.1955 476.6589,-211.784 466.9861,-216.107 471.7931,-221.1955"/>
+ <text text-anchor="middle" x="469.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  000</text>
+ </g>
+ <!-- 0x455 -->
+ <g id="node13" class="node">
+ <title>0x455</title>
+ <ellipse fill="none" stroke="#000000" cx="584.7906" cy="-195.6" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="584.7906" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x455</text>
+ </g>
+ <!-- 0x5BC->0x455 -->
+ <g id="edge12" class="edge">
+ <title>0x5BC->0x455</title>
+ <path fill="none" stroke="#000000" d="M429.4765,-273.1983C446.7051,-266.4645 468.7051,-257.4645 487.7906,-248.4 509.7186,-237.9855 533.6828,-224.9577 552.3055,-214.4487"/>
+ <polygon fill="#000000" stroke="#000000" points="554.288,-217.3478 561.2515,-209.3627 550.8284,-211.2625 554.288,-217.3478"/>
+ <text text-anchor="middle" x="533.7906" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000">  001</text>
+ </g>
+ <!-- 0x13B -->
+ <g id="node14" class="node">
+ <title>0x13B</title>
+ <ellipse fill="none" stroke="#000000" cx="400.7906" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="400.7906" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x13B</text>
+ </g>
+ <!-- 0x13A->0x13B -->
+ <g id="edge14" class="edge">
+ <title>0x13A->0x13B</title>
+ <path fill="none" stroke="#000000" d="M400.7906,-177.2006C400.7906,-165.0949 400.7906,-149.0076 400.7906,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="404.2907,-134.872 400.7906,-124.872 397.2907,-134.8721 404.2907,-134.872"/>
+ <text text-anchor="middle" x="409.3439" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000">  H</text>
+ </g>
+ <!-- contentArray[0] -->
+ <g id="node15" class="node">
+ <title>contentArray[0]</title>
+ <ellipse fill="none" stroke="#000000" cx="400.7906" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="400.7906" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- 0x13B->contentArray[0] -->
+ <g id="edge13" class="edge">
+ <title>0x13B->contentArray[0]</title>
+ <path fill="none" stroke="#000000" d="M400.7906,-88.4006C400.7906,-76.2949 400.7906,-60.2076 400.7906,-46.4674"/>
+ <polygon fill="#000000" stroke="#000000" points="404.2907,-46.072 400.7906,-36.072 397.2907,-46.0721 404.2907,-46.072"/>
+ <text text-anchor="middle" x="409.3439" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000">  A</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g4.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g4.svg
new file mode 100644
index 0000000..1523772
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.g4.svg
@@ -0,0 +1,290 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# http://www.graphviz.org/content/cluster
+
+strict digraph G {
+// "0x139" [color=grey,fontcolor=grey];
+
+ "0x13B" -> "contentArray[0]" [label=" A"];
+ "0x13A" -> "0x13B" [label=" H"];
+// "0x139" -> "0x13A" [label=" C"];
+// "0x139" -> "0x13A" [label=" C",color="grey",fontcolor="grey"];
+
+// "0x25E" -> "0x238" [label=" A"];
+// "0x25E" -> "0x13A" [label=" C"];
+// "0x25E" -> "0x33B" [label=" 5"];
+// "0x25E" -> "0x35C" [label=" B"];
+// "0x25E" -> "0x37A" [label=" P"];
+// "0x25E" -> "0x41B" [label=" 3"];
+
+ subgraph cluster_51F {
+ label = "Node 0x51F"
+ "0x51F" [label="Prefix 0x51F\ncontentArray[1]"]
+ "0x51F" -> "0x51C" [label="ε"];
+
+ subgraph cluster_51C {
+ label= "Split node 0x51C";
+ ranksep=1
+ color=grey;
+ node [style=dashed] "0x53C", "0x55C", "0x57C", "0x59C", "0x5BC";
+
+ "0x51C" -> "0x53C" [label=" 00"];
+ "0x51C" -> "0x57C" [label=" 01"];
+ "0x53C" -> "0x55C" [label=" 110"];
+ "0x57C" -> "0x59C" [label=" 000"];
+ "0x57C" -> "0x5BC" [label=" 010"];
+ }
+ }
+ "0x55C" -> "0x41B" [label=" 011",minlen=2];
+ "0x55C" -> "0x33B" [label=" 101",minlen=2];
+ "0x59C" -> "0x238" [label=" 001",minlen=2];
+ "0x59C" -> "0x35C" [label=" 010",minlen=2];
+ "0x59C" -> "0x13A" [label=" 011",minlen=2];
+ "0x5BC" -> "0x37A" [label=" 000",minlen=2];
+ "0x5BC" -> "0x455" [label=" 001",minlen=2];
+
+ { rank=same "0x238" -> "0x35C" -> "0x13A" [style=invis,constrain=false]}
+}
+-->
+<!-- Title: G Pages: 1 -->
+<svg width="663pt" height="739pt"
+ viewBox="0.00 0.00 662.97 739.23" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 735.2313)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-735.2313 658.9695,-735.2313 658.9695,4 -4,4"/>
+ <g id="clust1" class="cluster">
+ <title>cluster_51F</title>
+ <polygon fill="none" stroke="#000000" points="186.7906,-231.4 186.7906,-723.2313 469.7906,-723.2313 469.7906,-231.4 186.7906,-231.4"/>
+ <text text-anchor="middle" x="328.2906" y="-706.6313" font-family="Times,serif" font-size="14.00" fill="#000000">Node 0x51F</text>
+ </g>
+ <g id="clust2" class="cluster">
+ <title>cluster_51C</title>
+ <polygon fill="none" stroke="#c0c0c0" points="194.7906,-239.4 194.7906,-577.8 461.7906,-577.8 461.7906,-239.4 194.7906,-239.4"/>
+ <text text-anchor="middle" x="328.2906" y="-561.2" font-family="Times,serif" font-size="14.00" fill="#000000">Split node 0x51C</text>
+ </g>
+ <!-- 0x13B -->
+ <g id="node1" class="node">
+ <title>0x13B</title>
+ <ellipse fill="none" stroke="#000000" cx="435.7906" cy="-88.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="435.7906" y="-84.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x13B</text>
+ </g>
+ <!-- contentArray[0] -->
+ <g id="node2" class="node">
+ <title>contentArray[0]</title>
+ <ellipse fill="none" stroke="#000000" cx="435.7906" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="435.7906" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- 0x13B->contentArray[0] -->
+ <g id="edge1" class="edge">
+ <title>0x13B->contentArray[0]</title>
+ <path fill="none" stroke="#000000" d="M435.7906,-70.5672C435.7906,-63.2743 435.7906,-54.6987 435.7906,-46.6137"/>
+ <polygon fill="#000000" stroke="#000000" points="439.2907,-46.417 435.7906,-36.417 432.2907,-46.4171 439.2907,-46.417"/>
+ <text text-anchor="middle" x="444.3439" y="-49.2" font-family="Times,serif" font-size="14.00" fill="#000000">  A</text>
+ </g>
+ <!-- 0x13A -->
+ <g id="node3" class="node">
+ <title>0x13A</title>
+ <ellipse fill="none" stroke="#000000" cx="435.7906" cy="-159.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="435.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x13A</text>
+ </g>
+ <!-- 0x13A->0x13B -->
+ <g id="edge2" class="edge">
+ <title>0x13A->0x13B</title>
+ <path fill="none" stroke="#000000" d="M435.7906,-141.3672C435.7906,-134.0743 435.7906,-125.4987 435.7906,-117.4137"/>
+ <polygon fill="#000000" stroke="#000000" points="439.2907,-117.217 435.7906,-107.217 432.2907,-117.2171 439.2907,-117.217"/>
+ <text text-anchor="middle" x="444.3439" y="-120" font-family="Times,serif" font-size="14.00" fill="#000000">  H</text>
+ </g>
+ <!-- 0x51F -->
+ <g id="node4" class="node">
+ <title>0x51F</title>
+ <ellipse fill="none" stroke="#000000" cx="318.7906" cy="-661.0156" rx="117.2629" ry="29.3315"/>
+ <text text-anchor="middle" x="318.7906" y="-665.2156" font-family="Times,serif" font-size="14.00" fill="#000000">Prefix 0x51F</text>
+ <text text-anchor="middle" x="318.7906" y="-648.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- 0x51C -->
+ <g id="node5" class="node">
+ <title>0x51C</title>
+ <ellipse fill="none" stroke="#000000" cx="318.7906" cy="-527" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="318.7906" y="-522.8" font-family="Times,serif" font-size="14.00" fill="#000000">0x51C</text>
+ </g>
+ <!-- 0x51F->0x51C -->
+ <g id="edge3" class="edge">
+ <title>0x51F->0x51C</title>
+ <path fill="none" stroke="#000000" d="M318.7906,-631.2732C318.7906,-608.6091 318.7906,-577.6052 318.7906,-555.3044"/>
+ <polygon fill="#000000" stroke="#000000" points="322.2907,-555.1305 318.7906,-545.1306 315.2907,-555.1306 322.2907,-555.1305"/>
+ <text text-anchor="middle" x="322.8709" y="-600" font-family="Times,serif" font-size="14.00" fill="#000000">ε</text>
+ </g>
+ <!-- 0x53C -->
+ <g id="node6" class="node">
+ <title>0x53C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="237.7906" cy="-396.2" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="237.7906" y="-392" font-family="Times,serif" font-size="14.00" fill="#000000">0x53C</text>
+ </g>
+ <!-- 0x51C->0x53C -->
+ <g id="edge4" class="edge">
+ <title>0x51C->0x53C</title>
+ <path fill="none" stroke="#000000" d="M284.1131,-520.6804C267.7647,-515.4937 251.7906,-506.1624 251.7906,-489.5 251.7906,-489.5 251.7906,-489.5 251.7906,-433.7 251.7906,-430.2948 251.2777,-426.8193 250.4482,-423.4252"/>
+ <polygon fill="#000000" stroke="#000000" points="253.6734,-422.0307 247.2065,-413.6385 247.0285,-424.2318 253.6734,-422.0307"/>
+ <text text-anchor="middle" x="262.2906" y="-457.4" font-family="Times,serif" font-size="14.00" fill="#000000">  00</text>
+ </g>
+ <!-- 0x57C -->
+ <g id="node8" class="node">
+ <title>0x57C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="325.7906" cy="-396.2" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="325.7906" y="-392" font-family="Times,serif" font-size="14.00" fill="#000000">0x57C</text>
+ </g>
+ <!-- 0x51C->0x57C -->
+ <g id="edge5" class="edge">
+ <title>0x51C->0x57C</title>
+ <path fill="none" stroke="#000000" d="M320.8414,-508.7129C321.3687,-502.6109 321.7906,-495.77 321.7906,-489.5 321.7906,-489.5 321.7906,-489.5 321.7906,-433.7 321.7906,-430.7537 321.9142,-427.6845 322.118,-424.6266"/>
+ <polygon fill="#000000" stroke="#000000" points="325.6215,-424.7486 323.0562,-414.469 318.6512,-424.1046 325.6215,-424.7486"/>
+ <text text-anchor="middle" x="332.2906" y="-457.4" font-family="Times,serif" font-size="14.00" fill="#000000">  01</text>
+ </g>
+ <!-- 0x55C -->
+ <g id="node7" class="node">
+ <title>0x55C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="237.7906" cy="-265.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="237.7906" y="-261.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x55C</text>
+ </g>
+ <!-- 0x53C->0x55C -->
+ <g id="edge6" class="edge">
+ <title>0x53C->0x55C</title>
+ <path fill="none" stroke="#000000" d="M237.7906,-377.8895C237.7906,-371.7859 237.7906,-364.95 237.7906,-358.7 237.7906,-358.7 237.7906,-358.7 237.7906,-302.9 237.7906,-299.9703 237.7906,-296.9119 237.7906,-293.8605"/>
+ <polygon fill="#000000" stroke="#000000" points="241.2907,-293.7105 237.7906,-283.7105 234.2907,-293.7106 241.2907,-293.7105"/>
+ <text text-anchor="middle" x="251.7906" y="-326.6" font-family="Times,serif" font-size="14.00" fill="#000000">  110</text>
+ </g>
+ <!-- 0x41B -->
+ <g id="node11" class="node">
+ <title>0x41B</title>
+ <ellipse fill="none" stroke="#000000" cx="36.7906" cy="-159.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="36.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x41B</text>
+ </g>
+ <!-- 0x55C->0x41B -->
+ <g id="edge9" class="edge">
+ <title>0x55C->0x41B</title>
+ <path fill="none" stroke="#000000" d="M212.5449,-252.7617C191.8449,-242.3434 161.8208,-227.105 135.7906,-213.4 113.9245,-201.8874 89.5312,-188.6361 70.4296,-178.1656"/>
+ <polygon fill="#000000" stroke="#000000" points="71.9619,-175.014 61.5118,-173.2686 68.5926,-181.1498 71.9619,-175.014"/>
+ <text text-anchor="middle" x="149.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  011</text>
+ </g>
+ <!-- 0x33B -->
+ <g id="node12" class="node">
+ <title>0x33B</title>
+ <ellipse fill="none" stroke="#000000" cx="128.7906" cy="-159.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="128.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x33B</text>
+ </g>
+ <!-- 0x55C->0x33B -->
+ <g id="edge10" class="edge">
+ <title>0x55C->0x33B</title>
+ <path fill="none" stroke="#000000" d="M221.2359,-249.3313C202.9895,-231.6206 173.4782,-202.9757 152.7171,-182.824"/>
+ <polygon fill="#000000" stroke="#000000" points="155.1145,-180.2735 145.5012,-175.82 150.2391,-185.2964 155.1145,-180.2735"/>
+ <text text-anchor="middle" x="197.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  101</text>
+ </g>
+ <!-- 0x59C -->
+ <g id="node9" class="node">
+ <title>0x59C</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="325.7906" cy="-265.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="325.7906" y="-261.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x59C</text>
+ </g>
+ <!-- 0x57C->0x59C -->
+ <g id="edge7" class="edge">
+ <title>0x57C->0x59C</title>
+ <path fill="none" stroke="#000000" d="M325.7906,-377.8895C325.7906,-371.7859 325.7906,-364.95 325.7906,-358.7 325.7906,-358.7 325.7906,-358.7 325.7906,-302.9 325.7906,-299.9703 325.7906,-296.9119 325.7906,-293.8605"/>
+ <polygon fill="#000000" stroke="#000000" points="329.2907,-293.7105 325.7906,-283.7105 322.2907,-293.7106 329.2907,-293.7105"/>
+ <text text-anchor="middle" x="339.7906" y="-326.6" font-family="Times,serif" font-size="14.00" fill="#000000">  000</text>
+ </g>
+ <!-- 0x5BC -->
+ <g id="node10" class="node">
+ <title>0x5BC</title>
+ <ellipse fill="none" stroke="#000000" stroke-dasharray="5,2" cx="416.7906" cy="-265.4" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="416.7906" y="-261.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x5BC</text>
+ </g>
+ <!-- 0x57C->0x5BC -->
+ <g id="edge8" class="edge">
+ <title>0x57C->0x5BC</title>
+ <path fill="none" stroke="#000000" d="M356.6658,-387.2435C378.0878,-380.079 402.7906,-369.4801 402.7906,-358.7 402.7906,-358.7 402.7906,-358.7 402.7906,-302.9 402.7906,-299.4948 403.3035,-296.0193 404.133,-292.6252"/>
+ <polygon fill="#000000" stroke="#000000" points="407.5528,-293.4318 407.3747,-282.8385 400.9078,-291.2307 407.5528,-293.4318"/>
+ <text text-anchor="middle" x="416.7906" y="-326.6" font-family="Times,serif" font-size="14.00" fill="#000000">  010</text>
+ </g>
+ <!-- 0x59C->0x13A -->
+ <g id="edge13" class="edge">
+ <title>0x59C->0x13A</title>
+ <path fill="none" stroke="#000000" d="M342.4972,-249.3313C360.911,-231.6206 390.693,-202.9757 411.6447,-182.824"/>
+ <polygon fill="#000000" stroke="#000000" points="414.1457,-185.2747 418.9268,-175.82 409.2931,-180.2296 414.1457,-185.2747"/>
+ <text text-anchor="middle" x="410.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  011</text>
+ </g>
+ <!-- 0x238 -->
+ <g id="node13" class="node">
+ <title>0x238</title>
+ <ellipse fill="none" stroke="#000000" cx="218.7906" cy="-159.6" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="218.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x238</text>
+ </g>
+ <!-- 0x59C->0x238 -->
+ <g id="edge11" class="edge">
+ <title>0x59C->0x238</title>
+ <path fill="none" stroke="#000000" d="M309.5396,-249.3313C291.7067,-231.6983 262.9122,-203.2267 242.5467,-183.0897"/>
+ <polygon fill="#000000" stroke="#000000" points="244.7663,-180.3623 235.1945,-175.82 239.8445,-185.3399 244.7663,-180.3623"/>
+ <text text-anchor="middle" x="286.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  001</text>
+ </g>
+ <!-- 0x35C -->
+ <g id="node14" class="node">
+ <title>0x35C</title>
+ <ellipse fill="none" stroke="#000000" cx="325.7906" cy="-159.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="325.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x35C</text>
+ </g>
+ <!-- 0x59C->0x35C -->
+ <g id="edge12" class="edge">
+ <title>0x59C->0x35C</title>
+ <path fill="none" stroke="#000000" d="M325.7906,-246.971C325.7906,-230.6622 325.7906,-206.6111 325.7906,-187.8698"/>
+ <polygon fill="#000000" stroke="#000000" points="329.2907,-187.8177 325.7906,-177.8178 322.2907,-187.8178 329.2907,-187.8177"/>
+ <text text-anchor="middle" x="339.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  010</text>
+ </g>
+ <!-- 0x37A -->
+ <g id="node15" class="node">
+ <title>0x37A</title>
+ <ellipse fill="none" stroke="#000000" cx="528.7906" cy="-159.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="528.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x37A</text>
+ </g>
+ <!-- 0x5BC->0x37A -->
+ <g id="edge14" class="edge">
+ <title>0x5BC->0x37A</title>
+ <path fill="none" stroke="#000000" d="M433.801,-249.3313C452.5495,-231.6206 482.8731,-202.9757 504.2057,-182.824"/>
+ <polygon fill="#000000" stroke="#000000" points="506.7542,-185.2313 511.6201,-175.82 501.9472,-180.1427 506.7542,-185.2313"/>
+ <text text-anchor="middle" x="502.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  000</text>
+ </g>
+ <!-- 0x455 -->
+ <g id="node16" class="node">
+ <title>0x455</title>
+ <ellipse fill="none" stroke="#000000" cx="619.7906" cy="-159.6" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="619.7906" y="-155.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x455</text>
+ </g>
+ <!-- 0x5BC->0x455 -->
+ <g id="edge15" class="edge">
+ <title>0x5BC->0x455</title>
+ <path fill="none" stroke="#000000" d="M443.4197,-252.397C464.5065,-242.0254 494.6583,-227.0286 520.7906,-213.4 542.9996,-201.8175 567.7596,-188.3619 586.9624,-177.8047"/>
+ <polygon fill="#000000" stroke="#000000" points="588.838,-180.7674 595.9072,-172.8758 585.4597,-174.6365 588.838,-180.7674"/>
+ <text text-anchor="middle" x="564.7906" y="-200.8" font-family="Times,serif" font-size="14.00" fill="#000000">  001</text>
+ </g>
+ <!-- 0x238->0x35C -->
+ <!-- 0x35C->0x13A -->
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m1.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m1.svg
new file mode 100644
index 0000000..ff928a4
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m1.svg
@@ -0,0 +1,349 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+```plantuml
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+ node [color = "lightblue"; fontcolor="blue"]
+
+ start -> root
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trav [label = " v"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+
+ trav [label = "NONE"]
+ trave [label = "NONE"]
+ traver [label = "NONE"]
+ travers [label = "NONE"]
+ traverse [label = "NONE"]
+ }
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="575pt" height="823pt"
+ viewBox="0.00 0.00 575.47 822.89" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 818.8939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-818.8939 571.4738,-818.8939 571.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="279.2369" cy="-785.4782" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="279.2369" y="-789.6782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="279.2369" y="-772.8782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="392.2369" cy="-785.4782" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="392.2369" y="-781.2782" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="279.2369" cy="-685.2626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="279.2369" y="-681.0626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M269.3368,-757.0824C266.6104,-745.9368 264.8794,-733.0211 266.8463,-721.2626 267.3268,-718.39 268.009,-715.4398 268.8091,-712.5238"/>
+ <polygon fill="#000000" stroke="#000000" points="272.1732,-713.494 271.8627,-702.9037 265.5012,-711.3762 272.1732,-713.494"/>
+ <text text-anchor="middle" x="270.9322" y="-725.4626" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge16" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M279.2369,-755.9522C279.2369,-742.4671 279.2369,-726.5749 279.2369,-713.2779"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="279.2369,-703.2659 283.737,-713.2659 279.2369,-708.2659 279.237,-713.2659 279.237,-713.2659 279.237,-713.2659 279.2369,-708.2659 274.737,-713.2659 279.2369,-703.2659 279.2369,-703.2659"/>
+ <text text-anchor="middle" x="282.9322" y="-725.4626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge15" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M367.6112,-800.5185C350.7348,-807.9537 333.8585,-809.4839 316.9821,-805.109"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="307.0455,-801.8508 317.9498,-800.6907 311.7966,-803.4087 316.5477,-804.9667 316.5477,-804.9667 316.5477,-804.9667 311.7966,-803.4087 315.1456,-809.2427 307.0455,-801.8508 307.0455,-801.8508"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="91.2369" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="91.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge8" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M87.5943,-177.63C85.1221,-165.4338 81.802,-149.0543 78.9772,-135.1187"/>
+ <polygon fill="#000000" stroke="#000000" points="82.3528,-134.1534 76.9358,-125.0481 75.4923,-135.5441 82.3528,-134.1534"/>
+ <text text-anchor="middle" x="88.3172" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="91.2369" cy="-284.4" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="91.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge7" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M91.2369,-266.0006C91.2369,-253.8949 91.2369,-237.8076 91.2369,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="94.737,-223.672 91.2369,-213.672 87.737,-223.6721 94.737,-223.672"/>
+ <text text-anchor="middle" x="96.4869" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="92.2369" cy="-373.2" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="92.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge6" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M92.0297,-354.8006C91.8934,-342.6949 91.7122,-326.6076 91.5575,-312.8674"/>
+ <polygon fill="#000000" stroke="#000000" points="95.0529,-312.4319 91.4404,-302.472 88.0533,-312.5108 95.0529,-312.4319"/>
+ <text text-anchor="middle" x="95.9322" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="312.2369" cy="-373.2" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="312.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="203.2369" cy="-473.4156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="203.2369" y="-477.6156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="203.2369" y="-460.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge5" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#000000" d="M178.8775,-451.4229C160.5618,-434.8867 135.5561,-412.3105 117.1156,-395.6616"/>
+ <polygon fill="#000000" stroke="#000000" points="119.421,-393.0276 109.6531,-388.9241 114.7301,-398.2233 119.421,-393.0276"/>
+ <text text-anchor="middle" x="153.0942" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- trav -->
+ <g id="node14" class="node">
+ <title>trav</title>
+ <ellipse fill="none" stroke="#add8e6" cx="183.2369" cy="-373.2" rx="37.7006" ry="18"/>
+ <text text-anchor="middle" x="183.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#0000ff">NONE</text>
+ </g>
+ <!-- tra->trav -->
+ <g id="edge19" class="edge">
+ <title>tra->trav</title>
+ <path fill="none" stroke="#add8e6" d="M197.3983,-444.1597C194.7054,-430.666 191.5224,-414.7168 188.8568,-401.3604"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="186.8493,-391.3011 193.2195,-400.227 187.8279,-396.2044 188.8065,-401.1078 188.8065,-401.1078 188.8065,-401.1078 187.8279,-396.2044 184.3935,-401.9885 186.8493,-391.3011 186.8493,-391.3011"/>
+ <text text-anchor="middle" x="198.4869" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#0000ff"> v</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="494.2369" cy="-373.2" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="494.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="304.2369" cy="-473.4156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="304.2369" y="-477.6156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="304.2369" y="-460.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M306.5939,-443.8897C307.6704,-430.4046 308.939,-414.5123 310.0005,-401.2153"/>
+ <polygon fill="#000000" stroke="#000000" points="313.4928,-401.4502 310.7997,-391.2033 306.515,-400.8931 313.4928,-401.4502"/>
+ <text text-anchor="middle" x="314.0942" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="446.2369" cy="-473.4156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="446.2369" y="-477.6156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="446.2369" y="-460.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M459.4802,-445.766C466.2441,-431.6441 474.4452,-414.5215 481.1822,-400.456"/>
+ <polygon fill="#000000" stroke="#000000" points="484.4134,-401.812 485.5766,-391.2812 478.1002,-398.7881 484.4134,-401.812"/>
+ <text text-anchor="middle" x="480.0942" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="279.2369" cy="-585.0469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="279.2369" y="-589.2469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="279.2369" y="-572.4469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge4" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#000000" d="M252.4549,-564.0846C243.9686,-556.4495 235.0956,-547.3003 228.5223,-537.6313 223.1796,-529.7724 218.7204,-520.6194 215.11,-511.7284"/>
+ <polygon fill="#000000" stroke="#000000" points="218.3382,-510.37 211.5239,-502.2573 211.7918,-512.8487 218.3382,-510.37"/>
+ <text text-anchor="middle" x="234.0942" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge18" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M262.877,-558.5399C255.5232,-546.8584 246.6268,-533.0409 238.2369,-520.8313 235.1223,-516.2987 231.7727,-511.5792 228.424,-506.9511"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="222.4485,-498.7793 231.9836,-504.1952 225.3998,-502.8154 228.3512,-506.8514 228.3512,-506.8514 228.3512,-506.8514 225.3998,-502.8154 224.7187,-509.5077 222.4485,-498.7793 222.4485,-498.7793"/>
+ <text text-anchor="middle" x="254.0942" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M285.7401,-556.0085C288.6971,-542.8046 292.2409,-526.9807 295.4318,-512.7327"/>
+ <polygon fill="#000000" stroke="#000000" points="298.9093,-513.2196 297.6794,-502.6964 292.0785,-511.6898 298.9093,-513.2196"/>
+ <text text-anchor="middle" x="298.0942" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M307.9452,-565.8568C335.7854,-547.247 378.1751,-518.9116 408.6826,-498.5188"/>
+ <polygon fill="#000000" stroke="#000000" points="410.9757,-501.196 417.3443,-492.7289 407.0856,-495.3764 410.9757,-501.196"/>
+ <text text-anchor="middle" x="376.9322" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M271.4066,-667.6591C269.2309,-661.9119 267.1669,-655.422 266.0763,-649.2626 264.5952,-640.8975 265.0969,-631.9507 266.5509,-623.4728"/>
+ <polygon fill="#000000" stroke="#000000" points="270.0289,-623.9493 268.7277,-613.4347 263.1879,-622.4658 270.0289,-623.9493"/>
+ <text text-anchor="middle" x="270.3172" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge17" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M279.2369,-666.8815C279.2369,-655.1502 279.2369,-639.4774 279.2369,-624.9885"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="279.2369,-614.7209 283.737,-624.7208 279.2369,-619.7209 279.237,-624.7209 279.237,-624.7209 279.237,-624.7209 279.2369,-619.7209 274.737,-624.7209 279.2369,-614.7209 279.2369,-614.7209"/>
+ <text text-anchor="middle" x="283.3172" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trave -->
+ <g id="node15" class="node">
+ <title>trave</title>
+ <ellipse fill="none" stroke="#add8e6" cx="188.2369" cy="-284.4" rx="37.7006" ry="18"/>
+ <text text-anchor="middle" x="188.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">NONE</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge20" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#add8e6" d="M184.2729,-354.8006C184.9613,-342.575 185.8783,-326.2887 186.6569,-312.4599"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="187.2193,-302.472 191.1499,-312.7092 186.9382,-307.4641 186.6571,-312.4562 186.6571,-312.4562 186.6571,-312.4562 186.9382,-307.4641 182.1642,-312.2032 187.2193,-302.472 187.2193,-302.472"/>
+ <text text-anchor="middle" x="191.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- traver -->
+ <g id="node16" class="node">
+ <title>traver</title>
+ <ellipse fill="none" stroke="#add8e6" cx="193.2369" cy="-195.6" rx="37.7006" ry="18"/>
+ <text text-anchor="middle" x="193.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">NONE</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge21" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#add8e6" d="M189.2729,-266.0006C189.9613,-253.775 190.8783,-237.4887 191.6569,-223.6599"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="192.2193,-213.672 196.1499,-223.9092 191.9382,-218.6641 191.6571,-223.6562 191.6571,-223.6562 191.6571,-223.6562 191.9382,-218.6641 187.1642,-223.4032 192.2193,-213.672 192.2193,-213.672"/>
+ <text text-anchor="middle" x="195.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- travers -->
+ <g id="node17" class="node">
+ <title>travers</title>
+ <ellipse fill="none" stroke="#add8e6" cx="202.2369" cy="-106.8" rx="37.7006" ry="18"/>
+ <text text-anchor="middle" x="202.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">NONE</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge22" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#add8e6" d="M195.1017,-177.2006C196.3408,-164.975 197.9914,-148.6887 199.393,-134.8599"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="200.4053,-124.872 203.8739,-135.2748 199.901,-129.8465 199.3968,-134.821 199.3968,-134.821 199.3968,-134.821 199.901,-129.8465 194.9198,-134.3673 200.4053,-124.872 200.4053,-124.872"/>
+ <text text-anchor="middle" x="203.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+ </g>
+ <!-- traverse -->
+ <g id="node18" class="node">
+ <title>traverse</title>
+ <ellipse fill="none" stroke="#add8e6" cx="202.2369" cy="-18" rx="37.7006" ry="18"/>
+ <text text-anchor="middle" x="202.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#0000ff">NONE</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge23" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#add8e6" d="M202.2369,-88.4006C202.2369,-76.2949 202.2369,-60.2076 202.2369,-46.4674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="202.2369,-36.072 206.737,-46.072 202.2369,-41.072 202.237,-46.072 202.237,-46.072 202.237,-46.072 202.2369,-41.072 197.737,-46.0721 202.2369,-36.072 202.2369,-36.072"/>
+ <text text-anchor="middle" x="207.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m2.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m2.svg
new file mode 100644
index 0000000..ba33dd1
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m2.svg
@@ -0,0 +1,430 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+```plantuml
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+ node [color = "blue"; fontcolor="blue"]
+
+ start -> root
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trav [label = " v"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+
+ trav [label = "0x0B8"]
+ trave [label = "0x0B9"]
+ traver [label = "0x0BA"]
+ travers [label = "0x0BB"]
+ traverse [label = "contentArray[3]"]
+ }
+
+ {
+ edge [color = "blue"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+ }
+
+ subgraph back {
+ edge [color = "pink"; fontcolor="red"; arrowhead="vee"; constrain="false"]
+
+ traverse -> travers [label = " ~3"]
+ travers -> traver [label = "0x0BB"]
+ traver -> trave [label = "0x0BA"]
+ trave -> trav [label = "0x0B9"]
+ trav -> tra [label = "0x0B8"]
+ }
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="574pt" height="823pt"
+ viewBox="0.00 0.00 574.47 822.89" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 818.8939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-818.8939 570.4738,-818.8939 570.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="289.2369" cy="-785.4782" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="289.2369" y="-789.6782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="289.2369" y="-772.8782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="402.2369" cy="-785.4782" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="402.2369" y="-781.2782" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="289.2369" cy="-685.2626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="289.2369" y="-681.0626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M279.3368,-757.0824C276.6104,-745.9368 274.8794,-733.0211 276.8463,-721.2626 277.3268,-718.39 278.009,-715.4398 278.8091,-712.5238"/>
+ <polygon fill="#000000" stroke="#000000" points="282.1732,-713.494 281.8627,-702.9037 275.5012,-711.3762 282.1732,-713.494"/>
+ <text text-anchor="middle" x="280.9322" y="-725.4626" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge16" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M289.2369,-755.9522C289.2369,-742.4671 289.2369,-726.5749 289.2369,-713.2779"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="289.2369,-703.2659 293.737,-713.2659 289.2369,-708.2659 289.237,-713.2659 289.237,-713.2659 289.237,-713.2659 289.2369,-708.2659 284.737,-713.2659 289.2369,-703.2659 289.2369,-703.2659"/>
+ <text text-anchor="middle" x="292.9322" y="-725.4626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge15" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M377.6112,-800.5185C360.7348,-807.9537 343.8585,-809.4839 326.9821,-805.109"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="317.0455,-801.8508 327.9498,-800.6907 321.7966,-803.4087 326.5477,-804.9667 326.5477,-804.9667 326.5477,-804.9667 321.7966,-803.4087 325.1456,-809.2427 317.0455,-801.8508 317.0455,-801.8508"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="91.2369" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="91.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge8" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M87.5943,-177.63C85.1221,-165.4338 81.802,-149.0543 78.9772,-135.1187"/>
+ <polygon fill="#000000" stroke="#000000" points="82.3528,-134.1534 76.9358,-125.0481 75.4923,-135.5441 82.3528,-134.1534"/>
+ <text text-anchor="middle" x="88.3172" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="92.2369" cy="-284.4" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="92.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge7" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M92.0297,-266.0006C91.8934,-253.8949 91.7122,-237.8076 91.5575,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="95.0529,-223.6319 91.4404,-213.672 88.0533,-223.7108 95.0529,-223.6319"/>
+ <text text-anchor="middle" x="97.4869" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="93.2369" cy="-373.2" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="93.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge6" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M93.0297,-354.8006C92.8934,-342.6949 92.7122,-326.6076 92.5575,-312.8674"/>
+ <polygon fill="#000000" stroke="#000000" points="96.0529,-312.4319 92.4404,-302.472 89.0533,-312.5108 96.0529,-312.4319"/>
+ <text text-anchor="middle" x="96.9322" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="311.2369" cy="-373.2" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="311.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="219.2369" cy="-473.4156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="219.2369" y="-477.6156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="219.2369" y="-460.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge5" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#000000" d="M191.1461,-454.472C179.0394,-446.0482 164.8584,-435.8286 152.5223,-426 140.7539,-416.6237 128.241,-405.6422 117.7589,-396.1298"/>
+ <polygon fill="#000000" stroke="#000000" points="119.9234,-393.366 110.185,-389.1928 115.1954,-398.5281 119.9234,-393.366"/>
+ <text text-anchor="middle" x="158.0942" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- trav -->
+ <g id="node14" class="node">
+ <title>trav</title>
+ <ellipse fill="none" stroke="#0000ff" cx="183.2369" cy="-373.2" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="183.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B8</text>
+ </g>
+ <!-- tra->trav -->
+ <g id="edge19" class="edge">
+ <title>tra->trav</title>
+ <path fill="none" stroke="#add8e6" d="M217.5652,-443.8618C216.1848,-432.6862 213.6979,-420.0532 209.2369,-409.2 207.6981,-405.4564 205.697,-401.7342 203.4879,-398.182"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="197.713,-389.7873 207.0881,-395.4757 200.5468,-393.9067 203.3807,-398.0261 203.3807,-398.0261 203.3807,-398.0261 200.5468,-393.9067 199.6732,-400.5766 197.713,-389.7873 197.713,-389.7873"/>
+ <text text-anchor="middle" x="219.4869" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#0000ff"> v</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="493.2369" cy="-373.2" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="493.2369" y="-369" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="311.2369" cy="-473.4156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="311.2369" y="-477.6156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="311.2369" y="-460.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M311.2369,-443.8897C311.2369,-430.4046 311.2369,-414.5123 311.2369,-401.2153"/>
+ <polygon fill="#000000" stroke="#000000" points="314.737,-401.2033 311.2369,-391.2033 307.737,-401.2034 314.737,-401.2033"/>
+ <text text-anchor="middle" x="316.0942" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="448.2369" cy="-473.4156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="448.2369" y="-477.6156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="448.2369" y="-460.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M460.7719,-445.5C467.1109,-431.383 474.7723,-414.3209 481.0603,-400.3175"/>
+ <polygon fill="#000000" stroke="#000000" points="484.2575,-401.7415 485.1609,-391.1852 477.8717,-398.874 484.2575,-401.7415"/>
+ <text text-anchor="middle" x="480.0942" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="289.2369" cy="-585.0469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="289.2369" y="-589.2469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="289.2369" y="-572.4469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge4" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#000000" d="M262.3516,-564.0201C254.086,-556.4444 245.5803,-547.3475 239.5223,-537.6313 234.7133,-529.9184 230.9544,-520.9679 228.0512,-512.2428"/>
+ <polygon fill="#000000" stroke="#000000" points="231.37,-511.1256 225.1267,-502.5657 224.6692,-513.1506 231.37,-511.1256"/>
+ <text text-anchor="middle" x="245.0942" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge18" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M272.7366,-558.4643C265.5384,-546.8917 256.9887,-533.179 249.2369,-520.8313 246.7152,-516.8145 244.0675,-512.6105 241.4331,-508.4359"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="236.0886,-499.9773 245.2344,-506.0275 238.7594,-504.2043 241.4302,-508.4312 241.4302,-508.4312 241.4302,-508.4312 238.7594,-504.2043 237.6259,-510.8349 236.0886,-499.9773 236.0886,-499.9773"/>
+ <text text-anchor="middle" x="264.0942" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M295.0171,-555.7171C297.6192,-542.5138 300.7286,-526.7364 303.5256,-512.5441"/>
+ <polygon fill="#000000" stroke="#000000" points="306.9955,-513.0376 305.4953,-502.5496 300.1276,-511.6841 306.9955,-513.0376"/>
+ <text text-anchor="middle" x="307.0942" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M317.2969,-565.3465C343.5549,-546.9111 382.9334,-519.2642 411.6478,-499.1043"/>
+ <polygon fill="#000000" stroke="#000000" points="414.0173,-501.7172 420.1904,-493.1066 409.995,-495.9882 414.0173,-501.7172"/>
+ <text text-anchor="middle" x="381.9322" y="-525.0313" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M281.4066,-667.6591C279.2309,-661.9119 277.1669,-655.422 276.0763,-649.2626 274.5952,-640.8975 275.0969,-631.9507 276.5509,-623.4728"/>
+ <polygon fill="#000000" stroke="#000000" points="280.0289,-623.9493 278.7277,-613.4347 273.1879,-622.4658 280.0289,-623.9493"/>
+ <text text-anchor="middle" x="280.3172" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge17" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M289.2369,-666.8815C289.2369,-655.1502 289.2369,-639.4774 289.2369,-624.9885"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="289.2369,-614.7209 293.737,-624.7208 289.2369,-619.7209 289.237,-624.7209 289.237,-624.7209 289.237,-624.7209 289.2369,-619.7209 284.737,-624.7209 289.2369,-614.7209 289.2369,-614.7209"/>
+ <text text-anchor="middle" x="293.3172" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trav->tra -->
+ <g id="edge32" class="edge">
+ <title>trav->tra</title>
+ <path fill="none" stroke="#ffc0cb" d="M172.1325,-390.7196C166.9576,-401.1938 162.925,-414.6319 167.9003,-426 171.4315,-434.0686 177.0881,-441.3739 183.3808,-447.7"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="190.9707,-454.6637 180.5599,-451.219 187.2864,-451.2834 183.6021,-447.9031 183.6021,-447.9031 183.6021,-447.9031 187.2864,-451.2834 186.6444,-444.5873 190.9707,-454.6637 190.9707,-454.6637"/>
+ <text text-anchor="middle" x="186.9052" y="-413.4" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B8</text>
+ </g>
+ <!-- trave -->
+ <g id="node15" class="node">
+ <title>trave</title>
+ <ellipse fill="none" stroke="#0000ff" cx="189.2369" cy="-284.4" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="189.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B9</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge20" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#add8e6" d="M206.004,-359.0012C213.2194,-353.2102 220.3301,-345.8294 224.2369,-337.2 228.9378,-326.8164 224.0959,-316.1864 216.6675,-307.2114"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="209.6485,-299.798 219.7915,-303.9657 213.0861,-303.4288 216.5238,-307.0596 216.5238,-307.0596 216.5238,-307.0596 213.0861,-303.4288 213.2561,-310.1535 209.6485,-299.798 209.6485,-299.798"/>
+ <text text-anchor="middle" x="231.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge24" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#0000ff" d="M165.9239,-357.0271C157.1626,-346.8113 149.3672,-333.1725 154.5223,-320.4 156.3746,-315.8107 159.123,-311.4803 162.2767,-307.5211"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="165.0223,-309.7042 169.162,-299.9515 159.844,-304.994 165.0223,-309.7042"/>
+ <text text-anchor="middle" x="160.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- trave->trav -->
+ <g id="edge31" class="edge">
+ <title>trave->trav</title>
+ <path fill="none" stroke="#ffc0cb" d="M185.3926,-302.6325C184.3695,-308.2985 183.4083,-314.5837 182.9003,-320.4 182.2094,-328.3107 182.0246,-336.9134 182.0825,-344.8339"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="182.2872,-355.0829 177.5883,-345.1748 182.1873,-350.0839 182.0874,-345.0849 182.0874,-345.0849 182.0874,-345.0849 182.1873,-350.0839 186.5865,-344.995 182.2872,-355.0829 182.2872,-355.0829"/>
+ <text text-anchor="middle" x="201.9052" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B9</text>
+ </g>
+ <!-- traver -->
+ <g id="node16" class="node">
+ <title>traver</title>
+ <ellipse fill="none" stroke="#0000ff" cx="194.2369" cy="-195.6" rx="38.8671" ry="18"/>
+ <text text-anchor="middle" x="194.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BA</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge21" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#add8e6" d="M213.0898,-270.4154C220.6549,-264.6446 228.1173,-257.2217 232.2369,-248.4 237.2769,-237.6073 231.7142,-226.7456 223.4297,-217.7036"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="216.0035,-210.5778 226.3347,-214.2545 219.6113,-214.0396 223.219,-217.5015 223.219,-217.5015 223.219,-217.5015 219.6113,-214.0396 220.1034,-220.7485 216.0035,-210.5778 216.0035,-210.5778"/>
+ <text text-anchor="middle" x="238.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge25" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#0000ff" d="M172.8782,-268.1851C164.5976,-257.9534 157.2248,-244.3128 162.0763,-231.6 163.7001,-227.3449 166.0838,-223.2741 168.8315,-219.5011"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="171.5668,-221.6859 175.2747,-211.761 166.1869,-217.2074 171.5668,-221.6859"/>
+ <text text-anchor="middle" x="166.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- traver->trave -->
+ <g id="edge30" class="edge">
+ <title>traver->trave</title>
+ <path fill="none" stroke="#ffc0cb" d="M190.328,-213.8298C189.2877,-219.4957 188.3103,-225.7815 187.7937,-231.6 187.0912,-239.5125 187.0294,-248.1157 187.2551,-256.0363"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="187.7079,-266.2849 182.7708,-256.4934 187.4871,-261.2898 187.2664,-256.2947 187.2664,-256.2947 187.2664,-256.2947 187.4871,-261.2898 191.762,-256.096 187.7079,-266.2849 187.7079,-266.2849"/>
+ <text text-anchor="middle" x="208.4585" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BA</text>
+ </g>
+ <!-- travers -->
+ <g id="node17" class="node">
+ <title>travers</title>
+ <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-106.8" rx="38.305" ry="18"/>
+ <text text-anchor="middle" x="203.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BB</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge22" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#add8e6" d="M219.6485,-181.4864C227.3996,-175.7839 234.9896,-168.4371 239.2369,-159.6 244.2682,-149.1317 239.1644,-138.3954 231.3732,-129.361"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="224.3601,-122.2156 234.5764,-126.2003 227.8624,-125.784 231.3648,-129.3524 231.3648,-129.3524 231.3648,-129.3524 227.8624,-125.784 228.1532,-132.5046 224.3601,-122.2156 224.3601,-122.2156"/>
+ <text text-anchor="middle" x="245.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge26" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#0000ff" d="M177.7403,-179.2722C169.4246,-168.9981 162.0986,-155.3524 167.2923,-142.8 169.2677,-138.0258 172.2173,-133.5536 175.5926,-129.4948"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="178.1364,-131.8988 182.4642,-122.2282 173.0503,-127.0892 178.1364,-131.8988"/>
+ <text text-anchor="middle" x="172.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> s</text>
+ </g>
+ <!-- travers->traver -->
+ <g id="edge29" class="edge">
+ <title>travers->traver</title>
+ <path fill="none" stroke="#ffc0cb" d="M198.6734,-125.0051C197.4382,-130.6699 196.2533,-136.9614 195.5637,-142.8 194.632,-150.6883 194.1789,-159.2838 193.9873,-167.2046"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="193.8675,-177.4577 189.4847,-167.4058 193.9259,-172.4581 193.9844,-167.4584 193.9844,-167.4584 193.9844,-167.4584 193.9259,-172.4581 198.4841,-167.511 193.8675,-177.4577 193.8675,-177.4577"/>
+ <text text-anchor="middle" x="216.0735" y="-147" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BB</text>
+ </g>
+ <!-- traverse -->
+ <g id="node18" class="node">
+ <title>traverse</title>
+ <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="203.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[3]</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge23" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#add8e6" d="M184.8676,-90.9569C179.5135,-85.2132 174.3474,-78.2805 171.5223,-70.8 168.8843,-63.8149 168.8843,-60.9851 171.5223,-54 172.8668,-50.4399 174.7415,-47.004 176.9144,-43.7612"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="183.203,-35.676 180.6155,-46.3323 180.1332,-39.6227 177.0634,-43.5695 177.0634,-43.5695 177.0634,-43.5695 180.1332,-39.6227 173.5114,-40.8067 183.203,-35.676 183.203,-35.676"/>
+ <text text-anchor="middle" x="177.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge27" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#0000ff" d="M203.2369,-88.4006C203.2369,-76.2949 203.2369,-60.2076 203.2369,-46.4674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="206.737,-46.072 203.2369,-36.072 199.737,-46.0721 206.737,-46.072"/>
+ <text text-anchor="middle" x="208.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- traverse->travers -->
+ <g id="edge28" class="edge">
+ <title>traverse->travers</title>
+ <path fill="none" stroke="#ffc0cb" d="M209.9024,-36.0722C211.6782,-41.7317 213.3487,-48.0535 214.2369,-54 215.3399,-61.3847 215.3399,-63.4153 214.2369,-70.8 213.8344,-73.4945 213.2713,-76.2661 212.6106,-79.0181"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="209.9024,-88.7278 208.2546,-77.8864 211.2458,-83.9116 212.5891,-79.0954 212.5891,-79.0954 212.5891,-79.0954 211.2458,-83.9116 216.9237,-80.3044 209.9024,-88.7278 209.9024,-88.7278"/>
+ <text text-anchor="middle" x="224.2732" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#ff0000"> ~3</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m3.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m3.svg
new file mode 100644
index 0000000..e71114f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.m3.svg
@@ -0,0 +1,500 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+```plantuml
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"; color = "lightgrey"; fontcolor = lightgray]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tra [label = " a"; color = "lightgrey"; fontcolor = lightgray]
+ tra -> trac [label = " c"; color = "lightgrey"; fontcolor = lightgray]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+ node [color = "blue"; fontcolor="blue"]
+
+ start -> root
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trav [label = " v"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+
+ tra2 [label = "Sparse\n0x0DE"]
+ trav [label = "Chain\n0x0B8"]
+ trave [label = "0x0B9"]
+ traver [label = "0x0BA"]
+ travers [label = "0x0BB"]
+ traverse [label = "contentArray[3]"]
+ }
+
+ {rank=same tra -> tra2 -> tre -> tri [style=invis]}
+ {rank=same trac -> trav -> tree -> trie [style=invis]}
+
+ {
+ edge [color = "blue"]
+ tr -> tra2 [label = " a"]
+ tra2 -> trac [label = " c"]
+ tra2 -> trav [label = " v"]
+ trav -> trave [label = " e"]
+ trave -> traver [label = " r"]
+ traver -> travers [label = " s"]
+ travers -> traverse [label = " e"]
+ }
+
+ subgraph back {
+ edge [color = "pink"; fontcolor="red"; arrowhead="vee"; constrain="false"]
+
+ traverse -> travers [label = " ~3"]
+ travers -> traver [label = "0x0BB"]
+ traver -> trave [label = "0x0BA"]
+ trave -> trav [label = "0x0B9"]
+ trav -> tra2 [label = "0x0B8"]
+ tra2 -> tr [label = "0x0DE"]
+ tr -> t [label = "0x07E"]
+ t -> root [label = "0x09B"]
+ root -> start [label = "0x09A"]
+ }
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="611pt" height="846pt"
+ viewBox="0.00 0.00 611.47 845.73" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 841.7251)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-841.7251 607.4738,-841.7251 607.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="230.2369" cy="-808.3095" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="230.2369" y="-812.5095" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="230.2369" y="-795.7095" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="363.2369" cy="-808.3095" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="363.2369" y="-804.1095" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- root->start -->
+ <g id="edge43" class="edge">
+ <title>root->start</title>
+ <path fill="none" stroke="#ffc0cb" d="M263.7984,-808.3095C277.3037,-808.3095 293.2339,-808.3095 308.3398,-808.3095"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="318.6261,-808.3095 308.6262,-812.8096 313.6261,-808.3095 308.6261,-808.3096 308.6261,-808.3096 308.6261,-808.3096 313.6261,-808.3095 308.6261,-803.8096 318.6261,-808.3095 318.6261,-808.3095"/>
+ <text text-anchor="middle" x="291.2509" y="-815.5095" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09A</text>
+ </g>
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="230.2369" cy="-708.0939" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="230.2369" y="-703.8939" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M210.9564,-784.2612C203.7393,-772.3485 198.2768,-757.6462 202.8463,-744.0939 204.1636,-740.1869 206.0926,-736.3684 208.3269,-732.7663"/>
+ <polygon fill="#000000" stroke="#000000" points="211.3844,-734.5068 214.3357,-724.3313 205.683,-730.4454 211.3844,-734.5068"/>
+ <text text-anchor="middle" x="206.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge14" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M230.2369,-778.7835C230.2369,-765.2984 230.2369,-749.4062 230.2369,-736.1092"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="230.2369,-726.0972 234.737,-736.0971 230.2369,-731.0972 230.237,-736.0972 230.237,-736.0972 230.237,-736.0972 230.2369,-731.0972 225.737,-736.0972 230.2369,-726.0972 230.2369,-726.0972"/>
+ <text text-anchor="middle" x="233.9322" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge13" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M319.4042,-805.6333C307.2938,-805.1818 294.1479,-804.9584 282.0184,-805.3095 279.3264,-805.3874 276.5568,-805.4881 273.7643,-805.6051"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="263.6504,-806.089 273.4239,-801.1161 268.6447,-805.85 273.639,-805.611 273.639,-805.611 273.639,-805.611 268.6447,-805.85 273.8541,-810.1058 263.6504,-806.089 263.6504,-806.089"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="74.2369" cy="-195.6" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="74.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge6" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M74.0297,-177.2006C73.8934,-165.0949 73.7122,-149.0076 73.5575,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="77.0529,-134.8319 73.4404,-124.872 70.0533,-134.9108 77.0529,-134.8319"/>
+ <text text-anchor="middle" x="78.3172" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="78.2369" cy="-284.4" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="78.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge5" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M77.4081,-266.0006C76.8628,-253.8949 76.1381,-237.8076 75.5192,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="78.9975,-223.5044 75.0509,-213.672 72.0046,-223.8194 78.9975,-223.5044"/>
+ <text text-anchor="middle" x="82.4869" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="94.2369" cy="-384.6156" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="94.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge4" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M91.3022,-366.2345C88.9115,-351.2603 85.4955,-329.8643 82.756,-312.7055"/>
+ <polygon fill="#000000" stroke="#000000" points="86.1858,-311.9877 81.1529,-302.6646 79.2734,-313.0913 86.1858,-311.9877"/>
+ <text text-anchor="middle" x="89.9322" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- trav -->
+ <g id="node14" class="node">
+ <title>trav</title>
+ <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="202.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#0000ff">Chain</text>
+ <text text-anchor="middle" x="202.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B8</text>
+ </g>
+ <!-- trac->trav -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#d3d3d3" cx="124.2369" cy="-496.2469" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="124.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">Chain</text>
+ <text text-anchor="middle" x="124.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge8" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#d3d3d3" d="M108.792,-469.6104C105.5657,-463.008 102.5364,-455.818 100.5223,-448.8313 97.2068,-437.3303 95.5641,-424.2429 94.7752,-412.8969"/>
+ <polygon fill="#d3d3d3" stroke="#d3d3d3" points="98.2581,-412.4701 94.2539,-402.6611 91.2672,-412.8262 98.2581,-412.4701"/>
+ <text text-anchor="middle" x="106.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> c</text>
+ </g>
+ <!-- tra->trav -->
+ <g id="edge17" class="edge">
+ <title>tra->trav</title>
+ <path fill="none" stroke="#add8e6" d="M124.8024,-466.7151C126.1832,-455.1695 129.2555,-442.2712 135.7369,-432.0313 142.2539,-421.7351 151.8894,-413.0083 161.8152,-405.9576"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="170.1591,-400.4315 164.3065,-409.7051 165.9904,-403.1924 161.8218,-405.9533 161.8218,-405.9533 161.8218,-405.9533 165.9904,-403.1924 159.337,-402.2015 170.1591,-400.4315 170.1591,-400.4315"/>
+ <text text-anchor="middle" x="141.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> v</text>
+ </g>
+ <!-- tra2 -->
+ <g id="node19" class="node">
+ <title>tra2</title>
+ <ellipse fill="none" stroke="#0000ff" cx="235.2369" cy="-496.2469" rx="39.2145" ry="29.3315"/>
+ <text text-anchor="middle" x="235.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#0000ff">Sparse</text>
+ <text text-anchor="middle" x="235.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0DE</text>
+ </g>
+ <!-- tra->tra2 -->
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="348.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="348.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="530.2369" cy="-384.6156" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="530.2369" y="-380.4156" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="348.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="348.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="348.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M348.2369,-466.6249C348.2369,-449.9873 348.2369,-429.334 348.2369,-412.9163"/>
+ <polygon fill="#000000" stroke="#000000" points="351.737,-412.8239 348.2369,-402.8239 344.737,-412.8239 351.737,-412.8239"/>
+ <text text-anchor="middle" x="353.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="459.2369" cy="-496.2469" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="459.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="459.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tre->tri -->
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M476.0629,-469.7919C487.3072,-452.1128 502.0457,-428.9399 513.3106,-411.2283"/>
+ <polygon fill="#000000" stroke="#000000" points="516.4345,-412.8384 518.848,-402.5221 510.5279,-409.0817 516.4345,-412.8384"/>
+ <text text-anchor="middle" x="503.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="230.2369" cy="-607.8782" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="230.2369" y="-612.0782" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="230.2369" y="-595.2782" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge7" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#d3d3d3" d="M195.7438,-595.1807C178.8708,-587.4308 159.461,-575.9889 146.5223,-560.4626 140.4195,-553.1393 135.9491,-544.0436 132.6887,-535.0233"/>
+ <polygon fill="#d3d3d3" stroke="#d3d3d3" points="135.9777,-533.8168 129.6057,-525.3523 129.3084,-535.943 135.9777,-533.8168"/>
+ <text text-anchor="middle" x="152.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> a</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge16" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M207.757,-584.2041C191.9107,-567.5159 170.5426,-545.0127 153.387,-526.9457"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="146.3209,-519.5041 156.4699,-523.6571 149.7638,-523.13 153.2067,-526.7558 153.2067,-526.7558 153.2067,-526.7558 149.7638,-523.13 149.9435,-529.8544 146.3209,-519.5041 146.3209,-519.5041"/>
+ <text text-anchor="middle" x="189.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M254.4007,-585.0186C272.3773,-568.0122 297.076,-544.6465 316.6218,-526.1556"/>
+ <polygon fill="#000000" stroke="#000000" points="319.2463,-528.4909 324.1054,-519.076 314.4357,-523.4058 319.2463,-528.4909"/>
+ <text text-anchor="middle" x="302.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M262.5613,-592.1209C303.3074,-572.2583 373.3978,-538.0911 417.9723,-516.3623"/>
+ <polygon fill="#000000" stroke="#000000" points="419.6217,-519.452 427.0769,-511.924 416.5544,-513.1598 419.6217,-519.452"/>
+ <text text-anchor="middle" x="361.9322" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- tr->t -->
+ <g id="edge41" class="edge">
+ <title>tr->t</title>
+ <path fill="none" stroke="#ffc0cb" d="M228.1399,-637.287C227.605,-648.2159 227.2816,-660.7205 227.6843,-672.0939 227.7703,-674.5242 227.889,-677.0451 228.0283,-679.5733"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="228.6875,-689.7931 223.5531,-680.1035 228.3656,-684.8034 228.0438,-679.8138 228.0438,-679.8138 228.0438,-679.8138 228.3656,-684.8034 232.5344,-679.5241 228.6875,-689.7931 228.6875,-689.7931"/>
+ <text text-anchor="middle" x="246.5132" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x07E</text>
+ </g>
+ <!-- tr->tra2 -->
+ <g id="edge28" class="edge">
+ <title>tr->tra2</title>
+ <path fill="none" stroke="#0000ff" d="M218.5572,-579.8196C215.264,-568.6049 213.1491,-555.5518 215.5223,-543.6626 216.1971,-540.2816 217.118,-536.8402 218.1951,-533.4336"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="221.4953,-534.5993 221.57,-524.0047 214.9047,-532.2403 221.4953,-534.5993"/>
+ <text text-anchor="middle" x="221.0942" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- t->root -->
+ <g id="edge42" class="edge">
+ <title>t->root</title>
+ <path fill="none" stroke="#ffc0cb" d="M235.0886,-726.2786C236.3802,-731.9426 237.5942,-738.2388 238.2369,-744.0939 239.1247,-752.1817 238.87,-760.8327 238.048,-769.081"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="236.7469,-779.2407 233.5537,-768.7501 237.3821,-774.2812 238.0173,-769.3217 238.0173,-769.3217 238.0173,-769.3217 237.3821,-774.2812 242.4808,-769.8934 236.7469,-779.2407 236.7469,-779.2407"/>
+ <text text-anchor="middle" x="257.9052" y="-748.2939" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09B</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M251.7729,-693.2308C258.3244,-687.4682 264.7179,-680.2751 268.2369,-672.0939 271.1872,-665.2348 270.7115,-662.3385 268.2369,-655.2939 266.2659,-649.6828 263.3202,-644.2436 259.9247,-639.1727"/>
+ <polygon fill="#000000" stroke="#000000" points="262.6469,-636.9677 253.8782,-631.0211 257.0247,-641.138 262.6469,-636.9677"/>
+ <text text-anchor="middle" x="275.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge15" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M213.324,-692.0138C208.3995,-686.2543 203.6543,-679.3722 201.0763,-672.0939 197.4827,-661.9481 199.7954,-651.1983 204.382,-641.4556"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="209.3576,-632.4758 208.4471,-643.4038 206.9342,-636.8493 204.5109,-641.2228 204.5109,-641.2228 204.5109,-641.2228 206.9342,-636.8493 200.5747,-639.0418 209.3576,-632.4758 209.3576,-632.4758"/>
+ <text text-anchor="middle" x="205.3172" y="-659.4939" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trav->tree -->
+ <!-- trave -->
+ <g id="node15" class="node">
+ <title>trave</title>
+ <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-284.4" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="202.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0B9</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge18" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#add8e6" d="M181.4692,-359.4476C176.9833,-352.6281 172.8815,-344.9707 170.5223,-337.2 168.3531,-330.0554 167.8843,-327.3851 170.5223,-320.4 172.1555,-316.0753 174.5712,-311.9338 177.3533,-308.0992"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="183.8676,-300.2431 180.9485,-310.8134 180.6761,-304.0921 177.4845,-307.941 177.4845,-307.941 177.4845,-307.941 180.6761,-304.0921 174.0205,-305.0686 183.8676,-300.2431 183.8676,-300.2431"/>
+ <text text-anchor="middle" x="176.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- trav->trave -->
+ <g id="edge31" class="edge">
+ <title>trav->trave</title>
+ <path fill="none" stroke="#0000ff" d="M202.2369,-355.0897C202.2369,-341.6046 202.2369,-325.7123 202.2369,-312.4153"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-312.4033 202.2369,-302.4033 198.737,-312.4034 205.737,-312.4033"/>
+ <text text-anchor="middle" x="207.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- trav->tra2 -->
+ <g id="edge39" class="edge">
+ <title>trav->tra2</title>
+ <path fill="none" stroke="#ffc0cb" d="M210.8211,-413.654C214.805,-427.1305 219.5957,-443.3363 223.8744,-457.8102"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="226.7386,-467.4991 219.5882,-459.1851 225.3211,-462.7042 223.9036,-457.9093 223.9036,-457.9093 223.9036,-457.9093 225.3211,-462.7042 228.219,-456.6336 226.7386,-467.4991 226.7386,-467.4991"/>
+ <text text-anchor="middle" x="239.9052" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B8</text>
+ </g>
+ <!-- trave->trav -->
+ <g id="edge38" class="edge">
+ <title>trave->trav</title>
+ <path fill="none" stroke="#ffc0cb" d="M208.9024,-302.4722C210.6782,-308.1317 212.3487,-314.4535 213.2369,-320.4 214.4724,-328.6713 214.0766,-337.5213 212.8888,-345.926"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="211.1036,-355.888 208.4382,-345.2509 211.9856,-350.9664 212.8676,-346.0448 212.8676,-346.0448 212.8676,-346.0448 211.9856,-350.9664 217.297,-346.8386 211.1036,-355.888 211.1036,-355.888"/>
+ <text text-anchor="middle" x="232.9052" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0B9</text>
+ </g>
+ <!-- traver -->
+ <g id="node16" class="node">
+ <title>traver</title>
+ <ellipse fill="none" stroke="#0000ff" cx="202.2369" cy="-195.6" rx="38.8671" ry="18"/>
+ <text text-anchor="middle" x="202.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BA</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge19" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#add8e6" d="M185.324,-268.32C180.3995,-262.5605 175.6543,-255.6784 173.0763,-248.4 170.5834,-241.3618 170.5834,-238.6382 173.0763,-231.6 174.4962,-227.5912 176.5735,-223.7026 178.9762,-220.0556"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="185.0164,-212.0415 182.5911,-222.7358 182.007,-216.0344 178.9975,-220.0273 178.9975,-220.0273 178.9975,-220.0273 182.007,-216.0344 175.4039,-217.3188 185.0164,-212.0415 185.0164,-212.0415"/>
+ <text text-anchor="middle" x="177.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- trave->traver -->
+ <g id="edge32" class="edge">
+ <title>trave->traver</title>
+ <path fill="none" stroke="#0000ff" d="M202.2369,-266.0006C202.2369,-253.8949 202.2369,-237.8076 202.2369,-224.0674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="205.737,-223.672 202.2369,-213.672 198.737,-223.6721 205.737,-223.672"/>
+ <text text-anchor="middle" x="206.3172" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- traver->trave -->
+ <g id="edge37" class="edge">
+ <title>traver->trave</title>
+ <path fill="none" stroke="#ffc0cb" d="M207.6937,-213.7509C209.1467,-219.4136 210.5127,-225.7174 211.2369,-231.6 212.1492,-239.0107 212.1492,-240.9893 211.2369,-248.4 210.9201,-250.9736 210.4804,-253.6279 209.9643,-256.2738"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="207.6937,-266.2491 205.5255,-255.4997 208.8035,-261.3738 209.9132,-256.4985 209.9132,-256.4985 209.9132,-256.4985 208.8035,-261.3738 214.301,-257.4973 207.6937,-266.2491 207.6937,-266.2491"/>
+ <text text-anchor="middle" x="232.4585" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BA</text>
+ </g>
+ <!-- travers -->
+ <g id="node17" class="node">
+ <title>travers</title>
+ <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-106.8" rx="38.305" ry="18"/>
+ <text text-anchor="middle" x="203.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">0x0BB</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge20" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#add8e6" d="M183.9854,-179.6658C174.6965,-169.5397 166.3137,-155.9116 171.2923,-142.8 172.9385,-138.4645 175.3723,-134.3169 178.1749,-130.4793"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="184.7367,-122.6213 181.7812,-133.1814 181.5319,-126.4592 178.3271,-130.2971 178.3271,-130.2971 178.3271,-130.2971 181.5319,-126.4592 174.873,-127.4128 184.7367,-122.6213 184.7367,-122.6213"/>
+ <text text-anchor="middle" x="176.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+ </g>
+ <!-- traver->travers -->
+ <g id="edge33" class="edge">
+ <title>traver->travers</title>
+ <path fill="none" stroke="#0000ff" d="M202.4441,-177.2006C202.5804,-165.0949 202.7616,-149.0076 202.9163,-135.2674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="206.4205,-134.9108 203.0334,-124.872 199.4209,-134.8319 206.4205,-134.9108"/>
+ <text text-anchor="middle" x="207.7092" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> s</text>
+ </g>
+ <!-- travers->traver -->
+ <g id="edge36" class="edge">
+ <title>travers->traver</title>
+ <path fill="none" stroke="#ffc0cb" d="M209.0666,-124.8376C211.6638,-135.0186 213.8193,-147.9908 212.2369,-159.6 211.8717,-162.2793 211.3602,-165.0407 210.7599,-167.7863"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="208.2983,-177.4866 206.3963,-166.6869 209.5282,-172.6402 210.758,-167.7938 210.758,-167.7938 210.758,-167.7938 209.5282,-172.6402 215.1198,-168.9007 208.2983,-177.4866 208.2983,-177.4866"/>
+ <text text-anchor="middle" x="233.0735" y="-147" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BB</text>
+ </g>
+ <!-- traverse -->
+ <g id="node18" class="node">
+ <title>traverse</title>
+ <ellipse fill="none" stroke="#0000ff" cx="203.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="203.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#0000ff">contentArray[3]</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge21" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#add8e6" d="M184.8676,-90.9569C179.5135,-85.2132 174.3474,-78.2805 171.5223,-70.8 168.8843,-63.8149 168.8843,-60.9851 171.5223,-54 172.8668,-50.4399 174.7415,-47.004 176.9144,-43.7612"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="183.203,-35.676 180.6155,-46.3323 180.1332,-39.6227 177.0634,-43.5695 177.0634,-43.5695 177.0634,-43.5695 180.1332,-39.6227 173.5114,-40.8067 183.203,-35.676 183.203,-35.676"/>
+ <text text-anchor="middle" x="177.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- travers->traverse -->
+ <g id="edge34" class="edge">
+ <title>travers->traverse</title>
+ <path fill="none" stroke="#0000ff" d="M203.2369,-88.4006C203.2369,-76.2949 203.2369,-60.2076 203.2369,-46.4674"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="206.737,-46.072 203.2369,-36.072 199.737,-46.0721 206.737,-46.072"/>
+ <text text-anchor="middle" x="208.0942" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- traverse->travers -->
+ <g id="edge35" class="edge">
+ <title>traverse->travers</title>
+ <path fill="none" stroke="#ffc0cb" d="M209.9024,-36.0722C211.6782,-41.7317 213.3487,-48.0535 214.2369,-54 215.3399,-61.3847 215.3399,-63.4153 214.2369,-70.8 213.8344,-73.4945 213.2713,-76.2661 212.6106,-79.0181"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="209.9024,-88.7278 208.2546,-77.8864 211.2458,-83.9116 212.5891,-79.0954 212.5891,-79.0954 212.5891,-79.0954 211.2458,-83.9116 216.9237,-80.3044 209.9024,-88.7278 209.9024,-88.7278"/>
+ <text text-anchor="middle" x="224.2732" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#ff0000"> ~3</text>
+ </g>
+ <!-- tra2->trac -->
+ <g id="edge29" class="edge">
+ <title>tra2->trac</title>
+ <path fill="none" stroke="#0000ff" d="M208.0507,-474.7233C183.1741,-455.0282 146.5501,-426.0326 121.6137,-406.2901"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="123.6518,-403.4396 113.6389,-399.9764 119.3067,-408.9278 123.6518,-403.4396"/>
+ <text text-anchor="middle" x="178.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- tra2->tre -->
+ <!-- tra2->tr -->
+ <g id="edge40" class="edge">
+ <title>tra2->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M233.9039,-526.0074C233.3268,-538.8926 232.6427,-554.1663 232.0226,-568.0109"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="231.5637,-578.2562 227.5157,-568.0648 231.7874,-573.2612 232.0112,-568.2662 232.0112,-568.2662 232.0112,-568.2662 231.7874,-573.2612 236.5067,-568.4676 231.5637,-578.2562 231.5637,-578.2562"/>
+ <text text-anchor="middle" x="253.0665" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0DE</text>
+ </g>
+ <!-- tra2->trav -->
+ <g id="edge30" class="edge">
+ <title>tra2->trav</title>
+ <path fill="none" stroke="#0000ff" d="M214.7335,-471.0228C210.2756,-464.2044 206.1695,-456.5615 203.7369,-448.8313 201.3127,-441.1275 200.14,-432.6433 199.7048,-424.4366"/>
+ <polygon fill="#0000ff" stroke="#0000ff" points="203.201,-424.2005 199.5092,-414.2697 196.2023,-424.3352 203.201,-424.2005"/>
+ <text text-anchor="middle" x="209.4869" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> v</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.p1.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.p1.svg
new file mode 100644
index 0000000..c89c085
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.p1.svg
@@ -0,0 +1,405 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+digraph G {
+ { rank=same root -> start [style=invis] }
+ newrank = true
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"; color = "lightgrey"; fontcolor = lightgray]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"; color = "lightgrey"; fontcolor = lightgray]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ {
+ ranksep = 0.1
+ tree2 [label = "Chain\n0x0BB"]
+ tree2p [label = "Prefix\n0x0BF\ncontentArray[1]"]
+ tree2p -> tree2 [label = " ε"]
+ }
+
+ tre -> tree2p [label = " e"]
+ tree2 -> trees [label = " s"]
+
+ {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> tree2p -> trie [style=invis]}
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+ node [color = "lightblue"; fontcolor="blue"]
+
+ start -> root
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+ tree -> trees [label = " s"]
+
+ trees [label = "contentArray[3]"; constraint = false]
+ }
+
+
+ subgraph back {
+ edge [color = "pink"; fontcolor="red"; arrowhead="vee"; constraint="false"]
+
+ trees -> tree2 [label = " ~3"]
+ tree2 -> tree2p [label = "0x0BB"]
+ tree2p -> tre [label = "0x0BF"]
+ tre -> tr [label = "0x03B"]
+ tr -> t [label = "0x07E"]
+ t -> root [label = "0x09B"]
+ root -> start [label = "0x09A"]
+ }
+}
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="666pt" height="678pt"
+ viewBox="0.00 0.00 666.47 677.52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 673.5152)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-673.5152 662.4738,-673.5152 662.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="264.2369" cy="-640.0996" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="264.2369" y="-644.2996" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="264.2369" y="-627.4996" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="397.2369" cy="-640.0996" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="397.2369" y="-635.8996" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- root->start -->
+ <g id="edge33" class="edge">
+ <title>root->start</title>
+ <path fill="none" stroke="#ffc0cb" d="M297.7984,-640.0996C311.3037,-640.0996 327.2339,-640.0996 342.3398,-640.0996"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="352.6261,-640.0996 342.6262,-644.5997 347.6261,-640.0996 342.6261,-640.0997 342.6261,-640.0997 342.6261,-640.0997 347.6261,-640.0996 342.6261,-635.5997 352.6261,-640.0996 352.6261,-640.0996"/>
+ <text text-anchor="middle" x="325.2509" y="-647.2996" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09A</text>
+ </g>
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="264.2369" cy="-557.8839" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="264.2369" y="-553.6839" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M253.1918,-612.3067C251.4263,-605.2505 250.368,-597.5734 252.6036,-585.3125"/>
+ <polygon fill="#000000" stroke="#000000" points="256.0259,-586.0473 254.8506,-575.5179 249.2031,-584.4821 256.0259,-586.0473"/>
+ <text text-anchor="middle" x="255.9322" y="-589.0839" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge22" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M264.2369,-610.4178C264.2369,-602.5756 264.2369,-594.1307 264.2369,-586.3542"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="264.2369,-576.2835 268.737,-586.2835 264.2369,-581.2835 264.237,-586.2835 264.237,-586.2835 264.237,-586.2835 264.2369,-581.2835 259.737,-586.2836 264.2369,-576.2835 264.2369,-576.2835"/>
+ <text text-anchor="middle" x="267.9322" y="-589.0839" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge21" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M353.4042,-637.4233C341.2938,-636.9719 328.1479,-636.7485 316.0184,-637.0996 313.3264,-637.1775 310.5568,-637.2781 307.7643,-637.3951"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="297.6504,-637.8791 307.4239,-632.9062 302.6447,-637.6401 307.639,-637.401 307.639,-637.401 307.639,-637.401 302.6447,-637.6401 307.8541,-641.8959 297.6504,-637.8791 297.6504,-637.8791"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-88.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-84.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge8" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-70.5672C73.2369,-63.2743 73.2369,-54.6987 73.2369,-46.6137"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-46.417 73.2369,-36.417 69.737,-46.4171 76.737,-46.417"/>
+ <text text-anchor="middle" x="77.3172" y="-49.2" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-171.0156" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-166.8156" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge7" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-152.7693C73.2369,-142.338 73.2369,-129.027 73.2369,-117.2514"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-116.9953 73.2369,-106.9953 69.737,-116.9953 76.737,-116.9953"/>
+ <text text-anchor="middle" x="78.4869" y="-120" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-276.5263" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-272.3263" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge6" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-258.1478C73.2369,-241.8835 73.2369,-217.8982 73.2369,-199.2081"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-199.1836 73.2369,-189.1836 69.737,-199.1836 76.737,-199.1836"/>
+ <text text-anchor="middle" x="76.9322" y="-213.6313" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#d3d3d3" cx="217.2369" cy="-276.5263" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-272.3263" font-family="Times,serif" font-size="14.00" fill="#d3d3d3">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="141.2369" cy="-382.037" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="141.2369" y="-386.237" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="141.2369" y="-369.437" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge5" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#000000" d="M124.4279,-355.9557C113.8753,-339.582 100.3244,-318.5561 89.8158,-302.2506"/>
+ <polygon fill="#000000" stroke="#000000" points="92.6883,-300.2467 84.329,-293.7372 86.8044,-304.0388 92.6883,-300.2467"/>
+ <text text-anchor="middle" x="121.0942" y="-331.0214" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="250.2369" cy="-382.037" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="250.2369" y="-386.237" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="250.2369" y="-369.437" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tra->tre -->
+ <!-- tree2p -->
+ <g id="node15" class="node">
+ <title>tree2p</title>
+ <ellipse fill="none" stroke="#000000" cx="401.2369" cy="-276.5263" rx="75.1528" ry="41.0911"/>
+ <text text-anchor="middle" x="401.2369" y="-289.1263" font-family="Times,serif" font-size="14.00" fill="#000000">Prefix</text>
+ <text text-anchor="middle" x="401.2369" y="-272.3263" font-family="Times,serif" font-size="14.00" fill="#000000">0x0BF</text>
+ <text text-anchor="middle" x="401.2369" y="-255.5263" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- tree->tree2p -->
+ <!-- trees -->
+ <g id="node16" class="node">
+ <title>trees</title>
+ <ellipse fill="none" stroke="#000000" cx="295.2369" cy="-88.8" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="295.2369" y="-84.6" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[3]</text>
+ </g>
+ <!-- tree->trees -->
+ <g id="edge26" class="edge">
+ <title>tree->trees</title>
+ <path fill="none" stroke="#add8e6" d="M224.7159,-258.5263C238.4018,-225.5876 267.5561,-155.4207 283.7655,-116.4088"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="287.7806,-106.7455 288.0991,-117.7068 285.8621,-111.3628 283.9435,-115.9801 283.9435,-115.9801 283.9435,-115.9801 285.8621,-111.3628 279.788,-114.2534 287.7806,-106.7455 287.7806,-106.7455"/>
+ <text text-anchor="middle" x="277.7092" y="-166.8156" font-family="Times,serif" font-size="14.00" fill="#0000ff"> s</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#d3d3d3" d="M227.0349,-358.8106C223.5725,-354.1203 220.5044,-348.9742 218.5223,-343.6214 214.0163,-331.453 213.1878,-317.1731 213.6897,-304.9221"/>
+ <polygon fill="#d3d3d3" stroke="#d3d3d3" points="217.1925,-305.0065 214.4144,-294.7824 210.2104,-304.5074 217.1925,-305.0065"/>
+ <text text-anchor="middle" x="224.0942" y="-331.0214" font-family="Times,serif" font-size="14.00" fill="#d3d3d3"> e</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge25" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#add8e6" d="M241.2195,-353.2057C236.4749,-338.0358 230.683,-319.5176 225.9798,-304.4799"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="222.9239,-294.7093 230.2038,-302.9101 224.4164,-299.4813 225.909,-304.2534 225.909,-304.2534 225.909,-304.2534 224.4164,-299.4813 221.6142,-305.5967 222.9239,-294.7093 222.9239,-294.7093"/>
+ <text text-anchor="middle" x="243.0942" y="-331.0214" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="361.2369" cy="-382.037" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="361.2369" y="-386.237" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="361.2369" y="-369.437" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tre->tri -->
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="264.2369" cy="-475.6683" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="264.2369" y="-479.8683" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="264.2369" y="-463.0683" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tre->tr -->
+ <g id="edge30" class="edge">
+ <title>tre->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M220.1451,-399.8185C206.9606,-410.0222 196.0135,-423.5087 203.5777,-437.2526 207.8544,-445.0234 214.4401,-451.4422 221.7208,-456.6604"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="230.4902,-462.2435 219.638,-460.6689 226.2725,-459.5582 222.0547,-456.8729 222.0547,-456.8729 222.0547,-456.8729 226.2725,-459.5582 224.4715,-453.0769 230.4902,-462.2435 230.4902,-462.2435"/>
+ <text text-anchor="middle" x="224.0665" y="-424.6526" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x03B</text>
+ </g>
+ <!-- tre->tree2p -->
+ <g id="edge14" class="edge">
+ <title>tre->tree2p</title>
+ <path fill="none" stroke="#000000" d="M281.9441,-365.9135C292.0842,-361.18 303.4736,-356.2938 314.2369,-352.6214 331.0728,-346.877 338.4386,-353.4933 353.2369,-343.6214 360.9367,-338.4848 367.8078,-331.6659 373.7781,-324.3413"/>
+ <polygon fill="#000000" stroke="#000000" points="376.657,-326.3371 379.8994,-316.2506 371.0746,-322.1136 376.657,-326.3371"/>
+ <text text-anchor="middle" x="377.0942" y="-331.0214" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="585.2369" cy="-276.5263" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="585.2369" y="-272.3263" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M393.8084,-366.6948C432.9219,-348.2712 498.6628,-317.3053 541.999,-296.8927"/>
+ <polygon fill="#000000" stroke="#000000" points="543.6019,-300.0065 551.1571,-292.5789 540.619,-293.6739 543.6019,-300.0065"/>
+ <text text-anchor="middle" x="478.0942" y="-331.0214" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge4" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#000000" d="M230.3862,-462.3473C216.3498,-455.9318 200.378,-447.4106 187.5223,-437.2526 179.7381,-431.102 172.3346,-423.4626 165.8229,-415.8473"/>
+ <polygon fill="#000000" stroke="#000000" points="168.1714,-413.1902 159.1314,-407.6649 162.7526,-417.6217 168.1714,-413.1902"/>
+ <text text-anchor="middle" x="193.0942" y="-424.6526" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M262.3234,-446.0575C261.7354,-439.4979 260.9854,-432.4951 259.2059,-421.283"/>
+ <polygon fill="#000000" stroke="#000000" points="262.641,-420.6036 257.5414,-411.3168 255.7366,-421.7567 262.641,-420.6036"/>
+ <text text-anchor="middle" x="267.0942" y="-424.6526" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge24" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#add8e6" d="M251.4368,-447.9439C250.2405,-444.4076 249.2174,-440.7879 248.5223,-437.2526 247.53,-432.2063 247.0287,-426.8513 246.8562,-421.5383"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="246.8871,-411.4654 251.3563,-421.4792 246.8717,-416.4654 246.8563,-421.4654 246.8563,-421.4654 246.8563,-421.4654 246.8717,-416.4654 242.3563,-421.4516 246.8871,-411.4654 246.8871,-411.4654"/>
+ <text text-anchor="middle" x="254.0942" y="-424.6526" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M288.2144,-452.5235C300.8059,-440.3693 316.39,-425.3264 329.9234,-412.263"/>
+ <polygon fill="#000000" stroke="#000000" points="332.4349,-414.7033 337.199,-405.2401 327.5733,-409.6669 332.4349,-414.7033"/>
+ <text text-anchor="middle" x="325.9322" y="-424.6526" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- tr->t -->
+ <g id="edge31" class="edge">
+ <title>tr->t</title>
+ <path fill="none" stroke="#ffc0cb" d="M264.2369,-505.1649C264.2369,-513.0799 264.2369,-521.6204 264.2369,-529.4751"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="264.2369,-539.6376 259.737,-529.6376 264.2369,-534.6376 264.237,-529.6376 264.237,-529.6376 264.237,-529.6376 264.2369,-534.6376 268.737,-529.6377 264.2369,-539.6376 264.2369,-539.6376"/>
+ <text text-anchor="middle" x="282.5132" y="-518.2839" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x07E</text>
+ </g>
+ <!-- t->root -->
+ <g id="edge32" class="edge">
+ <title>t->root</title>
+ <path fill="none" stroke="#ffc0cb" d="M272.5745,-575.5942C273.6774,-578.6326 274.6336,-581.8043 275.2369,-584.8839 276.3056,-590.3394 276.3564,-596.0962 275.789,-601.7512"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="274.1862,-611.733 271.3286,-601.146 274.9789,-606.7962 275.7717,-601.8595 275.7717,-601.8595 275.7717,-601.8595 274.9789,-606.7962 280.2148,-602.573 274.1862,-611.733 274.1862,-611.733"/>
+ <text text-anchor="middle" x="294.9052" y="-589.0839" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x09B</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M254.2705,-540.2954C252.9499,-537.2423 251.8032,-534.0343 251.0763,-530.8839 249.8023,-525.3626 249.7646,-519.536 250.472,-513.8197"/>
+ <polygon fill="#000000" stroke="#000000" points="253.9589,-514.2255 252.4441,-503.7395 247.0891,-512.8814 253.9589,-514.2255"/>
+ <text text-anchor="middle" x="255.3172" y="-518.2839" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge23" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M296.3684,-548.8052C304.9495,-544.7547 313.2223,-538.9976 318.2369,-530.8839 325.8698,-518.5336 317.3847,-506.7525 305.051,-497.2687"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="296.521,-491.3854 307.3079,-493.3588 300.6369,-494.2243 304.7529,-497.0631 304.7529,-497.0631 304.7529,-497.0631 300.6369,-494.2243 302.1979,-500.7675 296.521,-491.3854 296.521,-491.3854"/>
+ <text text-anchor="middle" x="325.3172" y="-518.2839" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- tree2 -->
+ <g id="node14" class="node">
+ <title>tree2</title>
+ <ellipse fill="none" stroke="#000000" cx="354.2369" cy="-171.0156" rx="39.2342" ry="29.3315"/>
+ <text text-anchor="middle" x="354.2369" y="-175.2156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="354.2369" y="-158.4156" font-family="Times,serif" font-size="14.00" fill="#000000">0x0BB</text>
+ </g>
+ <!-- tree2->tree2p -->
+ <g id="edge28" class="edge">
+ <title>tree2->tree2p</title>
+ <path fill="none" stroke="#ffc0cb" d="M332.5998,-195.7225C326.8044,-205.3439 323.4112,-216.3994 328.5637,-226.2313 331.3469,-231.5422 334.9738,-236.4037 339.122,-240.8292"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="346.5383,-247.8677 336.1871,-244.2478 342.9116,-244.4257 339.2849,-240.9838 339.2849,-240.9838 339.2849,-240.9838 342.9116,-244.4257 342.3827,-237.7197 346.5383,-247.8677 346.5383,-247.8677"/>
+ <text text-anchor="middle" x="349.0735" y="-213.6313" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BB</text>
+ </g>
+ <!-- tree2->trees -->
+ <g id="edge15" class="edge">
+ <title>tree2->trees</title>
+ <path fill="none" stroke="#000000" d="M335.5689,-145.0021C328.6935,-135.4213 320.9064,-124.5701 314.0208,-114.9751"/>
+ <polygon fill="#000000" stroke="#000000" points="316.6766,-112.6728 308.0027,-106.5889 310.9895,-116.7541 316.6766,-112.6728"/>
+ <text text-anchor="middle" x="331.7092" y="-120" font-family="Times,serif" font-size="14.00" fill="#000000"> s</text>
+ </g>
+ <!-- tree2p->tre -->
+ <g id="edge29" class="edge">
+ <title>tree2p->tre</title>
+ <path fill="none" stroke="#ffc0cb" d="M346.159,-304.6442C334.2868,-311.4197 322.0324,-318.9824 311.1177,-326.8214 300.7955,-334.2347 290.2869,-343.1729 280.9708,-351.6776"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="273.4561,-358.6766 277.7068,-348.5681 277.1149,-355.2689 280.7738,-351.8611 280.7738,-351.8611 280.7738,-351.8611 277.1149,-355.2689 283.8408,-355.1541 273.4561,-358.6766 273.4561,-358.6766"/>
+ <text text-anchor="middle" x="330.2965" y="-331.0214" font-family="Times,serif" font-size="14.00" fill="#ff0000">0x0BF</text>
+ </g>
+ <!-- tree2p->trie -->
+ <!-- tree2p->tree2 -->
+ <g id="edge13" class="edge">
+ <title>tree2p->tree2</title>
+ <path fill="none" stroke="#000000" d="M383.3341,-236.3363C379.2138,-227.0866 374.8498,-217.2898 370.7898,-208.1754"/>
+ <polygon fill="#000000" stroke="#000000" points="373.9581,-206.6863 366.6918,-198.9757 367.5638,-209.5346 373.9581,-206.6863"/>
+ <text text-anchor="middle" x="383.0672" y="-213.6313" font-family="Times,serif" font-size="14.00" fill="#000000"> ε</text>
+ </g>
+ <!-- trees->tree2 -->
+ <g id="edge27" class="edge">
+ <title>trees->tree2</title>
+ <path fill="none" stroke="#ffc0cb" d="M290.8236,-106.8267C289.7499,-115.1419 289.9269,-124.8606 294.1643,-132.6 298.1354,-139.8531 304.124,-145.9366 310.807,-150.9671"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="319.5438,-156.7845 308.7261,-154.9878 315.382,-154.0133 311.2202,-151.2421 311.2202,-151.2421 311.2202,-151.2421 315.382,-154.0133 313.7143,-147.4965 319.5438,-156.7845 319.5438,-156.7845"/>
+ <text text-anchor="middle" x="303.2732" y="-120" font-family="Times,serif" font-size="14.00" fill="#ff0000"> ~3</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w1.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w1.svg
new file mode 100644
index 0000000..1be94ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w1.svg
@@ -0,0 +1,226 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+``` plantuml
+digraph G {
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ // edge [color="none", fontcolor="none"]
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+}
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="480pt" height="734pt"
+ viewBox="0.00 0.00 480.47 734.09" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 730.0939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-730.0939 476.4738,-730.0939 476.4738,4 -4,4"/>
+ <!-- tractor -->
+ <g id="node1" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node2" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge7" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-88.4006C73.2369,-76.2949 73.2369,-60.2076 73.2369,-46.4674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-46.072 73.2369,-36.072 69.737,-46.0721 76.737,-46.072"/>
+ <text text-anchor="middle" x="77.3172" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node3" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge6" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-177.2006C73.2369,-165.0949 73.2369,-149.0076 73.2369,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-134.872 73.2369,-124.872 69.737,-134.8721 76.737,-134.872"/>
+ <text text-anchor="middle" x="78.4869" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node4" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge5" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-266.0006C73.2369,-253.8949 73.2369,-237.8076 73.2369,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-223.672 73.2369,-213.672 69.737,-223.6721 76.737,-223.672"/>
+ <text text-anchor="middle" x="76.9322" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- tree -->
+ <g id="node6" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node5" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="112.2369" cy="-384.6156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="112.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="112.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge4" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#000000" d="M101.2694,-356.4333C95.8306,-342.4575 89.2887,-325.6472 83.8929,-311.7821"/>
+ <polygon fill="#000000" stroke="#000000" points="87.0872,-310.3393 80.1988,-302.2895 80.5638,-312.878 87.0872,-310.3393"/>
+ <text text-anchor="middle" x="98.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- trie -->
+ <g id="node8" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="399.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="399.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tre -->
+ <g id="node7" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge9" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-355.0897C217.2369,-341.6046 217.2369,-325.7123 217.2369,-312.4153"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-312.4033 217.2369,-302.4033 213.737,-312.4034 220.737,-312.4033"/>
+ <text text-anchor="middle" x="222.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri -->
+ <g id="node9" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="354.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="354.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="354.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge11" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M366.7719,-356.7C373.1109,-342.583 380.7723,-325.5209 387.0603,-311.5175"/>
+ <polygon fill="#000000" stroke="#000000" points="390.2575,-312.9415 391.1609,-302.3852 383.8717,-310.074 390.2575,-312.9415"/>
+ <text text-anchor="middle" x="386.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr -->
+ <g id="node10" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-496.2469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="217.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge3" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#000000" d="M194.7118,-472.2992C179.1047,-455.7065 158.1828,-433.4633 141.317,-415.5323"/>
+ <polygon fill="#000000" stroke="#000000" points="143.7653,-413.0267 134.3644,-408.1406 138.6664,-417.8227 143.7653,-413.0267"/>
+ <text text-anchor="middle" x="176.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge8" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-466.6249C217.2369,-453.7568 217.2369,-438.4867 217.2369,-424.6319"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-424.3761 217.2369,-414.3761 213.737,-424.3762 220.737,-424.3761"/>
+ <text text-anchor="middle" x="222.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge10" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M243.328,-474.9872C264.9956,-457.3319 295.9963,-432.0716 319.7169,-412.7435"/>
+ <polygon fill="#000000" stroke="#000000" points="322.2348,-415.2066 327.7763,-406.1765 317.813,-409.78 322.2348,-415.2066"/>
+ <text text-anchor="middle" x="296.9322" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- t -->
+ <g id="node11" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-596.4626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-592.2626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge2" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-578.0815C217.2369,-566.3502 217.2369,-550.6774 217.2369,-536.1885"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-535.9208 217.2369,-525.9209 213.737,-535.9209 220.737,-535.9208"/>
+ <text text-anchor="middle" x="221.3172" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- root -->
+ <g id="node12" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-696.6782" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-700.8782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-684.0782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge1" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-667.1522C217.2369,-653.6671 217.2369,-637.7749 217.2369,-624.4779"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-624.4659 217.2369,-614.4659 213.737,-624.4659 220.737,-624.4659"/>
+ <text text-anchor="middle" x="220.9322" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w2.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w2.svg
new file mode 100644
index 0000000..9d8ab22
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w2.svg
@@ -0,0 +1,326 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+```plantuml
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+
+ start -> root
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+ }
+
+ subgraph back {
+ edge [color = "pink"; fontcolor="blue"; constraint="false"; arrowhead="vee"]
+
+ tractor -> tracto -> tract -> trac -> tra -> tr
+ tree -> tre -> tr
+ trie -> tri -> tr -> t -> root -> start
+ }
+}
+
+```
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="480pt" height="734pt"
+ viewBox="0.00 0.00 480.47 734.09" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 730.0939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-730.0939 476.4738,-730.0939 476.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-696.6782" rx="33.1337" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-700.8782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-684.0782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="330.2369" cy="-696.6782" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="330.2369" y="-692.4782" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- root->start -->
+ <g id="edge27" class="edge">
+ <title>root->start</title>
+ <path fill="none" stroke="#ffc0cb" d="M245.0455,-713.0508C262.0796,-719.8061 279.1137,-720.5455 296.1478,-715.2689"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="305.6112,-711.7185 297.8292,-719.4444 300.9299,-713.4748 296.2485,-715.2312 296.2485,-715.2312 296.2485,-715.2312 300.9299,-713.4748 294.6678,-711.0179 305.6112,-711.7185 305.6112,-711.7185"/>
+ </g>
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-596.4626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-592.2626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge5" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-667.1522C217.2369,-653.6671 217.2369,-637.7749 217.2369,-624.4779"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-614.4659 221.737,-624.4659 217.2369,-619.4659 217.237,-624.4659 217.237,-624.4659 217.237,-624.4659 217.2369,-619.4659 212.737,-624.4659 217.2369,-614.4659 217.2369,-614.4659"/>
+ <text text-anchor="middle" x="220.9322" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge4" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M285.8549,-696.6782C277.5437,-696.6782 269.2326,-696.6782 260.9215,-696.6782"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="250.7838,-696.6782 260.7838,-692.1783 255.7838,-696.6783 260.7838,-696.6783 260.7838,-696.6783 260.7838,-696.6783 255.7838,-696.6783 260.7837,-701.1783 250.7838,-696.6782 250.7838,-696.6782"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tractor->tracto -->
+ <g id="edge16" class="edge">
+ <title>tractor->tracto</title>
+ <path fill="none" stroke="#ffc0cb" d="M78.6937,-36.1509C80.1467,-41.8136 81.5127,-48.1174 82.2369,-54 83.1492,-61.4107 83.1492,-63.3893 82.2369,-70.8 81.9201,-73.3736 81.4804,-76.0279 80.9643,-78.6738"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="78.6937,-88.6491 76.5255,-77.8997 79.8035,-83.7738 80.9132,-78.8985 80.9132,-78.8985 80.9132,-78.8985 79.8035,-83.7738 85.301,-79.8973 78.6937,-88.6491 78.6937,-88.6491"/>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge11" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#add8e6" d="M73.2369,-88.4006C73.2369,-76.2949 73.2369,-60.2076 73.2369,-46.4674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="73.2369,-36.072 77.737,-46.072 73.2369,-41.072 73.237,-46.072 73.237,-46.072 73.237,-46.072 73.2369,-41.072 68.737,-46.0721 73.2369,-36.072 73.2369,-36.072"/>
+ <text text-anchor="middle" x="77.3172" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tracto->tract -->
+ <g id="edge17" class="edge">
+ <title>tracto->tract</title>
+ <path fill="none" stroke="#ffc0cb" d="M80.3796,-124.4595C82.3635,-130.2097 84.2449,-136.6872 85.2369,-142.8 86.4329,-150.1703 86.4329,-152.2297 85.2369,-159.6 84.797,-162.3109 84.1821,-165.0936 83.461,-167.8527"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="80.5061,-177.5725 79.1093,-166.6959 81.9604,-172.7886 83.4148,-168.0048 83.4148,-168.0048 83.4148,-168.0048 81.9604,-172.7886 87.7202,-169.3137 80.5061,-177.5725 80.5061,-177.5725"/>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge10" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#add8e6" d="M73.2369,-177.2006C73.2369,-165.0949 73.2369,-149.0076 73.2369,-135.2674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="73.2369,-124.872 77.737,-134.872 73.2369,-129.872 73.237,-134.872 73.237,-134.872 73.237,-134.872 73.2369,-129.872 68.737,-134.8721 73.2369,-124.872 73.2369,-124.872"/>
+ <text text-anchor="middle" x="78.4869" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- tract->trac -->
+ <g id="edge18" class="edge">
+ <title>tract->trac</title>
+ <path fill="none" stroke="#ffc0cb" d="M78.0886,-213.7847C79.3802,-219.4487 80.5942,-225.7449 81.2369,-231.6 82.0516,-239.0221 82.0516,-240.9779 81.2369,-248.4 80.9557,-250.9616 80.5652,-253.6076 80.1066,-256.2482"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="78.0886,-266.2153 75.6626,-255.5211 79.0808,-261.3147 80.0731,-256.4142 80.0731,-256.4142 80.0731,-256.4142 79.0808,-261.3147 84.4836,-257.3072 78.0886,-266.2153 78.0886,-266.2153"/>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge9" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#add8e6" d="M73.2369,-266.0006C73.2369,-253.8949 73.2369,-237.8076 73.2369,-224.0674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="73.2369,-213.672 77.737,-223.672 73.2369,-218.672 73.237,-223.672 73.237,-223.672 73.237,-223.672 73.2369,-218.672 68.737,-223.6721 73.2369,-213.672 73.2369,-213.672"/>
+ <text text-anchor="middle" x="76.9322" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="112.2369" cy="-384.6156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="112.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="112.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- trac->tra -->
+ <g id="edge19" class="edge">
+ <title>trac->tra</title>
+ <path fill="none" stroke="#ffc0cb" d="M92.2023,-299.8107C98.0398,-305.6172 103.8226,-312.698 107.2369,-320.4 110.6198,-328.0312 112.455,-336.6721 113.3574,-345.0849"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="114.0369,-355.1505 108.8735,-345.4763 113.7001,-350.1618 113.3633,-345.1732 113.3633,-345.1732 113.3633,-345.1732 113.7001,-350.1618 117.853,-344.87 114.0369,-355.1505 114.0369,-355.1505"/>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra->trac -->
+ <g id="edge8" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#add8e6" d="M101.2694,-356.4333C95.8306,-342.4575 89.2887,-325.6472 83.8929,-311.7821"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="80.1988,-302.2895 88.0191,-309.9766 82.0121,-306.9491 83.8255,-311.6087 83.8255,-311.6087 83.8255,-311.6087 82.0121,-306.9491 79.6319,-313.2407 80.1988,-302.2895 80.1988,-302.2895"/>
+ <text text-anchor="middle" x="98.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> c</text>
+ </g>
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-496.2469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="217.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tra->tr -->
+ <g id="edge20" class="edge">
+ <title>tra->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M134.8244,-407.6467C142.3198,-415.356 150.6782,-424.0253 158.2369,-432.0313 168.0928,-442.4704 178.7426,-453.987 188.2457,-464.3504"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="195.2572,-472.0136 185.1867,-467.6734 191.882,-468.3247 188.5068,-464.6357 188.5068,-464.6357 188.5068,-464.6357 191.882,-468.3247 191.8268,-461.598 195.2572,-472.0136 195.2572,-472.0136"/>
+ </g>
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tree->tre -->
+ <g id="edge21" class="edge">
+ <title>tree->tre</title>
+ <path fill="none" stroke="#ffc0cb" d="M225.7116,-302.3279C227.9713,-307.9814 230.0993,-314.3363 231.2369,-320.4 232.8247,-328.8637 232.2596,-337.9179 230.673,-346.479"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="228.409,-356.2376 226.2855,-345.4793 229.5391,-351.3669 230.6691,-346.4963 230.6691,-346.4963 230.6691,-346.4963 229.5391,-351.3669 235.0527,-347.5133 228.409,-356.2376 228.409,-356.2376"/>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="399.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="399.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tre->tree -->
+ <g id="edge13" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-355.0897C217.2369,-341.6046 217.2369,-325.7123 217.2369,-312.4153"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-302.4033 221.737,-312.4033 217.2369,-307.4033 217.237,-312.4033 217.237,-312.4033 217.237,-312.4033 217.2369,-307.4033 212.737,-312.4034 217.2369,-302.4033 217.2369,-302.4033"/>
+ <text text-anchor="middle" x="222.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tre->tr -->
+ <g id="edge22" class="edge">
+ <title>tre->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M227.8923,-412.9079C230.9037,-424.04 232.9509,-436.9736 231.2369,-448.8313 230.8027,-451.835 230.2214,-454.9136 229.5418,-457.989"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="227.0968,-467.7057 225.1732,-456.9099 228.317,-462.8569 229.5371,-458.008 229.5371,-458.008 229.5371,-458.008 228.317,-462.8569 233.9011,-459.1062 227.0968,-467.7057 227.0968,-467.7057"/>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="326.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="326.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="326.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- trie->tri -->
+ <g id="edge23" class="edge">
+ <title>trie->tri</title>
+ <path fill="none" stroke="#ffc0cb" d="M393.8563,-302.4822C390.1724,-313.1558 384.6144,-326.606 377.2369,-337.2 372.5248,-343.9665 366.7036,-350.493 360.6987,-356.4272"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="353.0638,-363.6005 357.2704,-353.4736 356.7078,-360.1768 360.3518,-356.7531 360.3518,-356.7531 360.3518,-356.7531 356.7078,-360.1768 363.4331,-360.0327 353.0638,-363.6005 353.0638,-363.6005"/>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge15" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#add8e6" d="M342.2386,-357.7635C349.5562,-346.0182 358.5767,-332.2461 367.5223,-320.4 370.1199,-316.9602 372.9633,-313.4399 375.8469,-310.0124"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="382.5518,-302.2644 379.4108,-312.7708 379.2799,-306.0453 376.0081,-309.8261 376.0081,-309.8261 376.0081,-309.8261 379.2799,-306.0453 372.6053,-306.8815 382.5518,-302.2644 382.5518,-302.2644"/>
+ <text text-anchor="middle" x="372.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tri->tr -->
+ <g id="edge24" class="edge">
+ <title>tri->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M314.1697,-412.7968C308.1382,-424.7883 300.0242,-438.3853 290.2369,-448.8313 280.9326,-458.7618 269.1419,-467.5728 257.7651,-474.8231"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="249.1939,-480.056 255.3841,-471.0044 253.4614,-477.4506 257.729,-474.8451 257.729,-474.8451 257.729,-474.8451 253.4614,-477.4506 260.0739,-478.6859 249.1939,-480.056 249.1939,-480.056"/>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge7" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M186.1766,-479.0539C173.479,-471.0083 159.2794,-460.5991 148.5223,-448.8313 141.1239,-440.7377 134.615,-430.8736 129.2283,-421.329"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="124.4153,-412.3441 133.1041,-419.0341 126.7763,-416.7516 129.1373,-421.159 129.1373,-421.159 129.1373,-421.159 126.7763,-416.7516 125.1706,-423.284 124.4153,-412.3441 124.4153,-412.3441"/>
+ <text text-anchor="middle" x="154.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge12" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-466.6249C217.2369,-453.7568 217.2369,-438.4867 217.2369,-424.6319"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-414.3761 221.737,-424.3761 217.2369,-419.3761 217.237,-424.3761 217.237,-424.3761 217.237,-424.3761 217.2369,-419.3761 212.737,-424.3762 217.2369,-414.3761 217.2369,-414.3761"/>
+ <text text-anchor="middle" x="222.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge14" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#add8e6" d="M240.0869,-472.8454C256.2553,-456.2866 278.1111,-433.9032 295.7628,-415.8254"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="303.0416,-408.3709 299.275,-418.6696 299.5485,-411.9483 296.0553,-415.5258 296.0553,-415.5258 296.0553,-415.5258 299.5485,-411.9483 292.8356,-412.382 303.0416,-408.3709 303.0416,-408.3709"/>
+ <text text-anchor="middle" x="282.9322" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> i</text>
+ </g>
+ <!-- tr->t -->
+ <g id="edge25" class="edge">
+ <title>tr->t</title>
+ <path fill="none" stroke="#ffc0cb" d="M224.5566,-525.3096C226.4734,-536.3112 227.6536,-548.9542 226.2369,-560.4626 225.9201,-563.0362 225.4804,-565.6905 224.9643,-568.3364"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="222.6937,-578.3117 220.5255,-567.5623 223.8035,-573.4364 224.9132,-568.5611 224.9132,-568.5611 224.9132,-568.5611 223.8035,-573.4364 229.301,-569.5599 222.6937,-578.3117 222.6937,-578.3117"/>
+ </g>
+ <!-- t->root -->
+ <g id="edge26" class="edge">
+ <title>t->root</title>
+ <path fill="none" stroke="#ffc0cb" d="M222.0886,-614.6473C223.3802,-620.3113 224.5942,-626.6075 225.2369,-632.4626 226.1247,-640.5504 225.87,-649.2014 225.048,-657.4497"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="223.7469,-667.6094 220.5537,-657.1188 224.3821,-662.6499 225.0173,-657.6904 225.0173,-657.6904 225.0173,-657.6904 224.3821,-662.6499 229.4808,-658.2621 223.7469,-667.6094 223.7469,-667.6094"/>
+ </g>
+ <!-- t->tr -->
+ <g id="edge6" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-578.0815C217.2369,-566.3502 217.2369,-550.6774 217.2369,-536.1885"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-525.9209 221.737,-535.9208 217.2369,-530.9209 217.237,-535.9209 217.237,-535.9209 217.237,-535.9209 217.2369,-530.9209 212.737,-535.9209 217.2369,-525.9209 217.2369,-525.9209"/>
+ <text text-anchor="middle" x="221.3172" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w3.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w3.svg
new file mode 100644
index 0000000..2202ade
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.w3.svg
@@ -0,0 +1,269 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+# embedded DOT (plantuml) works in IDEA preview
+# but not on GitHub
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"]
+
+ start -> root
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+ }
+
+ subgraph back {
+ edge [color = "pink"; fontcolor="blue"; constraint="false"; arrowhead="vee"]
+ tractor -> tr
+ tree -> tr
+ trie -> start
+ }
+}
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="480pt" height="734pt"
+ viewBox="0.00 0.00 480.47 734.09" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 730.0939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-730.0939 476.4738,-730.0939 476.4738,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-696.6782" rx="34.9213" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-700.8782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-684.0782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="332.2369" cy="-696.6782" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="332.2369" y="-692.4782" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-596.4626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-592.2626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge3" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-667.1522C217.2369,-653.6671 217.2369,-637.7749 217.2369,-624.4779"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-614.4659 221.737,-624.4659 217.2369,-619.4659 217.237,-624.4659 217.237,-624.4659 217.237,-624.4659 217.2369,-619.4659 212.737,-624.4659 217.2369,-614.4659 217.2369,-614.4659"/>
+ <text text-anchor="middle" x="220.9322" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge2" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M307.4351,-711.6056C290.438,-719.0227 273.4409,-720.6565 256.4438,-716.507"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="246.4361,-713.3955 257.3213,-712.0674 251.2107,-714.88 255.9852,-716.3645 255.9852,-716.3645 255.9852,-716.3645 251.2107,-714.88 254.6492,-720.6616 246.4361,-713.3955 246.4361,-713.3955"/>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-496.2469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="217.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tractor->tr -->
+ <g id="edge16" class="edge">
+ <title>tractor->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M57.5135,-35.7428C46.6571,-49.2423 33.1507,-68.8431 27.2369,-88.8 26.2429,-92.1544 24.6138,-277.4611 29.2369,-302.4 32.9143,-322.2376 56.637,-420.9537 90.2369,-448.8313 113.2626,-467.9355 144.7374,-479.7585 170.592,-486.8207"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="180.558,-489.3874 169.7517,-491.2511 175.716,-488.1403 170.874,-486.8933 170.874,-486.8933 170.874,-486.8933 175.716,-488.1403 171.9963,-482.5355 180.558,-489.3874 180.558,-489.3874"/>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge9" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#add8e6" d="M73.2369,-88.4006C73.2369,-76.2949 73.2369,-60.2076 73.2369,-46.4674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="73.2369,-36.072 77.737,-46.072 73.2369,-41.072 73.237,-46.072 73.237,-46.072 73.237,-46.072 73.2369,-41.072 68.737,-46.0721 73.2369,-36.072 73.2369,-36.072"/>
+ <text text-anchor="middle" x="77.3172" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge8" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#add8e6" d="M73.2369,-177.2006C73.2369,-165.0949 73.2369,-149.0076 73.2369,-135.2674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="73.2369,-124.872 77.737,-134.872 73.2369,-129.872 73.237,-134.872 73.237,-134.872 73.237,-134.872 73.2369,-129.872 68.737,-134.8721 73.2369,-124.872 73.2369,-124.872"/>
+ <text text-anchor="middle" x="78.4869" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge7" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#add8e6" d="M73.2369,-266.0006C73.2369,-253.8949 73.2369,-237.8076 73.2369,-224.0674"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="73.2369,-213.672 77.737,-223.672 73.2369,-218.672 73.237,-223.672 73.237,-223.672 73.237,-223.672 73.2369,-218.672 68.737,-223.6721 73.2369,-213.672 73.2369,-213.672"/>
+ <text text-anchor="middle" x="76.9322" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> t</text>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="124.2369" cy="-384.6156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="124.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="124.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge6" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#add8e6" d="M110.3009,-357.2313C103.0163,-342.9169 94.1342,-325.4635 86.8914,-311.2313"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="82.1788,-301.9709 90.7249,-308.8422 84.4465,-306.4271 86.7143,-310.8832 86.7143,-310.8832 86.7143,-310.8832 84.4465,-306.4271 82.7038,-312.9242 82.1788,-301.9709 82.1788,-301.9709"/>
+ <text text-anchor="middle" x="104.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> c</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="399.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="399.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tree->tr -->
+ <g id="edge17" class="edge">
+ <title>tree->tr</title>
+ <path fill="none" stroke="#ffc0cb" d="M158.3327,-295.1942C129.7342,-301.5466 99.3858,-310.3083 89.2369,-320.4 77.8256,-331.747 81.481,-339.2647 79.2369,-355.2 75.5906,-381.0917 66.3187,-391.2981 79.2369,-414.0313 98.9655,-448.7494 139.7974,-470.3508 171.9533,-482.6472"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="181.6212,-486.1688 170.6849,-486.9744 176.9232,-484.4574 172.2251,-482.7461 172.2251,-482.7461 172.2251,-482.7461 176.9232,-484.4574 173.7653,-478.5179 181.6212,-486.1688 181.6212,-486.1688"/>
+ </g>
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge11" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-355.0897C217.2369,-341.6046 217.2369,-325.7123 217.2369,-312.4153"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-302.4033 221.737,-312.4033 217.2369,-307.4033 217.237,-312.4033 217.237,-312.4033 217.237,-312.4033 217.2369,-307.4033 212.737,-312.4034 217.2369,-302.4033 217.2369,-302.4033"/>
+ <text text-anchor="middle" x="222.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- trie->start -->
+ <g id="edge18" class="edge">
+ <title>trie->start</title>
+ <path fill="none" stroke="#ffc0cb" d="M398.5378,-302.5097C397.4092,-327.6379 394.7326,-374.4581 389.2369,-414.0313 376.1548,-508.2316 351.0965,-618.2848 339.0431,-668.779"/>
+ <polygon fill="#ffc0cb" stroke="#ffc0cb" points="336.6774,-678.6227 334.6388,-667.848 337.8458,-673.7612 339.0142,-668.8996 339.0142,-668.8996 339.0142,-668.8996 337.8458,-673.7612 343.3896,-669.9511 336.6774,-678.6227 336.6774,-678.6227"/>
+ </g>
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="342.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="342.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="342.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge13" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#add8e6" d="M351.1677,-355.794C355.3217,-344.3517 360.8359,-331.3387 367.5223,-320.4 369.7215,-316.8022 372.2911,-313.2194 374.9987,-309.7866"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="381.4573,-302.1119 378.4615,-312.6607 378.2379,-305.9375 375.0184,-309.7632 375.0184,-309.7632 375.0184,-309.7632 378.2379,-305.9375 371.5754,-306.8657 381.4573,-302.1119 381.4573,-302.1119"/>
+ <text text-anchor="middle" x="372.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge5" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M196.5972,-471.4723C183.2324,-455.43 165.6885,-434.3715 151.2227,-417.0077"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="144.8189,-409.321 154.6771,-414.1237 148.0193,-413.1625 151.2197,-417.0041 151.2197,-417.0041 151.2197,-417.0041 148.0193,-413.1625 147.7623,-419.8845 144.8189,-409.321 144.8189,-409.321"/>
+ <text text-anchor="middle" x="182.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge10" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-466.6249C217.2369,-453.7568 217.2369,-438.4867 217.2369,-424.6319"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-414.3761 221.737,-424.3761 217.2369,-419.3761 217.237,-424.3761 217.237,-424.3761 217.237,-424.3761 217.2369,-419.3761 212.737,-424.3762 217.2369,-414.3761 217.2369,-414.3761"/>
+ <text text-anchor="middle" x="222.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge12" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#add8e6" d="M241.0794,-473.3437C254.1439,-460.9437 270.7139,-445.4579 285.8463,-432.0313 293.0131,-425.6723 300.8138,-418.9751 308.2639,-412.682"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="316.1638,-406.0471 311.4004,-415.9244 312.335,-409.2628 308.5062,-412.4785 308.5062,-412.4785 308.5062,-412.4785 312.335,-409.2628 305.6121,-409.0326 316.1638,-406.0471 316.1638,-406.0471"/>
+ <text text-anchor="middle" x="288.9322" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> i</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge4" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M217.2369,-578.0815C217.2369,-566.3502 217.2369,-550.6774 217.2369,-536.1885"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="217.2369,-525.9209 221.737,-535.9208 217.2369,-530.9209 217.237,-535.9209 217.237,-535.9209 217.237,-535.9209 217.2369,-530.9209 212.737,-535.9209 217.2369,-525.9209 217.2369,-525.9209"/>
+ <text text-anchor="middle" x="221.3172" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> r</text>
+ </g>
+ </g>
+</svg>
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.wc1.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.wc1.svg
new file mode 100644
index 0000000..ae595f3
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.wc1.svg
@@ -0,0 +1,349 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+ - Licensed to the Apache Software Foundation (ASF) under one
+ - or more contributor license agreements. See the NOTICE file
+ - distributed with this work for additional information
+ - regarding copyright ownership. The ASF licenses this file
+ - to you under the Apache License, Version 2.0 (the
+ - "License"); you may not use this file except in compliance
+ - with the License. You may obtain a copy of the License at
+ -
+ - http://www.apache.org/licenses/LICENSE-2.0
+ -
+ - Unless required by applicable law or agreed to in writing, software
+ - distributed under the License is distributed on an "AS IS" BASIS,
+ - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ - See the License for the specific language governing permissions and
+ - limitations under the License.
+ -->
+
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"; constraint="false"]
+
+ start -> root [label = " 0, -1"]
+
+ root -> t [label = " 1, t"]
+ t -> tr [label = " 2, r"]
+ tr -> tra [label = " 3, a"]
+ tra -> trac [label = " 4, c"]
+ trac -> tract [label = " 5, t"]
+ tract -> tracto [label = " 6, o"]
+ tracto -> tractor [label = " 7, r"]
+
+ tractor -> tre [label = " 3, e"]
+
+ // tr -> tre [label = " e"]
+ tre -> tree [label = " 4, e"]
+
+ tree -> tri [label = "3, i"; ]
+
+ // tr -> tri [label = " i"]
+ tri -> trie [label = " 4, e"]
+
+ trie -> start [label = "-1, -1"]
+ }
+}
+
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="496pt" height="734pt"
+ viewBox="0.00 0.00 496.47 734.09" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 730.0939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-730.0939 492.4673,-730.0939 492.4673,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-696.6782" rx="34.9213" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-700.8782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-684.0782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="343.2369" cy="-696.6782" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="343.2369" y="-692.4782" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-596.4626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-592.2626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-667.1522C217.2369,-653.6671 217.2369,-637.7749 217.2369,-624.4779"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-624.4659 217.2369,-614.4659 213.737,-624.4659 220.737,-624.4659"/>
+ <text text-anchor="middle" x="220.9322" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge16" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#add8e6" d="M226.1036,-667.9505C228.4851,-656.8764 229.9747,-644.0969 228.2369,-632.4626 227.8344,-629.7681 227.2713,-626.9965 226.6106,-624.2444"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="223.9024,-614.5348 230.9237,-622.9581 225.2458,-619.351 226.5891,-624.1672 226.5891,-624.1672 226.5891,-624.1672 225.2458,-619.351 222.2546,-625.3762 223.9024,-614.5348 223.9024,-614.5348"/>
+ <text text-anchor="middle" x="239.9322" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 1, t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge15" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M298.8722,-696.6782C287.0707,-696.6782 274.3106,-696.6782 262.4637,-696.6782"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="252.3349,-696.6782 262.335,-692.1783 257.3349,-696.6783 262.3349,-696.6783 262.3349,-696.6783 262.3349,-696.6783 257.3349,-696.6783 262.3349,-701.1783 252.3349,-696.6782 252.3349,-696.6782"/>
+ <text text-anchor="middle" x="275.5731" y="-703.8782" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 0, -1</text>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="217.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tractor->tre -->
+ <g id="edge23" class="edge">
+ <title>tractor->tre</title>
+ <path fill="none" stroke="#add8e6" d="M123.782,-31.2033C214.7179,-58.0052 403.6425,-128.3147 481.2369,-266.4 489.075,-280.3486 492.1649,-290.7134 481.2369,-302.4 440.4686,-345.9983 254.0052,-276.8017 213.2369,-320.4 206.7937,-327.2904 205.037,-336.6232 205.5759,-346.0371"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="206.9485,-356.1722 201.1471,-346.8666 206.2775,-351.2174 205.6064,-346.2627 205.6064,-346.2627 205.6064,-346.2627 206.2775,-351.2174 210.0657,-345.6587 206.9485,-356.1722 206.9485,-356.1722"/>
+ <text text-anchor="middle" x="453.0942" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 3, e</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge8" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-88.4006C73.2369,-76.2949 73.2369,-60.2076 73.2369,-46.4674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-46.072 73.2369,-36.072 69.737,-46.0721 76.737,-46.072"/>
+ <text text-anchor="middle" x="77.3172" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge22" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#add8e6" d="M78.6937,-88.6491C80.1467,-82.9864 81.5127,-76.6826 82.2369,-70.8 83.1492,-63.3893 83.1492,-61.4107 82.2369,-54 81.9201,-51.4264 81.4804,-48.7721 80.9643,-46.1262"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="78.6937,-36.1509 85.301,-44.9027 79.8035,-41.0262 80.9132,-45.9015 80.9132,-45.9015 80.9132,-45.9015 79.8035,-41.0262 76.5255,-46.9003 78.6937,-36.1509 78.6937,-36.1509"/>
+ <text text-anchor="middle" x="94.3172" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 7, r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge7" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-177.2006C73.2369,-165.0949 73.2369,-149.0076 73.2369,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-134.872 73.2369,-124.872 69.737,-134.8721 76.737,-134.872"/>
+ <text text-anchor="middle" x="78.4869" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge21" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#add8e6" d="M80.5061,-177.5725C82.4432,-171.9148 84.266,-165.5828 85.2369,-159.6 86.4329,-152.2297 86.4329,-150.1703 85.2369,-142.8 84.7719,-139.9346 84.1115,-136.9891 83.3367,-134.0759"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="80.3796,-124.4595 87.6201,-132.6951 81.8492,-129.2387 83.3189,-134.0178 83.3189,-134.0178 83.3189,-134.0178 81.8492,-129.2387 79.0176,-135.3405 80.3796,-124.4595 80.3796,-124.4595"/>
+ <text text-anchor="middle" x="98.4869" y="-147" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 6, o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="73.2369" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="73.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge6" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M73.2369,-266.0006C73.2369,-253.8949 73.2369,-237.8076 73.2369,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="76.737,-223.672 73.2369,-213.672 69.737,-223.6721 76.737,-223.672"/>
+ <text text-anchor="middle" x="76.9322" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge20" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#add8e6" d="M79.7865,-266.6959C81.6052,-260.9434 83.3294,-254.4757 84.2369,-248.4 85.3399,-241.0153 85.3399,-238.9847 84.2369,-231.6 83.8344,-228.9055 83.2713,-226.1339 82.6106,-223.3819"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="79.9024,-213.6722 86.9237,-222.0956 81.2458,-218.4884 82.5891,-223.3046 82.5891,-223.3046 82.5891,-223.3046 81.2458,-218.4884 78.2546,-224.5136 79.9024,-213.6722 79.9024,-213.6722"/>
+ <text text-anchor="middle" x="95.9322" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 5, t</text>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="217.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="123.2369" cy="-384.6156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="123.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="123.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge5" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#000000" d="M102.7638,-360.0748C97.5807,-353.0088 92.3929,-345.074 88.5223,-337.2 84.7334,-329.4922 81.7193,-320.678 79.4036,-312.4881"/>
+ <polygon fill="#000000" stroke="#000000" points="82.7327,-311.3819 76.8381,-302.5783 75.9561,-313.1364 82.7327,-311.3819"/>
+ <text text-anchor="middle" x="94.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge19" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#add8e6" d="M113.8009,-356.1612C109.6057,-344.7723 104.2515,-331.7001 98.2369,-320.4 96.3599,-316.8736 94.2013,-313.2795 91.9556,-309.7938"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="86.3046,-301.4527 95.6391,-307.2076 89.1091,-305.5922 91.9136,-309.7316 91.9136,-309.7316 91.9136,-309.7316 89.1091,-305.5922 88.1881,-312.2557 86.3046,-301.4527 86.3046,-301.4527"/>
+ <text text-anchor="middle" x="118.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 4, c</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="399.2369" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="399.2369" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="333.2369" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="333.2369" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="333.2369" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tree->tri -->
+ <g id="edge25" class="edge">
+ <title>tree->tri</title>
+ <path fill="none" stroke="#add8e6" d="M238.3195,-301.876C245.324,-307.7286 253.1441,-314.3129 260.2369,-320.4 273.6022,-331.8703 288.1999,-344.6585 300.805,-355.7846"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="308.3671,-362.4707 297.8947,-359.2181 304.6213,-359.1588 300.8755,-355.8469 300.8755,-355.8469 300.8755,-355.8469 304.6213,-359.1588 303.8562,-352.4756 308.3671,-362.4707 308.3671,-362.4707"/>
+ <text text-anchor="middle" x="288.1822" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">3, i</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-355.0897C217.2369,-341.6046 217.2369,-325.7123 217.2369,-312.4153"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-312.4033 217.2369,-302.4033 213.737,-312.4034 220.737,-312.4033"/>
+ <text text-anchor="middle" x="222.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge24" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#add8e6" d="M228.409,-356.2376C231.4877,-345.0943 233.446,-332.1755 231.2369,-320.4 230.7037,-317.5576 229.9528,-314.6513 229.0727,-311.7832"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="225.7116,-302.3279 233.3012,-310.243 227.3863,-307.0391 229.0611,-311.7503 229.0611,-311.7503 229.0611,-311.7503 227.3863,-307.0391 224.821,-313.2575 225.7116,-302.3279 225.7116,-302.3279"/>
+ <text text-anchor="middle" x="244.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 4, e</text>
+ </g>
+ <!-- trie->start -->
+ <g id="edge27" class="edge">
+ <title>trie->start</title>
+ <path fill="none" stroke="#add8e6" d="M405.2983,-302.5134C406.9127,-308.1745 408.4309,-314.487 409.2369,-320.4 410.2453,-327.7983 409.9138,-329.7641 409.2369,-337.2 397.8673,-462.0872 364.7169,-608.394 350.1917,-668.6161"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="347.7856,-678.5093 345.7763,-667.729 348.9672,-673.6509 350.1489,-668.7925 350.1489,-668.7925 350.1489,-668.7925 348.9672,-673.6509 354.5214,-669.856 347.7856,-678.5093 347.7856,-678.5093"/>
+ <text text-anchor="middle" x="406.3975" y="-492.0469" font-family="Times,serif" font-size="14.00" fill="#0000ff">-1, -1</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M343.5308,-356.2283C348.4068,-344.6002 354.8542,-331.3353 362.5223,-320.4 365.189,-316.597 368.2884,-312.8528 371.5344,-309.3005"/>
+ <polygon fill="#000000" stroke="#000000" points="374.2909,-311.4868 378.7636,-301.8824 369.2777,-306.6013 374.2909,-311.4868"/>
+ <text text-anchor="middle" x="367.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge26" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#add8e6" d="M354.9336,-360.2898C360.8766,-353.0969 367.0926,-345.0431 372.2369,-337.2 377.4883,-329.1936 382.4875,-320.0361 386.7247,-311.6206"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="391.1577,-302.5353 390.8167,-313.4959 388.9651,-307.0289 386.7725,-311.5225 386.7725,-311.5225 386.7725,-311.5225 388.9651,-307.0289 382.7282,-309.5492 391.1577,-302.5353 391.1577,-302.5353"/>
+ <text text-anchor="middle" x="393.0942" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 4, e</text>
+ </g>
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="217.2369" cy="-496.2469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="217.2369" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="217.2369" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge4" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#000000" d="M182.6116,-484.1004C165.8889,-476.4961 147.0056,-465.018 135.5223,-448.8313 130.4241,-441.645 127.2882,-432.9125 125.3932,-424.2323"/>
+ <polygon fill="#000000" stroke="#000000" points="128.8093,-423.4413 123.6954,-414.1623 121.9067,-424.6051 128.8093,-423.4413"/>
+ <text text-anchor="middle" x="141.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge18" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M196.3752,-471.4723C182.7729,-455.3186 164.8877,-434.0788 150.2087,-416.6465"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="143.7168,-408.9369 153.6002,-413.6877 146.9374,-412.7616 150.158,-416.5862 150.158,-416.5862 150.158,-416.5862 146.9374,-412.7616 146.7158,-419.4848 143.7168,-408.9369 143.7168,-408.9369"/>
+ <text text-anchor="middle" x="188.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 3, a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-466.6249C217.2369,-453.7568 217.2369,-438.4867 217.2369,-424.6319"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-424.3761 217.2369,-414.3761 213.737,-424.3762 220.737,-424.3761"/>
+ <text text-anchor="middle" x="222.0942" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M234.0049,-469.6873C242.4241,-457.4678 253.2833,-443.2328 264.8463,-432.0313 273.7641,-423.3922 284.4016,-415.2101 294.5991,-408.1516"/>
+ <polygon fill="#000000" stroke="#000000" points="296.6631,-410.9815 303.0143,-402.5013 292.761,-405.17 296.6631,-410.9815"/>
+ <text text-anchor="middle" x="268.9322" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M217.2369,-578.0815C217.2369,-566.3502 217.2369,-550.6774 217.2369,-536.1885"/>
+ <polygon fill="#000000" stroke="#000000" points="220.737,-535.9208 217.2369,-525.9209 213.737,-535.9209 220.737,-535.9208"/>
+ <text text-anchor="middle" x="221.3172" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge17" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#add8e6" d="M224.3796,-578.803C226.3635,-573.0529 228.2449,-566.5754 229.2369,-560.4626 230.5801,-552.1852 230.1465,-543.3325 228.8503,-534.9273"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="226.9029,-524.9655 233.2379,-533.9164 227.8622,-529.8726 228.8215,-534.7798 228.8215,-534.7798 228.8215,-534.7798 227.8622,-529.8726 224.4051,-535.6431 226.9029,-524.9655 226.9029,-524.9655"/>
+ <text text-anchor="middle" x="241.3172" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 2, r</text>
+ </g>
+ </g>
+</svg>
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.wc2.svg b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.wc2.svg
new file mode 100644
index 0000000..5f63519
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/InMemoryTrie.md.wc2.svg
@@ -0,0 +1,314 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+ - Licensed to the Apache Software Foundation (ASF) under one
+ - or more contributor license agreements. See the NOTICE file
+ - distributed with this work for additional information
+ - regarding copyright ownership. The ASF licenses this file
+ - to you under the Apache License, Version 2.0 (the
+ - "License"); you may not use this file except in compliance
+ - with the License. You may obtain a copy of the License at
+ -
+ - http://www.apache.org/licenses/LICENSE-2.0
+ -
+ - Unless required by applicable law or agreed to in writing, software
+ - distributed under the License is distributed on an "AS IS" BASIS,
+ - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ - See the License for the specific language governing permissions and
+ - limitations under the License.
+ -->
+
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+digraph G {
+ { rank=same root -> start [style=invis] }
+ start [label="start/end"]
+
+ tractor [label = "contentArray[0]"]
+ tracto [label = "0x01B"]
+ tract [label = "0x01A"]
+ trac [label = "0x019"]
+ tra [label = "Chain\n0x018"]
+
+ tree [label = "contentArray[1]"]
+ tre [label = "Chain\n0x03B"]
+ trie [label = "contentArray[2]"]
+ tri [label = "Chain\n0x05B"]
+
+ tr [label = "Sparse\n0x07E"]
+
+ t [label = "0x09B"]
+ root [label = "Chain\n0x9A"]
+
+ root -> t [label = " t"]
+ t -> tr [label = " r"]
+ tr -> tra [label = " a"]
+ tra -> trac [label = " c"]
+ trac -> tract [label = " t"]
+ tract -> tracto [label = " o"]
+ tracto -> tractor [label = " r"]
+
+ tr -> tre [label = " e"]
+ tre -> tree [label = " e"]
+
+ tr -> tri [label = " i"]
+ tri -> trie [label = " e"]
+
+ // {rank=same tra -> tre -> tri [style=invis]}
+ {rank=same trac -> tree -> trie [style=invis]}
+
+ subgraph path {
+ edge [color = "lightblue"; fontcolor="blue"; arrowhead="vee"; constraint="false"]
+
+ start -> root [label = " 0, -1"]
+
+ root -> tr [label = "t, 2, r "]
+ tr -> tra [label = "3, a"]
+ tra -> tractor [label = "cto, 7, r"]
+
+ // tr -> tre [label = " e"]
+ tre -> tree [label = " 4, e"]
+
+ // tr -> tri [label = " i"]
+ tri -> trie [label = " 4, e"]
+
+ tractor -> tre [label = " 3, e"]
+ tree -> tri [label = "3, i"; ]
+ trie -> start [label = "-1, -1"]
+ }
+}
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="515pt" height="734pt"
+ viewBox="0.00 0.00 514.97 734.09" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 730.0939)">
+ <title>G</title>
+ <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-730.0939 510.9735,-730.0939 510.9735,4 -4,4"/>
+ <!-- root -->
+ <g id="node1" class="node">
+ <title>root</title>
+ <ellipse fill="none" stroke="#000000" cx="235.743" cy="-696.6782" rx="34.9213" ry="29.3315"/>
+ <text text-anchor="middle" x="235.743" y="-700.8782" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="235.743" y="-684.0782" font-family="Times,serif" font-size="14.00" fill="#000000">0x9A</text>
+ </g>
+ <!-- start -->
+ <g id="node2" class="node">
+ <title>start</title>
+ <ellipse fill="none" stroke="#000000" cx="361.743" cy="-696.6782" rx="44.0775" ry="18"/>
+ <text text-anchor="middle" x="361.743" y="-692.4782" font-family="Times,serif" font-size="14.00" fill="#000000">start/end</text>
+ </g>
+ <!-- root->start -->
+ <!-- tr -->
+ <g id="node12" class="node">
+ <title>tr</title>
+ <ellipse fill="none" stroke="#000000" cx="235.743" cy="-496.2469" rx="37.9027" ry="29.3315"/>
+ <text text-anchor="middle" x="235.743" y="-500.4469" font-family="Times,serif" font-size="14.00" fill="#000000">Sparse</text>
+ <text text-anchor="middle" x="235.743" y="-483.6469" font-family="Times,serif" font-size="14.00" fill="#000000">0x07E</text>
+ </g>
+ <!-- root->tr -->
+ <g id="edge16" class="edge">
+ <title>root->tr</title>
+ <path fill="none" stroke="#add8e6" d="M208.9256,-677.804C190.37,-662.9875 167.1966,-640.5442 156.6918,-614.4626 142.7976,-579.9656 172.623,-545.1366 199.529,-522.3097"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="207.3667,-515.9125 202.4651,-525.7219 203.4931,-519.0741 199.6196,-522.2357 199.6196,-522.2357 199.6196,-522.2357 203.4931,-519.0741 196.7741,-518.7495 207.3667,-515.9125 207.3667,-515.9125"/>
+ <text text-anchor="middle" x="173.2686" y="-592.2626" font-family="Times,serif" font-size="14.00" fill="#0000ff">t, 2, r </text>
+ </g>
+ <!-- t -->
+ <g id="node13" class="node">
+ <title>t</title>
+ <ellipse fill="none" stroke="#000000" cx="235.743" cy="-596.4626" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="235.743" y="-592.2626" font-family="Times,serif" font-size="14.00" fill="#000000">0x09B</text>
+ </g>
+ <!-- root->t -->
+ <g id="edge2" class="edge">
+ <title>root->t</title>
+ <path fill="none" stroke="#000000" d="M235.743,-667.1522C235.743,-653.6671 235.743,-637.7749 235.743,-624.4779"/>
+ <polygon fill="#000000" stroke="#000000" points="239.2431,-624.4659 235.743,-614.4659 232.2431,-624.4659 239.2431,-624.4659"/>
+ <text text-anchor="middle" x="239.4383" y="-636.6626" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- start->root -->
+ <g id="edge15" class="edge">
+ <title>start->root</title>
+ <path fill="none" stroke="#add8e6" d="M317.3783,-696.6782C305.5769,-696.6782 292.8167,-696.6782 280.9698,-696.6782"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="270.841,-696.6782 280.8411,-692.1783 275.841,-696.6783 280.841,-696.6783 280.841,-696.6783 280.841,-696.6783 275.841,-696.6783 280.841,-701.1783 270.841,-696.6782 270.841,-696.6782"/>
+ <text text-anchor="middle" x="294.0792" y="-703.8782" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 0, -1</text>
+ </g>
+ <!-- tractor -->
+ <g id="node3" class="node">
+ <title>tractor</title>
+ <ellipse fill="none" stroke="#000000" cx="91.743" cy="-18" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="91.743" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[0]</text>
+ </g>
+ <!-- tre -->
+ <g id="node9" class="node">
+ <title>tre</title>
+ <ellipse fill="none" stroke="#000000" cx="235.743" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="235.743" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="235.743" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x03B</text>
+ </g>
+ <!-- tractor->tre -->
+ <g id="edge21" class="edge">
+ <title>tractor->tre</title>
+ <path fill="none" stroke="#add8e6" d="M142.2881,-31.2033C233.224,-58.0052 422.1486,-128.3147 499.743,-266.4 507.5812,-280.3486 510.6711,-290.7134 499.743,-302.4 458.9747,-345.9983 272.5113,-276.8017 231.743,-320.4 225.2999,-327.2904 223.5432,-336.6232 224.0821,-346.0371"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="225.4547,-356.1722 219.6532,-346.8666 224.7836,-351.2174 224.1125,-346.2627 224.1125,-346.2627 224.1125,-346.2627 224.7836,-351.2174 228.5718,-345.6587 225.4547,-356.1722 225.4547,-356.1722"/>
+ <text text-anchor="middle" x="471.6003" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 3, e</text>
+ </g>
+ <!-- tracto -->
+ <g id="node4" class="node">
+ <title>tracto</title>
+ <ellipse fill="none" stroke="#000000" cx="91.743" cy="-106.8" rx="36.5824" ry="18"/>
+ <text text-anchor="middle" x="91.743" y="-102.6" font-family="Times,serif" font-size="14.00" fill="#000000">0x01B</text>
+ </g>
+ <!-- tracto->tractor -->
+ <g id="edge8" class="edge">
+ <title>tracto->tractor</title>
+ <path fill="none" stroke="#000000" d="M91.743,-88.4006C91.743,-76.2949 91.743,-60.2076 91.743,-46.4674"/>
+ <polygon fill="#000000" stroke="#000000" points="95.2431,-46.072 91.743,-36.072 88.2431,-46.0721 95.2431,-46.072"/>
+ <text text-anchor="middle" x="95.8233" y="-58.2" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ <!-- tract -->
+ <g id="node5" class="node">
+ <title>tract</title>
+ <ellipse fill="none" stroke="#000000" cx="91.743" cy="-195.6" rx="37.1443" ry="18"/>
+ <text text-anchor="middle" x="91.743" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#000000">0x01A</text>
+ </g>
+ <!-- tract->tracto -->
+ <g id="edge7" class="edge">
+ <title>tract->tracto</title>
+ <path fill="none" stroke="#000000" d="M91.743,-177.2006C91.743,-165.0949 91.743,-149.0076 91.743,-135.2674"/>
+ <polygon fill="#000000" stroke="#000000" points="95.2431,-134.872 91.743,-124.872 88.2431,-134.8721 95.2431,-134.872"/>
+ <text text-anchor="middle" x="96.993" y="-147" font-family="Times,serif" font-size="14.00" fill="#000000"> o</text>
+ </g>
+ <!-- trac -->
+ <g id="node6" class="node">
+ <title>trac</title>
+ <ellipse fill="none" stroke="#000000" cx="91.743" cy="-284.4" rx="35.3587" ry="18"/>
+ <text text-anchor="middle" x="91.743" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">0x019</text>
+ </g>
+ <!-- trac->tract -->
+ <g id="edge6" class="edge">
+ <title>trac->tract</title>
+ <path fill="none" stroke="#000000" d="M91.743,-266.0006C91.743,-253.8949 91.743,-237.8076 91.743,-224.0674"/>
+ <polygon fill="#000000" stroke="#000000" points="95.2431,-223.672 91.743,-213.672 88.2431,-223.6721 95.2431,-223.672"/>
+ <text text-anchor="middle" x="95.4383" y="-235.8" font-family="Times,serif" font-size="14.00" fill="#000000"> t</text>
+ </g>
+ <!-- tree -->
+ <g id="node8" class="node">
+ <title>tree</title>
+ <ellipse fill="none" stroke="#000000" cx="235.743" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="235.743" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[1]</text>
+ </g>
+ <!-- trac->tree -->
+ <!-- tra -->
+ <g id="node7" class="node">
+ <title>tra</title>
+ <ellipse fill="none" stroke="#000000" cx="117.743" cy="-384.6156" rx="36.125" ry="29.3315"/>
+ <text text-anchor="middle" x="117.743" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="117.743" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x018</text>
+ </g>
+ <!-- tra->tractor -->
+ <g id="edge18" class="edge">
+ <title>tra->tractor</title>
+ <path fill="none" stroke="#add8e6" d="M94.3918,-361.848C65.2172,-331.3805 17.371,-274.1277 2.9772,-213.6 -12.3828,-149.0092 36.553,-79.6557 67.7845,-43.4033"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="74.5314,-35.7623 71.2857,-46.2369 71.2219,-39.5104 67.9125,-43.2584 67.9125,-43.2584 67.9125,-43.2584 71.2219,-39.5104 64.5393,-40.2799 74.5314,-35.7623 74.5314,-35.7623"/>
+ <text text-anchor="middle" x="25.1259" y="-191.4" font-family="Times,serif" font-size="14.00" fill="#0000ff">cto, 7, r</text>
+ </g>
+ <!-- tra->trac -->
+ <g id="edge5" class="edge">
+ <title>tra->trac</title>
+ <path fill="none" stroke="#000000" d="M110.2227,-355.6291C106.6613,-341.9016 102.4282,-325.5854 98.9126,-312.0348"/>
+ <polygon fill="#000000" stroke="#000000" points="102.2863,-311.1009 96.3871,-302.3003 95.5106,-312.8588 102.2863,-311.1009"/>
+ <text text-anchor="middle" x="109.6003" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> c</text>
+ </g>
+ <!-- trie -->
+ <g id="node10" class="node">
+ <title>trie</title>
+ <ellipse fill="none" stroke="#000000" cx="417.743" cy="-284.4" rx="73.4745" ry="18"/>
+ <text text-anchor="middle" x="417.743" y="-280.2" font-family="Times,serif" font-size="14.00" fill="#000000">contentArray[2]</text>
+ </g>
+ <!-- tree->trie -->
+ <!-- tri -->
+ <g id="node11" class="node">
+ <title>tri</title>
+ <ellipse fill="none" stroke="#000000" cx="351.743" cy="-384.6156" rx="37.9306" ry="29.3315"/>
+ <text text-anchor="middle" x="351.743" y="-388.8156" font-family="Times,serif" font-size="14.00" fill="#000000">Chain</text>
+ <text text-anchor="middle" x="351.743" y="-372.0156" font-family="Times,serif" font-size="14.00" fill="#000000">0x05B</text>
+ </g>
+ <!-- tree->tri -->
+ <g id="edge22" class="edge">
+ <title>tree->tri</title>
+ <path fill="none" stroke="#add8e6" d="M256.2723,-301.9531C263.1097,-307.8109 270.7613,-314.3787 277.743,-320.4 291.1837,-331.9919 305.9615,-344.8025 318.7535,-355.9126"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="326.432,-362.5843 315.9318,-359.4223 322.6577,-359.3048 318.8834,-356.0254 318.8834,-356.0254 318.8834,-356.0254 322.6577,-359.3048 321.8349,-352.6285 326.432,-362.5843 326.432,-362.5843"/>
+ <text text-anchor="middle" x="304.6883" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff">3, i</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge10" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#000000" d="M235.743,-355.0897C235.743,-341.6046 235.743,-325.7123 235.743,-312.4153"/>
+ <polygon fill="#000000" stroke="#000000" points="239.2431,-312.4033 235.743,-302.4033 232.2431,-312.4034 239.2431,-312.4033"/>
+ <text text-anchor="middle" x="240.6003" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tre->tree -->
+ <g id="edge19" class="edge">
+ <title>tre->tree</title>
+ <path fill="none" stroke="#add8e6" d="M246.9152,-356.2376C249.9939,-345.0943 251.9521,-332.1755 249.743,-320.4 249.2098,-317.5576 248.4589,-314.6513 247.5789,-311.7832"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="244.2177,-302.3279 251.8073,-310.243 245.8925,-307.0391 247.5672,-311.7503 247.5672,-311.7503 247.5672,-311.7503 245.8925,-307.0391 243.3271,-313.2575 244.2177,-302.3279 244.2177,-302.3279"/>
+ <text text-anchor="middle" x="261.6003" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 4, e</text>
+ </g>
+ <!-- trie->start -->
+ <g id="edge23" class="edge">
+ <title>trie->start</title>
+ <path fill="none" stroke="#add8e6" d="M423.8044,-302.5134C425.4188,-308.1745 426.937,-314.487 427.743,-320.4 428.7515,-327.7983 428.42,-329.7641 427.743,-337.2 416.3734,-462.0872 383.223,-608.394 368.6978,-668.6161"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="366.2917,-678.5093 364.2825,-667.729 367.4734,-673.6509 368.655,-668.7925 368.655,-668.7925 368.655,-668.7925 367.4734,-673.6509 373.0275,-669.856 366.2917,-678.5093 366.2917,-678.5093"/>
+ <text text-anchor="middle" x="424.9036" y="-492.0469" font-family="Times,serif" font-size="14.00" fill="#0000ff">-1, -1</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge12" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#000000" d="M362.0369,-356.2283C366.9129,-344.6002 373.3603,-331.3353 381.0284,-320.4 383.6952,-316.597 386.7946,-312.8528 390.0406,-309.3005"/>
+ <polygon fill="#000000" stroke="#000000" points="392.7971,-311.4868 397.2698,-301.8824 387.7839,-306.6013 392.7971,-311.4868"/>
+ <text text-anchor="middle" x="385.6003" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tri->trie -->
+ <g id="edge20" class="edge">
+ <title>tri->trie</title>
+ <path fill="none" stroke="#add8e6" d="M373.4397,-360.2898C379.3828,-353.0969 385.5988,-345.0431 390.743,-337.2 395.9944,-329.1936 400.9936,-320.0361 405.2308,-311.6206"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="409.6638,-302.5353 409.3228,-313.4959 407.4712,-307.0289 405.2786,-311.5225 405.2786,-311.5225 405.2786,-311.5225 407.4712,-307.0289 401.2343,-309.5492 409.6638,-302.5353 409.6638,-302.5353"/>
+ <text text-anchor="middle" x="411.6003" y="-324.6" font-family="Times,serif" font-size="14.00" fill="#0000ff"> 4, e</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge4" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#000000" d="M201.7509,-482.8183C184.9175,-474.8752 165.2161,-463.4812 151.0284,-448.8313 143.6903,-441.2541 137.5841,-431.7356 132.6915,-422.3656"/>
+ <polygon fill="#000000" stroke="#000000" points="135.7196,-420.5888 128.2027,-413.1224 129.4228,-423.6468 135.7196,-420.5888"/>
+ <text text-anchor="middle" x="156.6003" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> a</text>
+ </g>
+ <!-- tr->tra -->
+ <g id="edge17" class="edge">
+ <title>tr->tra</title>
+ <path fill="none" stroke="#add8e6" d="M211.5792,-473.3873C193.4836,-456.2683 168.5764,-432.7055 148.9704,-414.1576"/>
+ <polygon fill="#add8e6" stroke="#add8e6" points="141.4713,-407.0632 151.8282,-410.6666 145.1035,-410.4994 148.7357,-413.9356 148.7357,-413.9356 148.7357,-413.9356 145.1035,-410.4994 145.6431,-417.2046 141.4713,-407.0632 141.4713,-407.0632"/>
+ <text text-anchor="middle" x="194.8503" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#0000ff">3, a</text>
+ </g>
+ <!-- tr->tre -->
+ <g id="edge9" class="edge">
+ <title>tr->tre</title>
+ <path fill="none" stroke="#000000" d="M235.743,-466.6249C235.743,-453.7568 235.743,-438.4867 235.743,-424.6319"/>
+ <polygon fill="#000000" stroke="#000000" points="239.2431,-424.3761 235.743,-414.3761 232.2431,-424.3762 239.2431,-424.3761"/>
+ <text text-anchor="middle" x="240.6003" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> e</text>
+ </g>
+ <!-- tr->tri -->
+ <g id="edge11" class="edge">
+ <title>tr->tri</title>
+ <path fill="none" stroke="#000000" d="M259.4973,-473.3873C277.0521,-456.4936 301.128,-433.3244 320.2814,-414.8924"/>
+ <polygon fill="#000000" stroke="#000000" points="322.8434,-417.2844 327.6219,-407.8283 317.9895,-412.2405 322.8434,-417.2844"/>
+ <text text-anchor="middle" x="303.4383" y="-436.2313" font-family="Times,serif" font-size="14.00" fill="#000000"> i</text>
+ </g>
+ <!-- t->tr -->
+ <g id="edge3" class="edge">
+ <title>t->tr</title>
+ <path fill="none" stroke="#000000" d="M235.743,-578.0815C235.743,-566.3502 235.743,-550.6774 235.743,-536.1885"/>
+ <polygon fill="#000000" stroke="#000000" points="239.2431,-535.9208 235.743,-525.9209 232.2431,-535.9209 239.2431,-535.9208"/>
+ <text text-anchor="middle" x="239.8233" y="-547.8626" font-family="Times,serif" font-size="14.00" fill="#000000"> r</text>
+ </g>
+ </g>
+</svg>
diff --git a/src/java/org/apache/cassandra/db/tries/MergeTrie.java b/src/java/org/apache/cassandra/db/tries/MergeTrie.java
new file mode 100644
index 0000000..f280776
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/MergeTrie.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * A merged view of two tries.
+ *
+ * This is accomplished by walking the two cursors in parallel; the merged cursor takes the position and features of the
+ * smaller and advances with it; when the two cursors are equal, both are advanced.
+ *
+ * Crucial for the efficiency of this is the fact that when they are advanced like this, we can compare cursors'
+ * positions by their depth descending and then incomingTransition ascending.
+ *
+ * See Trie.md for further details.
+ */
+class MergeTrie<T> extends Trie<T>
+{
+ private final MergeResolver<T> resolver;
+ protected final Trie<T> t1;
+ protected final Trie<T> t2;
+
+ MergeTrie(MergeResolver<T> resolver, Trie<T> t1, Trie<T> t2)
+ {
+ this.resolver = resolver;
+ this.t1 = t1;
+ this.t2 = t2;
+ }
+
+ @Override
+ protected Cursor<T> cursor()
+ {
+ return new MergeCursor<>(resolver, t1, t2);
+ }
+
+ static class MergeCursor<T> implements Cursor<T>
+ {
+ private final MergeResolver<T> resolver;
+ private final Cursor<T> c1;
+ private final Cursor<T> c2;
+
+ boolean atC1;
+ boolean atC2;
+
+ MergeCursor(MergeResolver<T> resolver, Trie<T> t1, Trie<T> t2)
+ {
+ this.resolver = resolver;
+ this.c1 = t1.cursor();
+ this.c2 = t2.cursor();
+ assert c1.depth() == 0;
+ assert c2.depth() == 0;
+ atC1 = atC2 = true;
+ }
+
+ @Override
+ public int advance()
+ {
+ return checkOrder(atC1 ? c1.advance() : c1.depth(),
+ atC2 ? c2.advance() : c2.depth());
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ return checkOrder(atC1 ? c1.skipChildren() : c1.depth(),
+ atC2 ? c2.skipChildren() : c2.depth());
+ }
+
+ @Override
+ public int advanceMultiple(TransitionsReceiver receiver)
+ {
+ // While we are on a shared position, we must descend one byte at a time to maintain the cursor ordering.
+ if (atC1 && atC2)
+ return checkOrder(c1.advance(), c2.advance());
+
+ // If we are in a branch that's only covered by one of the sources, we can use its advanceMultiple as it is
+ // only different from advance if it takes multiple steps down, which does not change the order of the
+ // cursors.
+ // Since it might ascend, we still have to check the order after the call.
+ if (atC1)
+ return checkOrder(c1.advanceMultiple(receiver), c2.depth());
+ else // atC2
+ return checkOrder(c1.depth(), c2.advanceMultiple(receiver));
+ }
+
+ private int checkOrder(int c1depth, int c2depth)
+ {
+ if (c1depth > c2depth)
+ {
+ atC1 = true;
+ atC2 = false;
+ return c1depth;
+ }
+ if (c1depth < c2depth)
+ {
+ atC1 = false;
+ atC2 = true;
+ return c2depth;
+ }
+ // c1depth == c2depth
+ int c1trans = c1.incomingTransition();
+ int c2trans = c2.incomingTransition();
+ atC1 = c1trans <= c2trans;
+ atC2 = c1trans >= c2trans;
+ return c1depth;
+ }
+
+ @Override
+ public int depth()
+ {
+ return atC1 ? c1.depth() : c2.depth();
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return atC1 ? c1.incomingTransition() : c2.incomingTransition();
+ }
+
+ public T content()
+ {
+ T mc = atC2 ? c2.content() : null;
+ T nc = atC1 ? c1.content() : null;
+ if (mc == null)
+ return nc;
+ else if (nc == null)
+ return mc;
+ else
+ return resolver.resolve(nc, mc);
+ }
+ }
+
+ /**
+ * Special instance for sources that are guaranteed (by the caller) distinct. The main difference is that we can
+ * form unordered value list by concatenating sources.
+ */
+ static class Distinct<T> extends MergeTrie<T>
+ {
+ Distinct(Trie<T> input1, Trie<T> input2)
+ {
+ super(throwingResolver(), input1, input2);
+ }
+
+ @Override
+ public Iterable<T> valuesUnordered()
+ {
+ return Iterables.concat(t1.valuesUnordered(), t2.valuesUnordered());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/SingletonTrie.java b/src/java/org/apache/cassandra/db/tries/SingletonTrie.java
new file mode 100644
index 0000000..0336a85
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/SingletonTrie.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Singleton trie, mapping the given key to value.
+ */
+class SingletonTrie<T> extends Trie<T>
+{
+ private final ByteComparable key;
+ private final T value;
+
+ SingletonTrie(ByteComparable key, T value)
+ {
+ this.key = key;
+ this.value = value;
+ }
+
+ public Cursor cursor()
+ {
+ return new Cursor();
+ }
+
+ class Cursor implements Trie.Cursor<T>
+ {
+ private final ByteSource src = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ private int currentDepth = 0;
+ private int currentTransition = -1;
+ private int nextTransition = src.next();
+
+ @Override
+ public int advance()
+ {
+ currentTransition = nextTransition;
+ if (currentTransition != ByteSource.END_OF_STREAM)
+ {
+ nextTransition = src.next();
+ return ++currentDepth;
+ }
+ else
+ {
+ return currentDepth = -1;
+ }
+ }
+
+ @Override
+ public int advanceMultiple(TransitionsReceiver receiver)
+ {
+ if (nextTransition == ByteSource.END_OF_STREAM)
+ return currentDepth = -1;
+ int current = nextTransition;
+ int depth = currentDepth;
+ int next = src.next();
+ while (next != ByteSource.END_OF_STREAM)
+ {
+ if (receiver != null)
+ receiver.addPathByte(current);
+ current = next;
+ next = src.next();
+ ++depth;
+ }
+ currentTransition = current;
+ nextTransition = next;
+ return currentDepth = ++depth;
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ return currentDepth = -1; // no alternatives
+ }
+
+ @Override
+ public int depth()
+ {
+ return currentDepth;
+ }
+
+ @Override
+ public T content()
+ {
+ return nextTransition == ByteSource.END_OF_STREAM ? value : null;
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return currentTransition;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/SlicedTrie.java b/src/java/org/apache/cassandra/db/tries/SlicedTrie.java
new file mode 100644
index 0000000..75ae3df
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/SlicedTrie.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Represents a sliced view of a trie, i.e. the content within the given pair of bounds.
+ *
+ * Applied by advancing three tries in parallel: the left bound, the source and the right bound. While the source
+ * bound is smallest, we don't issue any content and skip over any children. As soon as the left bound becomes strictly
+ * smaller, we stop processing it (as it's a singleton trie it will remain smaller until it's exhausted) and start
+ * issuing the nodes and content from the source. As soon as the right bound becomes strictly smaller, we finish the
+ * walk.
+ *
+ * We don't explicitly construct tries for the two bounds; tracking the current depth (= prefix length) and transition
+ * as characters are requested from the key is sufficient as it is a trie with just a single descent path. Because we
+ * need the next character to tell if it's been exhausted, we keep these one position ahead. The source is always
+ * advanced, thus this gives us the thing to compare it against after the advance.
+ *
+ * We also track the current state to make some decisions a little simpler.
+ *
+ * See Trie.md for further details.
+ */
+public class SlicedTrie<T> extends Trie<T>
+{
+ private final Trie<T> source;
+
+ /** Left-side boundary. The characters of this are requested as we descend along the left-side boundary. */
+ private final ByteComparable left;
+
+ /** Right-side boundary. The characters of this are requested as we descend along the right-side boundary. */
+ private final ByteComparable right;
+
+ private final boolean includeLeft;
+ private final boolean includeRight;
+
+ public SlicedTrie(Trie<T> source, ByteComparable left, boolean includeLeft, ByteComparable right, boolean includeRight)
+ {
+ this.source = source;
+ this.left = left;
+ this.right = right;
+ this.includeLeft = includeLeft;
+ this.includeRight = includeRight;
+ }
+
+ @Override
+ protected Cursor<T> cursor()
+ {
+ return new SlicedCursor<>(this);
+ }
+
+ private enum State
+ {
+ /** The cursor is still positioned on some prefix of the left bound. Content should not be produced. */
+ BEFORE_LEFT,
+ /** The cursor is positioned inside the range, i.e. beyond the left bound, possibly on a prefix of the right. */
+ INSIDE,
+ /** The cursor is positioned beyond the right bound. Exhaustion (depth -1) has been reported. */
+ AFTER_RIGHT
+ }
+
+ private static class SlicedCursor<T> implements Cursor<T>
+ {
+ private final ByteSource left;
+ private final ByteSource right;
+ private final boolean includeLeft;
+ private final boolean excludeRight;
+ private final Cursor<T> source;
+
+ private State state;
+ private int leftNext;
+ private int leftNextDepth;
+ private int rightNext;
+ private int rightNextDepth;
+
+ public SlicedCursor(SlicedTrie<T> slicedTrie)
+ {
+ source = slicedTrie.source.cursor();
+ if (slicedTrie.left != null)
+ {
+ left = slicedTrie.left.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ includeLeft = slicedTrie.includeLeft;
+ leftNext = left.next();
+ leftNextDepth = 1;
+ if (leftNext == ByteSource.END_OF_STREAM && includeLeft)
+ state = State.INSIDE;
+ else
+ state = State.BEFORE_LEFT;
+ }
+ else
+ {
+ left = null;
+ includeLeft = true;
+ state = State.INSIDE;
+ }
+
+ if (slicedTrie.right != null)
+ {
+ right = slicedTrie.right.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ excludeRight = !slicedTrie.includeRight;
+ rightNext = right.next();
+ rightNextDepth = 1;
+ if (rightNext == ByteSource.END_OF_STREAM && excludeRight)
+ state = State.BEFORE_LEFT; // This is a hack, we are after the right bound but we don't want to
+ // report depth -1 yet. So just make sure root's content is not reported.
+ }
+ else
+ {
+ right = null;
+ excludeRight = true;
+ rightNextDepth = 0;
+ }
+ }
+
+ @Override
+ public int advance()
+ {
+ assert (state != State.AFTER_RIGHT);
+
+ int newDepth = source.advance();
+ int transition = source.incomingTransition();
+
+ if (state == State.BEFORE_LEFT)
+ {
+ // Skip any transitions before the left bound
+ while (newDepth == leftNextDepth && transition < leftNext)
+ {
+ newDepth = source.skipChildren();
+ transition = source.incomingTransition();
+ }
+
+ // Check if we are still following the left bound
+ if (newDepth == leftNextDepth && transition == leftNext)
+ {
+ assert leftNext != ByteSource.END_OF_STREAM;
+ leftNext = left.next();
+ ++leftNextDepth;
+ if (leftNext == ByteSource.END_OF_STREAM && includeLeft)
+ state = State.INSIDE; // report the content on the left bound
+ }
+ else // otherwise we are beyond it
+ state = State.INSIDE;
+ }
+
+ return checkRightBound(newDepth, transition);
+ }
+
+ private int markDone()
+ {
+ state = State.AFTER_RIGHT;
+ return -1;
+ }
+
+ private int checkRightBound(int newDepth, int transition)
+ {
+ // Cursor positions compare by depth descending and transition ascending.
+ if (newDepth > rightNextDepth)
+ return newDepth;
+ if (newDepth < rightNextDepth)
+ return markDone();
+ // newDepth == rightDepth
+ if (transition < rightNext)
+ return newDepth;
+ if (transition > rightNext)
+ return markDone();
+
+ // Following right bound
+ rightNext = right.next();
+ ++rightNextDepth;
+ if (rightNext == ByteSource.END_OF_STREAM && excludeRight)
+ return markDone(); // do not report any content on the right bound
+ return newDepth;
+ }
+
+ @Override
+ public int advanceMultiple(TransitionsReceiver receiver)
+ {
+ switch (state)
+ {
+ case BEFORE_LEFT:
+ return advance(); // descend only one level to be able to compare cursors correctly
+ case INSIDE:
+ int depth = source.depth();
+ if (depth == rightNextDepth - 1) // this is possible because right is already advanced;
+ return advance(); // we need to check next byte against right boundary in this case
+ int newDepth = source.advanceMultiple(receiver);
+ if (newDepth > depth)
+ return newDepth; // successfully advanced
+ // we ascended, check if we are still within boundaries
+ return checkRightBound(newDepth, source.incomingTransition());
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ assert (state != State.AFTER_RIGHT);
+
+ // We are either inside or following the left bound. In the latter case ascend takes us beyond it.
+ state = State.INSIDE;
+ return checkRightBound(source.skipChildren(), source.incomingTransition());
+ }
+
+ @Override
+ public int depth()
+ {
+ return state == State.AFTER_RIGHT ? -1 : source.depth();
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return source.incomingTransition();
+ }
+
+ @Override
+ public T content()
+ {
+ return state == State.INSIDE ? source.content() : null;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/Trie.java b/src/java/org/apache/cassandra/db/tries/Trie.java
new file mode 100644
index 0000000..a139e08
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/Trie.java
@@ -0,0 +1,622 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+
+import org.agrona.DirectBuffer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Base class for tries.
+ *
+ * Normal users of tries will only use the public methods, which provide various transformations of the trie, conversion
+ * of its content to other formats (e.g. iterable of values), and several forms of processing.
+ *
+ * For any unimplemented data extraction operations one can build on the {@link TrieEntriesWalker} (for-each processing)
+ * and {@link TrieEntriesIterator} (to iterator) base classes, which provide the necessary mechanisms to handle walking
+ * the trie.
+ *
+ * The internal representation of tries using this interface is defined in the {@link Cursor} interface.
+ *
+ * Cursors are a method of presenting the internal structure of a trie without representing nodes as objects, which is
+ * still useful for performing the basic operations on tries (iteration, slicing/intersection and merging). A cursor
+ * will list the nodes of a trie in order, together with information about the path that was taken to reach them.
+ *
+ * To begin traversal over a trie, one must retrieve a cursor by calling {@link #cursor()}. Because cursors are
+ * stateful, the traversal must always proceed from one thread. Should concurrent reads be required, separate calls to
+ * {@link #cursor()} must be made. Any modification that has completed before the construction of a cursor must be
+ * visible, but any later concurrent modifications may be presented fully, partially or not at all; this also means that
+ * if multiple are made, the cursor may see any part of any subset of them.
+ *
+ * Note: This model only supports depth-first traversals. We do not currently have a need for breadth-first walks.
+ *
+ * See Trie.md for further description of the trie representation model.
+ *
+ * @param <T> The content type of the trie.
+ */
+public abstract class Trie<T>
+{
+ /**
+ * A trie cursor.
+ *
+ * This is the internal representation of the trie, which enables efficient walks and basic operations (merge,
+ * slice) on tries.
+ *
+ * The cursor represents the state of a walk over the nodes of trie. It provides three main features:
+ * - the current "depth" or descend-depth in the trie;
+ * - the "incomingTransition", i.e. the byte that was used to reach the current point;
+ * - the "content" associated with the current node,
+ * and provides methods for advancing to the next position. This is enough information to extract all paths, and
+ * also to easily compare cursors over different tries that are advanced together. Advancing is always done in
+ * order; if one imagines the set of nodes in the trie with their associated paths, a cursor may only advance from a
+ * node with a lexicographically smaller path to one with bigger. The "advance" operation moves to the immediate
+ * next, it is also possible to skip over some items e.g. all children of the current node ("skipChildren").
+ *
+ * Moving to the immediate next position in the lexicographic order is accomplished by:
+ * - if the current node has children, moving to its first child;
+ * - otherwise, ascend the parent chain and return the next child of the closest parent that still has any.
+ * As long as the trie is not exhausted, advancing always takes one step down, from the current node, or from a node
+ * on the parent chain. By comparing the new depth (which "advance" also returns) with the one before the advance,
+ * one can tell if the former was the case (if newDepth == oldDepth + 1) and how many steps up we had to take
+ * (oldDepth + 1 - newDepth). When following a path down, the cursor will stop on all prefixes.
+ *
+ * When it is created the cursor is placed on the root node with depth() = 0, incomingTransition() = -1. Since
+ * tries can have mappings for empty, content() can possibly be non-null. It is not allowed for a cursor to start
+ * in exhausted state (i.e. with depth() = -1).
+ *
+ * For example, the following trie:
+ * t
+ * r
+ * e
+ * e *
+ * i
+ * e *
+ * p *
+ * w
+ * i
+ * n *
+ * has nodes reachable with the paths
+ * "", t, tr, tre, tree*, tri, trie*, trip*, w, wi, win*
+ * and the cursor will list them with the following (depth, incomingTransition) pairs:
+ * (0, -1), (1, t), (2, r), (3, e), (4, e)*, (3, i), (4, e)*, (4, p)*, (1, w), (2, i), (3, n)*
+ *
+ * Because we exhaust transitions on bigger depths before we go the next transition on the smaller ones, when
+ * cursors are advanced together their positions can be easily compared using only the depth and incomingTransition:
+ * - one that is higher in depth is before one that is lower;
+ * - for equal depths, the one with smaller incomingTransition is first.
+ *
+ * If we consider walking the trie above in parallel with this:
+ * t
+ * r
+ * i
+ * c
+ * k *
+ * u
+ * p *
+ * the combined iteration will proceed as follows:
+ * (0, -1)+ (0, -1)+ cursors equal, advance both
+ * (1, t)+ (1, t)+ t cursors equal, advance both
+ * (2, r)+ (2, r)+ tr cursors equal, advance both
+ * (3, e)+ < (3, i) tre cursors not equal, advance smaller (3 = 3, e < i)
+ * (4, e)+ < (3, i) tree* cursors not equal, advance smaller (4 > 3)
+ * (3, i)+ (3, i)+ tri cursors equal, advance both
+ * (4, e) > (4, c)+ tric cursors not equal, advance smaller (4 = 4, e > c)
+ * (4, e) > (5, k)+ trick* cursors not equal, advance smaller (4 < 5)
+ * (4, e)+ < (1, u) trie* cursors not equal, advance smaller (4 > 1)
+ * (4, p)+ < (1, u) trip* cursors not equal, advance smaller (4 > 1)
+ * (1, w) > (1, u)+ u cursors not equal, advance smaller (1 = 1, w > u)
+ * (1, w) > (2, p)+ up* cursors not equal, advance smaller (1 < 2)
+ * (1, w)+ < (-1, -1) w cursors not equal, advance smaller (1 > -1)
+ * (2, i)+ < (-1, -1) wi cursors not equal, advance smaller (2 > -1)
+ * (3, n)+ < (-1, -1) win* cursors not equal, advance smaller (3 > -1)
+ * (-1, -1) (-1, -1) both exhasted
+ */
+ protected interface Cursor<T>
+ {
+
+ /**
+ * @return the current descend-depth; 0, if the cursor has just been created and is positioned on the root,
+ * and -1, if the trie has been exhausted.
+ */
+ int depth();
+
+ /**
+ * @return the last transition taken; if positioned on the root, return -1
+ */
+ int incomingTransition();
+
+ /**
+ * @return the content associated with the current node. This may be non-null for any presented node, including
+ * the root.
+ */
+ T content();
+
+ /**
+ * Advance one position to the node whose associated path is next lexicographically.
+ * This can be either:
+ * - descending one level to the first child of the current node,
+ * - ascending to the closest parent that has remaining children, and then descending one level to its next
+ * child.
+ *
+ * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
+ * for performance reasons we won't always check this.
+ *
+ * @return depth (can be prev+1 or <=prev), -1 means that the trie is exhausted
+ */
+ int advance();
+
+ /**
+ * Advance, descending multiple levels if the cursor can do this for the current position without extra work
+ * (e.g. when positioned on a chain node in a memtable trie). If the current node does not have children this
+ * is exactly the same as advance(), otherwise it may take multiple steps down (but will not necessarily, even
+ * if they exist).
+ *
+ * Note that if any positions are skipped, their content must be null.
+ *
+ * This is an optional optimization; the default implementation falls back to calling advance.
+ *
+ * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
+ * for performance reasons we won't always check this.
+ *
+ * @param receiver object that will receive all transitions taken except the last;
+ * on ascend, or if only one step down was taken, it will not receive any
+ * @return the new depth, -1 if the trie is exhausted
+ */
+ default int advanceMultiple(TransitionsReceiver receiver)
+ {
+ return advance();
+ }
+
+ /**
+ * Advance all the way to the next node with non-null content.
+ *
+ * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
+ * for performance reasons we won't always check this.
+ *
+ * @param receiver object that will receive all taken transitions
+ * @return the content, null if the trie is exhausted
+ */
+ default T advanceToContent(ResettingTransitionsReceiver receiver)
+ {
+ int prevDepth = depth();
+ while (true)
+ {
+ int currDepth = advanceMultiple(receiver);
+ if (currDepth <= 0)
+ return null;
+ if (receiver != null)
+ {
+ if (currDepth <= prevDepth)
+ receiver.resetPathLength(currDepth - 1);
+ receiver.addPathByte(incomingTransition());
+ }
+ T content = content();
+ if (content != null)
+ return content;
+ prevDepth = currDepth;
+ }
+ }
+
+ /**
+ * Ignore the current node's children and advance to the next child of the closest node on the parent chain that
+ * has any.
+ *
+ * It is an error to call this after the trie has already been exhausted (i.e. when depth() == -1);
+ * for performance reasons we won't always check this.
+ *
+ * @return the new depth, always <= previous depth; -1 if the trie is exhausted
+ */
+ int skipChildren();
+ }
+
+ protected abstract Cursor<T> cursor();
+
+ /**
+ * Used by {@link Cursor#advanceMultiple} to feed the transitions taken.
+ */
+ protected interface TransitionsReceiver
+ {
+ /** Add a single byte to the path. */
+ void addPathByte(int nextByte);
+ /** Add the count bytes from position pos in the given buffer. */
+ void addPathBytes(DirectBuffer buffer, int pos, int count);
+ }
+
+ /**
+ * Used by {@link Cursor#advanceToContent} to track the transitions and backtracking taken.
+ */
+ protected interface ResettingTransitionsReceiver extends TransitionsReceiver
+ {
+ /** Delete all bytes beyond the given length. */
+ void resetPathLength(int newLength);
+ }
+
+ /**
+ * A push interface for walking over the trie. Builds upon TransitionsReceiver to be given the bytes of the
+ * path, and adds methods called on encountering content and completion.
+ * See {@link TrieDumper} for an example of how this can be used, and {@link TrieEntriesWalker} as a base class
+ * for other common usages.
+ */
+ protected interface Walker<T, R> extends ResettingTransitionsReceiver
+ {
+ /** Called when content is found. */
+ void content(T content);
+
+ /** Called at the completion of the walk. */
+ R complete();
+ }
+
+ // Version of the byte comparable conversion to use for all operations
+ protected static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
+
+ /**
+ * Adapter interface providing the methods a {@link Walker} to a {@link Consumer}, so that the latter can be used
+ * with {@link #process}.
+ *
+ * This enables calls like
+ * trie.forEachEntry(x -> System.out.println(x));
+ * to be mapped directly to a single call to {@link #process} without extra allocations.
+ */
+ public interface ValueConsumer<T> extends Consumer<T>, Walker<T, Void>
+ {
+ @Override
+ default void content(T content)
+ {
+ accept(content);
+ }
+
+ @Override
+ default Void complete()
+ {
+ return null;
+ }
+
+ @Override
+ default void resetPathLength(int newDepth)
+ {
+ // not tracking path
+ }
+
+ @Override
+ default void addPathByte(int nextByte)
+ {
+ // not tracking path
+ }
+
+ @Override
+ default void addPathBytes(DirectBuffer buffer, int pos, int count)
+ {
+ // not tracking path
+ }
+ }
+
+ /**
+ * Call the given consumer on all content values in the trie in order.
+ */
+ public void forEachValue(ValueConsumer<T> consumer)
+ {
+ process(consumer);
+ }
+
+ /**
+ * Call the given consumer on all (path, content) pairs with non-null content in the trie in order.
+ */
+ public void forEachEntry(BiConsumer<ByteComparable, T> consumer)
+ {
+ process(new TrieEntriesWalker.WithConsumer<T>(consumer));
+ // Note: we can't do the ValueConsumer trick here, because the implementation requires state and cannot be
+ // implemented with default methods alone.
+ }
+
+ /**
+ * Process the trie using the given Walker.
+ */
+ public <R> R process(Walker<T, R> walker)
+ {
+ return process(walker, cursor());
+ }
+
+ static <T, R> R process(Walker<T, R> walker, Cursor<T> cursor)
+ {
+ assert cursor.depth() == 0 : "The provided cursor has already been advanced.";
+ T content = cursor.content(); // handle content on the root node
+ if (content == null)
+ content = cursor.advanceToContent(walker);
+
+ while (content != null)
+ {
+ walker.content(content);
+ content = cursor.advanceToContent(walker);
+ }
+ return walker.complete();
+ }
+
+ /**
+ * Constuct a textual representation of the trie.
+ */
+ public String dump()
+ {
+ return dump(Object::toString);
+ }
+
+ /**
+ * Constuct a textual representation of the trie using the given content-to-string mapper.
+ */
+ public String dump(Function<T, String> contentToString)
+ {
+ return process(new TrieDumper<>(contentToString));
+ }
+
+ /**
+ * Returns a singleton trie mapping the given byte path to content.
+ */
+ public static <T> Trie<T> singleton(ByteComparable b, T v)
+ {
+ return new SingletonTrie<>(b, v);
+ }
+
+ /**
+ * Returns a view of the subtrie containing everything in this trie whose keys fall between the given boundaries.
+ * The view is live, i.e. any write to the source will be reflected in the subtrie.
+ *
+ * This method will not check its arguments for correctness. The resulting trie may be empty or throw an exception
+ * if the right bound is smaller than the left.
+ *
+ * @param left the left bound for the returned subtrie. If {@code null}, the resulting subtrie is not left-bounded.
+ * @param includeLeft whether {@code left} is an inclusive bound of not.
+ * @param right the right bound for the returned subtrie. If {@code null}, the resulting subtrie is not right-bounded.
+ * @param includeRight whether {@code right} is an inclusive bound of not.
+ * @return a view of the subtrie containing all the keys of this trie falling between {@code left} (inclusively if
+ * {@code includeLeft}) and {@code right} (inclusively if {@code includeRight}).
+ */
+ public Trie<T> subtrie(ByteComparable left, boolean includeLeft, ByteComparable right, boolean includeRight)
+ {
+ if (left == null && right == null)
+ return this;
+ return new SlicedTrie<>(this, left, includeLeft, right, includeRight);
+ }
+
+ /**
+ * Returns a view of the subtrie containing everything in this trie whose keys fall between the given boundaries,
+ * left-inclusive and right-exclusive.
+ * The view is live, i.e. any write to the source will be reflected in the subtrie.
+ *
+ * This method will not check its arguments for correctness. The resulting trie may be empty or throw an exception
+ * if the right bound is smaller than the left.
+ *
+ * Equivalent to calling subtrie(left, true, right, false).
+ *
+ * @param left the left bound for the returned subtrie. If {@code null}, the resulting subtrie is not left-bounded.
+ * @param right the right bound for the returned subtrie. If {@code null}, the resulting subtrie is not right-bounded.
+ * @return a view of the subtrie containing all the keys of this trie falling between {@code left} (inclusively if
+ * {@code includeLeft}) and {@code right} (inclusively if {@code includeRight}).
+ */
+ public Trie<T> subtrie(ByteComparable left, ByteComparable right)
+ {
+ return subtrie(left, true, right, false);
+ }
+
+ /**
+ * Returns the ordered entry set of this trie's content as an iterable.
+ */
+ public Iterable<Map.Entry<ByteComparable, T>> entrySet()
+ {
+ return this::entryIterator;
+ }
+
+ /**
+ * Returns the ordered entry set of this trie's content in an iterator.
+ */
+ public Iterator<Map.Entry<ByteComparable, T>> entryIterator()
+ {
+ return new TrieEntriesIterator.AsEntries<>(this);
+ }
+
+ /**
+ * Returns the ordered set of values of this trie as an iterable.
+ */
+ public Iterable<T> values()
+ {
+ return this::valueIterator;
+ }
+
+ /**
+ * Returns the ordered set of values of this trie in an iterator.
+ */
+ public Iterator<T> valueIterator()
+ {
+ return new TrieValuesIterator<>(this);
+ }
+
+ /**
+ * Returns the values in any order. For some tries this is much faster than the ordered iterable.
+ */
+ public Iterable<T> valuesUnordered()
+ {
+ return values();
+ }
+
+ /**
+ * Resolver of content of merged nodes, used for two-source merges (i.e. mergeWith).
+ */
+ public interface MergeResolver<T>
+ {
+ // Note: No guarantees about argument order.
+ // E.g. during t1.mergeWith(t2, resolver), resolver may be called with t1 or t2's items as first argument.
+ T resolve(T b1, T b2);
+ }
+
+ /**
+ * Constructs a view of the merge of this trie with the given one. The view is live, i.e. any write to any of the
+ * sources will be reflected in the merged view.
+ *
+ * If there is content for a given key in both sources, the resolver will be called to obtain the combination.
+ * (The resolver will not be called if there's content from only one source.)
+ */
+ public Trie<T> mergeWith(Trie<T> other, MergeResolver<T> resolver)
+ {
+ return new MergeTrie<>(resolver, this, other);
+ }
+
+ /**
+ * Resolver of content of merged nodes.
+ *
+ * The resolver's methods are only called if more than one of the merged nodes contain content, and the
+ * order in which the arguments are given is not defined. Only present non-null values will be included in the
+ * collection passed to the resolving methods.
+ *
+ * Can also be used as a two-source resolver.
+ */
+ public interface CollectionMergeResolver<T> extends MergeResolver<T>
+ {
+ T resolve(Collection<T> contents);
+
+ @Override
+ default T resolve(T c1, T c2)
+ {
+ return resolve(ImmutableList.of(c1, c2));
+ }
+ }
+
+ private static final CollectionMergeResolver<Object> THROWING_RESOLVER = new CollectionMergeResolver<Object>()
+ {
+ @Override
+ public Object resolve(Collection<Object> contents)
+ {
+ throw error();
+ }
+
+ private AssertionError error()
+ {
+ throw new AssertionError("Entries must be distinct.");
+ }
+ };
+
+ /**
+ * Returns a resolver that throws whenever more than one of the merged nodes contains content.
+ * Can be used to merge tries that are known to have distinct content paths.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> CollectionMergeResolver<T> throwingResolver()
+ {
+ return (CollectionMergeResolver<T>) THROWING_RESOLVER;
+ }
+
+ /**
+ * Constructs a view of the merge of multiple tries. The view is live, i.e. any write to any of the
+ * sources will be reflected in the merged view.
+ *
+ * If there is content for a given key in more than one sources, the resolver will be called to obtain the
+ * combination. (The resolver will not be called if there's content from only one source.)
+ */
+ public static <T> Trie<T> merge(Collection<? extends Trie<T>> sources, CollectionMergeResolver<T> resolver)
+ {
+ switch (sources.size())
+ {
+ case 0:
+ return empty();
+ case 1:
+ return sources.iterator().next();
+ case 2:
+ {
+ Iterator<? extends Trie<T>> it = sources.iterator();
+ Trie<T> t1 = it.next();
+ Trie<T> t2 = it.next();
+ return t1.mergeWith(t2, resolver);
+ }
+ default:
+ return new CollectionMergeTrie<>(sources, resolver);
+ }
+ }
+
+ /**
+ * Constructs a view of the merge of multiple tries, where each source must have distinct keys. The view is live,
+ * i.e. any write to any of the sources will be reflected in the merged view.
+ *
+ * If there is content for a given key in more than one sources, the merge will throw an assertion error.
+ */
+ public static <T> Trie<T> mergeDistinct(Collection<? extends Trie<T>> sources)
+ {
+ switch (sources.size())
+ {
+ case 0:
+ return empty();
+ case 1:
+ return sources.iterator().next();
+ case 2:
+ {
+ Iterator<? extends Trie<T>> it = sources.iterator();
+ Trie<T> t1 = it.next();
+ Trie<T> t2 = it.next();
+ return new MergeTrie.Distinct<>(t1, t2);
+ }
+ default:
+ return new CollectionMergeTrie.Distinct<>(sources);
+ }
+ }
+
+ private static final Trie<Object> EMPTY = new Trie<Object>()
+ {
+ protected Cursor<Object> cursor()
+ {
+ return new Cursor<Object>()
+ {
+ int depth = 0;
+
+ public int advance()
+ {
+ return depth = -1;
+ }
+
+ public int skipChildren()
+ {
+ return depth = -1;
+ }
+
+ public int depth()
+ {
+ return depth;
+ }
+
+ public Object content()
+ {
+ return null;
+ }
+
+ public int incomingTransition()
+ {
+ return -1;
+ }
+ };
+ }
+ };
+
+ @SuppressWarnings("unchecked")
+ public static <T> Trie<T> empty()
+ {
+ return (Trie<T>) EMPTY;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/Trie.md b/src/java/org/apache/cassandra/db/tries/Trie.md
new file mode 100644
index 0000000..4265871
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/Trie.md
@@ -0,0 +1,252 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# `Trie` interface
+
+Tries in Cassandra are used to represent key-value mappings in an efficient way, currently only for the partition map
+in a memtable. The design we use is focussed on performing the equivalent of a read query, being able to most efficiently:
+- combine multiple sources while maintaining order,
+- restrict the combination to a range of keys,
+- efficiently walk the result and extract all covered key-value combinations.
+
+For this the `Trie` interface provides the following public methods:
+- Consuming the content using `forEachValue` or `forEachEntry`.
+- Conversion to iterable and iterator using `values/valuesIterator` and `entrySet/entryIterator`.
+- Getting a view of the subtrie between a given pair of bounds using `subtrie`.
+- Merging two or multiple tries together using `mergeWith` and the static `merge`.
+- Constructing `singleton` tries representing a single key-to-value mapping.
+
+The internal representation of a trie is given by a `Cursor`, which provides a method of walking the nodes of the trie
+in order, and to which various transformations like merges and intersections can be easily and efficiently applied.
+The sections below detail the motivation behind this design as well as the implementations of the basic operations.
+
+## Walking a trie
+
+Walking a `Trie` is achieved using a cursor. Before we describe it in detail, let's give a quick example of what a
+classic trie walk looks like and how it can be optimized. Suppose we want to walk the following trie:
+
+
+
+(Note: the node labels are `InMemoryTrie` node IDs which can be ignored here, with the exception of `contentArray[x]`
+ones which specify that the relevant node has some associated content.)
+
+The classic walk descends (<span style="color:lightblue">light blue</span>) on every character and backtracks
+(<span style="color:pink">pink</span>) to the parent, resulting in the following walk:
+
+
+
+One can see from this graph that many of the backtracking steps are only taken so that they can immediately be followed
+by another backtracking step. We often also know in advance that a node does not need to be examined further on the way
+back: if it only has one child (which is always the case for all nodes in a `Chain`), or if we are descending into
+its last child (which is easy to check for `Sparse` nodes). This simplifies the walk to:
+
+
+
+In addition to making the walk simpler, shortening the backtracking paths means a smaller walk state representation,
+which is quite helpful in keeping the garbage collection cost down. In this example, the backtracking state of the walk
+at the "tractor" node is only `[("tr", child 2)]`, changes to `[("tr", child 3)]` on descent to "tre", and becomes empty
+(as "tr" has no further children and can be removed) on descent to "tri".
+
+One further optimization of the walk is to jump directly to the next child without stopping at a branching parent (note:
+the black arrows represent the trie structure):
+
+
+
+This graph is what a cursor walk over this trie is. Looking closely at the graph, one can see that it stops exactly once
+on each node, and that the nodes are visited in lexicographic order. There is no longer a need for a separate backtrack
+or ascend operation, because all arrows _advance_ in the representation of the trie. However, to be able to understand
+where the next transition character sits in the path, every transition now also comes with information about the
+descend-depth it ends in.
+
+To see how this can be used to map the state to a path, we can imagine an array being filled at its `depth-1`th
+position on every transition, and the current path being the `depth`-long sequence. This, e.g. the array would hold
+[t, r, a, c, t, o, r] at the "tractor" node and change to [t, r, e, c, t, o, r] for the next advance, where the new
+current path is the first 3 characters of the array.
+
+Cursors are stateful objects that store the backtracking state of the walk. That is, a list of all parent nodes in the
+path to reach the current node that have further children, together with information which child backtracking should go
+to. Cursors do not store paths as these are often not needed — for example, in a walk over a merged trie it makes
+better sense for the consumer to construct the path instead of having them duplicated in every source cursor. Multiple
+cursors can be constructed and operated in parallel over a trie.
+
+Cursor walks of this type are very efficient but still carry enough information to make it very easy and efficient to
+merge and intersect tries. If we are walking a single trie (or a single-source branch in a union trie), we can
+improve the efficiency even further by taking multiple steps down in `Chain` nodes, provided we have a suitable
+mechanism of passing additional transition characters:
+
+
+
+This is supported by `Cursor.advanceMultiple`.
+
+### Why cursors instead of nodes?
+
+The most straightforward representation of a trie is done by giving users every `Node` visited as an object.
+Then the consumer can query its transitions, get its children, decide to walk over them in any order it sees
+fit, and retain those that it actually needs. This is a very natural and cheap represention if the nodes are actually
+the objects in memory that represent the trie.
+
+The latter is not the case for us: we store tries in integer blobs or files on disk and we present transformed views of
+tries. Thus every such `Node` object to give to consumers must be constructed. Also, we only do depth-first walks and it
+does not make that much sense to provide the full flexibility of that kind of interface.
+
+When doing only depth-first walks, a cursor needs far fewer objects to represent its state than a node representation.
+Consider the following for an approach presenting nodes:
+- In a process that requires single-step descends (i.e. in a merge or intersection) the iteration state must create an
+ object for every intermediate node even when they are known to require no backtracking because they have only one
+ child.
+- Childless final states require a node.
+- A transformation such as a merge must present the result as a transformed node, but it also requires a node for each
+ input. If the transformed node is retained, so must be the sources.
+
+Cursors can represent the first two in their internal state without additional backtracking state, and require only one
+transformed cursor to be constructed for the entire walk. Additionally, cursors' representation of backtracking state
+may be closely tied to the specific trie implementation, which also gives further improvement opportunities (e.g. the
+`Split` node treatment in `InMemoryTrie`).
+
+### Why not visitors?
+
+A visitor or a push alternative is one where the trie drives the iteration and the caller provides a visitor or a
+consumer. This can work well if the trie to walk is single-source, but requires some form of stop/restart or pull
+mechanism to implement ordered merges.
+
+Push-style walks are still a useful way to consume the final transformed/merged content, thus `Trie` provides
+the `Walker` interface and `process` method. The implementations of `forEachEntry` and `dump` are straightforward
+applications of this.
+
+### The `Cursor` interface
+
+The cursor represents the state of a walk over the nodes of trie. It provides three main features:
+- the current `depth` or descend-depth in the trie;
+- the `incomingTransition`, i.e. the byte that was used to reach the current point;
+- the `content` associated with the current node,
+
+and provides methods for advancing to the next position. This is enough information to extract all paths, and
+also to easily compare cursors over different tries that are advanced together. Advancing is always done in
+order; if one imagines the set of nodes in the trie with their associated paths, a cursor may only advance from a
+node with a lexicographically smaller path to one with bigger. The `advance` operation moves to the immediate
+next, it is also possible to skip over some items e.g. all children of the current node (`skipChildren`).
+
+Moving to the immediate next position in the lexicographic order is accomplished by:
+- if the current node has children, moving to its first child;
+- otherwise, ascend the parent chain and return the next child of the closest parent that still has any.
+
+As long as the trie is not exhausted, advancing always takes one step down, from the current node, or from a node
+on the parent chain. By comparing the new depth (which `advance` also returns) with the one before the advance,
+one can tell if the former was the case (if `newDepth == oldDepth + 1`) and how many steps up we had to take
+(`oldDepth + 1 - newDepth`). When following a path down, the cursor will stop on all prefixes.
+
+In addition to the single-step `advance` method, the cursor also provides an `advanceMultiple` method for descending
+multiple steps down when this is known to be efficient. If it is not feasible to descend (e.g. because there are no
+children, or because getting to the child of the first child requires a page fetch from disk), `advanceMultiple` will
+act just like `advance`.
+
+For convenience, the interface also provides an `advanceToContent` method for walking to the next node with non-null
+content. This is implemented via `advanceMultiple`.
+
+When the cursor is created it is placed on the root node with `depth() = 0`, `incomingTransition() = -1`. Since
+tries can have mappings for empty, `content()` can possibly be non-null in the starting position. It is not allowed
+for a cursor to start in exhausted state (i.e. with `depth() = -1`).
+
+### Using cursors in parallel
+
+One important feature of cursors is the fact that we can easily walk them in parallel. More precisely, when we use a
+procedure where we only advance the smaller, or both if they are equal, we can compare the cursors' state by:
+- the reverse order of their current depth (i.e. higher depth first),
+- if depths are equal, the order of their current incoming transition (lexicographically smaller first).
+
+We can prove this by induction, where for two cursors `a` and `b` we maintain that:
+1. for any depth `i < mindepth - 1`, `path(a)[i] == path(b)[i]`
+2. if `a.depth < b.depth`, then `path(a)[mindepth - 1] > path(b)[mindepth - 1]`
+3. if `a.depth > b.depth`, then `path(a)[mindepth - 1] < path(b)[mindepth - 1]`
+
+where `mindepth = min(a.depth, b.depth)` and `path(cursor)` is the path corresponding to the node the cursor is
+positioned at. Note that `path(cursor)[cursor.depth - 1] == cursor.incomingTransition`.
+
+These conditions ensure that `path(a) < path(b)` if and only if `a.depth > b.depth` or `a.depth == b.depth` and
+`a.incomingTransition < b.incomingTransition`. Indeed, if the second part is true then 1 and 3 enforce the first,
+and if the second part is not true, i.e. `a.depth <= b.depth` and (`a.depth != b.depth` or
+`a.incomingTransition >= b.incomingTransition`), which entails `a.depth < b.depth` or `a.depth == b.depth` and
+`a.incomingTransition >= b.incomingTransition`, then by 2 and 1 we can conclude that `path(a) >= path(b)`, i.e. the
+first part is not true either.
+
+The conditions are trivially true for the initial state where both cursors are positioned at the root with depth 0.
+Also, when we advance a cursor, it is always the case that the path of the previous state and the path of the new state
+must be the same in all positions before its new depth minus one'th. Thus, if the two cursors are equal before
+advancing, i.e. they are positioned on exactly the same path, the state after advancing must satisfy condition 1 above
+because the earliest byte in either path that can have changed is the one at position `min(a.depth, b.depth) - 1`.
+Moreover, if the depths are different, the cursor with the lower one will have advanced its character in position
+`depth - 1` while the other cursor's character at that position will have remained the same, thus conditions 2 and 3 are
+also satisfied.
+
+If `path(a)` was smaller before advancing we have that `a.depth >= b.depth`. The parallel walk will then only advance
+`a`. If the new depth of `a` is higher than `b`'s, nothing changes in conditions 1-3 (the bytes before `b.depth` do
+not change at all in either cursor). If the new depth of `a` is the same as `b`'s, condition 1 is still satisfied
+because these bytes cannot have changed, and the premises in 2 and 3 are false. If the new depth of `a` is lower
+than `b`'s, however, `a` must have advanced the byte at index `depth - 1`, and because (due to 1) it was previously
+equal to `b`'s at this index, it must now be higher, proving 2. Condition 1 is still true because these bytes cannot
+have changed, and 3 is true because it has a false premise.
+
+The same argument holds when `b` is the smaller cursor to be advanced.
+
+## Merging two tries
+
+Two tries can be merged using `Trie.mergeWith`, which is implemented using the class `MergeTrie`. The implementation
+is a straightforward application of the parallel walking scheme above, where the merged cursor presents the depth
+and incoming transition of the currently smaller cursor, and advances by advancing the smaller cursor, or both if they
+are equal.
+
+If the cursors are not equal, we can also apply `advanceMultiple`, because it can only be different from `advance`
+if it descends. When a cursor is known to be smaller it is guaranteed to remain smaller when it descends as its
+new depth will be larger than before and thus larger than the other cursor's. This cannot be done to advance both
+cursors when they are equal, because that can violate the conditions. (Simple example: one descends through "a" and
+the other through "bb" — condition 2. is violated, the latter will have higher depth but will not be smaller.)
+
+## Merging an arbitrary number of tries
+
+Merging is extended to an arbitrary number of sources in `CollectionMergeTrie`, used through the static `Trie.merge`.
+The process is a generalization of the above, implemented using the min-heap solution from `MergeIterator` applied
+to cursors.
+
+In this solution an extra head element is maintained _before_ the min-heap to optimize for single-source branches
+where we prefer to advance using just one comparison (head to heap top) instead of two (heap top to its two
+descendants) at the expense of possibly adding one additional comparison in the general case.
+
+As above, when we know that the head element is not equal to the heap top (i.e. it's necessarily smaller) we can
+use its `advanceMultiple` safely.
+
+## Slicing tries
+
+Slicing, implemented in `SlicedTrie` and used via `Trie.subtrie`, can also be seen as a variation of the parallel
+walk. In this case we walk the source as well as singletons of the two bounds.
+
+More precisely, while the source cursor is smaller than the left bound, we don't produce any output. That is, we
+keep advancing in a loop, but to avoid walking subtries unnecessarily, we use `skipChildren` instead of `advance`.
+As we saw above, a smaller cursor that descends remains smaller, thus there is no point to do so when we are
+ahead of the left bound. When the source matches a node from the left bound, we descend both and pass the
+state to the consumer. As soon as the source becomes known greater than the left bound, we can stop processing
+the latter and pass any state we see to the consumer.
+
+Throughout this we also process the right bound cursor and we stop the iteration (by returning `depth = -1`)
+as soon as the source becomes larger than the right bound.
+
+`SlicedTrie` does not use singleton tries and cursors over them but opts to implement them directly, using an
+implicit representation using a pair of `depth` and `incomingTransition` for each bound.
+
+In slices we can also use `advanceMultiple` when we are certain to be strictly inside the slice, i.e. beyond the
+left bound and before a prefix of the right bound. As above, descending to any depth in this case is safe as the
+result will remain smaller than the right bound.
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/db/tries/TrieDumper.java b/src/java/org/apache/cassandra/db/tries/TrieDumper.java
new file mode 100644
index 0000000..9dfb2c1
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TrieDumper.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.function.Function;
+
+import org.agrona.DirectBuffer;
+
+/**
+ * Simple utility class for dumping the structure of a trie to string.
+ */
+class TrieDumper<T> implements Trie.Walker<T, String>
+{
+ private final StringBuilder b;
+ private final Function<T, String> contentToString;
+ int needsIndent = -1;
+ int currentLength = 0;
+
+ public TrieDumper(Function<T, String> contentToString)
+ {
+ this.contentToString = contentToString;
+ this.b = new StringBuilder();
+ }
+
+ private void endLineAndSetIndent(int newIndent)
+ {
+ needsIndent = newIndent;
+ }
+
+ @Override
+ public void resetPathLength(int newLength)
+ {
+ currentLength = newLength;
+ endLineAndSetIndent(newLength);
+ }
+
+ private void maybeIndent()
+ {
+ if (needsIndent >= 0)
+ {
+ b.append('\n');
+ for (int i = 0; i < needsIndent; ++i)
+ b.append(" ");
+ needsIndent = -1;
+ }
+ }
+
+ @Override
+ public void addPathByte(int nextByte)
+ {
+ maybeIndent();
+ ++currentLength;
+ b.append(String.format("%02x", nextByte));
+ }
+
+ @Override
+ public void addPathBytes(DirectBuffer buffer, int pos, int count)
+ {
+ maybeIndent();
+ for (int i = 0; i < count; ++i)
+ b.append(String.format("%02x", buffer.getByte(pos + i) & 0xFF));
+ currentLength += count;
+ }
+
+ @Override
+ public void content(T content)
+ {
+ b.append(" -> ");
+ b.append(contentToString.apply(content));
+ endLineAndSetIndent(currentLength);
+ }
+
+ @Override
+ public String complete()
+ {
+ return b.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java b/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java
new file mode 100644
index 0000000..7ab3e7d
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.AbstractMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Convertor of trie entries to iterator where each entry is passed through {@link #mapContent} (to be implemented by
+ * descendants).
+ */
+public abstract class TrieEntriesIterator<T, V> extends TriePathReconstructor implements Iterator<V>
+{
+ private final Trie.Cursor<T> cursor;
+ T next;
+ boolean gotNext;
+
+ protected TrieEntriesIterator(Trie<T> trie)
+ {
+ cursor = trie.cursor();
+ assert cursor.depth() == 0;
+ next = cursor.content();
+ gotNext = next != null;
+ }
+
+ public boolean hasNext()
+ {
+ if (!gotNext)
+ {
+ next = cursor.advanceToContent(this);
+ gotNext = true;
+ }
+
+ return next != null;
+ }
+
+ public V next()
+ {
+ gotNext = false;
+ T v = next;
+ next = null;
+ return mapContent(v, keyBytes, keyPos);
+ }
+
+ protected abstract V mapContent(T content, byte[] bytes, int byteLength);
+
+ /**
+ * Iterator representing the content of the trie a sequence of (path, content) pairs.
+ */
+ static class AsEntries<T> extends TrieEntriesIterator<T, Map.Entry<ByteComparable, T>>
+ {
+ public AsEntries(Trie<T> trie)
+ {
+ super(trie);
+ }
+
+ @Override
+ protected Map.Entry<ByteComparable, T> mapContent(T content, byte[] bytes, int byteLength)
+ {
+ return toEntry(content, bytes, byteLength);
+ }
+ }
+
+ static <T> java.util.Map.Entry<ByteComparable, T> toEntry(T content, byte[] bytes, int byteLength)
+ {
+ return new AbstractMap.SimpleImmutableEntry<>(toByteComparable(bytes, byteLength), content);
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java b/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java
new file mode 100644
index 0000000..ca06015
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.function.BiConsumer;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Walker of trie entries, used with Trie.process to walk all content in order and provide the path through which values
+ * are reached.
+ */
+public abstract class TrieEntriesWalker<T, V> extends TriePathReconstructor implements Trie.Walker<T, V>
+{
+ @Override
+ public void content(T content)
+ {
+ content(content, keyBytes, keyPos);
+ }
+
+ protected abstract void content(T content, byte[] bytes, int byteLength);
+
+ /**
+ * Iterator representing the content of the trie a sequence of (path, content) pairs.
+ */
+ static class WithConsumer<T> extends TrieEntriesWalker<T, Void>
+ {
+ private final BiConsumer<ByteComparable, T> consumer;
+
+ public WithConsumer(BiConsumer<ByteComparable, T> consumer)
+ {
+ this.consumer = consumer;
+ }
+
+ @Override
+ protected void content(T content, byte[] bytes, int byteLength)
+ {
+ consumer.accept(toByteComparable(bytes, byteLength), content);
+ }
+
+ @Override
+ public Void complete()
+ {
+ return null;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java b/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java
new file mode 100644
index 0000000..4a9883f
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.util.Arrays;
+
+import org.agrona.DirectBuffer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+public class TriePathReconstructor implements Trie.ResettingTransitionsReceiver
+{
+ protected byte[] keyBytes = new byte[32];
+ protected int keyPos = 0;
+
+ public void addPathByte(int nextByte)
+ {
+ if (keyPos >= keyBytes.length)
+ keyBytes = Arrays.copyOf(keyBytes, keyPos * 2);
+ keyBytes[keyPos++] = (byte) nextByte;
+ }
+
+ public void addPathBytes(DirectBuffer buffer, int pos, int count)
+ {
+ int newPos = keyPos + count;
+ if (newPos > keyBytes.length)
+ keyBytes = Arrays.copyOf(keyBytes, Math.max(newPos + 16, keyBytes.length * 2));
+ buffer.getBytes(pos, keyBytes, keyPos, count);
+ keyPos = newPos;
+ }
+
+ public void resetPathLength(int newLength)
+ {
+ keyPos = newLength;
+ }
+
+ static ByteComparable toByteComparable(byte[] bytes, int byteLength)
+ {
+ return ByteComparable.fixedLength(Arrays.copyOf(bytes, byteLength));
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java b/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java
new file mode 100644
index 0000000..29d3642
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/tries/TrieValuesIterator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.Iterator;
+
+/**
+ * Ordered iterator of trie content.
+ */
+class TrieValuesIterator<T> implements Iterator<T>
+{
+ private final Trie.Cursor<T> cursor;
+ T next;
+ boolean gotNext;
+
+ protected TrieValuesIterator(Trie<T> trie)
+ {
+ cursor = trie.cursor();
+ assert cursor.depth() == 0;
+ next = cursor.content();
+ gotNext = next != null;
+ }
+
+ public boolean hasNext()
+ {
+ if (!gotNext)
+ {
+ next = cursor.advanceToContent(null);
+ gotNext = true;
+ }
+
+ return next != null;
+ }
+
+ public T next()
+ {
+ gotNext = false;
+ T v = next;
+ next = null;
+ return v;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/view/View.java b/src/java/org/apache/cassandra/db/view/View.java
index d813d0e..a3ecc33 100644
--- a/src/java/org/apache/cassandra/db/view/View.java
+++ b/src/java/org/apache/cassandra/db/view/View.java
@@ -35,6 +35,7 @@
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.schema.ViewMetadata;
+import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.utils.FBUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,7 +178,7 @@
rawSelect.setBindVariables(Collections.emptyList());
- select = rawSelect.prepare(true);
+ select = rawSelect.prepare(ClientState.forInternalCalls(), true);
}
return select;
diff --git a/src/java/org/apache/cassandra/db/view/ViewManager.java b/src/java/org/apache/cassandra/db/view/ViewManager.java
index 111f96a..106a15f 100644
--- a/src/java/org/apache/cassandra/db/view/ViewManager.java
+++ b/src/java/org/apache/cassandra/db/view/ViewManager.java
@@ -36,6 +36,8 @@
import org.apache.cassandra.schema.Views;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ENABLE_COORDINATOR_BATCHLOG;
+
/**
* Manages {@link View}'s for a single {@link ColumnFamilyStore}. All of the views for that table are created when this
* manager is initialized.
@@ -57,7 +59,7 @@
private static final Striped<Lock> LOCKS = Striped.lazyWeakLock(DatabaseDescriptor.getConcurrentViewWriters() * 1024);
- private static final boolean enableCoordinatorBatchlog = Boolean.getBoolean("cassandra.mv_enable_coordinator_batchlog");
+ private static final boolean enableCoordinatorBatchlog = MV_ENABLE_COORDINATOR_BATCHLOG.getBoolean();
private final ConcurrentMap<String, View> viewsByName = new ConcurrentHashMap<>();
private final ConcurrentMap<TableId, TableViews> viewsByBaseTable = new ConcurrentHashMap<>();
diff --git a/src/java/org/apache/cassandra/db/virtual/ClientsTable.java b/src/java/org/apache/cassandra/db/virtual/ClientsTable.java
index d39c269..027bb9b 100644
--- a/src/java/org/apache/cassandra/db/virtual/ClientsTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/ClientsTable.java
@@ -40,6 +40,7 @@
private static final String SSL_ENABLED = "ssl_enabled";
private static final String SSL_PROTOCOL = "ssl_protocol";
private static final String SSL_CIPHER_SUITE = "ssl_cipher_suite";
+ private static final String KEYSPACE_NAME = "keyspace_name";
ClientsTable(String keyspace)
{
@@ -60,6 +61,7 @@
.addRegularColumn(SSL_ENABLED, BooleanType.instance)
.addRegularColumn(SSL_PROTOCOL, UTF8Type.instance)
.addRegularColumn(SSL_CIPHER_SUITE, UTF8Type.instance)
+ .addRegularColumn(KEYSPACE_NAME, UTF8Type.instance)
.build());
}
@@ -83,7 +85,8 @@
.column(REQUEST_COUNT, client.requestCount())
.column(SSL_ENABLED, client.sslEnabled())
.column(SSL_PROTOCOL, client.sslProtocol().orElse(null))
- .column(SSL_CIPHER_SUITE, client.sslCipherSuite().orElse(null));
+ .column(SSL_CIPHER_SUITE, client.sslCipherSuite().orElse(null))
+ .column(KEYSPACE_NAME, client.keyspace().orElse(null));
}
return result;
diff --git a/src/java/org/apache/cassandra/db/virtual/LogMessagesTable.java b/src/java/org/apache/cassandra/db/virtual/LogMessagesTable.java
new file mode 100644
index 0000000..5903ac2
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/LogMessagesTable.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import ch.qos.logback.classic.spi.LoggingEvent;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Virtual table for holding Cassandra logs. Entries to this table are added via log appender.
+ * <p>
+ * The virtual table is bounded in its size. If a new log message is appended to virtual table,
+ * the oldest one is removed.
+ * <p>
+ * This virtual table can be truncated.
+ * <p>
+ * This table does not enable {@code ALLOW FILTERING} implicitly.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/CASSANDRA-18238">CASSANDRA-18238</a>
+ * @see org.apache.cassandra.utils.logging.VirtualTableAppender
+ */
+public final class LogMessagesTable extends AbstractMutableVirtualTable
+{
+ private static final Logger logger = LoggerFactory.getLogger(LogMessagesTable.class);
+
+ public static final int LOGS_VIRTUAL_TABLE_MIN_ROWS = 1000;
+ public static final int LOGS_VIRTUAL_TABLE_DEFAULT_ROWS = 50_000;
+ public static final int LOGS_VIRTUAL_TABLE_MAX_ROWS = 100_000;
+
+ public static final String TABLE_NAME = "system_logs";
+ private static final String TABLE_COMMENT = "Cassandra logs";
+
+ public static final String TIMESTAMP_COLUMN_NAME = "timestamp";
+ public static final String LOGGER_COLUMN_NAME = "logger";
+ public static final String ORDER_IN_MILLISECOND_COLUMN_NAME = "order_in_millisecond";
+ public static final String LEVEL_COLUMN_NAME = "level";
+ public static final String MESSAGE_COLUMN_NAME = "message";
+
+ private final List<LogMessage> buffer;
+
+ LogMessagesTable(String keyspace)
+ {
+ this(keyspace, resolveBufferSize());
+ }
+
+ @VisibleForTesting
+ LogMessagesTable(String keyspace, int size)
+ {
+ super(TableMetadata.builder(keyspace, TABLE_NAME)
+ .comment(TABLE_COMMENT)
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(TimestampType.instance))
+ .addPartitionKeyColumn(TIMESTAMP_COLUMN_NAME, TimestampType.instance)
+ .addClusteringColumn(ORDER_IN_MILLISECOND_COLUMN_NAME, Int32Type.instance)
+ .addRegularColumn(LOGGER_COLUMN_NAME, UTF8Type.instance)
+ .addRegularColumn(LEVEL_COLUMN_NAME, UTF8Type.instance)
+ .addRegularColumn(MESSAGE_COLUMN_NAME, UTF8Type.instance).build());
+
+ logger.debug("capacity of virtual table {} is set to be at most {} rows", metadata().toString(), size);
+ buffer = BoundedLinkedList.create(size);
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata(), DecoratedKey.comparator.reversed());
+
+ synchronized (buffer)
+ {
+ long milliSecondsOfPreviousLog = 0;
+ long milliSecondsOfCurrentLog;
+
+ int index = 0;
+
+ Iterator<LogMessage> iterator = buffer.listIterator();
+ while (iterator.hasNext())
+ {
+ LogMessage log = iterator.next();
+
+ milliSecondsOfCurrentLog = log.timestamp;
+ if (milliSecondsOfPreviousLog == milliSecondsOfCurrentLog)
+ ++index;
+ else
+ index = 0;
+
+ milliSecondsOfPreviousLog = milliSecondsOfCurrentLog;
+
+ result.row(new Date(log.timestamp), index)
+ .column(LOGGER_COLUMN_NAME, log.logger)
+ .column(LEVEL_COLUMN_NAME, log.level)
+ .column(MESSAGE_COLUMN_NAME, log.message);
+ }
+ }
+
+ return result;
+ }
+
+ public void add(LoggingEvent event)
+ {
+ buffer.add(new LogMessage(event));
+ }
+
+ @Override
+ public void truncate()
+ {
+ buffer.clear();
+ }
+
+ @Override
+ public boolean allowFilteringImplicitly()
+ {
+ return false;
+ }
+
+ @VisibleForTesting
+ static int resolveBufferSize()
+ {
+ int size = CassandraRelevantProperties.LOGS_VIRTUAL_TABLE_MAX_ROWS.getInt();
+ return (size < LOGS_VIRTUAL_TABLE_MIN_ROWS || size > LOGS_VIRTUAL_TABLE_MAX_ROWS)
+ ? LOGS_VIRTUAL_TABLE_DEFAULT_ROWS : size;
+ }
+
+ @VisibleForTesting
+ public static class LogMessage
+ {
+ public final long timestamp;
+ public final String logger;
+ public final String level;
+ public final String message;
+
+ public LogMessage(LoggingEvent event)
+ {
+ this(event.getTimeStamp(), event.getLoggerName(), event.getLevel().toString(), event.getFormattedMessage());
+ }
+
+ public LogMessage(long timestamp, String logger, String level, String message)
+ {
+ this.timestamp = timestamp;
+ this.logger = logger;
+ this.level = level;
+ this.message = message;
+ }
+ }
+
+ private static final class BoundedLinkedList<T> extends LinkedList<T>
+ {
+ private final int maxSize;
+
+ public static <T> List<T> create(int size)
+ {
+ return Collections.synchronizedList(new BoundedLinkedList<>(size));
+ }
+
+ private BoundedLinkedList(int maxSize)
+ {
+ this.maxSize = maxSize;
+ }
+
+ @Override
+ public boolean add(T t)
+ {
+ if (size() == maxSize)
+ removeLast();
+
+ addFirst(t);
+
+ return true;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java
new file mode 100644
index 0000000..aeba61c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import org.apache.cassandra.concurrent.DebuggableTask;
+import org.apache.cassandra.concurrent.SharedExecutorPool;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static java.lang.Long.max;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
+
+/**
+ * Virtual table that lists currently running queries on the NTR (coordinator) and Read/Mutation (local) stages
+ *
+ * Example:
+ * <pre>
+ * cqlsh> SELECT * FROM system_views.queries;
+ *
+ * thread_id | queued_micros | running_micros | task
+ * ------------------------------+---------------+-----------------+--------------------------------------------------------------------------------
+ * Native-Transport-Requests-7 | 72923 | 7611 | QUERY select * from system_views.queries; [pageSize = 100]
+ * MutationStage-2 | 18249 | 2084 | Mutation(keyspace='distributed_test_keyspace', key='000000f8', modifications...
+ * ReadStage-2 | 72447 | 10121 | SELECT * FROM keyspace.table LIMIT 5000
+ * </pre>
+ */
+final class QueriesTable extends AbstractVirtualTable
+{
+ private static final String TABLE_NAME = "queries";
+ private static final String ID = "thread_id";
+ private static final String QUEUED = "queued_micros";
+ private static final String RUNNING = "running_micros";
+ private static final String DESC = "task";
+
+ QueriesTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, TABLE_NAME)
+ .comment("Lists currently running queries")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ // The thread name is unique since the id given to each SEPWorker is unique
+ .addPartitionKeyColumn(ID, UTF8Type.instance)
+ .addRegularColumn(QUEUED, LongType.instance)
+ .addRegularColumn(RUNNING, LongType.instance)
+ .addRegularColumn(DESC, UTF8Type.instance)
+ .build());
+ }
+
+ /**
+ * Walks the {@link SharedExecutorPool} workers for any {@link DebuggableTask} instances and populates the table.
+ */
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ for (DebuggableTask.RunningDebuggableTask task : SharedExecutorPool.SHARED.runningTasks())
+ {
+ if (!task.hasTask()) continue;
+
+ long creationTimeNanos = task.creationTimeNanos();
+ long startTimeNanos = task.startTimeNanos();
+ long now = approxTime.now();
+
+ long queuedMicros = NANOSECONDS.toMicros(max((startTimeNanos > 0 ? startTimeNanos : now) - creationTimeNanos, 0));
+ long runningMicros = startTimeNanos > 0 ? NANOSECONDS.toMicros(now - startTimeNanos) : 0;
+
+ result.row(task.threadId())
+ .column(QUEUED, queuedMicros)
+ .column(RUNNING, runningMicros)
+ .column(DESC, task.description());
+ }
+
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
index 488b580..e2f38f8 100644
--- a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
@@ -38,6 +38,7 @@
private final static String SSTABLES = "sstables";
private final static String TOTAL = "total";
private final static String UNIT = "unit";
+ private final static String TARGET_DIRECTORY = "target_directory";
SSTableTasksTable(String keyspace)
{
@@ -54,6 +55,7 @@
.addRegularColumn(SSTABLES, Int32Type.instance)
.addRegularColumn(TOTAL, LongType.instance)
.addRegularColumn(UNIT, UTF8Type.instance)
+ .addRegularColumn(TARGET_DIRECTORY, UTF8Type.instance)
.build());
}
@@ -76,7 +78,8 @@
.column(PROGRESS, completed)
.column(SSTABLES, task.getSSTables().size())
.column(TOTAL, total)
- .column(UNIT, task.getUnit().toString().toLowerCase());
+ .column(UNIT, task.getUnit().toString().toLowerCase())
+ .column(TARGET_DIRECTORY, task.targetDirectory());
}
return result;
diff --git a/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java b/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java
index 715f4f8..6f3052d 100644
--- a/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java
+++ b/src/java/org/apache/cassandra/db/virtual/SimpleDataSet.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.virtual;
import java.nio.ByteBuffer;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -54,10 +55,15 @@
private Row currentRow;
+ public SimpleDataSet(TableMetadata metadata, Comparator<DecoratedKey> comparator)
+ {
+ super(new TreeMap<>(comparator));
+ this.metadata = metadata;
+ }
+
public SimpleDataSet(TableMetadata metadata)
{
- super(new TreeMap<>(DecoratedKey.comparator));
- this.metadata = metadata;
+ this(metadata, DecoratedKey.comparator);
}
public SimpleDataSet row(Object... primaryKeyValues)
diff --git a/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java b/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java
new file mode 100644
index 0000000..d3df293
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.util.Date;
+
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.snapshot.TableSnapshot;
+
+public class SnapshotsTable extends AbstractVirtualTable
+{
+ private static final String NAME = "name";
+ private static final String KEYSPACE_NAME = "keyspace_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String TRUE_SIZE = "true_size";
+ private static final String SIZE_ON_DISK = "size_on_disk";
+ private static final String CREATED_AT = "created_at";
+ private static final String EXPIRES_AT = "expires_at";
+ private static final String EPHEMERAL = "ephemeral";
+
+ SnapshotsTable(String keyspace)
+ {
+ super(TableMetadata.builder(keyspace, "snapshots")
+ .comment("available snapshots")
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .partitioner(new LocalPartitioner(UTF8Type.instance))
+ .addPartitionKeyColumn(NAME, UTF8Type.instance)
+ .addClusteringColumn(KEYSPACE_NAME, UTF8Type.instance)
+ .addClusteringColumn(TABLE_NAME, UTF8Type.instance)
+ .addRegularColumn(TRUE_SIZE, LongType.instance)
+ .addRegularColumn(SIZE_ON_DISK, LongType.instance)
+ .addRegularColumn(CREATED_AT, TimestampType.instance)
+ .addRegularColumn(EXPIRES_AT, TimestampType.instance)
+ .addRegularColumn(EPHEMERAL, BooleanType.instance)
+ .build());
+ }
+
+ @Override
+ public DataSet data()
+ {
+ SimpleDataSet result = new SimpleDataSet(metadata());
+
+ for (TableSnapshot tableSnapshot : StorageService.instance.snapshotManager.loadSnapshots())
+ {
+ SimpleDataSet row = result.row(tableSnapshot.getTag(),
+ tableSnapshot.getKeyspaceName(),
+ tableSnapshot.getTableName())
+ .column(TRUE_SIZE, tableSnapshot.computeTrueSizeBytes())
+ .column(SIZE_ON_DISK, tableSnapshot.computeSizeOnDiskBytes())
+ .column(CREATED_AT, new Date(tableSnapshot.getCreatedAt().toEpochMilli()));
+
+ if (tableSnapshot.isExpiring())
+ row.column(EXPIRES_AT, new Date(tableSnapshot.getExpiresAt().toEpochMilli()));
+
+ row.column(EPHEMERAL, tableSnapshot.isEphemeral());
+ }
+
+ return result;
+ }
+}
diff --git a/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java b/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java
index d08d614..f1d0d7c 100644
--- a/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/SystemPropertiesTable.java
@@ -47,15 +47,15 @@
{
SimpleDataSet result = new SimpleDataSet(metadata());
- System.getenv().keySet()
+ System.getenv().keySet() // checkstyle: suppress nearby 'blockSystemPropertyUsage'
.stream()
.filter(SystemPropertiesTable::isCassandraRelevant)
- .forEach(name -> addRow(result, name, System.getenv(name)));
+ .forEach(name -> addRow(result, name, System.getenv(name))); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
System.getProperties().stringPropertyNames()
.stream()
.filter(SystemPropertiesTable::isCassandraRelevant)
- .forEach(name -> addRow(result, name, System.getProperty(name)));
+ .forEach(name -> addRow(result, name, System.getProperty(name))); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
return result;
}
@@ -66,7 +66,7 @@
SimpleDataSet result = new SimpleDataSet(metadata());
String name = UTF8Type.instance.compose(partitionKey.getKey());
if (isCassandraRelevant(name))
- addRow(result, name, System.getProperty(name, System.getenv(name)));
+ addRow(result, name, System.getProperty(name, System.getenv(name))); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
return result;
}
diff --git a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
index f13e61c..12df6c7 100644
--- a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
+++ b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
@@ -47,6 +47,9 @@
.add(new BatchMetricsTable(VIRTUAL_VIEWS))
.add(new StreamingVirtualTable(VIRTUAL_VIEWS))
.add(new GossipInfoTable(VIRTUAL_VIEWS))
+ .add(new QueriesTable(VIRTUAL_VIEWS))
+ .add(new LogMessagesTable(VIRTUAL_VIEWS))
+ .add(new SnapshotsTable(VIRTUAL_VIEWS))
.addAll(LocalRepairTables.getAll(VIRTUAL_VIEWS))
.build());
}
diff --git a/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java b/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
index 9ff421c..8368fd9 100644
--- a/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
+++ b/src/java/org/apache/cassandra/db/virtual/TableMetricTables.java
@@ -77,7 +77,9 @@
new HistogramTableMetric(name, "tombstones_per_read", t -> t.tombstoneScannedHistogram.cf),
new HistogramTableMetric(name, "rows_per_read", t -> t.liveScannedHistogram.cf),
new StorageTableMetric(name, "disk_usage", (TableMetrics t) -> t.totalDiskSpaceUsed),
- new StorageTableMetric(name, "max_partition_size", (TableMetrics t) -> t.maxPartitionSize));
+ new StorageTableMetric(name, "max_partition_size", (TableMetrics t) -> t.maxPartitionSize),
+ new StorageTableMetric(name, "max_sstable_size", (TableMetrics t) -> t.maxSSTableSize),
+ new TableMetricTable(name, "max_sstable_duration", t -> t.maxSSTableDuration, "max_sstable_duration", LongType.instance, ""));
}
/**
diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualTable.java b/src/java/org/apache/cassandra/db/virtual/VirtualTable.java
index 5373f4c..53a9f2a 100644
--- a/src/java/org/apache/cassandra/db/virtual/VirtualTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/VirtualTable.java
@@ -76,4 +76,15 @@
* Truncates data from the underlying source, if supported.
*/
void truncate();
+
+ /**
+ * Tells whether {@code ALLOW FILTERING} is implicitly added to select statement
+ * which requires it. Defaults to true.
+ *
+ * @return true if {@code ALLOW FILTERING} is implicitly added to select statements when required, false otherwise.
+ */
+ default boolean allowFilteringImplicitly()
+ {
+ return true;
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java b/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
index 3a5db52..ae929c8 100644
--- a/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
@@ -26,6 +26,9 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Hex;
import org.apache.cassandra.utils.ObjectSizes;
@@ -102,6 +105,12 @@
}
@Override
+ public ByteSource asComparableBytes(ByteComparable.Version version)
+ {
+ return ByteSource.of(token, version);
+ }
+
+ @Override
public IPartitioner getPartitioner()
{
return instance;
@@ -222,6 +231,11 @@
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
+ public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return new BytesToken(ByteSourceInverse.getUnescapedBytes(comparableBytes));
+ }
+
public ByteBuffer toByteArray(Token token)
{
BytesToken bytesToken = (BytesToken) token;
@@ -287,7 +301,7 @@
Token lastToken = sortedTokens.get(sortedTokens.size() - 1);
for (Token node : sortedTokens)
{
- allTokens.put(node, new Float(0.0));
+ allTokens.put(node, 0.0F);
sortedRanges.add(new Range<Token>(lastToken, node));
lastToken = node;
}
@@ -305,7 +319,7 @@
}
// Sum every count up and divide count/total for the fractional ownership.
- Float total = new Float(0.0);
+ Float total = 0.0F;
for (Float f : allTokens.values())
total += f;
for (Map.Entry<Token, Float> row : allTokens.entrySet())
diff --git a/src/java/org/apache/cassandra/dht/LocalPartitioner.java b/src/java/org/apache/cassandra/dht/LocalPartitioner.java
index 09cd2b7..74a1264 100644
--- a/src/java/org/apache/cassandra/dht/LocalPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/LocalPartitioner.java
@@ -26,7 +26,10 @@
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.CachedHashDecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.memory.HeapCloner;
@@ -83,6 +86,12 @@
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
+ public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ ByteBuffer tokenData = comparator.fromComparableBytes(ByteBufferAccessor.instance, comparableBytes, version);
+ return new LocalToken(tokenData);
+ }
+
public ByteBuffer toByteArray(Token token)
{
return ((LocalToken)token).token;
@@ -116,7 +125,7 @@
public Map<Token, Float> describeOwnership(List<Token> sortedTokens)
{
- return Collections.singletonMap((Token)getMinimumToken(), new Float(1.0));
+ return Collections.singletonMap((Token)getMinimumToken(), 1.0F);
}
public AbstractType<?> getTokenValidator()
@@ -175,6 +184,12 @@
}
@Override
+ public ByteSource asComparableBytes(ByteComparable.Version version)
+ {
+ return comparator.asComparableBytes(ByteBufferAccessor.instance, token, version);
+ }
+
+ @Override
public IPartitioner getPartitioner()
{
return LocalPartitioner.this;
diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
index e2daac4..57993a6 100644
--- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
+++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
@@ -33,6 +33,9 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.cassandra.utils.MurmurHash;
import org.apache.cassandra.utils.ObjectSizes;
@@ -177,6 +180,12 @@
}
@Override
+ public ByteSource asComparableBytes(ByteComparable.Version version)
+ {
+ return ByteSource.of(token);
+ }
+
+ @Override
public IPartitioner getPartitioner()
{
return instance;
@@ -294,7 +303,7 @@
throw new RuntimeException("No nodes present in the cluster. Has this node finished starting up?");
// 1-case
if (sortedTokens.size() == 1)
- ownerships.put(i.next(), new Float(1.0));
+ ownerships.put(i.next(), 1.0F);
// n-case
else
{
@@ -326,6 +335,12 @@
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
+ public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ long tokenData = ByteSourceInverse.getSignedLong(comparableBytes);
+ return new LongToken(tokenData);
+ }
+
public ByteBuffer toByteArray(Token token)
{
LongToken longToken = (LongToken) token;
diff --git a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
index 16c5db1..2566cbf 100644
--- a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
@@ -33,6 +33,9 @@
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.Pair;
@@ -128,6 +131,11 @@
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
+ public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return new StringToken(ByteSourceInverse.getString(comparableBytes));
+ }
+
public ByteBuffer toByteArray(Token token)
{
StringToken stringToken = (StringToken) token;
@@ -194,6 +202,12 @@
{
return EMPTY_SIZE + ObjectSizes.sizeOf(token);
}
+
+ @Override
+ public ByteSource asComparableBytes(ByteComparable.Version version)
+ {
+ return ByteSource.of(token, version);
+ }
}
public StringToken getToken(ByteBuffer key)
@@ -220,7 +234,7 @@
Token lastToken = sortedTokens.get(sortedTokens.size() - 1);
for (Token node : sortedTokens)
{
- allTokens.put(node, new Float(0.0));
+ allTokens.put(node, 0.0F);
sortedRanges.add(new Range<Token>(lastToken, node));
lastToken = node;
}
@@ -238,7 +252,7 @@
}
// Sum every count up and divide count/total for the fractional ownership.
- Float total = new Float(0.0);
+ Float total = 0.0F;
for (Float f : allTokens.values())
total += f;
for (Map.Entry<Token, Float> row : allTokens.entrySet())
diff --git a/src/java/org/apache/cassandra/dht/RandomPartitioner.java b/src/java/org/apache/cassandra/dht/RandomPartitioner.java
index 241b785..1574301 100644
--- a/src/java/org/apache/cassandra/dht/RandomPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/RandomPartitioner.java
@@ -27,6 +27,8 @@
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.db.CachedHashDecoratedKey;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -34,6 +36,8 @@
import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.GuidGenerator;
import org.apache.cassandra.utils.ObjectSizes;
@@ -158,6 +162,11 @@
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
+ public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return fromByteArray(IntegerType.instance.fromComparableBytes(ByteBufferAccessor.instance, comparableBytes, version));
+ }
+
public ByteBuffer toByteArray(Token token)
{
BigIntegerToken bigIntegerToken = (BigIntegerToken) token;
@@ -245,6 +254,12 @@
}
@Override
+ public ByteSource asComparableBytes(ByteComparable.Version version)
+ {
+ return IntegerType.instance.asComparableBytes(ByteArrayAccessor.instance, token.toByteArray(), version);
+ }
+
+ @Override
public IPartitioner getPartitioner()
{
return instance;
@@ -293,7 +308,7 @@
// 1-case
if (sortedTokens.size() == 1)
{
- ownerships.put(i.next(), new Float(1.0));
+ ownerships.put(i.next(), 1.0F);
}
// n-case
else
diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java
index dda6863..9b7833b 100644
--- a/src/java/org/apache/cassandra/dht/RangeStreamer.java
+++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java
@@ -71,6 +71,7 @@
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.Iterables.all;
import static com.google.common.collect.Iterables.any;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
import static org.apache.cassandra.locator.Replica.fullReplica;
/**
@@ -203,6 +204,33 @@
}
/**
+ * Source filter which excludes nodes from local DC.
+ */
+ public static class ExcludeLocalDatacenterFilter implements SourceFilter
+ {
+ private final IEndpointSnitch snitch;
+ private final String localDc;
+
+ public ExcludeLocalDatacenterFilter(IEndpointSnitch snitch)
+ {
+ this.snitch = snitch;
+ this.localDc = snitch.getLocalDatacenter();
+ }
+
+ @Override
+ public boolean apply(Replica replica)
+ {
+ return !snitch.getDatacenter(replica).equals(localDc);
+ }
+
+ @Override
+ public String message(Replica replica)
+ {
+ return "Filtered " + replica + " out because it belongs to the local datacenter";
+ }
+ }
+
+ /**
* Source filter which excludes the current node from source calculations
*/
public static class ExcludeLocalNodeFilter implements SourceFilter
@@ -667,32 +695,62 @@
logger.debug("Keyspace {} Sources {}", keyspace, sources);
sources.asMap().forEach((source, fetchReplicas) -> {
- // filter out already streamed ranges
- SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
+ List<FetchReplica> remaining;
- Predicate<FetchReplica> isAvailable = fetch -> {
- boolean isInFull = available.full.contains(fetch.local.range());
- boolean isInTrans = available.trans.contains(fetch.local.range());
-
- if (!isInFull && !isInTrans)
- //Range is unavailable
- return false;
-
- if (fetch.local.isFull())
- //For full, pick only replicas with matching transientness
- return isInFull == fetch.remote.isFull();
-
- // Any transient or full will do
- return true;
- };
-
- List<FetchReplica> remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
-
- if (remaining.size() < available.full.size() + available.trans.size())
+ // If the operator's specified they want to reset bootstrap progress, we don't check previous attempted
+ // bootstraps and just restart with all.
+ if (RESET_BOOTSTRAP_PROGRESS.getBoolean())
{
- List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
- logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}",
- fetchReplicas, skipped, available.full, available.trans);
+ // TODO: Also remove the files on disk. See discussion in CASSANDRA-17679
+ SystemKeyspace.resetAvailableStreamedRangesForKeyspace(keyspace);
+ remaining = new ArrayList<>(fetchReplicas);
+ }
+ else
+ {
+ // Filter out already streamed ranges
+ SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
+
+ Predicate<FetchReplica> isAvailable = fetch -> {
+ boolean isInFull = available.full.contains(fetch.local.range());
+ boolean isInTrans = available.trans.contains(fetch.local.range());
+
+ if (!isInFull && !isInTrans)
+ // Range is unavailable
+ return false;
+
+ if (fetch.local.isFull())
+ // For full, pick only replicas with matching transientness
+ return isInFull == fetch.remote.isFull();
+
+ // Any transient or full will do
+ return true;
+ };
+
+ remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
+
+ if (remaining.size() < available.full.size() + available.trans.size())
+ {
+ // If the operator hasn't specified what to do when we discover a previous partially successful bootstrap,
+ // we error out and tell them to manually reconcile it. See CASSANDRA-17679.
+ if (!RESET_BOOTSTRAP_PROGRESS.isPresent())
+ {
+ List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
+ String msg = String.format("Discovered existing bootstrap data and %s " +
+ "is not configured; aborting bootstrap. Please clean up local files manually " +
+ "and try again or set cassandra.reset_bootstrap_progress=true to ignore. " +
+ "Found: %s. Fully available: %s. Transiently available: %s",
+ RESET_BOOTSTRAP_PROGRESS.getKey(), skipped, available.full, available.trans);
+ logger.error(msg);
+ throw new IllegalStateException(msg);
+ }
+
+ if (!RESET_BOOTSTRAP_PROGRESS.getBoolean())
+ {
+ List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
+ logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}",
+ fetchReplicas, skipped, available.full, available.trans);
+ }
+ }
}
if (logger.isTraceEnabled())
diff --git a/src/java/org/apache/cassandra/dht/Token.java b/src/java/org/apache/cassandra/dht/Token.java
index d8e82f8..3543dab 100644
--- a/src/java/org/apache/cassandra/dht/Token.java
+++ b/src/java/org/apache/cassandra/dht/Token.java
@@ -26,6 +26,8 @@
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
public abstract class Token implements RingPosition<Token>, Serializable
{
@@ -37,8 +39,31 @@
{
public abstract ByteBuffer toByteArray(Token token);
public abstract Token fromByteArray(ByteBuffer bytes);
+
+ /**
+ * Produce a byte-comparable representation of the token.
+ * See {@link Token#asComparableBytes}
+ */
+ public ByteSource asComparableBytes(Token token, ByteComparable.Version version)
+ {
+ return token.asComparableBytes(version);
+ }
+
+ /**
+ * Translates the given byte-comparable representation to a token instance. If the given bytes don't correspond
+ * to the encoding of an instance of the expected token type, an {@link IllegalArgumentException} may be thrown.
+ *
+ * @param comparableBytes A byte-comparable representation (presumably of a token of some expected token type).
+ * @return A new {@link Token} instance, corresponding to the given byte-ordered representation. If we were
+ * to call {@link #asComparableBytes(ByteComparable.Version)} on the returned object, we should get a
+ * {@link ByteSource} equal to the input one as a result.
+ * @throws IllegalArgumentException if the bytes do not encode a valid token.
+ */
+ public abstract Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version);
+
public abstract String toString(Token token); // serialize as string, not necessarily human-readable
public abstract Token fromString(String string); // deserialize
+
public abstract void validate(String token) throws ConfigurationException;
public void serialize(Token token, DataOutputPlus out) throws IOException
@@ -100,6 +125,20 @@
abstract public Object getTokenValue();
/**
+ * Produce a weakly prefix-free byte-comparable representation of the token, i.e. such a sequence of bytes that any
+ * pair x, y of valid tokens of this type and any bytes b1, b2 between 0x10 and 0xEF,
+ * (+ stands for concatenation)
+ * compare(x, y) == compareLexicographicallyUnsigned(asByteComparable(x)+b1, asByteComparable(y)+b2)
+ * (i.e. the values compare like the original type, and an added 0x10-0xEF byte at the end does not change that) and:
+ * asByteComparable(x)+b1 is not a prefix of asByteComparable(y) (weakly prefix free)
+ * (i.e. a valid representation of a value may be a prefix of another valid representation of a value only if the
+ * following byte in the latter is smaller than 0x10 or larger than 0xEF). These properties are trivially true if
+ * the encoding compares correctly and is prefix free, but also permits a little more freedom that enables somewhat
+ * more efficient encoding of arbitrary-length byte-comparable blobs.
+ */
+ abstract public ByteSource asComparableBytes(ByteComparable.Version version);
+
+ /**
* Returns a measure for the token space covered between this token and next.
* Used by the token allocation algorithm (see CASSANDRA-7032).
*/
@@ -128,7 +167,7 @@
/*
* A token corresponds to the range of all the keys having this token.
- * A token is thus no comparable directly to a key. But to be able to select
+ * A token is thus not comparable directly to a key. But to be able to select
* keys given tokens, we introduce two "fake" keys for each token T:
* - lowerBoundKey: a "fake" key representing the lower bound T represents.
* In other words, lowerBoundKey is the smallest key that
@@ -190,6 +229,20 @@
return ((pos instanceof KeyBound) && !((KeyBound)pos).isMinimumBound) ? 0 : 1;
}
+ @Override
+ public ByteSource asComparableBytes(Version version)
+ {
+ int terminator = isMinimumBound ? ByteSource.LT_NEXT_COMPONENT : ByteSource.GT_NEXT_COMPONENT;
+ return ByteSource.withTerminator(terminator, token.asComparableBytes(version));
+ }
+
+ @Override
+ public ByteComparable asComparableBound(boolean before)
+ {
+ // This class is already a bound thus nothing needs to be changed from its representation
+ return this;
+ }
+
public IPartitioner getPartitioner()
{
return getToken().getPartitioner();
diff --git a/src/java/org/apache/cassandra/exceptions/QueryCancelledException.java b/src/java/org/apache/cassandra/exceptions/QueryCancelledException.java
new file mode 100644
index 0000000..45b6334
--- /dev/null
+++ b/src/java/org/apache/cassandra/exceptions/QueryCancelledException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.exceptions;
+
+import org.apache.cassandra.db.ReadCommand;
+
+public class QueryCancelledException extends RuntimeException
+{
+ public QueryCancelledException(ReadCommand command)
+ {
+ super("Query cancelled for taking too long: " + command.toCQLString());
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
index 3d3476a..ca64298 100644
--- a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
+++ b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
@@ -19,8 +19,6 @@
import java.io.IOException;
-import com.google.common.primitives.Ints;
-
import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -101,12 +99,12 @@
if (version < VERSION_40)
out.writeShort(reason.code);
else
- out.writeUnsignedVInt(reason.code);
+ out.writeUnsignedVInt32(reason.code);
}
public RequestFailureReason deserialize(DataInputPlus in, int version) throws IOException
{
- return fromCode(version < VERSION_40 ? in.readUnsignedShort() : Ints.checkedCast(in.readUnsignedVInt()));
+ return fromCode(version < VERSION_40 ? in.readUnsignedShort() : in.readUnsignedVInt32());
}
public long serializedSize(RequestFailureReason reason, int version)
diff --git a/src/java/org/apache/cassandra/gms/ApplicationState.java b/src/java/org/apache/cassandra/gms/ApplicationState.java
index c45d3c2..3f44bcc 100644
--- a/src/java/org/apache/cassandra/gms/ApplicationState.java
+++ b/src/java/org/apache/cassandra/gms/ApplicationState.java
@@ -52,7 +52,7 @@
* which new sstables are written), but may contain more on newly upgraded nodes before `upgradesstable` has been
* run.
*
- * <p>The value (a set of sstable {@link org.apache.cassandra.io.sstable.format.VersionAndType}) is serialized as
+ * <p>The value (a set of sstable {@link org.apache.cassandra.io.sstable.format.Version}) is serialized as
* a comma-separated list.
**/
SSTABLE_VERSIONS,
diff --git a/src/java/org/apache/cassandra/gms/EndpointState.java b/src/java/org/apache/cassandra/gms/EndpointState.java
index e756744..4243c75 100644
--- a/src/java/org/apache/cassandra/gms/EndpointState.java
+++ b/src/java/org/apache/cassandra/gms/EndpointState.java
@@ -29,6 +29,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -46,6 +47,8 @@
{
protected static final Logger logger = LoggerFactory.getLogger(EndpointState.class);
+ static volatile boolean LOOSE_DEF_OF_EMPTY_ENABLED = CassandraRelevantProperties.LOOSE_DEF_OF_EMPTY_ENABLED.getBoolean();
+
public final static IVersionedSerializer<EndpointState> serializer = new EndpointStateSerializer();
public final static IVersionedSerializer<EndpointState> nullableSerializer = NullableSerializer.wrap(serializer);
@@ -186,17 +189,25 @@
updateTimestamp = nanoTime();
}
+ @VisibleForTesting
+ public void unsafeSetUpdateTimestamp(long value)
+ {
+ updateTimestamp = value;
+ }
+
public boolean isAlive()
{
return isAlive;
}
- void markAlive()
+ @VisibleForTesting
+ public void markAlive()
{
isAlive = true;
}
- void markDead()
+ @VisibleForTesting
+ public void markDead()
{
isAlive = false;
}
@@ -207,7 +218,17 @@
public boolean isEmptyWithoutStatus()
{
Map<ApplicationState, VersionedValue> state = applicationState.get();
- return hbState.isEmpty() && !(state.containsKey(ApplicationState.STATUS_WITH_PORT) || state.containsKey(ApplicationState.STATUS));
+ boolean hasStatus = state.containsKey(ApplicationState.STATUS_WITH_PORT) || state.containsKey(ApplicationState.STATUS);
+ return hbState.isEmpty() && !hasStatus
+ // In the very specific case where hbState.isEmpty and STATUS is missing, this is known to be safe to "fake"
+ // the data, as this happens when the gossip state isn't coming from the node but instead from a peer who
+ // restarted and is missing the node's state.
+ //
+ // When hbState is not empty, then the node gossiped an empty STATUS; this happens during bootstrap and it's not
+ // possible to tell if this is ok or not (we can't really tell if the node is dead or having networking issues).
+ // For these cases allow an external actor to verify and inform Cassandra that it is safe - this is done by
+ // updating the LOOSE_DEF_OF_EMPTY_ENABLED field.
+ || (LOOSE_DEF_OF_EMPTY_ENABLED && !hasStatus);
}
public boolean isRpcReady()
diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java
index 5177154..9cf7ad0 100644
--- a/src/java/org/apache/cassandra/gms/FailureDetector.java
+++ b/src/java/org/apache/cassandra/gms/FailureDetector.java
@@ -17,29 +17,46 @@
*/
package org.apache.cassandra.gms;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
-import java.nio.file.Path;
-import java.io.*;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.net.UnknownHostException;
-import java.util.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
-import javax.management.openmbean.*;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.CompositeDataSupport;
+import javax.management.openmbean.CompositeType;
+import javax.management.openmbean.OpenDataException;
+import javax.management.openmbean.OpenType;
+import javax.management.openmbean.SimpleType;
+import javax.management.openmbean.TabularData;
+import javax.management.openmbean.TabularDataSupport;
+import javax.management.openmbean.TabularType;
-import org.apache.cassandra.locator.Replica;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FD_INITIAL_VALUE_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FD_MAX_INTERVAL_MS;
import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_LOCAL_PAUSE_IN_MS;
import static org.apache.cassandra.config.DatabaseDescriptor.newFailureDetector;
import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
@@ -55,21 +72,19 @@
private static final int SAMPLE_SIZE = 1000;
protected static final long INITIAL_VALUE_NANOS = TimeUnit.NANOSECONDS.convert(getInitialValue(), TimeUnit.MILLISECONDS);
private static final int DEBUG_PERCENTAGE = 80; // if the phi is larger than this percentage of the max, log a debug message
- private static final long DEFAULT_MAX_PAUSE = 5000L * 1000000L; // 5 seconds
private static final long MAX_LOCAL_PAUSE_IN_NANOS = getMaxLocalPause();
private long lastInterpret = preciseTime.now();
private long lastPause = 0L;
private static long getMaxLocalPause()
{
- if (System.getProperty("cassandra.max_local_pause_in_ms") != null)
- {
- long pause = Long.parseLong(System.getProperty("cassandra.max_local_pause_in_ms"));
- logger.warn("Overriding max local pause time to {}ms", pause);
- return pause * 1000000L;
- }
- else
- return DEFAULT_MAX_PAUSE;
+ long pause = MAX_LOCAL_PAUSE_IN_MS.getLong();
+
+ if (!String.valueOf(pause).equals(MAX_LOCAL_PAUSE_IN_MS.getDefaultValue()))
+ logger.warn("Overriding {} max local pause time from {}ms to {}ms",
+ MAX_LOCAL_PAUSE_IN_MS.getKey(), MAX_LOCAL_PAUSE_IN_MS.getDefaultValue(), pause);
+
+ return pause * 1000000L;
}
public static final IFailureDetector instance = newFailureDetector();
@@ -93,16 +108,12 @@
private static long getInitialValue()
{
- String newvalue = System.getProperty("cassandra.fd_initial_value_ms");
- if (newvalue == null)
- {
- return Gossiper.intervalInMillis * 2;
- }
- else
- {
- logger.info("Overriding FD INITIAL_VALUE to {}ms", newvalue);
- return Integer.parseInt(newvalue);
- }
+ long newValue = FD_INITIAL_VALUE_MS.getLong(Gossiper.intervalInMillis * 2L);
+
+ if (newValue != Gossiper.intervalInMillis * 2)
+ logger.info("Overriding {} from {}ms to {}ms", FD_INITIAL_VALUE_MS.getKey(), Gossiper.intervalInMillis * 2, newValue);
+
+ return newValue;
}
public String getAllEndpointStates()
@@ -476,16 +487,10 @@
private static long getMaxInterval()
{
- String newvalue = System.getProperty("cassandra.fd_max_interval_ms");
- if (newvalue == null)
- {
- return FailureDetector.INITIAL_VALUE_NANOS;
- }
- else
- {
- logger.info("Overriding FD MAX_INTERVAL to {}ms", newvalue);
- return TimeUnit.NANOSECONDS.convert(Integer.parseInt(newvalue), TimeUnit.MILLISECONDS);
- }
+ long newValue = FD_MAX_INTERVAL_MS.getLong(FailureDetector.INITIAL_VALUE_NANOS);
+ if (newValue != FailureDetector.INITIAL_VALUE_NANOS)
+ logger.info("Overriding {} from {}ms to {}ms", FD_MAX_INTERVAL_MS.getKey(), FailureDetector.INITIAL_VALUE_NANOS, newValue);
+ return TimeUnit.NANOSECONDS.convert(newValue, TimeUnit.MILLISECONDS);
}
synchronized void add(long value, InetAddressAndPort ep)
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index ae213c0..adff044 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -18,8 +18,21 @@
package org.apache.cassandra.gms;
import java.net.UnknownHostException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListSet;
@@ -32,59 +45,60 @@
import java.util.function.BooleanSupplier;
import java.util.function.Supplier;
import java.util.stream.Collectors;
-
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Uninterruptibles;
-
-import org.apache.cassandra.concurrent.*;
-import org.apache.cassandra.concurrent.FutureTask;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.NoPayload;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.utils.CassandraVersion;
-import org.apache.cassandra.utils.ExecutorUtils;
-import org.apache.cassandra.utils.ExpiringMemoizingSupplier;
-import org.apache.cassandra.utils.MBeanWrapper;
-import org.apache.cassandra.utils.NoSpamLogger;
-import org.apache.cassandra.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.FutureTask;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.ExpiringMemoizingSupplier;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.RecomputingSupplier;
-import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import org.apache.cassandra.utils.concurrent.NotScheduledFuture;
+import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_GOSSIP_ENDPOINT_REMOVAL;
import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_QUARANTINE_DELAY;
import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_SKIP_WAITING_TO_SETTLE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.apache.cassandra.config.CassandraRelevantProperties.SHUTDOWN_ANNOUNCE_DELAY_IN_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.VERY_LONG_TIME_MS;
import static org.apache.cassandra.config.DatabaseDescriptor.getClusterName;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitionerName;
+import static org.apache.cassandra.gms.VersionedValue.BOOTSTRAPPING_STATUS;
import static org.apache.cassandra.net.NoPayload.noPayload;
import static org.apache.cassandra.net.Verb.ECHO_REQ;
import static org.apache.cassandra.net.Verb.GOSSIP_DIGEST_SYN;
-import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
/**
* This module is responsible for Gossiping information for the local endpoint. This abstraction
@@ -105,11 +119,6 @@
{
public static final String MBEAN_NAME = "org.apache.cassandra.net:type=Gossiper";
- public static class Props
- {
- public static final String DISABLE_THREAD_VALIDATION = "cassandra.gossip.disable_thread_validation";
- }
-
private static final ScheduledExecutorPlus executor = executorFactory().scheduled("GossipTasks");
static final ApplicationState[] STATES = ApplicationState.values();
@@ -204,11 +213,13 @@
return epStates.isEmpty() || epStates.keySet().equals(Collections.singleton(FBUtilities.getBroadcastAddressAndPort()));
}
+ private static final ExpiringMemoizingSupplier.Memoized<CassandraVersion> NO_UPGRADE_IN_PROGRESS = new ExpiringMemoizingSupplier.Memoized<>(null);
+ private static final ExpiringMemoizingSupplier.NotMemoized<CassandraVersion> CURRENT_NODE_VERSION = new ExpiringMemoizingSupplier.NotMemoized<>(SystemKeyspace.CURRENT_VERSION);
final Supplier<ExpiringMemoizingSupplier.ReturnValue<CassandraVersion>> upgradeFromVersionSupplier = () ->
{
// Once there are no prior version nodes we don't need to keep rechecking
if (!upgradeInProgressPossible)
- return new ExpiringMemoizingSupplier.Memoized<>(null);
+ return NO_UPGRADE_IN_PROGRESS;
CassandraVersion minVersion = SystemKeyspace.CURRENT_VERSION;
@@ -217,14 +228,15 @@
// If we don't know any epstate we don't know anything about the cluster.
// If we only know about ourselves, we can assume that version is CURRENT_VERSION
if (!isEnabled() || isLoneNode(endpointStateMap))
- {
- return new ExpiringMemoizingSupplier.NotMemoized<>(minVersion);
- }
+ return CURRENT_NODE_VERSION;
// Check the release version of all the peers it heard of. Not necessary the peer that it has/had contacted with.
boolean allHostsHaveKnownVersion = true;
for (InetAddressAndPort host : endpointStateMap.keySet())
{
+ if (justRemovedEndpoints.containsKey(host))
+ continue;
+
CassandraVersion version = getReleaseVersion(host);
//Raced with changes to gossip state, wait until next iteration
@@ -241,7 +253,7 @@
return new ExpiringMemoizingSupplier.NotMemoized<>(minVersion);
upgradeInProgressPossible = false;
- return new ExpiringMemoizingSupplier.Memoized<>(null);
+ return NO_UPGRADE_IN_PROGRESS;
};
private final Supplier<CassandraVersion> upgradeFromVersionMemoized = ExpiringMemoizingSupplier.memoizeWithExpiration(upgradeFromVersionSupplier, 1, TimeUnit.MINUTES);
@@ -253,18 +265,18 @@
((ExpiringMemoizingSupplier<CassandraVersion>) upgradeFromVersionMemoized).expire();
}
- private static final boolean disableThreadValidation = Boolean.getBoolean(Props.DISABLE_THREAD_VALIDATION);
+ private static final boolean disableThreadValidation = GOSSIP_DISABLE_THREAD_VALIDATION.getBoolean();
private static volatile boolean disableEndpointRemoval = DISABLE_GOSSIP_ENDPOINT_REMOVAL.getBoolean();
private static long getVeryLongTime()
{
- String newVLT = System.getProperty("cassandra.very_long_time_ms");
- if (newVLT != null)
- {
- logger.info("Overriding aVeryLongTime to {}ms", newVLT);
- return Long.parseLong(newVLT);
- }
- return 259200 * 1000; // 3 days
+ long time = VERY_LONG_TIME_MS.getLong();
+ String defaultValue = VERY_LONG_TIME_MS.getDefaultValue();
+
+ if (!String.valueOf(time).equals(defaultValue))
+ logger.info("Overriding {} from {} to {}ms", VERY_LONG_TIME_MS.getKey(), defaultValue, time);
+
+ return time;
}
private static boolean isInGossipStage()
@@ -1502,7 +1514,7 @@
return isAdministrativelyInactiveState(epState);
}
- private static String getGossipStatus(EndpointState epState)
+ public static String getGossipStatus(EndpointState epState)
{
if (epState == null)
{
@@ -1525,11 +1537,78 @@
return pieces[0];
}
+ /**
+ * Gossip offers no happens-before relationship, but downstream subscribers assume a happens-before relationship
+ * before being notified! To attempt to be nicer to subscribers, this {@link Comparator} attempts to order EndpointState
+ * within a map based off a few heuristics:
+ * <ol>
+ * <li>STATUS - some STATUS depends on other instance STATUS, so make sure they are last; eg. BOOT, and BOOT_REPLACE</li>
+ * <li>generation - normally defined as system clock millis, this can be skewed and is a best effort</li>
+ * <li>address - tie breaker to make sure order is consistent</li>
+ * </ol>
+ * <p>
+ * Problems:
+ * Generation is normally defined as system clock millis, which can be skewed and in-consistent cross nodes
+ * (generations do not have a happens-before relationship, so ordering is sketchy at best).
+ * <p>
+ * Motivations:
+ * {@link Map#entrySet()} returns data in effectivlly random order, so can get into a situation such as the following example.
+ * {@code
+ * 3 node cluster: n1, n2, and n3
+ * n2 goes down and n4 does host replacement and fails before completion
+ * h5 tries to do a host replacement against n4 (ignore the fact this doesn't make sense)
+ * }
+ * In that case above, the {@link Map#entrySet()} ordering can be random, causing h4 to apply before h2, which will
+ * be rejected by subscripers (only after updating gossip causing zero retries).
+ */
+ @VisibleForTesting
+ static Comparator<Entry<InetAddressAndPort, EndpointState>> stateOrderMap()
+ {
+ // There apears to be some edge cases where the state we are ordering get added to the global state causing
+ // ordering to change... to avoid that rely on a cache
+ // see CASSANDRA-17908
+ class Cache extends HashMap<InetAddressAndPort, EndpointState>
+ {
+ EndpointState get(Entry<InetAddressAndPort, EndpointState> e)
+ {
+ if (containsKey(e.getKey()))
+ return get(e.getKey());
+ put(e.getKey(), new EndpointState(e.getValue()));
+ return get(e.getKey());
+ }
+ }
+ Cache cache = new Cache();
+ return ((Comparator<Entry<InetAddressAndPort, EndpointState>>) (e1, e2) -> {
+ String e1status = getGossipStatus(cache.get(e1));
+ String e2status = getGossipStatus(cache.get(e2));
+
+ if (Objects.equals(e1status, e2status) || (BOOTSTRAPPING_STATUS.contains(e1status) && BOOTSTRAPPING_STATUS.contains(e2status)))
+ return 0;
+
+ // check status first, make sure bootstrap status happens-after all others
+ if (BOOTSTRAPPING_STATUS.contains(e1status))
+ return 1;
+ if (BOOTSTRAPPING_STATUS.contains(e2status))
+ return -1;
+ return 0;
+ })
+ .thenComparingInt((Entry<InetAddressAndPort, EndpointState> e) -> cache.get(e).getHeartBeatState().getGeneration())
+ .thenComparing(Entry::getKey);
+ }
+
+ private static Iterable<Entry<InetAddressAndPort, EndpointState>> order(Map<InetAddressAndPort, EndpointState> epStateMap)
+ {
+ List<Entry<InetAddressAndPort, EndpointState>> list = new ArrayList<>(epStateMap.entrySet());
+ Collections.sort(list, stateOrderMap());
+ return list;
+ }
+
@VisibleForTesting
public void applyStateLocally(Map<InetAddressAndPort, EndpointState> epStateMap)
{
checkProperThreadForStateMutation();
- for (Entry<InetAddressAndPort, EndpointState> entry : epStateMap.entrySet())
+ boolean hasMajorVersion3Nodes = hasMajorVersion3Nodes();
+ for (Entry<InetAddressAndPort, EndpointState> entry : order(epStateMap))
{
InetAddressAndPort ep = entry.getKey();
if (ep.equals(getBroadcastAddressAndPort()) && !isInShadowRound())
@@ -1543,7 +1622,7 @@
EndpointState localEpStatePtr = endpointStateMap.get(ep);
EndpointState remoteState = entry.getValue();
- if (!hasMajorVersion3Nodes())
+ if (!hasMajorVersion3Nodes)
remoteState.removeMajorVersion3LegacyApplicationStates();
/*
@@ -1579,7 +1658,7 @@
if (remoteMaxVersion > localMaxVersion)
{
// apply states, but do not notify since there is no major change
- applyNewStates(ep, localEpStatePtr, remoteState);
+ applyNewStates(ep, localEpStatePtr, remoteState, hasMajorVersion3Nodes);
}
else if (logger.isTraceEnabled())
logger.trace("Ignoring remote version {} <= {} for {}", remoteMaxVersion, localMaxVersion, ep);
@@ -1602,7 +1681,7 @@
}
}
- private void applyNewStates(InetAddressAndPort addr, EndpointState localState, EndpointState remoteState)
+ private void applyNewStates(InetAddressAndPort addr, EndpointState localState, EndpointState remoteState, boolean hasMajorVersion3Nodes)
{
// don't assert here, since if the node restarts the version will go back to zero
int oldVersion = localState.getHeartBeatState().getHeartBeatVersion();
@@ -1631,7 +1710,7 @@
localState.addApplicationStates(updatedStates);
// get rid of legacy fields once the cluster is not in mixed mode
- if (!hasMajorVersion3Nodes())
+ if (!hasMajorVersion3Nodes)
localState.removeMajorVersion3LegacyApplicationStates();
// need to run STATUS or STATUS_WITH_PORT first to handle BOOT_REPLACE correctly (else won't be a member, so TOKENS won't be processed)
@@ -2490,6 +2569,19 @@
return minVersion;
}
+ @Override
+ public boolean getLooseEmptyEnabled()
+ {
+ return EndpointState.LOOSE_DEF_OF_EMPTY_ENABLED;
+ }
+
+ @Override
+ public void setLooseEmptyEnabled(boolean enabled)
+ {
+ logger.info("Setting loose definition of empty to {}", enabled);
+ EndpointState.LOOSE_DEF_OF_EMPTY_ENABLED = enabled;
+ }
+
public void unsafeSetEnabled()
{
scheduledGossipTask = new NotScheduledFuture<>();
diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java
index 47d7207..2d59e37 100644
--- a/src/java/org/apache/cassandra/gms/GossiperMBean.java
+++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java
@@ -38,4 +38,7 @@
/** Returns each node's database release version */
public Map<String, List<String>> getReleaseVersionsWithPort();
+ public boolean getLooseEmptyEnabled();
+
+ public void setLooseEmptyEnabled(boolean enabled);
}
diff --git a/src/java/org/apache/cassandra/gms/VersionedValue.java b/src/java/org/apache/cassandra/gms/VersionedValue.java
index 26644e1..7c64278 100644
--- a/src/java/org/apache/cassandra/gms/VersionedValue.java
+++ b/src/java/org/apache/cassandra/gms/VersionedValue.java
@@ -17,29 +17,32 @@
*/
package org.apache.cassandra.gms;
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.net.InetAddress;
import java.util.Collection;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
-import static java.nio.charset.StandardCharsets.ISO_8859_1;
-
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import org.apache.commons.lang3.StringUtils;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.format.VersionAndType;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.commons.lang3.StringUtils;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
/**
@@ -84,6 +87,8 @@
// values for ApplicationState.REMOVAL_COORDINATOR
public final static String REMOVAL_COORDINATOR = "REMOVER";
+ public static Set<String> BOOTSTRAPPING_STATUS = ImmutableSet.of(STATUS_BOOTSTRAPPING, STATUS_BOOTSTRAPPING_REPLACE);
+
public final int version;
public final String value;
@@ -99,6 +104,12 @@
this(value, VersionGenerator.getNextVersion());
}
+ @VisibleForTesting
+ public VersionedValue withVersion(int version)
+ {
+ return new VersionedValue(value, version);
+ }
+
public static VersionedValue unsafeMakeVersionedValue(String value, int version)
{
return new VersionedValue(value, version);
@@ -315,10 +326,10 @@
return new VersionedValue(String.valueOf(value));
}
- public VersionedValue sstableVersions(Set<VersionAndType> versions)
+ public VersionedValue sstableVersions(Set<Version> versions)
{
return new VersionedValue(versions.stream()
- .map(VersionAndType::toString)
+ .map(Version::toFormatAndVersionString)
.collect(Collectors.joining(",")));
}
}
diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java
deleted file mode 100644
index 3625685..0000000
--- a/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputSplit;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.Arrays;
-
-public class ColumnFamilySplit extends InputSplit implements Writable, org.apache.hadoop.mapred.InputSplit
-{
- private String startToken;
- private String endToken;
- private long length;
- private String[] dataNodes;
-
- @Deprecated
- public ColumnFamilySplit(String startToken, String endToken, String[] dataNodes)
- {
- this(startToken, endToken, Long.MAX_VALUE, dataNodes);
- }
-
- public ColumnFamilySplit(String startToken, String endToken, long length, String[] dataNodes)
- {
- assert startToken != null;
- assert endToken != null;
- this.startToken = startToken;
- this.endToken = endToken;
- this.length = length;
- this.dataNodes = dataNodes;
- }
-
- public String getStartToken()
- {
- return startToken;
- }
-
- public String getEndToken()
- {
- return endToken;
- }
-
- // getLength and getLocations satisfy the InputSplit abstraction
-
- public long getLength()
- {
- return length;
- }
-
- public String[] getLocations()
- {
- return dataNodes;
- }
-
- // This should only be used by KeyspaceSplit.read();
- protected ColumnFamilySplit() {}
-
- // These three methods are for serializing and deserializing
- // KeyspaceSplits as needed by the Writable interface.
- public void write(DataOutput out) throws IOException
- {
- out.writeUTF(startToken);
- out.writeUTF(endToken);
- out.writeInt(dataNodes.length);
- for (String endpoint : dataNodes)
- {
- out.writeUTF(endpoint);
- }
- out.writeLong(length);
- }
-
- public void readFields(DataInput in) throws IOException
- {
- startToken = in.readUTF();
- endToken = in.readUTF();
- int numOfEndpoints = in.readInt();
- dataNodes = new String[numOfEndpoints];
- for(int i = 0; i < numOfEndpoints; i++)
- {
- dataNodes[i] = in.readUTF();
- }
- try
- {
- length = in.readLong();
- }
- catch (EOFException e)
- {
- //We must be deserializing in a mixed-version cluster.
- }
- }
-
- @Override
- public String toString()
- {
- return "ColumnFamilySplit(" +
- "(" + startToken
- + ", '" + endToken + ']'
- + " @" + (dataNodes == null ? null : Arrays.asList(dataNodes)) + ')';
- }
-
- public static ColumnFamilySplit read(DataInput in) throws IOException
- {
- ColumnFamilySplit w = new ColumnFamilySplit();
- w.readFields(in);
- return w;
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/ConfigHelper.java b/src/java/org/apache/cassandra/hadoop/ConfigHelper.java
deleted file mode 100644
index cc539b1..0000000
--- a/src/java/org/apache/cassandra/hadoop/ConfigHelper.java
+++ /dev/null
@@ -1,408 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.hadoop;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-import org.apache.hadoop.conf.Configuration;
-
-public class ConfigHelper
-{
- private static final String INPUT_PARTITIONER_CONFIG = "cassandra.input.partitioner.class";
- private static final String OUTPUT_PARTITIONER_CONFIG = "cassandra.output.partitioner.class";
- private static final String INPUT_KEYSPACE_CONFIG = "cassandra.input.keyspace";
- private static final String OUTPUT_KEYSPACE_CONFIG = "cassandra.output.keyspace";
- private static final String INPUT_KEYSPACE_USERNAME_CONFIG = "cassandra.input.keyspace.username";
- private static final String INPUT_KEYSPACE_PASSWD_CONFIG = "cassandra.input.keyspace.passwd";
- private static final String OUTPUT_KEYSPACE_USERNAME_CONFIG = "cassandra.output.keyspace.username";
- private static final String OUTPUT_KEYSPACE_PASSWD_CONFIG = "cassandra.output.keyspace.passwd";
- private static final String INPUT_COLUMNFAMILY_CONFIG = "cassandra.input.columnfamily";
- private static final String OUTPUT_COLUMNFAMILY_CONFIG = "mapreduce.output.basename"; //this must == OutputFormat.BASE_OUTPUT_NAME
- private static final String INPUT_PREDICATE_CONFIG = "cassandra.input.predicate";
- private static final String INPUT_KEYRANGE_CONFIG = "cassandra.input.keyRange";
- private static final String INPUT_SPLIT_SIZE_CONFIG = "cassandra.input.split.size";
- private static final String INPUT_SPLIT_SIZE_IN_MIB_CONFIG = "cassandra.input.split.size_mb";
- private static final String INPUT_WIDEROWS_CONFIG = "cassandra.input.widerows";
- private static final int DEFAULT_SPLIT_SIZE = 64 * 1024;
- private static final String RANGE_BATCH_SIZE_CONFIG = "cassandra.range.batch.size";
- private static final int DEFAULT_RANGE_BATCH_SIZE = 4096;
- private static final String INPUT_INITIAL_ADDRESS = "cassandra.input.address";
- private static final String OUTPUT_INITIAL_ADDRESS = "cassandra.output.address";
- private static final String OUTPUT_INITIAL_PORT = "cassandra.output.port";
- private static final String READ_CONSISTENCY_LEVEL = "cassandra.consistencylevel.read";
- private static final String WRITE_CONSISTENCY_LEVEL = "cassandra.consistencylevel.write";
- private static final String OUTPUT_COMPRESSION_CLASS = "cassandra.output.compression.class";
- private static final String OUTPUT_COMPRESSION_CHUNK_LENGTH = "cassandra.output.compression.length";
- private static final String OUTPUT_LOCAL_DC_ONLY = "cassandra.output.local.dc.only";
- private static final String DEFAULT_CASSANDRA_NATIVE_PORT = "7000";
-
- private static final Logger logger = LoggerFactory.getLogger(ConfigHelper.class);
-
- /**
- * Set the keyspace and column family for the input of this job.
- *
- * @param conf Job configuration you are about to run
- * @param keyspace
- * @param columnFamily
- * @param widerows
- */
- public static void setInputColumnFamily(Configuration conf, String keyspace, String columnFamily, boolean widerows)
- {
- if (keyspace == null)
- throw new UnsupportedOperationException("keyspace may not be null");
-
- if (columnFamily == null)
- throw new UnsupportedOperationException("table may not be null");
-
- conf.set(INPUT_KEYSPACE_CONFIG, keyspace);
- conf.set(INPUT_COLUMNFAMILY_CONFIG, columnFamily);
- conf.set(INPUT_WIDEROWS_CONFIG, String.valueOf(widerows));
- }
-
- /**
- * Set the keyspace and column family for the input of this job.
- *
- * @param conf Job configuration you are about to run
- * @param keyspace
- * @param columnFamily
- */
- public static void setInputColumnFamily(Configuration conf, String keyspace, String columnFamily)
- {
- setInputColumnFamily(conf, keyspace, columnFamily, false);
- }
-
- /**
- * Set the keyspace for the output of this job.
- *
- * @param conf Job configuration you are about to run
- * @param keyspace
- */
- public static void setOutputKeyspace(Configuration conf, String keyspace)
- {
- if (keyspace == null)
- throw new UnsupportedOperationException("keyspace may not be null");
-
- conf.set(OUTPUT_KEYSPACE_CONFIG, keyspace);
- }
-
- /**
- * Set the column family for the output of this job.
- *
- * @param conf Job configuration you are about to run
- * @param columnFamily
- */
- public static void setOutputColumnFamily(Configuration conf, String columnFamily)
- {
- conf.set(OUTPUT_COLUMNFAMILY_CONFIG, columnFamily);
- }
-
- /**
- * Set the column family for the output of this job.
- *
- * @param conf Job configuration you are about to run
- * @param keyspace
- * @param columnFamily
- */
- public static void setOutputColumnFamily(Configuration conf, String keyspace, String columnFamily)
- {
- setOutputKeyspace(conf, keyspace);
- setOutputColumnFamily(conf, columnFamily);
- }
-
- /**
- * The number of rows to request with each get range slices request.
- * Too big and you can either get timeouts when it takes Cassandra too
- * long to fetch all the data. Too small and the performance
- * will be eaten up by the overhead of each request.
- *
- * @param conf Job configuration you are about to run
- * @param batchsize Number of rows to request each time
- */
- public static void setRangeBatchSize(Configuration conf, int batchsize)
- {
- conf.setInt(RANGE_BATCH_SIZE_CONFIG, batchsize);
- }
-
- /**
- * The number of rows to request with each get range slices request.
- * Too big and you can either get timeouts when it takes Cassandra too
- * long to fetch all the data. Too small and the performance
- * will be eaten up by the overhead of each request.
- *
- * @param conf Job configuration you are about to run
- * @return Number of rows to request each time
- */
- public static int getRangeBatchSize(Configuration conf)
- {
- return conf.getInt(RANGE_BATCH_SIZE_CONFIG, DEFAULT_RANGE_BATCH_SIZE);
- }
-
- /**
- * Set the size of the input split.
- * This affects the number of maps created, if the number is too small
- * the overhead of each map will take up the bulk of the job time.
- *
- * @param conf Job configuration you are about to run
- * @param splitsize Number of partitions in the input split
- */
- public static void setInputSplitSize(Configuration conf, int splitsize)
- {
- conf.setInt(INPUT_SPLIT_SIZE_CONFIG, splitsize);
- }
-
- public static int getInputSplitSize(Configuration conf)
- {
- return conf.getInt(INPUT_SPLIT_SIZE_CONFIG, DEFAULT_SPLIT_SIZE);
- }
-
- /**
- * Set the size of the input split. setInputSplitSize value is used if this is not set.
- * This affects the number of maps created, if the number is too small
- * the overhead of each map will take up the bulk of the job time.
- *
- * @param conf Job configuration you are about to run
- * @param splitSizeMb Input split size in MiB
- */
- public static void setInputSplitSizeInMb(Configuration conf, int splitSizeMb)
- {
- conf.setInt(INPUT_SPLIT_SIZE_IN_MIB_CONFIG, splitSizeMb);
- }
-
- /**
- * cassandra.input.split.size will be used if the value is undefined or negative.
- * @param conf Job configuration you are about to run
- * @return split size in MiB or -1 if it is undefined.
- */
- public static int getInputSplitSizeInMb(Configuration conf)
- {
- return conf.getInt(INPUT_SPLIT_SIZE_IN_MIB_CONFIG, -1);
- }
-
- /**
- * Set the KeyRange to limit the rows.
- * @param conf Job configuration you are about to run
- */
- public static void setInputRange(Configuration conf, String startToken, String endToken)
- {
- conf.set(INPUT_KEYRANGE_CONFIG, startToken + "," + endToken);
- }
-
- /**
- * The start and end token of the input key range as a pair.
- *
- * may be null if unset.
- */
- public static Pair<String, String> getInputKeyRange(Configuration conf)
- {
- String str = conf.get(INPUT_KEYRANGE_CONFIG);
- if (str == null)
- return null;
-
- String[] parts = str.split(",");
- assert parts.length == 2;
- return Pair.create(parts[0], parts[1]);
- }
-
- public static String getInputKeyspace(Configuration conf)
- {
- return conf.get(INPUT_KEYSPACE_CONFIG);
- }
-
- public static String getOutputKeyspace(Configuration conf)
- {
- return conf.get(OUTPUT_KEYSPACE_CONFIG);
- }
-
- public static void setInputKeyspaceUserNameAndPassword(Configuration conf, String username, String password)
- {
- setInputKeyspaceUserName(conf, username);
- setInputKeyspacePassword(conf, password);
- }
-
- public static void setInputKeyspaceUserName(Configuration conf, String username)
- {
- conf.set(INPUT_KEYSPACE_USERNAME_CONFIG, username);
- }
-
- public static String getInputKeyspaceUserName(Configuration conf)
- {
- return conf.get(INPUT_KEYSPACE_USERNAME_CONFIG);
- }
-
- public static void setInputKeyspacePassword(Configuration conf, String password)
- {
- conf.set(INPUT_KEYSPACE_PASSWD_CONFIG, password);
- }
-
- public static String getInputKeyspacePassword(Configuration conf)
- {
- return conf.get(INPUT_KEYSPACE_PASSWD_CONFIG);
- }
-
- public static void setOutputKeyspaceUserNameAndPassword(Configuration conf, String username, String password)
- {
- setOutputKeyspaceUserName(conf, username);
- setOutputKeyspacePassword(conf, password);
- }
-
- public static void setOutputKeyspaceUserName(Configuration conf, String username)
- {
- conf.set(OUTPUT_KEYSPACE_USERNAME_CONFIG, username);
- }
-
- public static String getOutputKeyspaceUserName(Configuration conf)
- {
- return conf.get(OUTPUT_KEYSPACE_USERNAME_CONFIG);
- }
-
- public static void setOutputKeyspacePassword(Configuration conf, String password)
- {
- conf.set(OUTPUT_KEYSPACE_PASSWD_CONFIG, password);
- }
-
- public static String getOutputKeyspacePassword(Configuration conf)
- {
- return conf.get(OUTPUT_KEYSPACE_PASSWD_CONFIG);
- }
-
- public static String getInputColumnFamily(Configuration conf)
- {
- return conf.get(INPUT_COLUMNFAMILY_CONFIG);
- }
-
- public static String getOutputColumnFamily(Configuration conf)
- {
- if (conf.get(OUTPUT_COLUMNFAMILY_CONFIG) != null)
- return conf.get(OUTPUT_COLUMNFAMILY_CONFIG);
- else
- throw new UnsupportedOperationException("You must set the output column family using either setOutputColumnFamily or by adding a named output with MultipleOutputs");
- }
-
- public static boolean getInputIsWide(Configuration conf)
- {
- return Boolean.parseBoolean(conf.get(INPUT_WIDEROWS_CONFIG));
- }
-
- public static String getReadConsistencyLevel(Configuration conf)
- {
- return conf.get(READ_CONSISTENCY_LEVEL, "LOCAL_ONE");
- }
-
- public static void setReadConsistencyLevel(Configuration conf, String consistencyLevel)
- {
- conf.set(READ_CONSISTENCY_LEVEL, consistencyLevel);
- }
-
- public static String getWriteConsistencyLevel(Configuration conf)
- {
- return conf.get(WRITE_CONSISTENCY_LEVEL, "LOCAL_ONE");
- }
-
- public static void setWriteConsistencyLevel(Configuration conf, String consistencyLevel)
- {
- conf.set(WRITE_CONSISTENCY_LEVEL, consistencyLevel);
- }
-
- public static String getInputInitialAddress(Configuration conf)
- {
- return conf.get(INPUT_INITIAL_ADDRESS);
- }
-
- public static void setInputInitialAddress(Configuration conf, String address)
- {
- conf.set(INPUT_INITIAL_ADDRESS, address);
- }
- public static void setInputPartitioner(Configuration conf, String classname)
- {
- conf.set(INPUT_PARTITIONER_CONFIG, classname);
- }
-
- public static IPartitioner getInputPartitioner(Configuration conf)
- {
- return FBUtilities.newPartitioner(conf.get(INPUT_PARTITIONER_CONFIG));
- }
-
- public static String getOutputInitialAddress(Configuration conf)
- {
- return conf.get(OUTPUT_INITIAL_ADDRESS);
- }
-
- public static void setOutputInitialPort(Configuration conf, Integer port)
- {
- conf.set(OUTPUT_INITIAL_PORT, port.toString());
- }
-
- public static Integer getOutputInitialPort(Configuration conf)
- {
- return Integer.valueOf(conf.get(OUTPUT_INITIAL_PORT, DEFAULT_CASSANDRA_NATIVE_PORT));
- }
-
- public static void setOutputInitialAddress(Configuration conf, String address)
- {
- conf.set(OUTPUT_INITIAL_ADDRESS, address);
- }
-
- public static void setOutputPartitioner(Configuration conf, String classname)
- {
- conf.set(OUTPUT_PARTITIONER_CONFIG, classname);
- }
-
- public static IPartitioner getOutputPartitioner(Configuration conf)
- {
- return FBUtilities.newPartitioner(conf.get(OUTPUT_PARTITIONER_CONFIG));
- }
-
- public static String getOutputCompressionClass(Configuration conf)
- {
- return conf.get(OUTPUT_COMPRESSION_CLASS);
- }
-
- public static String getOutputCompressionChunkLength(Configuration conf)
- {
- return conf.get(OUTPUT_COMPRESSION_CHUNK_LENGTH, String.valueOf(CompressionParams.DEFAULT_CHUNK_LENGTH));
- }
-
- public static void setOutputCompressionClass(Configuration conf, String classname)
- {
- conf.set(OUTPUT_COMPRESSION_CLASS, classname);
- }
-
- public static void setOutputCompressionChunkLength(Configuration conf, String length)
- {
- conf.set(OUTPUT_COMPRESSION_CHUNK_LENGTH, length);
- }
-
- public static boolean getOutputLocalDCOnly(Configuration conf)
- {
- return Boolean.parseBoolean(conf.get(OUTPUT_LOCAL_DC_ONLY, "false"));
- }
-
- public static void setOutputLocalDCOnly(Configuration conf, boolean localDCOnly)
- {
- conf.set(OUTPUT_LOCAL_DC_ONLY, Boolean.toString(localDCOnly));
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/HadoopCompat.java b/src/java/org/apache/cassandra/hadoop/HadoopCompat.java
deleted file mode 100644
index 479f948..0000000
--- a/src/java/org/apache/cassandra/hadoop/HadoopCompat.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.hadoop;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.MapContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.StatusReporter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskInputOutputContext;
-
-/*
- * This is based on ContextFactory.java from hadoop-2.0.x sources.
- */
-
-/**
- * Utility methods to allow applications to deal with inconsistencies between
- * MapReduce Context Objects API between Hadoop 1.x and 2.x.
- */
-public class HadoopCompat
-{
-
- private static final boolean useV21;
-
- private static final Constructor<?> JOB_CONTEXT_CONSTRUCTOR;
- private static final Constructor<?> TASK_CONTEXT_CONSTRUCTOR;
- private static final Constructor<?> MAP_CONTEXT_CONSTRUCTOR;
- private static final Constructor<?> GENERIC_COUNTER_CONSTRUCTOR;
-
- private static final Field READER_FIELD;
- private static final Field WRITER_FIELD;
-
- private static final Method GET_CONFIGURATION_METHOD;
- private static final Method SET_STATUS_METHOD;
- private static final Method GET_COUNTER_METHOD;
- private static final Method INCREMENT_COUNTER_METHOD;
- private static final Method GET_TASK_ATTEMPT_ID;
- private static final Method PROGRESS_METHOD;
-
- static
- {
- boolean v21 = true;
- final String PACKAGE = "org.apache.hadoop.mapreduce";
- try
- {
- Class.forName(PACKAGE + ".task.JobContextImpl");
- } catch (ClassNotFoundException cnfe)
- {
- v21 = false;
- }
- useV21 = v21;
- Class<?> jobContextCls;
- Class<?> taskContextCls;
- Class<?> taskIOContextCls;
- Class<?> mapContextCls;
- Class<?> genericCounterCls;
- try
- {
- if (v21)
- {
- jobContextCls =
- Class.forName(PACKAGE+".task.JobContextImpl");
- taskContextCls =
- Class.forName(PACKAGE+".task.TaskAttemptContextImpl");
- taskIOContextCls =
- Class.forName(PACKAGE+".task.TaskInputOutputContextImpl");
- mapContextCls = Class.forName(PACKAGE + ".task.MapContextImpl");
- genericCounterCls = Class.forName(PACKAGE+".counters.GenericCounter");
- }
- else
- {
- jobContextCls =
- Class.forName(PACKAGE+".JobContext");
- taskContextCls =
- Class.forName(PACKAGE+".TaskAttemptContext");
- taskIOContextCls =
- Class.forName(PACKAGE+".TaskInputOutputContext");
- mapContextCls = Class.forName(PACKAGE + ".MapContext");
- genericCounterCls =
- Class.forName("org.apache.hadoop.mapred.Counters$Counter");
-
- }
- } catch (ClassNotFoundException e)
- {
- throw new IllegalArgumentException("Can't find class", e);
- }
- try
- {
- JOB_CONTEXT_CONSTRUCTOR =
- jobContextCls.getConstructor(Configuration.class, JobID.class);
- JOB_CONTEXT_CONSTRUCTOR.setAccessible(true);
- TASK_CONTEXT_CONSTRUCTOR =
- taskContextCls.getConstructor(Configuration.class,
- TaskAttemptID.class);
- TASK_CONTEXT_CONSTRUCTOR.setAccessible(true);
- GENERIC_COUNTER_CONSTRUCTOR =
- genericCounterCls.getDeclaredConstructor(String.class,
- String.class,
- Long.TYPE);
- GENERIC_COUNTER_CONSTRUCTOR.setAccessible(true);
-
- if (useV21)
- {
- MAP_CONTEXT_CONSTRUCTOR =
- mapContextCls.getDeclaredConstructor(Configuration.class,
- TaskAttemptID.class,
- RecordReader.class,
- RecordWriter.class,
- OutputCommitter.class,
- StatusReporter.class,
- InputSplit.class);
- Method get_counter;
- try
- {
- get_counter = Class.forName(PACKAGE + ".TaskAttemptContext").getMethod("getCounter", String.class,
- String.class);
- }
- catch (Exception e)
- {
- get_counter = Class.forName(PACKAGE + ".TaskInputOutputContext").getMethod("getCounter",
- String.class, String.class);
- }
- GET_COUNTER_METHOD = get_counter;
- }
- else
- {
- MAP_CONTEXT_CONSTRUCTOR =
- mapContextCls.getConstructor(Configuration.class,
- TaskAttemptID.class,
- RecordReader.class,
- RecordWriter.class,
- OutputCommitter.class,
- StatusReporter.class,
- InputSplit.class);
- GET_COUNTER_METHOD = Class.forName(PACKAGE+".TaskInputOutputContext")
- .getMethod("getCounter", String.class, String.class);
- }
- MAP_CONTEXT_CONSTRUCTOR.setAccessible(true);
- READER_FIELD = mapContextCls.getDeclaredField("reader");
- READER_FIELD.setAccessible(true);
- WRITER_FIELD = taskIOContextCls.getDeclaredField("output");
- WRITER_FIELD.setAccessible(true);
- GET_CONFIGURATION_METHOD = Class.forName(PACKAGE+".JobContext")
- .getMethod("getConfiguration");
- SET_STATUS_METHOD = Class.forName(PACKAGE+".TaskAttemptContext")
- .getMethod("setStatus", String.class);
- GET_TASK_ATTEMPT_ID = Class.forName(PACKAGE+".TaskAttemptContext")
- .getMethod("getTaskAttemptID");
- INCREMENT_COUNTER_METHOD = Class.forName(PACKAGE+".Counter")
- .getMethod("increment", Long.TYPE);
- PROGRESS_METHOD = Class.forName(PACKAGE+".TaskAttemptContext")
- .getMethod("progress");
-
- }
- catch (SecurityException e)
- {
- throw new IllegalArgumentException("Can't run constructor ", e);
- }
- catch (NoSuchMethodException e)
- {
- throw new IllegalArgumentException("Can't find constructor ", e);
- }
- catch (NoSuchFieldException e)
- {
- throw new IllegalArgumentException("Can't find field ", e);
- }
- catch (ClassNotFoundException e)
- {
- throw new IllegalArgumentException("Can't find class", e);
- }
- }
-
- /**
- * True if runtime Hadoop version is 2.x, false otherwise.
- */
- public static boolean isVersion2x()
- {
- return useV21;
- }
-
- private static Object newInstance(Constructor<?> constructor, Object...args)
- {
- try
- {
- return constructor.newInstance(args);
- }
- catch (InstantiationException e)
- {
- throw new IllegalArgumentException("Can't instantiate " + constructor, e);
- }
- catch (IllegalAccessException e)
- {
- throw new IllegalArgumentException("Can't instantiate " + constructor, e);
- }
- catch (InvocationTargetException e)
- {
- throw new IllegalArgumentException("Can't instantiate " + constructor, e);
- }
- }
-
- /**
- * Creates JobContext from a JobConf and jobId using the correct constructor
- * for based on Hadoop version. <code>jobId</code> could be null.
- */
- public static JobContext newJobContext(Configuration conf, JobID jobId) {
- return (JobContext) newInstance(JOB_CONTEXT_CONSTRUCTOR, conf, jobId);
- }
-
- /**
- * Creates TaskAttempContext from a JobConf and jobId using the correct
- * constructor for based on Hadoop version.
- */
- public static TaskAttemptContext newTaskAttemptContext(
- Configuration conf, TaskAttemptID taskAttemptId) {
- return (TaskAttemptContext)
- newInstance(TASK_CONTEXT_CONSTRUCTOR, conf, taskAttemptId);
- }
-
- /**
- * Instantiates MapContext under Hadoop 1 and MapContextImpl under Hadoop 2.
- */
- public static MapContext newMapContext(Configuration conf,
- TaskAttemptID taskAttemptID,
- RecordReader recordReader,
- RecordWriter recordWriter,
- OutputCommitter outputCommitter,
- StatusReporter statusReporter,
- InputSplit inputSplit) {
- return (MapContext) newInstance(MAP_CONTEXT_CONSTRUCTOR,
- conf, taskAttemptID, recordReader, recordWriter, outputCommitter,
- statusReporter, inputSplit);
- }
-
- /**
- * @return with Hadoop 2 : <code>new GenericCounter(args)</code>,<br>
- * with Hadoop 1 : <code>new Counter(args)</code>
- */
- public static Counter newGenericCounter(String name, String displayName, long value)
- {
- try
- {
- return (Counter)
- GENERIC_COUNTER_CONSTRUCTOR.newInstance(name, displayName, value);
- }
- catch (InstantiationException | IllegalAccessException | InvocationTargetException e)
- {
- throw new IllegalArgumentException("Can't instantiate Counter", e);
- }
- }
-
- /**
- * Invokes a method and rethrows any exception as runtime excetpions.
- */
- private static Object invoke(Method method, Object obj, Object... args)
- {
- try
- {
- return method.invoke(obj, args);
- }
- catch (IllegalAccessException | InvocationTargetException e)
- {
- throw new IllegalArgumentException("Can't invoke method " + method.getName(), e);
- }
- }
-
- /**
- * Invoke getConfiguration() on JobContext. Works with both
- * Hadoop 1 and 2.
- */
- public static Configuration getConfiguration(JobContext context)
- {
- return (Configuration) invoke(GET_CONFIGURATION_METHOD, context);
- }
-
- /**
- * Invoke setStatus() on TaskAttemptContext. Works with both
- * Hadoop 1 and 2.
- */
- public static void setStatus(TaskAttemptContext context, String status)
- {
- invoke(SET_STATUS_METHOD, context, status);
- }
-
- /**
- * returns TaskAttemptContext.getTaskAttemptID(). Works with both
- * Hadoop 1 and 2.
- */
- public static TaskAttemptID getTaskAttemptID(TaskAttemptContext taskContext)
- {
- return (TaskAttemptID) invoke(GET_TASK_ATTEMPT_ID, taskContext);
- }
-
- /**
- * Invoke getCounter() on TaskInputOutputContext. Works with both
- * Hadoop 1 and 2.
- */
- public static Counter getCounter(TaskInputOutputContext context,
- String groupName, String counterName)
- {
- return (Counter) invoke(GET_COUNTER_METHOD, context, groupName, counterName);
- }
-
- /**
- * Invoke TaskAttemptContext.progress(). Works with both
- * Hadoop 1 and 2.
- */
- public static void progress(TaskAttemptContext context)
- {
- invoke(PROGRESS_METHOD, context);
- }
-
- /**
- * Increment the counter. Works with both Hadoop 1 and 2
- */
- public static void incrementCounter(Counter counter, long increment)
- {
- // incrementing a count might be called often. Might be affected by
- // cost of invoke(). might be good candidate to handle in a shim.
- // (TODO Raghu) figure out how achieve such a build with maven
- invoke(INCREMENT_COUNTER_METHOD, counter, increment);
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/ReporterWrapper.java b/src/java/org/apache/cassandra/hadoop/ReporterWrapper.java
deleted file mode 100644
index d2cc769..0000000
--- a/src/java/org/apache/cassandra/hadoop/ReporterWrapper.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.hadoop;
-
-import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapreduce.StatusReporter;
-
-/**
- * A reporter that works with both mapred and mapreduce APIs.
- */
-public class ReporterWrapper extends StatusReporter implements Reporter
-{
- private Reporter wrappedReporter;
-
- public ReporterWrapper(Reporter reporter)
- {
- wrappedReporter = reporter;
- }
-
- @Override
- public Counters.Counter getCounter(Enum<?> anEnum)
- {
- return wrappedReporter.getCounter(anEnum);
- }
-
- @Override
- public Counters.Counter getCounter(String s, String s1)
- {
- return wrappedReporter.getCounter(s, s1);
- }
-
- @Override
- public void incrCounter(Enum<?> anEnum, long l)
- {
- wrappedReporter.incrCounter(anEnum, l);
- }
-
- @Override
- public void incrCounter(String s, String s1, long l)
- {
- wrappedReporter.incrCounter(s, s1, l);
- }
-
- @Override
- public InputSplit getInputSplit() throws UnsupportedOperationException
- {
- return wrappedReporter.getInputSplit();
- }
-
- @Override
- public void progress()
- {
- wrappedReporter.progress();
- }
-
- // @Override
- public float getProgress()
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setStatus(String s)
- {
- wrappedReporter.setStatus(s);
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkOutputFormat.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkOutputFormat.java
deleted file mode 100644
index dfdf855..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkOutputFormat.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.hadoop.HadoopCompat;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.Progressable;
-
-/**
- * The <code>CqlBulkOutputFormat</code> acts as a Hadoop-specific
- * OutputFormat that allows reduce tasks to store keys (and corresponding
- * bound variable values) as CQL rows (and respective columns) in a given
- * table.
- *
- * <p>
- * As is the case with the {@link org.apache.cassandra.hadoop.cql3.CqlOutputFormat},
- * you need to set the prepared statement in your
- * Hadoop job Configuration. The {@link CqlConfigHelper} class, through its
- * {@link org.apache.cassandra.hadoop.ConfigHelper#setOutputPreparedStatement} method, is provided to make this
- * simple.
- * you need to set the Keyspace. The {@link ConfigHelper} class, through its
- * {@link org.apache.cassandra.hadoop.ConfigHelper#setOutputColumnFamily} method, is provided to make this
- * simple.
- * </p>
- */
-public class CqlBulkOutputFormat extends OutputFormat<Object, List<ByteBuffer>>
- implements org.apache.hadoop.mapred.OutputFormat<Object, List<ByteBuffer>>
-{
-
- private static final String OUTPUT_CQL_SCHEMA_PREFIX = "cassandra.table.schema.";
- private static final String OUTPUT_CQL_INSERT_PREFIX = "cassandra.table.insert.";
- private static final String DELETE_SOURCE = "cassandra.output.delete.source";
- private static final String TABLE_ALIAS_PREFIX = "cqlbulkoutputformat.table.alias.";
-
- /** Fills the deprecated OutputFormat interface for streaming. */
- @Deprecated
- public CqlBulkRecordWriter getRecordWriter(FileSystem filesystem, JobConf job, String name, Progressable progress) throws IOException
- {
- return new CqlBulkRecordWriter(job, progress);
- }
-
- /**
- * Get the {@link RecordWriter} for the given task.
- *
- * @param context
- * the information about the current task.
- * @return a {@link RecordWriter} to write the output for the job.
- * @throws IOException
- */
- public CqlBulkRecordWriter getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException
- {
- return new CqlBulkRecordWriter(context);
- }
-
- @Override
- public void checkOutputSpecs(JobContext context)
- {
- checkOutputSpecs(HadoopCompat.getConfiguration(context));
- }
-
- private void checkOutputSpecs(Configuration conf)
- {
- if (ConfigHelper.getOutputKeyspace(conf) == null)
- {
- throw new UnsupportedOperationException("you must set the keyspace with setTable()");
- }
- }
-
- /** Fills the deprecated OutputFormat interface for streaming. */
- @Deprecated
- public void checkOutputSpecs(org.apache.hadoop.fs.FileSystem filesystem, org.apache.hadoop.mapred.JobConf job) throws IOException
- {
- checkOutputSpecs(job);
- }
-
- @Override
- public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException
- {
- return new NullOutputCommitter();
- }
-
- public static void setTableSchema(Configuration conf, String columnFamily, String schema)
- {
- conf.set(OUTPUT_CQL_SCHEMA_PREFIX + columnFamily, schema);
- }
-
- public static void setTableInsertStatement(Configuration conf, String columnFamily, String insertStatement)
- {
- conf.set(OUTPUT_CQL_INSERT_PREFIX + columnFamily, insertStatement);
- }
-
- public static String getTableSchema(Configuration conf, String columnFamily)
- {
- String schema = conf.get(OUTPUT_CQL_SCHEMA_PREFIX + columnFamily);
- if (schema == null)
- {
- throw new UnsupportedOperationException("You must set the Table schema using setTableSchema.");
- }
- return schema;
- }
-
- public static String getTableInsertStatement(Configuration conf, String columnFamily)
- {
- String insert = conf.get(OUTPUT_CQL_INSERT_PREFIX + columnFamily);
- if (insert == null)
- {
- throw new UnsupportedOperationException("You must set the Table insert statement using setTableSchema.");
- }
- return insert;
- }
-
- public static void setDeleteSourceOnSuccess(Configuration conf, boolean deleteSrc)
- {
- conf.setBoolean(DELETE_SOURCE, deleteSrc);
- }
-
- public static boolean getDeleteSourceOnSuccess(Configuration conf)
- {
- return conf.getBoolean(DELETE_SOURCE, false);
- }
-
- public static void setTableAlias(Configuration conf, String alias, String columnFamily)
- {
- conf.set(TABLE_ALIAS_PREFIX + alias, columnFamily);
- }
-
- public static String getTableForAlias(Configuration conf, String alias)
- {
- return conf.get(TABLE_ALIAS_PREFIX + alias);
- }
-
- /**
- * Set the hosts to ignore as comma delimited values.
- * Data will not be bulk loaded onto the ignored nodes.
- * @param conf job configuration
- * @param ignoreNodesCsv a comma delimited list of nodes to ignore
- */
- public static void setIgnoreHosts(Configuration conf, String ignoreNodesCsv)
- {
- conf.set(CqlBulkRecordWriter.IGNORE_HOSTS, ignoreNodesCsv);
- }
-
- /**
- * Set the hosts to ignore. Data will not be bulk loaded onto the ignored nodes.
- * @param conf job configuration
- * @param ignoreNodes the nodes to ignore
- */
- public static void setIgnoreHosts(Configuration conf, String... ignoreNodes)
- {
- conf.setStrings(CqlBulkRecordWriter.IGNORE_HOSTS, ignoreNodes);
- }
-
- /**
- * Get the hosts to ignore as a collection of strings
- * @param conf job configuration
- * @return the nodes to ignore as a collection of stirngs
- */
- public static Collection<String> getIgnoreHosts(Configuration conf)
- {
- return conf.getStringCollection(CqlBulkRecordWriter.IGNORE_HOSTS);
- }
-
- public static class NullOutputCommitter extends OutputCommitter
- {
- public void abortTask(TaskAttemptContext taskContext) { }
-
- public void cleanupJob(JobContext jobContext) { }
-
- public void commitTask(TaskAttemptContext taskContext) { }
-
- public boolean needsTaskCommit(TaskAttemptContext taskContext)
- {
- return false;
- }
-
- public void setupJob(JobContext jobContext) { }
-
- public void setupTask(TaskAttemptContext taskContext) { }
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java
deleted file mode 100644
index 82d5e8a..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
-
-import com.google.common.net.HostAndPort;
-import org.apache.cassandra.io.util.File;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.hadoop.HadoopCompat;
-import org.apache.cassandra.io.sstable.CQLSSTableWriter;
-import org.apache.cassandra.io.sstable.SSTableLoader;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.streaming.StreamState;
-import org.apache.cassandra.utils.NativeSSTableLoaderClient;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.Progressable;
-
-import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
-
-/**
- * The <code>CqlBulkRecordWriter</code> maps the output <key, value>
- * pairs to a Cassandra column family. In particular, it applies the binded variables
- * in the value to the prepared statement, which it associates with the key, and in
- * turn the responsible endpoint.
- *
- * <p>
- * Furthermore, this writer groups the cql queries by the endpoint responsible for
- * the rows being affected. This allows the cql queries to be executed in parallel,
- * directly to a responsible endpoint.
- * </p>
- *
- * @see CqlBulkOutputFormat
- */
-public class CqlBulkRecordWriter extends RecordWriter<Object, List<ByteBuffer>>
- implements org.apache.hadoop.mapred.RecordWriter<Object, List<ByteBuffer>>
-{
- public final static String OUTPUT_LOCATION = "mapreduce.output.bulkoutputformat.localdir";
- public final static String BUFFER_SIZE_IN_MB = "mapreduce.output.bulkoutputformat.buffersize";
- public final static String STREAM_THROTTLE_MBITS = "mapreduce.output.bulkoutputformat.streamthrottlembits";
- public final static String MAX_FAILED_HOSTS = "mapreduce.output.bulkoutputformat.maxfailedhosts";
- public final static String IGNORE_HOSTS = "mapreduce.output.bulkoutputformat.ignorehosts";
-
- private final Logger logger = LoggerFactory.getLogger(CqlBulkRecordWriter.class);
-
- protected final Configuration conf;
- protected final int maxFailures;
- protected final int bufferSize;
- protected Closeable writer;
- protected SSTableLoader loader;
- protected Progressable progress;
- protected TaskAttemptContext context;
- protected final Set<InetAddressAndPort> ignores = new HashSet<>();
-
- private String keyspace;
- private String table;
- private String schema;
- private String insertStatement;
- private File outputDir;
- private boolean deleteSrc;
- private IPartitioner partitioner;
-
- CqlBulkRecordWriter(TaskAttemptContext context) throws IOException
- {
- this(HadoopCompat.getConfiguration(context));
- this.context = context;
- setConfigs();
- }
-
- CqlBulkRecordWriter(Configuration conf, Progressable progress) throws IOException
- {
- this(conf);
- this.progress = progress;
- setConfigs();
- }
-
- CqlBulkRecordWriter(Configuration conf) throws IOException
- {
- this.conf = conf;
- DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(Integer.parseInt(conf.get(STREAM_THROTTLE_MBITS, "0")));
- maxFailures = Integer.parseInt(conf.get(MAX_FAILED_HOSTS, "0"));
- bufferSize = Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64"));
- setConfigs();
- }
-
- private void setConfigs() throws IOException
- {
- // if anything is missing, exceptions will be thrown here, instead of on write()
- keyspace = ConfigHelper.getOutputKeyspace(conf);
- table = ConfigHelper.getOutputColumnFamily(conf);
-
- // check if table is aliased
- String aliasedCf = CqlBulkOutputFormat.getTableForAlias(conf, table);
- if (aliasedCf != null)
- table = aliasedCf;
-
- schema = CqlBulkOutputFormat.getTableSchema(conf, table);
- insertStatement = CqlBulkOutputFormat.getTableInsertStatement(conf, table);
- outputDir = getTableDirectory();
- deleteSrc = CqlBulkOutputFormat.getDeleteSourceOnSuccess(conf);
- try
- {
- partitioner = ConfigHelper.getInputPartitioner(conf);
- }
- catch (Exception e)
- {
- partitioner = Murmur3Partitioner.instance;
- }
- try
- {
- for (String hostToIgnore : CqlBulkOutputFormat.getIgnoreHosts(conf))
- ignores.add(InetAddressAndPort.getByName(hostToIgnore));
- }
- catch (UnknownHostException e)
- {
- throw new RuntimeException(("Unknown host: " + e.getMessage()));
- }
- }
-
- protected String getOutputLocation() throws IOException
- {
- String dir = conf.get(OUTPUT_LOCATION, JAVA_IO_TMPDIR.getString());
- if (dir == null)
- throw new IOException("Output directory not defined, if hadoop is not setting java.io.tmpdir then define " + OUTPUT_LOCATION);
- return dir;
- }
-
- private void prepareWriter() throws IOException
- {
- if (writer == null)
- {
- writer = CQLSSTableWriter.builder()
- .forTable(schema)
- .using(insertStatement)
- .withPartitioner(ConfigHelper.getOutputPartitioner(conf))
- .inDirectory(outputDir)
- .withBufferSizeInMiB(Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")))
- .withPartitioner(partitioner)
- .build();
- }
-
- if (loader == null)
- {
- ExternalClient externalClient = new ExternalClient(conf);
- externalClient.setTableMetadata(TableMetadataRef.forOfflineTools(CreateTableStatement.parse(schema, keyspace).build()));
-
- loader = new SSTableLoader(outputDir, externalClient, new NullOutputHandler())
- {
- @Override
- public void onSuccess(StreamState finalState)
- {
- if (deleteSrc)
- FileUtils.deleteRecursive(outputDir);
- }
- };
- }
- }
-
- /**
- * <p>
- * The column values must correspond to the order in which
- * they appear in the insert stored procedure.
- *
- * Key is not used, so it can be null or any object.
- * </p>
- *
- * @param key
- * any object or null.
- * @param values
- * the values to write.
- * @throws IOException
- */
- @Override
- public void write(Object key, List<ByteBuffer> values) throws IOException
- {
- prepareWriter();
- try
- {
- ((CQLSSTableWriter) writer).rawAddRow(values);
-
- if (null != progress)
- progress.progress();
- if (null != context)
- HadoopCompat.progress(context);
- }
- catch (InvalidRequestException e)
- {
- throw new IOException("Error adding row with key: " + key, e);
- }
- }
-
- private File getTableDirectory() throws IOException
- {
- File dir = new File(String.format("%s%s%s%s%s-%s", getOutputLocation(), File.pathSeparator(), keyspace, File.pathSeparator(), table, UUID.randomUUID().toString()));
-
- if (!dir.exists() && !dir.tryCreateDirectories())
- {
- throw new IOException("Failed to created output directory: " + dir);
- }
-
- return dir;
- }
-
- @Override
- public void close(TaskAttemptContext context) throws IOException, InterruptedException
- {
- close();
- }
-
- /** Fills the deprecated RecordWriter interface for streaming. */
- @Deprecated
- public void close(org.apache.hadoop.mapred.Reporter reporter) throws IOException
- {
- close();
- }
-
- private void close() throws IOException
- {
- if (writer != null)
- {
- writer.close();
- Future<StreamState> future = loader.stream(ignores);
- while (true)
- {
- try
- {
- future.get(1000, TimeUnit.MILLISECONDS);
- break;
- }
- catch (ExecutionException | TimeoutException te)
- {
- if (null != progress)
- progress.progress();
- if (null != context)
- HadoopCompat.progress(context);
- }
- catch (InterruptedException e)
- {
- throw new IOException(e);
- }
- }
- if (loader.getFailedHosts().size() > 0)
- {
- if (loader.getFailedHosts().size() > maxFailures)
- throw new IOException("Too many hosts failed: " + loader.getFailedHosts());
- else
- logger.warn("Some hosts failed: {}", loader.getFailedHosts());
- }
- }
- }
-
- public static class ExternalClient extends NativeSSTableLoaderClient
- {
- public ExternalClient(Configuration conf)
- {
- super(resolveHostAddresses(conf),
- ConfigHelper.getOutputInitialPort(conf),
- ConfigHelper.getOutputKeyspaceUserName(conf),
- ConfigHelper.getOutputKeyspacePassword(conf),
- CqlConfigHelper.getSSLOptions(conf).orNull());
- }
-
- private static Collection<InetSocketAddress> resolveHostAddresses(Configuration conf)
- {
- Set<InetSocketAddress> addresses = new HashSet<>();
- int port = CqlConfigHelper.getOutputNativePort(conf);
- for (String host : ConfigHelper.getOutputInitialAddress(conf).split(","))
- {
- try
- {
- HostAndPort hap = HostAndPort.fromString(host);
- addresses.add(new InetSocketAddress(InetAddress.getByName(hap.getHost()), hap.getPortOrDefault(port)));
- }
- catch (UnknownHostException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- return addresses;
- }
- }
-
- public static class NullOutputHandler implements OutputHandler
- {
- public void output(String msg) {}
- public void debug(String msg) {}
- public void warn(String msg) {}
- public void warn(String msg, Throwable th) {}
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlClientHelper.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlClientHelper.java
deleted file mode 100644
index e0a6384..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlClientHelper.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.hadoop.cql3;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import com.datastax.driver.core.Host;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.Token;
-import com.datastax.driver.core.TokenRange;
-
-public class CqlClientHelper
-{
- private CqlClientHelper()
- {
- }
-
- public static Map<TokenRange, List<Host>> getLocalPrimaryRangeForDC(String keyspace, Metadata metadata, String targetDC)
- {
- Objects.requireNonNull(keyspace, "keyspace");
- Objects.requireNonNull(metadata, "metadata");
- Objects.requireNonNull(targetDC, "targetDC");
-
- // In 2.1 the logic was to have a set of nodes used as a seed, they were used to query
- // client.describe_local_ring(keyspace) -> List<TokenRange>; this should include all nodes in the local dc.
- // TokenRange contained the endpoints in order, so .endpoints.get(0) is the primary owner
- // Client does not have a similar API, instead it returns Set<Host>. To replicate this we first need
- // to compute the primary owners, then add in the replicas
-
- List<Token> tokens = new ArrayList<>();
- Map<Token, Host> tokenToHost = new HashMap<>();
- for (Host host : metadata.getAllHosts())
- {
- if (!targetDC.equals(host.getDatacenter()))
- continue;
-
- for (Token token : host.getTokens())
- {
- Host previous = tokenToHost.putIfAbsent(token, host);
- if (previous != null)
- throw new IllegalStateException("Two hosts share the same token; hosts " + host.getHostId() + ":"
- + host.getTokens() + ", " + previous.getHostId() + ":" + previous.getTokens());
- tokens.add(token);
- }
- }
- Collections.sort(tokens);
-
- Map<TokenRange, List<Host>> rangeToReplicas = new HashMap<>();
-
- // The first token in the ring uses the last token as its 'start', handle this here to simplify the loop
- Token start = tokens.get(tokens.size() - 1);
- Token end = tokens.get(0);
-
- addRange(keyspace, metadata, tokenToHost, rangeToReplicas, start, end);
- for (int i = 1; i < tokens.size(); i++)
- {
- start = tokens.get(i - 1);
- end = tokens.get(i);
-
- addRange(keyspace, metadata, tokenToHost, rangeToReplicas, start, end);
- }
-
- return rangeToReplicas;
- }
-
- private static void addRange(String keyspace,
- Metadata metadata,
- Map<Token, Host> tokenToHost,
- Map<TokenRange, List<Host>> rangeToReplicas,
- Token start, Token end)
- {
- Host host = tokenToHost.get(end);
- String dc = host.getDatacenter();
-
- TokenRange range = metadata.newTokenRange(start, end);
- List<Host> replicas = new ArrayList<>();
- replicas.add(host);
- // get all the replicas for the specific DC
- for (Host replica : metadata.getReplicas(keyspace, range))
- {
- if (dc.equals(replica.getDatacenter()) && !host.equals(replica))
- replicas.add(replica);
- }
- List<Host> previous = rangeToReplicas.put(range, replicas);
- if (previous != null)
- throw new IllegalStateException("Two hosts (" + host + ", " + previous + ") map to the same token range: " + range);
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java
deleted file mode 100644
index 998a754..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java
+++ /dev/null
@@ -1,654 +0,0 @@
-package org.apache.cassandra.hadoop.cql3;
-/*
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*/
-import java.nio.file.Files;
-import java.io.InputStream;
-import java.io.IOException;
-import java.security.KeyManagementException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.SecureRandom;
-import java.security.UnrecoverableKeyException;
-import java.security.cert.CertificateException;
-import java.util.Arrays;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
-
-import com.google.common.base.Optional;
-import org.apache.commons.lang3.StringUtils;
-
-import com.datastax.driver.core.AuthProvider;
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.HostDistance;
-import com.datastax.driver.core.JdkSSLOptions;
-import com.datastax.driver.core.PlainTextAuthProvider;
-import com.datastax.driver.core.ProtocolVersion;
-import com.datastax.driver.core.policies.LoadBalancingPolicy;
-import com.datastax.driver.core.PoolingOptions;
-import com.datastax.driver.core.ProtocolOptions;
-import com.datastax.driver.core.QueryOptions;
-import com.datastax.driver.core.SSLOptions;
-import com.datastax.driver.core.SocketOptions;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.io.util.File;
-import org.apache.hadoop.conf.Configuration;
-
-
-public class CqlConfigHelper
-{
- private static final String INPUT_CQL_COLUMNS_CONFIG = "cassandra.input.columnfamily.columns";
- private static final String INPUT_CQL_PAGE_ROW_SIZE_CONFIG = "cassandra.input.page.row.size";
- private static final String INPUT_CQL_WHERE_CLAUSE_CONFIG = "cassandra.input.where.clause";
- private static final String INPUT_CQL = "cassandra.input.cql";
-
- private static final String USERNAME = "cassandra.username";
- private static final String PASSWORD = "cassandra.password";
-
- private static final String INPUT_NATIVE_PORT = "cassandra.input.native.port";
- private static final String INPUT_NATIVE_CORE_CONNECTIONS_PER_HOST = "cassandra.input.native.core.connections.per.host";
- private static final String INPUT_NATIVE_MAX_CONNECTIONS_PER_HOST = "cassandra.input.native.max.connections.per.host";
- private static final String INPUT_NATIVE_MAX_SIMULT_REQ_PER_CONNECTION = "cassandra.input.native.max.simult.reqs.per.connection";
- private static final String INPUT_NATIVE_CONNECTION_TIMEOUT = "cassandra.input.native.connection.timeout";
- private static final String INPUT_NATIVE_READ_CONNECTION_TIMEOUT = "cassandra.input.native.read.connection.timeout";
- private static final String INPUT_NATIVE_RECEIVE_BUFFER_SIZE = "cassandra.input.native.receive.buffer.size";
- private static final String INPUT_NATIVE_SEND_BUFFER_SIZE = "cassandra.input.native.send.buffer.size";
- private static final String INPUT_NATIVE_SOLINGER = "cassandra.input.native.solinger";
- private static final String INPUT_NATIVE_TCP_NODELAY = "cassandra.input.native.tcp.nodelay";
- private static final String INPUT_NATIVE_REUSE_ADDRESS = "cassandra.input.native.reuse.address";
- private static final String INPUT_NATIVE_KEEP_ALIVE = "cassandra.input.native.keep.alive";
- private static final String INPUT_NATIVE_AUTH_PROVIDER = "cassandra.input.native.auth.provider";
- private static final String INPUT_NATIVE_SSL_TRUST_STORE_PATH = "cassandra.input.native.ssl.trust.store.path";
- private static final String INPUT_NATIVE_SSL_KEY_STORE_PATH = "cassandra.input.native.ssl.key.store.path";
- private static final String INPUT_NATIVE_SSL_TRUST_STORE_PASSWARD = "cassandra.input.native.ssl.trust.store.password";
- private static final String INPUT_NATIVE_SSL_KEY_STORE_PASSWARD = "cassandra.input.native.ssl.key.store.password";
- private static final String INPUT_NATIVE_SSL_CIPHER_SUITES = "cassandra.input.native.ssl.cipher.suites";
-
- private static final String INPUT_NATIVE_PROTOCOL_VERSION = "cassandra.input.native.protocol.version";
-
- private static final String OUTPUT_CQL = "cassandra.output.cql";
- private static final String OUTPUT_NATIVE_PORT = "cassandra.output.native.port";
-
- /**
- * Set the CQL columns for the input of this job.
- *
- * @param conf Job configuration you are about to run
- * @param columns
- */
- public static void setInputColumns(Configuration conf, String columns)
- {
- if (columns == null || columns.isEmpty())
- return;
-
- conf.set(INPUT_CQL_COLUMNS_CONFIG, columns);
- }
-
- /**
- * Set the CQL query Limit for the input of this job.
- *
- * @param conf Job configuration you are about to run
- * @param cqlPageRowSize
- */
- public static void setInputCQLPageRowSize(Configuration conf, String cqlPageRowSize)
- {
- if (cqlPageRowSize == null)
- {
- throw new UnsupportedOperationException("cql page row size may not be null");
- }
-
- conf.set(INPUT_CQL_PAGE_ROW_SIZE_CONFIG, cqlPageRowSize);
- }
-
- /**
- * Set the CQL user defined where clauses for the input of this job.
- *
- * @param conf Job configuration you are about to run
- * @param clauses
- */
- public static void setInputWhereClauses(Configuration conf, String clauses)
- {
- if (clauses == null || clauses.isEmpty())
- return;
-
- conf.set(INPUT_CQL_WHERE_CLAUSE_CONFIG, clauses);
- }
-
- /**
- * Set the CQL prepared statement for the output of this job.
- *
- * @param conf Job configuration you are about to run
- * @param cql
- */
- public static void setOutputCql(Configuration conf, String cql)
- {
- if (cql == null || cql.isEmpty())
- return;
-
- conf.set(OUTPUT_CQL, cql);
- }
-
- public static void setInputCql(Configuration conf, String cql)
- {
- if (cql == null || cql.isEmpty())
- return;
-
- conf.set(INPUT_CQL, cql);
- }
-
- public static void setUserNameAndPassword(Configuration conf, String username, String password)
- {
- if (StringUtils.isNotBlank(username))
- {
- conf.set(INPUT_NATIVE_AUTH_PROVIDER, PlainTextAuthProvider.class.getName());
- conf.set(USERNAME, username);
- conf.set(PASSWORD, password);
- }
- }
-
- public static Optional<Integer> getInputCoreConnections(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_CORE_CONNECTIONS_PER_HOST, conf);
- }
-
- public static Optional<Integer> getInputMaxConnections(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_MAX_CONNECTIONS_PER_HOST, conf);
- }
-
- public static int getInputNativePort(Configuration conf)
- {
- return Integer.parseInt(conf.get(INPUT_NATIVE_PORT, "9042"));
- }
-
- public static int getOutputNativePort(Configuration conf)
- {
- return Integer.parseInt(conf.get(OUTPUT_NATIVE_PORT, "9042"));
- }
-
- public static Optional<Integer> getInputMaxSimultReqPerConnections(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_MAX_SIMULT_REQ_PER_CONNECTION, conf);
- }
-
- public static Optional<Integer> getInputNativeConnectionTimeout(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_CONNECTION_TIMEOUT, conf);
- }
-
- public static Optional<Integer> getInputNativeReadConnectionTimeout(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_READ_CONNECTION_TIMEOUT, conf);
- }
-
- public static Optional<Integer> getInputNativeReceiveBufferSize(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_RECEIVE_BUFFER_SIZE, conf);
- }
-
- public static Optional<Integer> getInputNativeSendBufferSize(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_SEND_BUFFER_SIZE, conf);
- }
-
- public static Optional<Integer> getInputNativeSolinger(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_SOLINGER, conf);
- }
-
- public static Optional<Boolean> getInputNativeTcpNodelay(Configuration conf)
- {
- return getBooleanSetting(INPUT_NATIVE_TCP_NODELAY, conf);
- }
-
- public static Optional<Boolean> getInputNativeReuseAddress(Configuration conf)
- {
- return getBooleanSetting(INPUT_NATIVE_REUSE_ADDRESS, conf);
- }
-
- public static Optional<String> getInputNativeAuthProvider(Configuration conf)
- {
- return getStringSetting(INPUT_NATIVE_AUTH_PROVIDER, conf);
- }
-
- public static Optional<String> getInputNativeSSLTruststorePath(Configuration conf)
- {
- return getStringSetting(INPUT_NATIVE_SSL_TRUST_STORE_PATH, conf);
- }
-
- public static Optional<String> getInputNativeSSLKeystorePath(Configuration conf)
- {
- return getStringSetting(INPUT_NATIVE_SSL_KEY_STORE_PATH, conf);
- }
-
- public static Optional<String> getInputNativeSSLKeystorePassword(Configuration conf)
- {
- return getStringSetting(INPUT_NATIVE_SSL_KEY_STORE_PASSWARD, conf);
- }
-
- public static Optional<String> getInputNativeSSLTruststorePassword(Configuration conf)
- {
- return getStringSetting(INPUT_NATIVE_SSL_TRUST_STORE_PASSWARD, conf);
- }
-
- public static Optional<String> getInputNativeSSLCipherSuites(Configuration conf)
- {
- return getStringSetting(INPUT_NATIVE_SSL_CIPHER_SUITES, conf);
- }
-
- public static Optional<Boolean> getInputNativeKeepAlive(Configuration conf)
- {
- return getBooleanSetting(INPUT_NATIVE_KEEP_ALIVE, conf);
- }
-
- public static String getInputcolumns(Configuration conf)
- {
- return conf.get(INPUT_CQL_COLUMNS_CONFIG);
- }
-
- public static Optional<Integer> getInputPageRowSize(Configuration conf)
- {
- return getIntSetting(INPUT_CQL_PAGE_ROW_SIZE_CONFIG, conf);
- }
-
- public static String getInputWhereClauses(Configuration conf)
- {
- return conf.get(INPUT_CQL_WHERE_CLAUSE_CONFIG);
- }
-
- public static String getInputCql(Configuration conf)
- {
- return conf.get(INPUT_CQL);
- }
-
- public static String getOutputCql(Configuration conf)
- {
- return conf.get(OUTPUT_CQL);
- }
-
- private static Optional<Integer> getProtocolVersion(Configuration conf)
- {
- return getIntSetting(INPUT_NATIVE_PROTOCOL_VERSION, conf);
- }
-
- public static Cluster getInputCluster(String host, Configuration conf)
- {
- // this method has been left for backward compatibility
- return getInputCluster(new String[] {host}, conf);
- }
-
- public static Cluster getInputCluster(String[] hosts, Configuration conf)
- {
- int port = getInputNativePort(conf);
- return getCluster(hosts, conf, port);
- }
-
- public static Cluster getOutputCluster(String host, Configuration conf)
- {
- return getOutputCluster(new String[]{host}, conf);
- }
-
- public static Cluster getOutputCluster(String[] hosts, Configuration conf)
- {
- int port = getOutputNativePort(conf);
- return getCluster(hosts, conf, port);
- }
-
- public static Cluster getCluster(String[] hosts, Configuration conf, int port)
- {
- Optional<AuthProvider> authProvider = getAuthProvider(conf);
- Optional<SSLOptions> sslOptions = getSSLOptions(conf);
- Optional<Integer> protocolVersion = getProtocolVersion(conf);
- LoadBalancingPolicy loadBalancingPolicy = getReadLoadBalancingPolicy(hosts);
- SocketOptions socketOptions = getReadSocketOptions(conf);
- QueryOptions queryOptions = getReadQueryOptions(conf);
- PoolingOptions poolingOptions = getReadPoolingOptions(conf);
-
- Cluster.Builder builder = Cluster.builder()
- .addContactPoints(hosts)
- .withPort(port)
- .withCompression(ProtocolOptions.Compression.NONE);
-
- if (authProvider.isPresent())
- builder.withAuthProvider(authProvider.get());
- if (sslOptions.isPresent())
- builder.withSSL(sslOptions.get());
-
- if (protocolVersion.isPresent())
- {
- builder.withProtocolVersion(ProtocolVersion.fromInt(protocolVersion.get()));
- }
- builder.withLoadBalancingPolicy(loadBalancingPolicy)
- .withSocketOptions(socketOptions)
- .withQueryOptions(queryOptions)
- .withPoolingOptions(poolingOptions);
-
- return builder.build();
- }
-
- public static void setInputCoreConnections(Configuration conf, String connections)
- {
- conf.set(INPUT_NATIVE_CORE_CONNECTIONS_PER_HOST, connections);
- }
-
- public static void setInputMaxConnections(Configuration conf, String connections)
- {
- conf.set(INPUT_NATIVE_MAX_CONNECTIONS_PER_HOST, connections);
- }
-
- public static void setInputMaxSimultReqPerConnections(Configuration conf, String reqs)
- {
- conf.set(INPUT_NATIVE_MAX_SIMULT_REQ_PER_CONNECTION, reqs);
- }
-
- public static void setInputNativeConnectionTimeout(Configuration conf, String timeout)
- {
- conf.set(INPUT_NATIVE_CONNECTION_TIMEOUT, timeout);
- }
-
- public static void setInputNativeReadConnectionTimeout(Configuration conf, String timeout)
- {
- conf.set(INPUT_NATIVE_READ_CONNECTION_TIMEOUT, timeout);
- }
-
- public static void setInputNativeReceiveBufferSize(Configuration conf, String size)
- {
- conf.set(INPUT_NATIVE_RECEIVE_BUFFER_SIZE, size);
- }
-
- public static void setInputNativeSendBufferSize(Configuration conf, String size)
- {
- conf.set(INPUT_NATIVE_SEND_BUFFER_SIZE, size);
- }
-
- public static void setInputNativeSolinger(Configuration conf, String solinger)
- {
- conf.set(INPUT_NATIVE_SOLINGER, solinger);
- }
-
- public static void setInputNativeTcpNodelay(Configuration conf, String tcpNodelay)
- {
- conf.set(INPUT_NATIVE_TCP_NODELAY, tcpNodelay);
- }
-
- public static void setInputNativeAuthProvider(Configuration conf, String authProvider)
- {
- conf.set(INPUT_NATIVE_AUTH_PROVIDER, authProvider);
- }
-
- public static void setInputNativeSSLTruststorePath(Configuration conf, String path)
- {
- conf.set(INPUT_NATIVE_SSL_TRUST_STORE_PATH, path);
- }
-
- public static void setInputNativeSSLKeystorePath(Configuration conf, String path)
- {
- conf.set(INPUT_NATIVE_SSL_KEY_STORE_PATH, path);
- }
-
- public static void setInputNativeSSLKeystorePassword(Configuration conf, String pass)
- {
- conf.set(INPUT_NATIVE_SSL_KEY_STORE_PASSWARD, pass);
- }
-
- public static void setInputNativeSSLTruststorePassword(Configuration conf, String pass)
- {
- conf.set(INPUT_NATIVE_SSL_TRUST_STORE_PASSWARD, pass);
- }
-
- public static void setInputNativeSSLCipherSuites(Configuration conf, String suites)
- {
- conf.set(INPUT_NATIVE_SSL_CIPHER_SUITES, suites);
- }
-
- public static void setInputNativeReuseAddress(Configuration conf, String reuseAddress)
- {
- conf.set(INPUT_NATIVE_REUSE_ADDRESS, reuseAddress);
- }
-
- public static void setInputNativeKeepAlive(Configuration conf, String keepAlive)
- {
- conf.set(INPUT_NATIVE_KEEP_ALIVE, keepAlive);
- }
-
- public static void setInputNativePort(Configuration conf, String port)
- {
- conf.set(INPUT_NATIVE_PORT, port);
- }
-
- private static PoolingOptions getReadPoolingOptions(Configuration conf)
- {
- Optional<Integer> coreConnections = getInputCoreConnections(conf);
- Optional<Integer> maxConnections = getInputMaxConnections(conf);
- Optional<Integer> maxSimultaneousRequests = getInputMaxSimultReqPerConnections(conf);
-
- PoolingOptions poolingOptions = new PoolingOptions();
-
- for (HostDistance hostDistance : Arrays.asList(HostDistance.LOCAL, HostDistance.REMOTE))
- {
- if (coreConnections.isPresent())
- poolingOptions.setCoreConnectionsPerHost(hostDistance, coreConnections.get());
- if (maxConnections.isPresent())
- poolingOptions.setMaxConnectionsPerHost(hostDistance, maxConnections.get());
- if (maxSimultaneousRequests.isPresent())
- poolingOptions.setNewConnectionThreshold(hostDistance, maxSimultaneousRequests.get());
- }
-
- return poolingOptions;
- }
-
- private static QueryOptions getReadQueryOptions(Configuration conf)
- {
- String CL = ConfigHelper.getReadConsistencyLevel(conf);
- Optional<Integer> fetchSize = getInputPageRowSize(conf);
- QueryOptions queryOptions = new QueryOptions();
- if (CL != null && !CL.isEmpty())
- queryOptions.setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.valueOf(CL));
-
- if (fetchSize.isPresent())
- queryOptions.setFetchSize(fetchSize.get());
- return queryOptions;
- }
-
- private static SocketOptions getReadSocketOptions(Configuration conf)
- {
- SocketOptions socketOptions = new SocketOptions();
- Optional<Integer> connectTimeoutMillis = getInputNativeConnectionTimeout(conf);
- Optional<Integer> readTimeoutMillis = getInputNativeReadConnectionTimeout(conf);
- Optional<Integer> receiveBufferSize = getInputNativeReceiveBufferSize(conf);
- Optional<Integer> sendBufferSize = getInputNativeSendBufferSize(conf);
- Optional<Integer> soLinger = getInputNativeSolinger(conf);
- Optional<Boolean> tcpNoDelay = getInputNativeTcpNodelay(conf);
- Optional<Boolean> reuseAddress = getInputNativeReuseAddress(conf);
- Optional<Boolean> keepAlive = getInputNativeKeepAlive(conf);
-
- if (connectTimeoutMillis.isPresent())
- socketOptions.setConnectTimeoutMillis(connectTimeoutMillis.get());
- if (readTimeoutMillis.isPresent())
- socketOptions.setReadTimeoutMillis(readTimeoutMillis.get());
- if (receiveBufferSize.isPresent())
- socketOptions.setReceiveBufferSize(receiveBufferSize.get());
- if (sendBufferSize.isPresent())
- socketOptions.setSendBufferSize(sendBufferSize.get());
- if (soLinger.isPresent())
- socketOptions.setSoLinger(soLinger.get());
- if (tcpNoDelay.isPresent())
- socketOptions.setTcpNoDelay(tcpNoDelay.get());
- if (reuseAddress.isPresent())
- socketOptions.setReuseAddress(reuseAddress.get());
- if (keepAlive.isPresent())
- socketOptions.setKeepAlive(keepAlive.get());
-
- return socketOptions;
- }
-
- private static LoadBalancingPolicy getReadLoadBalancingPolicy(final String[] stickHosts)
- {
- return new LimitedLocalNodeFirstLocalBalancingPolicy(stickHosts);
- }
-
- private static Optional<AuthProvider> getDefaultAuthProvider(Configuration conf)
- {
- Optional<String> username = getStringSetting(USERNAME, conf);
- Optional<String> password = getStringSetting(PASSWORD, conf);
-
- if (username.isPresent() && password.isPresent())
- {
- return Optional.of(new PlainTextAuthProvider(username.get(), password.get()));
- }
- else
- {
- return Optional.absent();
- }
- }
-
- private static Optional<AuthProvider> getAuthProvider(Configuration conf)
- {
- Optional<String> authProvider = getInputNativeAuthProvider(conf);
- if (!authProvider.isPresent())
- return getDefaultAuthProvider(conf);
-
- return Optional.of(getClientAuthProvider(authProvider.get(), conf));
- }
-
- public static Optional<SSLOptions> getSSLOptions(Configuration conf)
- {
- Optional<String> truststorePath = getInputNativeSSLTruststorePath(conf);
-
- if (truststorePath.isPresent())
- {
- Optional<String> keystorePath = getInputNativeSSLKeystorePath(conf);
- Optional<String> truststorePassword = getInputNativeSSLTruststorePassword(conf);
- Optional<String> keystorePassword = getInputNativeSSLKeystorePassword(conf);
- Optional<String> cipherSuites = getInputNativeSSLCipherSuites(conf);
-
- SSLContext context;
- try
- {
- context = getSSLContext(truststorePath, truststorePassword, keystorePath, keystorePassword);
- }
- catch (UnrecoverableKeyException | KeyManagementException |
- NoSuchAlgorithmException | KeyStoreException | CertificateException | IOException e)
- {
- throw new RuntimeException(e);
- }
- String[] css = null;
- if (cipherSuites.isPresent())
- css = cipherSuites.get().split(",");
- return Optional.of(JdkSSLOptions.builder()
- .withSSLContext(context)
- .withCipherSuites(css)
- .build());
- }
- return Optional.absent();
- }
-
- private static Optional<Integer> getIntSetting(String parameter, Configuration conf)
- {
- String setting = conf.get(parameter);
- if (setting == null)
- return Optional.absent();
- return Optional.of(Integer.valueOf(setting));
- }
-
- private static Optional<Boolean> getBooleanSetting(String parameter, Configuration conf)
- {
- String setting = conf.get(parameter);
- if (setting == null)
- return Optional.absent();
- return Optional.of(Boolean.valueOf(setting));
- }
-
- private static Optional<String> getStringSetting(String parameter, Configuration conf)
- {
- String setting = conf.get(parameter);
- if (setting == null)
- return Optional.absent();
- return Optional.of(setting);
- }
-
- private static AuthProvider getClientAuthProvider(String factoryClassName, Configuration conf)
- {
- try
- {
- Class<?> c = Class.forName(factoryClassName);
- if (PlainTextAuthProvider.class.equals(c))
- {
- String username = getStringSetting(USERNAME, conf).or("");
- String password = getStringSetting(PASSWORD, conf).or("");
- return (AuthProvider) c.getConstructor(String.class, String.class)
- .newInstance(username, password);
- }
- else
- {
- return (AuthProvider) c.newInstance();
- }
- }
- catch (Exception e)
- {
- throw new RuntimeException("Failed to instantiate auth provider:" + factoryClassName, e);
- }
- }
-
- private static SSLContext getSSLContext(Optional<String> truststorePath,
- Optional<String> truststorePassword,
- Optional<String> keystorePath,
- Optional<String> keystorePassword)
- throws NoSuchAlgorithmException,
- KeyStoreException,
- CertificateException,
- IOException,
- UnrecoverableKeyException,
- KeyManagementException
- {
- SSLContext ctx = SSLContext.getInstance("SSL");
-
- TrustManagerFactory tmf = null;
- if (truststorePath.isPresent())
- {
- try (InputStream tsf = Files.newInputStream(File.getPath(truststorePath.get())))
- {
- KeyStore ts = KeyStore.getInstance("JKS");
- ts.load(tsf, truststorePassword.isPresent() ? truststorePassword.get().toCharArray() : null);
- tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- tmf.init(ts);
- }
- }
-
- KeyManagerFactory kmf = null;
- if (keystorePath.isPresent())
- {
- try (InputStream ksf = Files.newInputStream(File.getPath(keystorePath.get())))
- {
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(ksf, keystorePassword.isPresent() ? keystorePassword.get().toCharArray() : null);
- kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- kmf.init(ks, keystorePassword.isPresent() ? keystorePassword.get().toCharArray() : null);
- }
- }
-
- ctx.init(kmf != null ? kmf.getKeyManagers() : null,
- tmf != null ? tmf.getTrustManagers() : null,
- new SecureRandom());
- return ctx;
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java
deleted file mode 100644
index 0755684..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java
+++ /dev/null
@@ -1,502 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.*;
-import java.util.concurrent.*;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Host;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.SimpleStatement;
-import com.datastax.driver.core.Statement;
-import com.datastax.driver.core.TokenRange;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-
-import com.datastax.driver.core.exceptions.InvalidQueryException;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.dht.*;
-import org.apache.cassandra.hadoop.*;
-import org.apache.cassandra.utils.*;
-
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
-/**
- * Hadoop InputFormat allowing map/reduce against Cassandra rows within one ColumnFamily.
- *
- * At minimum, you need to set the KS and CF in your Hadoop job Configuration.
- * The ConfigHelper class is provided to make this
- * simple:
- * ConfigHelper.setInputColumnFamily
- *
- * You can also configure the number of rows per InputSplit with
- * 1: ConfigHelper.setInputSplitSize. The default split size is 64k rows.
- * or
- * 2: ConfigHelper.setInputSplitSizeInMb. InputSplit size in MB with new, more precise method
- * If no value is provided for InputSplitSizeInMb, we default to using InputSplitSize.
- *
- * CQLConfigHelper.setInputCQLPageRowSize. The default page row size is 1000. You
- * should set it to "as big as possible, but no bigger." It set the LIMIT for the CQL
- * query, so you need set it big enough to minimize the network overhead, and also
- * not too big to avoid out of memory issue.
- *
- * other native protocol connection parameters in CqlConfigHelper
- */
-public class CqlInputFormat extends org.apache.hadoop.mapreduce.InputFormat<Long, Row> implements org.apache.hadoop.mapred.InputFormat<Long, Row>
-{
- public static final String MAPRED_TASK_ID = "mapred.task.id";
- private static final Logger logger = LoggerFactory.getLogger(CqlInputFormat.class);
- private String keyspace;
- private String cfName;
- private IPartitioner partitioner;
-
- public RecordReader<Long, Row> getRecordReader(InputSplit split, JobConf jobConf, final Reporter reporter)
- throws IOException
- {
- TaskAttemptContext tac = HadoopCompat.newMapContext(
- jobConf,
- TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)),
- null,
- null,
- null,
- new ReporterWrapper(reporter),
- null);
-
-
- CqlRecordReader recordReader = new CqlRecordReader();
- recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit)split, tac);
- return recordReader;
- }
-
- @Override
- public org.apache.hadoop.mapreduce.RecordReader<Long, Row> createRecordReader(
- org.apache.hadoop.mapreduce.InputSplit arg0, TaskAttemptContext arg1) throws IOException,
- InterruptedException
- {
- return new CqlRecordReader();
- }
-
- protected void validateConfiguration(Configuration conf)
- {
- if (ConfigHelper.getInputKeyspace(conf) == null || ConfigHelper.getInputColumnFamily(conf) == null)
- {
- throw new UnsupportedOperationException("you must set the keyspace and table with setInputColumnFamily()");
- }
- if (ConfigHelper.getInputInitialAddress(conf) == null)
- throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node with setInputInitialAddress");
- if (ConfigHelper.getInputPartitioner(conf) == null)
- throw new UnsupportedOperationException("You must set the Cassandra partitioner class with setInputPartitioner");
- }
-
- public List<org.apache.hadoop.mapreduce.InputSplit> getSplits(JobContext context) throws IOException
- {
- Configuration conf = HadoopCompat.getConfiguration(context);
-
- validateConfiguration(conf);
-
- keyspace = ConfigHelper.getInputKeyspace(conf);
- cfName = ConfigHelper.getInputColumnFamily(conf);
- partitioner = ConfigHelper.getInputPartitioner(conf);
- logger.trace("partitioner is {}", partitioner);
-
- // canonical ranges, split into pieces, fetching the splits in parallel
- ExecutorService executor = executorFactory().pooled("HadoopInput", 128);
- List<org.apache.hadoop.mapreduce.InputSplit> splits = new ArrayList<>();
-
- String[] inputInitialAddress = ConfigHelper.getInputInitialAddress(conf).split(",");
- try (Cluster cluster = CqlConfigHelper.getInputCluster(inputInitialAddress, conf);
- Session session = cluster.connect())
- {
- List<SplitFuture> splitfutures = new ArrayList<>();
- //TODO if the job range is defined and does perfectly match tokens, then the logic will be unable to get estimates since they are pre-computed
- // tokens: [0, 10, 20]
- // job range: [0, 10) - able to get estimate
- // job range: [5, 15) - unable to get estimate
- Pair<String, String> jobKeyRange = ConfigHelper.getInputKeyRange(conf);
- Range<Token> jobRange = null;
- if (jobKeyRange != null)
- {
- jobRange = new Range<>(partitioner.getTokenFactory().fromString(jobKeyRange.left),
- partitioner.getTokenFactory().fromString(jobKeyRange.right));
- }
-
- Metadata metadata = cluster.getMetadata();
-
- // canonical ranges and nodes holding replicas
- Map<TokenRange, List<Host>> masterRangeNodes = getRangeMap(keyspace, metadata, getTargetDC(metadata, inputInitialAddress));
- for (TokenRange range : masterRangeNodes.keySet())
- {
- if (jobRange == null)
- {
- for (TokenRange unwrapped : range.unwrap())
- {
- // for each tokenRange, pick a live owner and ask it for the byte-sized splits
- SplitFuture task = new SplitFuture(new SplitCallable(unwrapped, masterRangeNodes.get(range), conf, session));
- executor.submit(task);
- splitfutures.add(task);
- }
- }
- else
- {
- TokenRange jobTokenRange = rangeToTokenRange(metadata, jobRange);
- if (range.intersects(jobTokenRange))
- {
- for (TokenRange intersection: range.intersectWith(jobTokenRange))
- {
- for (TokenRange unwrapped : intersection.unwrap())
- {
- // for each tokenRange, pick a live owner and ask it for the byte-sized splits
- SplitFuture task = new SplitFuture(new SplitCallable(unwrapped, masterRangeNodes.get(range), conf, session));
- executor.submit(task);
- splitfutures.add(task);
- }
- }
- }
- }
- }
-
- // wait until we have all the results back
- List<SplitFuture> failedTasks = new ArrayList<>();
- int maxSplits = 0;
- long expectedPartionsForFailedRanges = 0;
- for (SplitFuture task : splitfutures)
- {
- try
- {
- List<ColumnFamilySplit> tokenRangeSplits = task.get();
- if (tokenRangeSplits.size() > maxSplits)
- {
- maxSplits = tokenRangeSplits.size();
- expectedPartionsForFailedRanges = tokenRangeSplits.get(0).getLength();
- }
- splits.addAll(tokenRangeSplits);
- }
- catch (Exception e)
- {
- failedTasks.add(task);
- }
- }
- // The estimate is only stored on a single host, if that host is down then can not get the estimate
- // its more than likely that a single host could be "too large" for one split but there is no way of
- // knowning!
- // This logic attempts to guess the estimate from all the successful ranges
- if (!failedTasks.isEmpty())
- {
- // if every split failed this will be 0
- if (maxSplits == 0)
- throwAllSplitsFailed(failedTasks);
- for (SplitFuture task : failedTasks)
- {
- try
- {
- // the task failed, so this should throw
- task.get();
- }
- catch (Exception cause)
- {
- logger.warn("Unable to get estimate for {}, the host {} had a exception; falling back to default estimate", task.splitCallable.tokenRange, task.splitCallable.hosts.get(0), cause);
- }
- }
- for (SplitFuture task : failedTasks)
- splits.addAll(toSplit(task.splitCallable.hosts, splitTokenRange(task.splitCallable.tokenRange, maxSplits, expectedPartionsForFailedRanges)));
- }
- }
- finally
- {
- executor.shutdownNow();
- }
-
- assert splits.size() > 0;
- Collections.shuffle(splits, new Random(nanoTime()));
- return splits;
- }
-
- private static IllegalStateException throwAllSplitsFailed(List<SplitFuture> failedTasks)
- {
- IllegalStateException exception = new IllegalStateException("No successful tasks found");
- for (SplitFuture task : failedTasks)
- {
- try
- {
- // the task failed, so this should throw
- task.get();
- }
- catch (Exception cause)
- {
- exception.addSuppressed(cause);
- }
- }
- throw exception;
- }
-
- private static String getTargetDC(Metadata metadata, String[] inputInitialAddress)
- {
- BiMultiValMap<InetAddress, String> addressToDc = new BiMultiValMap<>();
- Multimap<String, InetAddress> dcToAddresses = addressToDc.inverse();
-
- // only way to match is off the broadcast addresses, so for all hosts do a existence check
- Set<InetAddress> addresses = new HashSet<>(inputInitialAddress.length);
- for (String inputAddress : inputInitialAddress)
- addresses.addAll(parseAddress(inputAddress));
-
- for (Host host : metadata.getAllHosts())
- {
- InetAddress address = host.getBroadcastAddress();
- if (addresses.contains(address))
- addressToDc.put(address, host.getDatacenter());
- }
-
- switch (dcToAddresses.keySet().size())
- {
- case 1:
- return Iterables.getOnlyElement(dcToAddresses.keySet());
- case 0:
- throw new IllegalStateException("Input addresses could not be used to find DC; non match client metadata");
- default:
- // Mutliple DCs found, attempt to pick the first based off address list. This is to mimic the 2.1
- // behavior which would connect in order and the first node successfully able to connect to was the
- // local DC to use; since client abstracts this, we rely on existence as a proxy for connect.
- for (String inputAddress : inputInitialAddress)
- {
- for (InetAddress add : parseAddress(inputAddress))
- {
- String dc = addressToDc.get(add);
- // possible the address isn't in the cluster and the client dropped, so ignore null
- if (dc != null)
- return dc;
- }
- }
- // some how we were able to connect to the cluster, find multiple DCs using matching, and yet couldn't
- // match again...
- throw new AssertionError("Unable to infer datacenter from initial addresses; multiple datacenters found "
- + dcToAddresses.keySet() + ", should only use addresses from one datacenter");
- }
- }
-
- private static List<InetAddress> parseAddress(String str)
- {
- try
- {
- return Arrays.asList(InetAddress.getAllByName(str));
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
-
- private TokenRange rangeToTokenRange(Metadata metadata, Range<Token> range)
- {
- return metadata.newTokenRange(metadata.newToken(partitioner.getTokenFactory().toString(range.left)),
- metadata.newToken(partitioner.getTokenFactory().toString(range.right)));
- }
-
- private Map<TokenRange, Long> getSubSplits(String keyspace, String cfName, TokenRange range, Host host, Configuration conf, Session session)
- {
- int splitSize = ConfigHelper.getInputSplitSize(conf);
- int splitSizeMiB = ConfigHelper.getInputSplitSizeInMb(conf);
- return describeSplits(keyspace, cfName, range, host, splitSize, splitSizeMiB, session);
- }
-
- private static Map<TokenRange, List<Host>> getRangeMap(String keyspace, Metadata metadata, String targetDC)
- {
- return CqlClientHelper.getLocalPrimaryRangeForDC(keyspace, metadata, targetDC);
- }
-
- private Map<TokenRange, Long> describeSplits(String keyspace, String table, TokenRange tokenRange, Host host, int splitSize, int splitSizeMb, Session session)
- {
- // In 2.1 the host list was walked in-order (only move to next if IOException) and calls
- // org.apache.cassandra.service.StorageService.getSplits(java.lang.String, java.lang.String, org.apache.cassandra.dht.Range<org.apache.cassandra.dht.Token>, int)
- // that call computes totalRowCountEstimate (used to compute #splits) then splits the ring based off those estimates
- //
- // The main difference is that the estimates in 2.1 were computed based off the data, so replicas could answer the estimates
- // In 3.0 we rely on the below CQL query which is local and only computes estimates for the primary range; this
- // puts us in a sticky spot to answer, if the node fails what do we do? 3.0 behavior only matches 2.1 IFF all
- // nodes are up and healthy
- ResultSet resultSet = queryTableEstimates(session, host, keyspace, table, tokenRange);
-
- Row row = resultSet.one();
-
- long meanPartitionSize = 0;
- long partitionCount = 0;
- int splitCount = 0;
-
- if (row != null)
- {
- meanPartitionSize = row.getLong("mean_partition_size");
- partitionCount = row.getLong("partitions_count");
-
- splitCount = splitSizeMb > 0
- ? (int)(meanPartitionSize * partitionCount / splitSizeMb / 1024 / 1024)
- : (int)(partitionCount / splitSize);
- }
-
- // If we have no data on this split or the size estimate is 0,
- // return the full split i.e., do not sub-split
- // Assume smallest granularity of partition count available from CASSANDRA-7688
- if (splitCount == 0)
- {
- Map<TokenRange, Long> wrappedTokenRange = new HashMap<>();
- wrappedTokenRange.put(tokenRange, partitionCount == 0 ? 128L : partitionCount);
- return wrappedTokenRange;
- }
-
- return splitTokenRange(tokenRange, splitCount, partitionCount / splitCount);
- }
-
- private static ResultSet queryTableEstimates(Session session, Host host, String keyspace, String table, TokenRange tokenRange)
- {
- try
- {
- String query = String.format("SELECT mean_partition_size, partitions_count " +
- "FROM %s.%s " +
- "WHERE keyspace_name = ? AND table_name = ? AND range_type = '%s' AND range_start = ? AND range_end = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.TABLE_ESTIMATES,
- SystemKeyspace.TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY);
- Statement stmt = new SimpleStatement(query, keyspace, table, tokenRange.getStart().toString(), tokenRange.getEnd().toString()).setHost(host);
- return session.execute(stmt);
- }
- catch (InvalidQueryException e)
- {
- // if the table doesn't exist, fall back to old table. This is likely to return no records in a multi
- // DC setup, but should work fine in a single DC setup.
- String query = String.format("SELECT mean_partition_size, partitions_count " +
- "FROM %s.%s " +
- "WHERE keyspace_name = ? AND table_name = ? AND range_start = ? AND range_end = ?",
- SchemaConstants.SYSTEM_KEYSPACE_NAME,
- SystemKeyspace.LEGACY_SIZE_ESTIMATES);
-
- Statement stmt = new SimpleStatement(query, keyspace, table, tokenRange.getStart().toString(), tokenRange.getEnd().toString()).setHost(host);
- return session.execute(stmt);
- }
- }
-
- private static Map<TokenRange, Long> splitTokenRange(TokenRange tokenRange, int splitCount, long partitionCount)
- {
- List<TokenRange> splitRanges = tokenRange.splitEvenly(splitCount);
- Map<TokenRange, Long> rangesWithLength = Maps.newHashMapWithExpectedSize(splitRanges.size());
- for (TokenRange range : splitRanges)
- rangesWithLength.put(range, partitionCount);
-
- return rangesWithLength;
- }
-
- // Old Hadoop API
- public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException
- {
- TaskAttemptContext tac = HadoopCompat.newTaskAttemptContext(jobConf, new TaskAttemptID());
- List<org.apache.hadoop.mapreduce.InputSplit> newInputSplits = this.getSplits(tac);
- InputSplit[] oldInputSplits = new InputSplit[newInputSplits.size()];
- for (int i = 0; i < newInputSplits.size(); i++)
- oldInputSplits[i] = (ColumnFamilySplit)newInputSplits.get(i);
- return oldInputSplits;
- }
-
- /**
- * Gets a token tokenRange and splits it up according to the suggested
- * size into input splits that Hadoop can use.
- */
- class SplitCallable implements Callable<List<ColumnFamilySplit>>
- {
-
- private final TokenRange tokenRange;
- private final List<Host> hosts;
- private final Configuration conf;
- private final Session session;
-
- public SplitCallable(TokenRange tokenRange, List<Host> hosts, Configuration conf, Session session)
- {
- Preconditions.checkArgument(!hosts.isEmpty(), "hosts list requires at least 1 host but was empty");
- this.tokenRange = tokenRange;
- this.hosts = hosts;
- this.conf = conf;
- this.session = session;
- }
-
- public List<ColumnFamilySplit> call() throws Exception
- {
- Map<TokenRange, Long> subSplits = getSubSplits(keyspace, cfName, tokenRange, hosts.get(0), conf, session);
- return toSplit(hosts, subSplits);
- }
-
- }
-
- private static class SplitFuture extends FutureTask<List<ColumnFamilySplit>>
- {
- private final SplitCallable splitCallable;
-
- SplitFuture(SplitCallable splitCallable)
- {
- super(splitCallable);
- this.splitCallable = splitCallable;
- }
- }
-
- private List<ColumnFamilySplit> toSplit(List<Host> hosts, Map<TokenRange, Long> subSplits)
- {
- // turn the sub-ranges into InputSplits
- String[] endpoints = new String[hosts.size()];
-
- // hadoop needs hostname, not ip
- int endpointIndex = 0;
- for (Host endpoint : hosts)
- endpoints[endpointIndex++] = endpoint.getAddress().getHostName();
-
- boolean partitionerIsOpp = partitioner instanceof OrderPreservingPartitioner || partitioner instanceof ByteOrderedPartitioner;
-
- ArrayList<ColumnFamilySplit> splits = new ArrayList<>();
- for (Map.Entry<TokenRange, Long> subSplitEntry : subSplits.entrySet())
- {
- TokenRange subrange = subSplitEntry.getKey();
- ColumnFamilySplit split =
- new ColumnFamilySplit(
- partitionerIsOpp ?
- subrange.getStart().toString().substring(2) : subrange.getStart().toString(),
- partitionerIsOpp ?
- subrange.getEnd().toString().substring(2) : subrange.getEnd().toString(),
- subSplitEntry.getValue(),
- endpoints);
-
- logger.trace("adding {}", split);
- splits.add(split);
- }
- return splits;
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlOutputFormat.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlOutputFormat.java
deleted file mode 100644
index cc0a6b1..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlOutputFormat.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.hadoop.*;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.mapreduce.*;
-
-/**
- * The <code>CqlOutputFormat</code> acts as a Hadoop-specific
- * OutputFormat that allows reduce tasks to store keys (and corresponding
- * bound variable values) as CQL rows (and respective columns) in a given
- * table.
- *
- * <p>
- * As is the case with the {@link org.apache.cassandra.hadoop.ColumnFamilyInputFormat},
- * you need to set the prepared statement in your
- * Hadoop job Configuration. The {@link CqlConfigHelper} class, through its
- * {@link CqlConfigHelper#setOutputCql} method, is provided to make this
- * simple.
- * you need to set the Keyspace. The {@link ConfigHelper} class, through its
- * {@link ConfigHelper#setOutputColumnFamily} method, is provided to make this
- * simple.
- * </p>
- *
- * <p>
- * For the sake of performance, this class employs a lazy write-back caching
- * mechanism, where its record writer prepared statement binded variable values
- * created based on the reduce's inputs (in a task-specific map), and periodically
- * makes the changes official by sending a execution of prepared statement request
- * to Cassandra.
- * </p>
- */
-public class CqlOutputFormat extends OutputFormat<Map<String, ByteBuffer>, List<ByteBuffer>>
- implements org.apache.hadoop.mapred.OutputFormat<Map<String, ByteBuffer>, List<ByteBuffer>>
-{
- public static final String BATCH_THRESHOLD = "mapreduce.output.columnfamilyoutputformat.batch.threshold";
- public static final String QUEUE_SIZE = "mapreduce.output.columnfamilyoutputformat.queue.size";
-
- /**
- * Check for validity of the output-specification for the job.
- *
- * @param context
- * information about the job
- */
- public void checkOutputSpecs(JobContext context)
- {
- checkOutputSpecs(HadoopCompat.getConfiguration(context));
- }
-
- protected void checkOutputSpecs(Configuration conf)
- {
- if (ConfigHelper.getOutputKeyspace(conf) == null)
- throw new UnsupportedOperationException("You must set the keyspace with setOutputKeyspace()");
- if (ConfigHelper.getOutputPartitioner(conf) == null)
- throw new UnsupportedOperationException("You must set the output partitioner to the one used by your Cassandra cluster");
- if (ConfigHelper.getOutputInitialAddress(conf) == null)
- throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node");
- }
-
- /** Fills the deprecated OutputFormat interface for streaming. */
- @Deprecated
- public void checkOutputSpecs(org.apache.hadoop.fs.FileSystem filesystem, org.apache.hadoop.mapred.JobConf job) throws IOException
- {
- checkOutputSpecs(job);
- }
-
- /**
- * The OutputCommitter for this format does not write any data to the DFS.
- *
- * @param context
- * the task context
- * @return an output committer
- * @throws IOException
- * @throws InterruptedException
- */
- public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException
- {
- return new NullOutputCommitter();
- }
-
- /** Fills the deprecated OutputFormat interface for streaming. */
- @Deprecated
- public CqlRecordWriter getRecordWriter(org.apache.hadoop.fs.FileSystem filesystem, org.apache.hadoop.mapred.JobConf job, String name, org.apache.hadoop.util.Progressable progress) throws IOException
- {
- return new CqlRecordWriter(job, progress);
- }
-
- /**
- * Get the {@link RecordWriter} for the given task.
- *
- * @param context
- * the information about the current task.
- * @return a {@link RecordWriter} to write the output for the job.
- * @throws IOException
- */
- public CqlRecordWriter getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException
- {
- return new CqlRecordWriter(context);
- }
-
- /**
- * An {@link OutputCommitter} that does nothing.
- */
- private static class NullOutputCommitter extends OutputCommitter
- {
- public void abortTask(TaskAttemptContext taskContext) { }
-
- public void cleanupJob(JobContext jobContext) { }
-
- public void commitTask(TaskAttemptContext taskContext) { }
-
- public boolean needsTaskCommit(TaskAttemptContext taskContext)
- {
- return false;
- }
-
- public void setupJob(JobContext jobContext) { }
-
- public void setupTask(TaskAttemptContext taskContext) { }
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordReader.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordReader.java
deleted file mode 100644
index d9aad19..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordReader.java
+++ /dev/null
@@ -1,784 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-
-import com.datastax.driver.core.TypeCodec;
-import org.apache.cassandra.utils.AbstractIterator;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ColumnDefinitions;
-import com.datastax.driver.core.ColumnMetadata;
-import com.datastax.driver.core.LocalDate;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.TableMetadata;
-import com.datastax.driver.core.Token;
-import com.datastax.driver.core.TupleValue;
-import com.datastax.driver.core.UDTValue;
-import com.google.common.reflect.TypeToken;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.hadoop.ColumnFamilySplit;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.hadoop.HadoopCompat;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-/**
- * <p>
- * CqlRecordReader reads the rows return from the CQL query
- * It uses CQL auto-paging.
- * </p>
- * <p>
- * Return a Long as a local CQL row key starts from 0;
- * </p>
- * {@code
- * Row as C* java driver CQL result set row
- * 1) select clause must include partition key columns (to calculate the progress based on the actual CF row processed)
- * 2) where clause must include token(partition_key1, ... , partition_keyn) > ? and
- * token(partition_key1, ... , partition_keyn) <= ? (in the right order)
- * }
- */
-public class CqlRecordReader extends RecordReader<Long, Row>
- implements org.apache.hadoop.mapred.RecordReader<Long, Row>, AutoCloseable
-{
- private static final Logger logger = LoggerFactory.getLogger(CqlRecordReader.class);
-
- private ColumnFamilySplit split;
- private RowIterator rowIterator;
-
- private Pair<Long, Row> currentRow;
- private int totalRowCount; // total number of rows to fetch
- private String keyspace;
- private String cfName;
- private String cqlQuery;
- private Cluster cluster;
- private Session session;
- private IPartitioner partitioner;
- private String inputColumns;
- private String userDefinedWhereClauses;
-
- private List<String> partitionKeys = new ArrayList<>();
-
- // partition keys -- key aliases
- private LinkedHashMap<String, Boolean> partitionBoundColumns = Maps.newLinkedHashMap();
- protected int nativeProtocolVersion = 1;
-
- public CqlRecordReader()
- {
- super();
- }
-
- @Override
- public void initialize(InputSplit split, TaskAttemptContext context) throws IOException
- {
- this.split = (ColumnFamilySplit) split;
- Configuration conf = HadoopCompat.getConfiguration(context);
- totalRowCount = (this.split.getLength() < Long.MAX_VALUE)
- ? (int) this.split.getLength()
- : ConfigHelper.getInputSplitSize(conf);
- cfName = ConfigHelper.getInputColumnFamily(conf);
- keyspace = ConfigHelper.getInputKeyspace(conf);
- partitioner = ConfigHelper.getInputPartitioner(conf);
- inputColumns = CqlConfigHelper.getInputcolumns(conf);
- userDefinedWhereClauses = CqlConfigHelper.getInputWhereClauses(conf);
-
- try
- {
- if (cluster != null)
- return;
-
- // create a Cluster instance
- String[] locations = split.getLocations();
- cluster = CqlConfigHelper.getInputCluster(locations, conf);
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
-
- if (cluster != null)
- session = cluster.connect(quote(keyspace));
-
- if (session == null)
- throw new RuntimeException("Can't create connection session");
-
- //get negotiated serialization protocol
- nativeProtocolVersion = cluster.getConfiguration().getProtocolOptions().getProtocolVersion().toInt();
-
- // If the user provides a CQL query then we will use it without validation
- // otherwise we will fall back to building a query using the:
- // inputColumns
- // whereClauses
- cqlQuery = CqlConfigHelper.getInputCql(conf);
- // validate that the user hasn't tried to give us a custom query along with input columns
- // and where clauses
- if (StringUtils.isNotEmpty(cqlQuery) && (StringUtils.isNotEmpty(inputColumns) ||
- StringUtils.isNotEmpty(userDefinedWhereClauses)))
- {
- throw new AssertionError("Cannot define a custom query with input columns and / or where clauses");
- }
-
- if (StringUtils.isEmpty(cqlQuery))
- cqlQuery = buildQuery();
- logger.trace("cqlQuery {}", cqlQuery);
-
- rowIterator = new RowIterator();
- logger.trace("created {}", rowIterator);
- }
-
- public void close()
- {
- if (session != null)
- session.close();
- if (cluster != null)
- cluster.close();
- }
-
- public Long getCurrentKey()
- {
- return currentRow.left;
- }
-
- public Row getCurrentValue()
- {
- return currentRow.right;
- }
-
- public float getProgress()
- {
- if (!rowIterator.hasNext())
- return 1.0F;
-
- // the progress is likely to be reported slightly off the actual but close enough
- float progress = ((float) rowIterator.totalRead / totalRowCount);
- return progress > 1.0F ? 1.0F : progress;
- }
-
- public boolean nextKeyValue() throws IOException
- {
- if (!rowIterator.hasNext())
- {
- logger.trace("Finished scanning {} rows (estimate was: {})", rowIterator.totalRead, totalRowCount);
- return false;
- }
-
- try
- {
- currentRow = rowIterator.next();
- }
- catch (Exception e)
- {
- // throw it as IOException, so client can catch it and handle it at client side
- IOException ioe = new IOException(e.getMessage());
- ioe.initCause(ioe.getCause());
- throw ioe;
- }
- return true;
- }
-
- // Because the old Hadoop API wants us to write to the key and value
- // and the new asks for them, we need to copy the output of the new API
- // to the old. Thus, expect a small performance hit.
- // And obviously this wouldn't work for wide rows. But since ColumnFamilyInputFormat
- // and ColumnFamilyRecordReader don't support them, it should be fine for now.
- public boolean next(Long key, Row value) throws IOException
- {
- if (nextKeyValue())
- {
- ((WrappedRow)value).setRow(getCurrentValue());
- return true;
- }
- return false;
- }
-
- public long getPos() throws IOException
- {
- return rowIterator.totalRead;
- }
-
- public Long createKey()
- {
- return Long.valueOf(0L);
- }
-
- public Row createValue()
- {
- return new WrappedRow();
- }
-
- /**
- * Return native version protocol of the cluster connection
- * @return serialization protocol version.
- */
- public int getNativeProtocolVersion()
- {
- return nativeProtocolVersion;
- }
-
- /** CQL row iterator
- * Input cql query
- * 1) select clause must include key columns (if we use partition key based row count)
- * 2) where clause must include token(partition_key1 ... partition_keyn) > ? and
- * token(partition_key1 ... partition_keyn) <= ?
- */
- private class RowIterator extends AbstractIterator<Pair<Long, Row>>
- {
- private long keyId = 0L;
- protected int totalRead = 0; // total number of cf rows read
- protected Iterator<Row> rows;
- private Map<String, ByteBuffer> previousRowKey = new HashMap<String, ByteBuffer>(); // previous CF row key
-
- public RowIterator()
- {
- AbstractType type = partitioner.getTokenValidator();
- ResultSet rs = session.execute(cqlQuery, type.compose(type.fromString(split.getStartToken())), type.compose(type.fromString(split.getEndToken())) );
- for (ColumnMetadata meta : cluster.getMetadata().getKeyspace(quote(keyspace)).getTable(quote(cfName)).getPartitionKey())
- partitionBoundColumns.put(meta.getName(), Boolean.TRUE);
- rows = rs.iterator();
- }
-
- protected Pair<Long, Row> computeNext()
- {
- if (rows == null || !rows.hasNext())
- return endOfData();
-
- Row row = rows.next();
- Map<String, ByteBuffer> keyColumns = new HashMap<String, ByteBuffer>(partitionBoundColumns.size());
- for (String column : partitionBoundColumns.keySet())
- keyColumns.put(column, row.getBytesUnsafe(column));
-
- // increase total CF row read
- if (previousRowKey.isEmpty() && !keyColumns.isEmpty())
- {
- previousRowKey = keyColumns;
- totalRead++;
- }
- else
- {
- for (String column : partitionBoundColumns.keySet())
- {
- // this is not correct - but we don't seem to have easy access to better type information here
- if (ByteBufferUtil.compareUnsigned(keyColumns.get(column), previousRowKey.get(column)) != 0)
- {
- previousRowKey = keyColumns;
- totalRead++;
- break;
- }
- }
- }
- keyId ++;
- return Pair.create(keyId, row);
- }
- }
-
- private static class WrappedRow implements Row
- {
- private Row row;
-
- public void setRow(Row row)
- {
- this.row = row;
- }
-
- @Override
- public ColumnDefinitions getColumnDefinitions()
- {
- return row.getColumnDefinitions();
- }
-
- @Override
- public boolean isNull(int i)
- {
- return row.isNull(i);
- }
-
- @Override
- public boolean isNull(String name)
- {
- return row.isNull(name);
- }
-
- @Override
- public Object getObject(int i)
- {
- return row.getObject(i);
- }
-
- @Override
- public <T> T get(int i, Class<T> aClass)
- {
- return row.get(i, aClass);
- }
-
- @Override
- public <T> T get(int i, TypeToken<T> typeToken)
- {
- return row.get(i, typeToken);
- }
-
- @Override
- public <T> T get(int i, TypeCodec<T> typeCodec)
- {
- return row.get(i, typeCodec);
- }
-
- @Override
- public Object getObject(String s)
- {
- return row.getObject(s);
- }
-
- @Override
- public <T> T get(String s, Class<T> aClass)
- {
- return row.get(s, aClass);
- }
-
- @Override
- public <T> T get(String s, TypeToken<T> typeToken)
- {
- return row.get(s, typeToken);
- }
-
- @Override
- public <T> T get(String s, TypeCodec<T> typeCodec)
- {
- return row.get(s, typeCodec);
- }
-
- @Override
- public boolean getBool(int i)
- {
- return row.getBool(i);
- }
-
- @Override
- public boolean getBool(String name)
- {
- return row.getBool(name);
- }
-
- @Override
- public short getShort(int i)
- {
- return row.getShort(i);
- }
-
- @Override
- public short getShort(String s)
- {
- return row.getShort(s);
- }
-
- @Override
- public byte getByte(int i)
- {
- return row.getByte(i);
- }
-
- @Override
- public byte getByte(String s)
- {
- return row.getByte(s);
- }
-
- @Override
- public int getInt(int i)
- {
- return row.getInt(i);
- }
-
- @Override
- public int getInt(String name)
- {
- return row.getInt(name);
- }
-
- @Override
- public long getLong(int i)
- {
- return row.getLong(i);
- }
-
- @Override
- public long getLong(String name)
- {
- return row.getLong(name);
- }
-
- @Override
- public Date getTimestamp(int i)
- {
- return row.getTimestamp(i);
- }
-
- @Override
- public Date getTimestamp(String s)
- {
- return row.getTimestamp(s);
- }
-
- @Override
- public LocalDate getDate(int i)
- {
- return row.getDate(i);
- }
-
- @Override
- public LocalDate getDate(String s)
- {
- return row.getDate(s);
- }
-
- @Override
- public long getTime(int i)
- {
- return row.getTime(i);
- }
-
- @Override
- public long getTime(String s)
- {
- return row.getTime(s);
- }
-
- @Override
- public float getFloat(int i)
- {
- return row.getFloat(i);
- }
-
- @Override
- public float getFloat(String name)
- {
- return row.getFloat(name);
- }
-
- @Override
- public double getDouble(int i)
- {
- return row.getDouble(i);
- }
-
- @Override
- public double getDouble(String name)
- {
- return row.getDouble(name);
- }
-
- @Override
- public ByteBuffer getBytesUnsafe(int i)
- {
- return row.getBytesUnsafe(i);
- }
-
- @Override
- public ByteBuffer getBytesUnsafe(String name)
- {
- return row.getBytesUnsafe(name);
- }
-
- @Override
- public ByteBuffer getBytes(int i)
- {
- return row.getBytes(i);
- }
-
- @Override
- public ByteBuffer getBytes(String name)
- {
- return row.getBytes(name);
- }
-
- @Override
- public String getString(int i)
- {
- return row.getString(i);
- }
-
- @Override
- public String getString(String name)
- {
- return row.getString(name);
- }
-
- @Override
- public BigInteger getVarint(int i)
- {
- return row.getVarint(i);
- }
-
- @Override
- public BigInteger getVarint(String name)
- {
- return row.getVarint(name);
- }
-
- @Override
- public BigDecimal getDecimal(int i)
- {
- return row.getDecimal(i);
- }
-
- @Override
- public BigDecimal getDecimal(String name)
- {
- return row.getDecimal(name);
- }
-
- @Override
- public UUID getUUID(int i)
- {
- return row.getUUID(i);
- }
-
- @Override
- public UUID getUUID(String name)
- {
- return row.getUUID(name);
- }
-
- @Override
- public InetAddress getInet(int i)
- {
- return row.getInet(i);
- }
-
- @Override
- public InetAddress getInet(String name)
- {
- return row.getInet(name);
- }
-
- @Override
- public <T> List<T> getList(int i, Class<T> elementsClass)
- {
- return row.getList(i, elementsClass);
- }
-
- @Override
- public <T> List<T> getList(int i, TypeToken<T> typeToken)
- {
- return row.getList(i, typeToken);
- }
-
- @Override
- public <T> List<T> getList(String name, Class<T> elementsClass)
- {
- return row.getList(name, elementsClass);
- }
-
- @Override
- public <T> List<T> getList(String s, TypeToken<T> typeToken)
- {
- return row.getList(s, typeToken);
- }
-
- @Override
- public <T> Set<T> getSet(int i, Class<T> elementsClass)
- {
- return row.getSet(i, elementsClass);
- }
-
- @Override
- public <T> Set<T> getSet(int i, TypeToken<T> typeToken)
- {
- return row.getSet(i, typeToken);
- }
-
- @Override
- public <T> Set<T> getSet(String name, Class<T> elementsClass)
- {
- return row.getSet(name, elementsClass);
- }
-
- @Override
- public <T> Set<T> getSet(String s, TypeToken<T> typeToken)
- {
- return row.getSet(s, typeToken);
- }
-
- @Override
- public <K, V> Map<K, V> getMap(int i, Class<K> keysClass, Class<V> valuesClass)
- {
- return row.getMap(i, keysClass, valuesClass);
- }
-
- @Override
- public <K, V> Map<K, V> getMap(int i, TypeToken<K> typeToken, TypeToken<V> typeToken1)
- {
- return row.getMap(i, typeToken, typeToken1);
- }
-
- @Override
- public <K, V> Map<K, V> getMap(String name, Class<K> keysClass, Class<V> valuesClass)
- {
- return row.getMap(name, keysClass, valuesClass);
- }
-
- @Override
- public <K, V> Map<K, V> getMap(String s, TypeToken<K> typeToken, TypeToken<V> typeToken1)
- {
- return row.getMap(s, typeToken, typeToken1);
- }
-
- @Override
- public UDTValue getUDTValue(int i)
- {
- return row.getUDTValue(i);
- }
-
- @Override
- public UDTValue getUDTValue(String name)
- {
- return row.getUDTValue(name);
- }
-
- @Override
- public TupleValue getTupleValue(int i)
- {
- return row.getTupleValue(i);
- }
-
- @Override
- public TupleValue getTupleValue(String name)
- {
- return row.getTupleValue(name);
- }
-
- @Override
- public Token getToken(int i)
- {
- return row.getToken(i);
- }
-
- @Override
- public Token getToken(String name)
- {
- return row.getToken(name);
- }
-
- @Override
- public Token getPartitionKeyToken()
- {
- return row.getPartitionKeyToken();
- }
- }
-
- /**
- * Build a query for the reader of the form:
- *
- * SELECT * FROM ks>cf token(pk1,...pkn)>? AND token(pk1,...pkn)<=? [AND user where clauses] [ALLOW FILTERING]
- */
- private String buildQuery()
- {
- fetchKeys();
-
- List<String> columns = getSelectColumns();
- String selectColumnList = columns.size() == 0 ? "*" : makeColumnList(columns);
- String partitionKeyList = makeColumnList(partitionKeys);
-
- return String.format("SELECT %s FROM %s.%s WHERE token(%s)>? AND token(%s)<=?" + getAdditionalWhereClauses(),
- selectColumnList, quote(keyspace), quote(cfName), partitionKeyList, partitionKeyList);
- }
-
- private String getAdditionalWhereClauses()
- {
- String whereClause = "";
- if (StringUtils.isNotEmpty(userDefinedWhereClauses))
- whereClause += " AND " + userDefinedWhereClauses;
- if (StringUtils.isNotEmpty(userDefinedWhereClauses))
- whereClause += " ALLOW FILTERING";
- return whereClause;
- }
-
- private List<String> getSelectColumns()
- {
- List<String> selectColumns = new ArrayList<>();
-
- if (StringUtils.isNotEmpty(inputColumns))
- {
- // We must select all the partition keys plus any other columns the user wants
- selectColumns.addAll(partitionKeys);
- for (String column : Splitter.on(',').split(inputColumns))
- {
- if (!partitionKeys.contains(column))
- selectColumns.add(column);
- }
- }
- return selectColumns;
- }
-
- private String makeColumnList(Collection<String> columns)
- {
- return Joiner.on(',').join(Iterables.transform(columns, new Function<String, String>()
- {
- public String apply(String column)
- {
- return quote(column);
- }
- }));
- }
-
- private void fetchKeys()
- {
- // get CF meta data
- TableMetadata tableMetadata = session.getCluster()
- .getMetadata()
- .getKeyspace(Metadata.quote(keyspace))
- .getTable(Metadata.quote(cfName));
- if (tableMetadata == null)
- {
- throw new RuntimeException("No table metadata found for " + keyspace + "." + cfName);
- }
- //Here we assume that tableMetadata.getPartitionKey() always
- //returns the list of columns in order of component_index
- for (ColumnMetadata partitionKey : tableMetadata.getPartitionKey())
- {
- partitionKeys.add(partitionKey.getName());
- }
- }
-
- private String quote(String identifier)
- {
- return "\"" + identifier.replaceAll("\"", "\"\"") + "\"";
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
deleted file mode 100644
index 4bce989..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
+++ /dev/null
@@ -1,536 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datastax.driver.core.*;
-import com.datastax.driver.core.exceptions.*;
-
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.marshal.CompositeType;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.hadoop.HadoopCompat;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.Progressable;
-
-import static java.util.stream.Collectors.toMap;
-
-/**
- * The <code>CqlRecordWriter</code> maps the output <key, value>
- * pairs to a Cassandra table. In particular, it applies the binded variables
- * in the value to the prepared statement, which it associates with the key, and in
- * turn the responsible endpoint.
- *
- * <p>
- * Furthermore, this writer groups the cql queries by the endpoint responsible for
- * the rows being affected. This allows the cql queries to be executed in parallel,
- * directly to a responsible endpoint.
- * </p>
- *
- * @see CqlOutputFormat
- */
-class CqlRecordWriter extends RecordWriter<Map<String, ByteBuffer>, List<ByteBuffer>> implements
- org.apache.hadoop.mapred.RecordWriter<Map<String, ByteBuffer>, List<ByteBuffer>>, AutoCloseable
-{
- private static final Logger logger = LoggerFactory.getLogger(CqlRecordWriter.class);
-
- // The configuration this writer is associated with.
- protected final Configuration conf;
- // The number of mutations to buffer per endpoint
- protected final int queueSize;
-
- protected final long batchThreshold;
-
- protected Progressable progressable;
- protected TaskAttemptContext context;
-
- // The ring cache that describes the token ranges each node in the ring is
- // responsible for. This is what allows us to group the mutations by
- // the endpoints they should be targeted at. The targeted endpoint
- // essentially
- // acts as the primary replica for the rows being affected by the mutations.
- private final NativeRingCache ringCache;
-
- // handles for clients for each range running in the threadpool
- protected final Map<InetAddress, RangeClient> clients;
-
- // host to prepared statement id mappings
- protected final ConcurrentHashMap<Session, PreparedStatement> preparedStatements = new ConcurrentHashMap<Session, PreparedStatement>();
-
- protected final String cql;
-
- protected List<ColumnMetadata> partitionKeyColumns;
- protected List<ColumnMetadata> clusterColumns;
-
- /**
- * Upon construction, obtain the map that this writer will use to collect
- * mutations, and the ring cache for the given keyspace.
- *
- * @param context the task attempt context
- * @throws IOException
- */
- CqlRecordWriter(TaskAttemptContext context) throws IOException
- {
- this(HadoopCompat.getConfiguration(context));
- this.context = context;
- }
-
- CqlRecordWriter(Configuration conf, Progressable progressable)
- {
- this(conf);
- this.progressable = progressable;
- }
-
- CqlRecordWriter(Configuration conf)
- {
- this.conf = conf;
- this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors());
- batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32);
- this.clients = new HashMap<>();
- String keyspace = ConfigHelper.getOutputKeyspace(conf);
-
- try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf))
- {
- Metadata metadata = cluster.getMetadata();
- ringCache = new NativeRingCache(conf, metadata);
- TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf));
- clusterColumns = tableMetadata.getClusteringColumns();
- partitionKeyColumns = tableMetadata.getPartitionKey();
-
- String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim();
- if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert"))
- throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement");
- cql = appendKeyWhereClauses(cqlQuery);
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Close this <code>RecordWriter</code> to future operations, but not before
- * flushing out the batched mutations.
- *
- * @param context the context of the task
- * @throws IOException
- */
- public void close(TaskAttemptContext context) throws IOException, InterruptedException
- {
- close();
- }
-
- /** Fills the deprecated RecordWriter interface for streaming. */
- @Deprecated
- public void close(org.apache.hadoop.mapred.Reporter reporter) throws IOException
- {
- close();
- }
-
- @Override
- public void close() throws IOException
- {
- // close all the clients before throwing anything
- IOException clientException = null;
- for (RangeClient client : clients.values())
- {
- try
- {
- client.close();
- }
- catch (IOException e)
- {
- clientException = e;
- }
- }
-
- if (clientException != null)
- throw clientException;
- }
-
- /**
- * If the key is to be associated with a valid value, a mutation is created
- * for it with the given table and columns. In the event the value
- * in the column is missing (i.e., null), then it is marked for
- * {@link Deletion}. Similarly, if the entire value for a key is missing
- * (i.e., null), then the entire key is marked for {@link Deletion}.
- * </p>
- *
- * @param keyColumns
- * the key to write.
- * @param values
- * the values to write.
- * @throws IOException
- */
- @Override
- public void write(Map<String, ByteBuffer> keyColumns, List<ByteBuffer> values) throws IOException
- {
- TokenRange range = ringCache.getRange(getPartitionKey(keyColumns));
-
- // get the client for the given range, or create a new one
- final InetAddress address = ringCache.getEndpoints(range).get(0);
- RangeClient client = clients.get(address);
- if (client == null)
- {
- // haven't seen keys for this range: create new client
- client = new RangeClient(ringCache.getEndpoints(range));
- client.start();
- clients.put(address, client);
- }
-
- // add primary key columns to the bind variables
- List<ByteBuffer> allValues = new ArrayList<ByteBuffer>(values);
- for (ColumnMetadata column : partitionKeyColumns)
- allValues.add(keyColumns.get(column.getName()));
- for (ColumnMetadata column : clusterColumns)
- allValues.add(keyColumns.get(column.getName()));
-
- client.put(allValues);
-
- if (progressable != null)
- progressable.progress();
- if (context != null)
- HadoopCompat.progress(context);
- }
-
- private static void closeSession(Session session)
- {
- //Close the session to satisfy to avoid warnings for the resource not being closed
- try
- {
- if (session != null)
- session.getCluster().closeAsync();
- }
- catch (Throwable t)
- {
- logger.warn("Error closing connection", t);
- }
- }
-
- /**
- * A client that runs in a threadpool and connects to the list of endpoints for a particular
- * range. Bound variables for keys in that range are sent to this client via a queue.
- */
- public class RangeClient extends Thread
- {
- // The list of endpoints for this range
- protected final List<InetAddress> endpoints;
- protected Cluster cluster = null;
- // A bounded queue of incoming mutations for this range
- protected final BlockingQueue<List<ByteBuffer>> queue = new ArrayBlockingQueue<List<ByteBuffer>>(queueSize);
-
- protected volatile boolean run = true;
- // we want the caller to know if something went wrong, so we record any unrecoverable exception while writing
- // so we can throw it on the caller's stack when he calls put() again, or if there are no more put calls,
- // when the client is closed.
- protected volatile IOException lastException;
-
- /**
- * Constructs an {@link RangeClient} for the given endpoints.
- * @param endpoints the possible endpoints to execute the mutations on
- */
- public RangeClient(List<InetAddress> endpoints)
- {
- super("client-" + endpoints);
- this.endpoints = endpoints;
- }
-
- /**
- * enqueues the given value to Cassandra
- */
- public void put(List<ByteBuffer> value) throws IOException
- {
- while (true)
- {
- if (lastException != null)
- throw lastException;
- try
- {
- if (queue.offer(value, 100, TimeUnit.MILLISECONDS))
- break;
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
- }
- }
-
- /**
- * Loops collecting cql binded variable values from the queue and sending to Cassandra
- */
- @SuppressWarnings("resource")
- public void run()
- {
- Session session = null;
-
- try
- {
- outer:
- while (run || !queue.isEmpty())
- {
- List<ByteBuffer> bindVariables;
- try
- {
- bindVariables = queue.take();
- }
- catch (InterruptedException e)
- {
- // re-check loop condition after interrupt
- continue;
- }
-
- ListIterator<InetAddress> iter = endpoints.listIterator();
- while (true)
- {
- // send the mutation to the last-used endpoint. first time through, this will NPE harmlessly.
- if (session != null)
- {
- try
- {
- int i = 0;
- PreparedStatement statement = preparedStatement(session);
- while (bindVariables != null)
- {
- BoundStatement boundStatement = new BoundStatement(statement);
- for (int columnPosition = 0; columnPosition < bindVariables.size(); columnPosition++)
- {
- boundStatement.setBytesUnsafe(columnPosition, bindVariables.get(columnPosition));
- }
- session.execute(boundStatement);
- i++;
-
- if (i >= batchThreshold)
- break;
- bindVariables = queue.poll();
- }
- break;
- }
- catch (Exception e)
- {
- closeInternal();
- if (!iter.hasNext())
- {
- lastException = new IOException(e);
- break outer;
- }
- }
- }
-
- // attempt to connect to a different endpoint
- try
- {
- InetAddress address = iter.next();
- String host = address.getHostName();
- cluster = CqlConfigHelper.getOutputCluster(host, conf);
- closeSession(session);
- session = cluster.connect();
- }
- catch (Exception e)
- {
- //If connection died due to Interrupt, just try connecting to the endpoint again.
- //There are too many ways for the Thread.interrupted() state to be cleared, so
- //we can't rely on that here. Until the java driver gives us a better way of knowing
- //that this exception came from an InterruptedException, this is the best solution.
- if (canRetryDriverConnection(e))
- {
- iter.previous();
- }
- closeInternal();
-
- // Most exceptions mean something unexpected went wrong to that endpoint, so
- // we should try again to another. Other exceptions (auth or invalid request) are fatal.
- if ((e instanceof AuthenticationException || e instanceof InvalidQueryException) || !iter.hasNext())
- {
- lastException = new IOException(e);
- break outer;
- }
- }
- }
- }
- }
- finally
- {
- closeSession(session);
- // close all our connections once we are done.
- closeInternal();
- }
- }
-
- /** get prepared statement id from cache, otherwise prepare it from Cassandra server*/
- private PreparedStatement preparedStatement(Session client)
- {
- PreparedStatement statement = preparedStatements.get(client);
- if (statement == null)
- {
- PreparedStatement result;
- try
- {
- result = client.prepare(cql);
- }
- catch (NoHostAvailableException e)
- {
- throw new RuntimeException("failed to prepare cql query " + cql, e);
- }
-
- PreparedStatement previousId = preparedStatements.putIfAbsent(client, result);
- statement = previousId == null ? result : previousId;
- }
- return statement;
- }
-
- public void close() throws IOException
- {
- // stop the run loop. this will result in closeInternal being called by the time join() finishes.
- run = false;
- interrupt();
- try
- {
- this.join();
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
-
- if (lastException != null)
- throw lastException;
- }
-
- protected void closeInternal()
- {
- if (cluster != null)
- {
- cluster.close();
- }
- }
-
- private boolean canRetryDriverConnection(Exception e)
- {
- if (e instanceof DriverException && e.getMessage().contains("Connection thread interrupted"))
- return true;
- if (e instanceof NoHostAvailableException)
- {
- if (((NoHostAvailableException) e).getErrors().size() == 1)
- {
- Throwable cause = ((NoHostAvailableException) e).getErrors().values().iterator().next();
- if (cause != null && cause.getCause() instanceof java.nio.channels.ClosedByInterruptException)
- {
- return true;
- }
- }
- }
- return false;
- }
- }
-
- private ByteBuffer getPartitionKey(Map<String, ByteBuffer> keyColumns)
- {
- ByteBuffer partitionKey;
- if (partitionKeyColumns.size() > 1)
- {
- ByteBuffer[] keys = new ByteBuffer[partitionKeyColumns.size()];
- for (int i = 0; i< keys.length; i++)
- keys[i] = keyColumns.get(partitionKeyColumns.get(i).getName());
-
- partitionKey = CompositeType.build(ByteBufferAccessor.instance, keys);
- }
- else
- {
- partitionKey = keyColumns.get(partitionKeyColumns.get(0).getName());
- }
- return partitionKey;
- }
-
- /**
- * add where clauses for partition keys and cluster columns
- */
- private String appendKeyWhereClauses(String cqlQuery)
- {
- String keyWhereClause = "";
-
- for (ColumnMetadata partitionKey : partitionKeyColumns)
- keyWhereClause += String.format("%s = ?", keyWhereClause.isEmpty() ? quote(partitionKey.getName()) : (" AND " + quote(partitionKey.getName())));
- for (ColumnMetadata clusterColumn : clusterColumns)
- keyWhereClause += " AND " + quote(clusterColumn.getName()) + " = ?";
-
- return cqlQuery + " WHERE " + keyWhereClause;
- }
-
- /** Quoting for working with uppercase */
- private String quote(String identifier)
- {
- return "\"" + identifier.replaceAll("\"", "\"\"") + "\"";
- }
-
- static class NativeRingCache
- {
- private final Map<TokenRange, Set<Host>> rangeMap;
- private final Metadata metadata;
- private final IPartitioner partitioner;
-
- public NativeRingCache(Configuration conf, Metadata metadata)
- {
- this.partitioner = ConfigHelper.getOutputPartitioner(conf);
- this.metadata = metadata;
- String keyspace = ConfigHelper.getOutputKeyspace(conf);
- this.rangeMap = metadata.getTokenRanges()
- .stream()
- .collect(toMap(p -> p, p -> metadata.getReplicas('"' + keyspace + '"', p)));
- }
-
- public TokenRange getRange(ByteBuffer key)
- {
- Token t = partitioner.getToken(key);
- com.datastax.driver.core.Token driverToken = metadata.newToken(partitioner.getTokenFactory().toString(t));
- for (TokenRange range : rangeMap.keySet())
- {
- if (range.contains(driverToken))
- {
- return range;
- }
- }
-
- throw new RuntimeException("Invalid token information returned by describe_ring: " + rangeMap);
- }
-
- public List<InetAddress> getEndpoints(TokenRange range)
- {
- Set<Host> hostSet = rangeMap.get(range);
- List<InetAddress> addresses = new ArrayList<>(hostSet.size());
- for (Host host: hostSet)
- {
- addresses.add(host.getAddress());
- }
- return addresses;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java b/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java
deleted file mode 100644
index 59b4eca..0000000
--- a/src/java/org/apache/cassandra/hadoop/cql3/LimitedLocalNodeFirstLocalBalancingPolicy.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.cql3;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Host;
-import com.datastax.driver.core.HostDistance;
-import com.datastax.driver.core.Statement;
-import com.datastax.driver.core.policies.LoadBalancingPolicy;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-import java.util.*;
-import java.util.concurrent.CopyOnWriteArraySet;
-
-/**
- * This load balancing policy is intended to be used only for CqlRecordReader when it fetches a particular split.
- * <p/>
- * It chooses alive hosts only from the set of the given replicas - because the connection is used to load the data from
- * the particular split, with a strictly defined list of the replicas, it is pointless to try the other nodes.
- * The policy tracks which of the replicas are alive, and when a new query plan is requested, it returns those replicas
- * in the following order:
- * <ul>
- * <li>the local node</li>
- * <li>the collection of the remaining hosts (which is shuffled on each request)</li>
- * </ul>
- */
-class LimitedLocalNodeFirstLocalBalancingPolicy implements LoadBalancingPolicy
-{
- private final static Logger logger = LoggerFactory.getLogger(LimitedLocalNodeFirstLocalBalancingPolicy.class);
-
- private final static Set<InetAddress> localAddresses = Collections.unmodifiableSet(getLocalInetAddresses());
-
- private final CopyOnWriteArraySet<Host> liveReplicaHosts = new CopyOnWriteArraySet<>();
-
- private final Set<InetAddress> replicaAddresses = new HashSet<>();
- private final Set<String> allowedDCs = new CopyOnWriteArraySet<>();
-
- public LimitedLocalNodeFirstLocalBalancingPolicy(String[] replicas)
- {
- for (String replica : replicas)
- {
- try
- {
- InetAddress[] addresses = InetAddress.getAllByName(replica);
- Collections.addAll(replicaAddresses, addresses);
- }
- catch (UnknownHostException e)
- {
- logger.warn("Invalid replica host name: {}, skipping it", replica);
- }
- }
- if (logger.isTraceEnabled())
- logger.trace("Created instance with the following replicas: {}", Arrays.asList(replicas));
- }
-
- @Override
- public void init(Cluster cluster, Collection<Host> hosts)
- {
- // first find which DCs the user defined
- Set<String> dcs = new HashSet<>();
- for (Host host : hosts)
- {
- if (replicaAddresses.contains(host.getAddress()))
- dcs.add(host.getDatacenter());
- }
- // filter to all nodes within the targeted DCs
- List<Host> replicaHosts = new ArrayList<>();
- for (Host host : hosts)
- {
- if (dcs.contains(host.getDatacenter()))
- replicaHosts.add(host);
- }
- liveReplicaHosts.addAll(replicaHosts);
- allowedDCs.addAll(dcs);
- logger.trace("Initialized with replica hosts: {}", replicaHosts);
- }
-
- @Override
- public void close()
- {
- //
- }
-
- @Override
- public HostDistance distance(Host host)
- {
- if (isLocalHost(host))
- {
- return HostDistance.LOCAL;
- }
- else
- {
- return HostDistance.REMOTE;
- }
- }
-
- @Override
- public Iterator<Host> newQueryPlan(String keyspace, Statement statement)
- {
- List<Host> local = new ArrayList<>(1);
- List<Host> remote = new ArrayList<>(liveReplicaHosts.size());
- for (Host liveReplicaHost : liveReplicaHosts)
- {
- if (isLocalHost(liveReplicaHost))
- {
- local.add(liveReplicaHost);
- }
- else
- {
- remote.add(liveReplicaHost);
- }
- }
-
- Collections.shuffle(remote);
-
- logger.trace("Using the following hosts order for the new query plan: {} | {}", local, remote);
-
- return Iterators.concat(local.iterator(), remote.iterator());
- }
-
- @Override
- public void onAdd(Host host)
- {
- if (liveReplicaHosts.contains(host))
- {
- liveReplicaHosts.add(host);
- logger.trace("Added a new host {}", host);
- }
- }
-
- @Override
- public void onUp(Host host)
- {
- if (liveReplicaHosts.contains(host))
- {
- liveReplicaHosts.add(host);
- logger.trace("The host {} is now up", host);
- }
- }
-
- @Override
- public void onDown(Host host)
- {
- if (liveReplicaHosts.remove(host))
- {
- logger.trace("The host {} is now down", host);
- }
- }
-
-
- @Override
- public void onRemove(Host host)
- {
- if (liveReplicaHosts.remove(host))
- {
- logger.trace("Removed the host {}", host);
- }
- }
-
- public void onSuspected(Host host)
- {
- // not supported by this load balancing policy
- }
-
- private static boolean isLocalHost(Host host)
- {
- InetAddress hostAddress = host.getAddress();
- return hostAddress.isLoopbackAddress() || localAddresses.contains(hostAddress);
- }
-
- private static Set<InetAddress> getLocalInetAddresses()
- {
- try
- {
- return Sets.newHashSet(Iterators.concat(
- Iterators.transform(
- Iterators.forEnumeration(NetworkInterface.getNetworkInterfaces()),
- new Function<NetworkInterface, Iterator<InetAddress>>()
- {
- @Override
- public Iterator<InetAddress> apply(NetworkInterface netIface)
- {
- return Iterators.forEnumeration(netIface.getInetAddresses());
- }
- })));
- }
- catch (SocketException e)
- {
- logger.warn("Could not retrieve local network interfaces.", e);
- return Collections.emptySet();
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/hadoop/package-info.java b/src/java/org/apache/cassandra/hadoop/package-info.java
deleted file mode 100644
index 835165b..0000000
--- a/src/java/org/apache/cassandra/hadoop/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package was deprecated. See CASSANDRA-16984.
- */
-@Deprecated
-package org.apache.cassandra.hadoop;
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java
index 3089894..c68c266 100644
--- a/src/java/org/apache/cassandra/hints/Hint.java
+++ b/src/java/org/apache/cassandra/hints/Hint.java
@@ -22,12 +22,11 @@
import java.util.concurrent.TimeUnit;
import com.google.common.base.Throwables;
-
-import javax.annotation.Nullable;
-
import com.google.common.primitives.Ints;
-import org.apache.cassandra.db.*;
+import javax.annotation.Nullable;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
@@ -38,6 +37,7 @@
import org.apache.cassandra.utils.vint.VIntCoding;
import org.assertj.core.util.VisibleForTesting;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_MAX_HINT_TTL;
import static org.apache.cassandra.db.TypeSizes.sizeof;
import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -60,7 +60,7 @@
public final class Hint
{
public static final Serializer serializer = new Serializer();
- static final int maxHintTTL = Integer.getInteger("cassandra.maxHintTTL", Integer.MAX_VALUE);
+ static final int maxHintTTL = CASSANDRA_MAX_HINT_TTL.getInt();
final Mutation mutation;
final long creationTime; // time of hint creation (in milliseconds)
@@ -120,7 +120,8 @@
}
catch (Exception e)
{
- throw Throwables.propagate(e.getCause());
+ Throwables.throwIfUnchecked(e.getCause());
+ throw new RuntimeException(e.getCause());
}
}
@@ -170,14 +171,14 @@
public void serialize(Hint hint, DataOutputPlus out, int version) throws IOException
{
out.writeLong(hint.creationTime);
- out.writeUnsignedVInt(hint.gcgs);
+ out.writeUnsignedVInt32(hint.gcgs);
Mutation.serializer.serialize(hint.mutation, out, version);
}
public Hint deserialize(DataInputPlus in, int version) throws IOException
{
long creationTime = in.readLong();
- int gcgs = (int) in.readUnsignedVInt();
+ int gcgs = in.readUnsignedVInt32();
return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
}
@@ -198,7 +199,7 @@
Hint deserializeIfLive(DataInputPlus in, long now, long size, int version) throws IOException
{
long creationTime = in.readLong();
- int gcgs = (int) in.readUnsignedVInt();
+ int gcgs = in.readUnsignedVInt32();
int bytesRead = sizeof(creationTime) + sizeofUnsignedVInt(gcgs);
if (isLive(creationTime, now, gcgs))
@@ -226,7 +227,7 @@
try (DataInputBuffer input = new DataInputBuffer(header))
{
long creationTime = input.readLong();
- int gcgs = (int) input.readUnsignedVInt();
+ int gcgs = input.readUnsignedVInt32();
if (!isLive(creationTime, now, gcgs))
{
diff --git a/src/java/org/apache/cassandra/hints/HintMessage.java b/src/java/org/apache/cassandra/hints/HintMessage.java
index 60d7641..978ab41 100644
--- a/src/java/org/apache/cassandra/hints/HintMessage.java
+++ b/src/java/org/apache/cassandra/hints/HintMessage.java
@@ -22,10 +22,10 @@
import java.nio.ByteBuffer;
import java.util.Objects;
import java.util.UUID;
-import javax.annotation.Nullable;
import com.google.common.primitives.Ints;
+import javax.annotation.Nullable;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.UnknownTableException;
import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
@@ -135,7 +135,7 @@
throw new IllegalArgumentException("serialize() called with non-matching version " + version);
UUIDSerializer.serializer.serialize(message.hostId, out, version);
- out.writeUnsignedVInt(message.hint.remaining());
+ out.writeUnsignedVInt32(message.hint.remaining());
out.write(message.hint);
}
else
diff --git a/src/java/org/apache/cassandra/hints/HintsBufferPool.java b/src/java/org/apache/cassandra/hints/HintsBufferPool.java
index 78f07dd..dd5829d 100644
--- a/src/java/org/apache/cassandra/hints/HintsBufferPool.java
+++ b/src/java/org/apache/cassandra/hints/HintsBufferPool.java
@@ -22,10 +22,10 @@
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_HINT_BUFFERS;
import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQueue;
/**
@@ -39,7 +39,7 @@
void flush(HintsBuffer buffer, HintsBufferPool pool);
}
- static final int MAX_ALLOCATED_BUFFERS = Integer.getInteger(Config.PROPERTY_PREFIX + "MAX_HINT_BUFFERS", 3);
+ static final int MAX_ALLOCATED_BUFFERS = MAX_HINT_BUFFERS.getInt();
private volatile HintsBuffer currentBuffer;
private final BlockingQueue<HintsBuffer> reserveBuffers;
private final int bufferSize;
diff --git a/src/java/org/apache/cassandra/hints/HintsCatalog.java b/src/java/org/apache/cassandra/hints/HintsCatalog.java
index 859252f..6bc0030 100644
--- a/src/java/org/apache/cassandra/hints/HintsCatalog.java
+++ b/src/java/org/apache/cassandra/hints/HintsCatalog.java
@@ -26,10 +26,11 @@
import javax.annotation.Nullable;
import com.google.common.collect.ImmutableMap;
-import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
@@ -161,8 +162,14 @@
FileUtils.handleFSErrorAndPropagate(e);
}
}
+ else if (DatabaseDescriptor.isClientInitialized())
+ {
+ logger.warn("Unable to open hint directory using Native library. Skipping sync.");
+ }
else
{
+ if (SyncUtil.SKIP_SYNC)
+ return;
logger.error("Unable to open directory {}", hintsDirectory.absolutePath());
FileUtils.handleFSErrorAndPropagate(new FSWriteError(new IOException(String.format("Unable to open hint directory %s", hintsDirectory.absolutePath())), hintsDirectory.absolutePath()));
}
diff --git a/src/java/org/apache/cassandra/hints/HintsDescriptor.java b/src/java/org/apache/cassandra/hints/HintsDescriptor.java
index 8e1f782..8aa8b7b 100644
--- a/src/java/org/apache/cassandra/hints/HintsDescriptor.java
+++ b/src/java/org/apache/cassandra/hints/HintsDescriptor.java
@@ -19,7 +19,6 @@
import java.io.DataInput;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
@@ -37,6 +36,8 @@
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileInputStreamPlus;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CountingOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,8 +50,9 @@
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.security.EncryptionContext;
+import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.Hex;
-import org.json.simple.JSONValue;
+import org.apache.cassandra.utils.JsonUtils;
import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
@@ -368,7 +370,7 @@
out.writeLong(hostId.getLeastSignificantBits());
updateChecksumLong(crc, hostId.getLeastSignificantBits());
- byte[] paramsBytes = JSONValue.toJSONString(parameters).getBytes(StandardCharsets.UTF_8);
+ byte[] paramsBytes = JsonUtils.writeAsJsonBytes(parameters);
out.writeInt(paramsBytes.length);
updateChecksumInt(crc, paramsBytes.length);
out.writeInt((int) crc.getValue());
@@ -387,10 +389,20 @@
size += TypeSizes.sizeof(hostId.getMostSignificantBits());
size += TypeSizes.sizeof(hostId.getLeastSignificantBits());
- byte[] paramsBytes = JSONValue.toJSONString(parameters).getBytes(StandardCharsets.UTF_8);
- size += TypeSizes.sizeof(paramsBytes.length);
+ // Let's avoid allocation of serialized output, use counting output stream
+ int serializedParamsLength;
+ try (CountingOutputStream out = new CountingOutputStream(ByteStreams.nullOutputStream()))
+ {
+ JsonUtils.JSON_OBJECT_MAPPER.writeValue(out, parameters);
+ serializedParamsLength = (int) out.getCount();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e); // should never happen
+ }
+ size += TypeSizes.sizeof(serializedParamsLength);
size += 4; // size checksum
- size += paramsBytes.length;
+ size += serializedParamsLength;
size += 4; // total checksum
return size;
@@ -428,7 +440,18 @@
@SuppressWarnings("unchecked")
private static ImmutableMap<String, Object> decodeJSONBytes(byte[] bytes)
{
- return ImmutableMap.copyOf((Map<String, Object>) JSONValue.parse(new String(bytes, StandardCharsets.UTF_8)));
+ // note: There is a Jackson module (datatype-guava) for directly reading into ImmutableMap,
+ // but would require adding dependency to that
+ try
+ {
+ return ImmutableMap.copyOf(JsonUtils.fromJsonMap(bytes));
+ }
+ catch (MarshalException e)
+ {
+ // Couple of options here: up to 4.0 simply returned null and caller failed with NPE.
+ // Seems cleaner to throw an exception
+ throw new MarshalException("Corrupt HintsDescriptor serialization, problem: " + e.getMessage(), e);
+ }
}
private static void updateChecksumLong(CRC32 crc, long value)
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
index 0f34db6..540f5bd 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
@@ -192,7 +192,7 @@
private boolean transfer(UUID hostId)
{
catalog.stores()
- .map(store -> new DispatchHintsTask(store, hostId))
+ .map(store -> new DispatchHintsTask(store, hostId, true))
.forEach(Runnable::run);
return !catalog.hasFiles();
@@ -205,21 +205,27 @@
private final UUID hostId;
private final RateLimiter rateLimiter;
- DispatchHintsTask(HintsStore store, UUID hostId)
+ DispatchHintsTask(HintsStore store, UUID hostId, boolean isTransfer)
{
this.store = store;
this.hostId = hostId;
- // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
- // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272).
- // the goal is to bound maximum hints traffic going towards a particular node from the rest of the cluster,
- // not total outgoing hints traffic from this node - this is why the rate limiter is not shared between
+ // Rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
+ // Max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272), unless we are transferring
+ // hints during decomission rather than dispatching them to their final destination.
+ // The goal is to bound maximum hints traffic going towards a particular node from the rest of the cluster,
+ // not total outgoing hints traffic from this node. This is why the rate limiter is not shared between
// all the dispatch tasks (as there will be at most one dispatch task for a particular host id at a time).
- int nodesCount = Math.max(1, StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1);
+ int nodesCount = isTransfer ? 1 : Math.max(1, StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1);
double throttleInBytes = DatabaseDescriptor.getHintedHandoffThrottleInKiB() * 1024.0 / nodesCount;
this.rateLimiter = RateLimiter.create(throttleInBytes == 0 ? Double.MAX_VALUE : throttleInBytes);
}
+ DispatchHintsTask(HintsStore store, UUID hostId)
+ {
+ this(store, hostId, false);
+ }
+
public void run()
{
try
diff --git a/src/java/org/apache/cassandra/hints/HintsWriter.java b/src/java/org/apache/cassandra/hints/HintsWriter.java
index 663427a..591e03d 100644
--- a/src/java/org/apache/cassandra/hints/HintsWriter.java
+++ b/src/java/org/apache/cassandra/hints/HintsWriter.java
@@ -37,6 +37,7 @@
import org.apache.cassandra.utils.SyncUtil;
import org.apache.cassandra.utils.Throwables;
+import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
import static org.apache.cassandra.utils.Throwables.perform;
@@ -78,7 +79,7 @@
{
// write the descriptor
descriptor.serialize(dob);
- ByteBuffer descriptorBytes = dob.buffer();
+ ByteBuffer descriptorBytes = dob.unsafeGetBufferAndFlip();
updateChecksum(crc, descriptorBytes);
channel.write(descriptorBytes);
@@ -248,7 +249,10 @@
updateChecksumInt(crc, hintSize);
out.writeInt((int) crc.getValue());
+ long startPosition = out.position();
Hint.serializer.serialize(hint, out, descriptor.messagingVersion());
+ long actualSize = out.position() - startPosition;
+ checkState(actualSize == hintSize, "Serialized hint size doesn't match calculated hint size");
updateChecksum(crc, hintBuffer, hintBuffer.position() - hintSize, hintSize);
out.writeInt((int) crc.getValue());
}
diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java
index 9f51b16..f2307b9 100644
--- a/src/java/org/apache/cassandra/index/Index.java
+++ b/src/java/org/apache/cassandra/index/Index.java
@@ -26,13 +26,19 @@
import java.util.concurrent.Callable;
import java.util.function.BiFunction;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -42,8 +48,9 @@
import org.apache.cassandra.index.transactions.IndexTransaction;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
-import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
/**
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
index 93ecd59..4c412a2 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
@@ -73,6 +73,7 @@
import org.apache.cassandra.utils.concurrent.*;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FORCE_DEFAULT_INDEXING_PAGE_SIZE;
import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination;
import static org.apache.cassandra.utils.ExecutorUtils.shutdown;
@@ -991,7 +992,7 @@
*/
public int calculateIndexingPageSize()
{
- if (Boolean.getBoolean("cassandra.force_default_indexing_page_size"))
+ if (FORCE_DEFAULT_INDEXING_PAGE_SIZE.getBoolean())
return DEFAULT_PAGE_SIZE;
double targetPageSizeInBytes = 32 * 1024 * 1024;
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index 1e86bc6..38588d3 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@ -17,16 +17,30 @@
*/
package org.apache.cassandra.index.sasi;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.function.BiFunction;
import com.googlecode.concurrenttrees.common.Iterables;
-
-import org.apache.cassandra.config.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.CassandraWriteContext;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.filter.RowFilter;
@@ -49,9 +63,15 @@
import org.apache.cassandra.index.sasi.plan.QueryPlan;
import org.apache.cassandra.index.transactions.IndexTransaction;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.MemtableDiscardedNotification;
+import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.notifications.MemtableSwitchedNotification;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Schema;
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
index 57a3f51..555bce1 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
@@ -21,13 +21,13 @@
package org.apache.cassandra.index.sasi;
import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.SortedMap;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionInterruptedException;
import org.apache.cassandra.db.compaction.OperationType;
@@ -36,11 +36,12 @@
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter;
import org.apache.cassandra.io.FSReadError;
-import org.apache.cassandra.io.sstable.KeyIterator;
-import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.KeyReader;
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.TimeUUID;
@@ -51,25 +52,29 @@
private final ColumnFamilyStore cfs;
private final TimeUUID compactionId = nextTimeUUID();
+ // Keep targetDirectory for compactions, needed for `nodetool compactionstats`
+ private String targetDirectory;
+
private final SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables;
private long bytesProcessed = 0;
- private final long totalSizeInBytes;
+ private final long totalBytesToProcess;
public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables)
{
- long totalIndexBytes = 0;
+ long totalBytesToProcess = 0;
for (SSTableReader sstable : sstables.keySet())
- totalIndexBytes += getPrimaryIndexLength(sstable);
+ totalBytesToProcess += sstable.uncompressedLength();
this.cfs = cfs;
this.sstables = sstables;
- this.totalSizeInBytes = totalIndexBytes;
+ this.totalBytesToProcess = totalBytesToProcess;
}
public void build()
{
AbstractType<?> keyValidator = cfs.metadata().partitionKeyType;
+ long processedBytesInFinishedSSTables = 0;
for (Map.Entry<SSTableReader, Map<ColumnMetadata, ColumnIndex>> e : sstables.entrySet())
{
SSTableReader sstable = e.getKey();
@@ -78,47 +83,47 @@
try (RandomAccessReader dataFile = sstable.openDataReader())
{
PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION);
+ targetDirectory = indexWriter.getDescriptor().directory.path();
- long previousKeyPosition = 0;
- try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata()))
+ try (KeyReader keys = sstable.keyReader())
{
- while (keys.hasNext())
+ while (!keys.isExhausted())
{
if (isStopRequested())
throw new CompactionInterruptedException(getCompactionInfo());
- final DecoratedKey key = keys.next();
- final long keyPosition = keys.getKeyPosition();
+ final DecoratedKey key = sstable.decorateKey(keys.key());
+ final long keyPosition = keys.keyPositionForSecondaryIndex();
- indexWriter.startPartition(key, keyPosition);
+ indexWriter.startPartition(key, keys.dataPosition(), keyPosition);
- try
+ dataFile.seek(keys.dataPosition());
+ ByteBufferUtil.readWithShortLength(dataFile); // key
+
+ try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key))
{
- RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
- dataFile.seek(indexEntry.position);
- ByteBufferUtil.readWithShortLength(dataFile); // key
-
- try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key))
+ // if the row has statics attached, it has to be indexed separately
+ if (cfs.metadata().hasStaticColumns())
{
- // if the row has statics attached, it has to be indexed separately
- if (cfs.metadata().hasStaticColumns())
- indexWriter.nextUnfilteredCluster(partition.staticRow());
-
- while (partition.hasNext())
- indexWriter.nextUnfilteredCluster(partition.next());
+ indexWriter.nextUnfilteredCluster(partition.staticRow());
}
- }
- catch (IOException ex)
- {
- throw new FSReadError(ex, sstable.getFilename());
+
+ while (partition.hasNext())
+ indexWriter.nextUnfilteredCluster(partition.next());
}
- bytesProcessed += keyPosition - previousKeyPosition;
- previousKeyPosition = keyPosition;
+ keys.advance();
+ long dataPosition = keys.isExhausted() ? sstable.uncompressedLength() : keys.dataPosition();
+ bytesProcessed = processedBytesInFinishedSSTables + dataPosition;
}
completeSSTable(indexWriter, sstable, indexes.values());
}
+ catch (IOException ex)
+ {
+ throw new FSReadError(ex, sstable.getFilename());
+ }
+ processedBytesInFinishedSSTables += sstable.uncompressedLength();
}
}
}
@@ -128,15 +133,10 @@
return new CompactionInfo(cfs.metadata(),
OperationType.INDEX_BUILD,
bytesProcessed,
- totalSizeInBytes,
+ totalBytesToProcess,
compactionId,
- sstables.keySet());
- }
-
- private long getPrimaryIndexLength(SSTable sstable)
- {
- File primaryIndex = new File(sstable.getIndexFilename());
- return primaryIndex.exists() ? primaryIndex.length() : 0;
+ sstables.keySet(),
+ targetDirectory);
}
private void completeSSTable(PerSSTableIndexWriter indexWriter, SSTableReader sstable, Collection<ColumnIndex> indexes)
@@ -145,7 +145,7 @@
for (ColumnIndex index : indexes)
{
- File tmpIndex = new File(sstable.descriptor.filenameFor(index.getComponent()));
+ File tmpIndex = sstable.descriptor.fileFor(index.getComponent());
if (!tmpIndex.exists()) // no data was inserted into the index for given sstable
continue;
diff --git a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java
index d756737..de9c0c2 100644
--- a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java
@@ -22,6 +22,9 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.base.Function;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
@@ -36,10 +39,6 @@
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
-import com.google.common.base.Function;
-
public class SSTableIndex
{
private final ColumnIndex columnIndex;
@@ -176,7 +175,7 @@
{
try
{
- return sstable.keyAt(offset);
+ return sstable.keyAtPositionFromSecondaryIndex(offset);
}
catch (IOException e)
{
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
index 81b776d..7d82878 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java
@@ -28,13 +28,12 @@
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
@@ -47,7 +46,9 @@
import org.apache.cassandra.index.sasi.utils.RangeIterator;
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.FBUtilities;
@@ -78,7 +79,7 @@
this.mode = IndexMode.getMode(column, config);
this.memtable = new AtomicReference<>(new IndexMemtable(this));
this.tracker = new DataTracker(keyValidator, this);
- this.component = new Component(Component.Type.SECONDARY_INDEX, String.format(FILE_NAME_FORMAT, getIndexName()));
+ this.component = Components.Types.SECONDARY_INDEX.createComponent(String.format(FILE_NAME_FORMAT, getIndexName()));
this.isTokenized = getAnalyzer().isTokenizing();
}
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java b/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java
index bf2293f..9b3e9ab 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java
@@ -24,6 +24,9 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sasi.SSTableIndex;
import org.apache.cassandra.index.sasi.conf.view.View;
@@ -31,9 +34,6 @@
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/** a pared-down version of DataTracker and DT.View. need one for each index of each column family */
public class DataTracker
{
@@ -145,7 +145,7 @@
if (sstable.isMarkedCompacted())
continue;
- File indexFile = new File(sstable.descriptor.filenameFor(columnIndex.getComponent()));
+ File indexFile = sstable.descriptor.fileFor(columnIndex.getComponent());
if (!indexFile.exists())
continue;
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
index e438079..88b272a 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
@@ -17,21 +17,33 @@
*/
package org.apache.cassandra.index.sasi.disk;
-import java.io.*;
+import java.io.Closeable;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
import java.util.stream.Collectors;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sasi.Term;
import org.apache.cassandra.index.sasi.plan.Expression;
import org.apache.cassandra.index.sasi.plan.Expression.Op;
-import org.apache.cassandra.index.sasi.utils.MappedBuffer;
-import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
import org.apache.cassandra.index.sasi.utils.AbstractIterator;
+import org.apache.cassandra.index.sasi.utils.MappedBuffer;
import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.File;
@@ -40,11 +52,6 @@
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-
import static org.apache.cassandra.index.sasi.disk.OnDiskBlock.SearchResult;
public class OnDiskIndex implements Iterable<OnDiskIndex.DataTerm>, Closeable
@@ -144,7 +151,7 @@
FileChannel channel = index.newReadChannel();
indexSize = channel.size();
- indexFile = new MappedBuffer(new ChannelProxy(indexPath, channel));
+ indexFile = new MappedBuffer(new ChannelProxy(index, channel));
}
catch (IOException e)
{
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
index fb5e9b9..f89ff62 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
@@ -21,35 +21,37 @@
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.io.util.File;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
import org.apache.cassandra.index.sasi.utils.CombinedTermIterator;
import org.apache.cassandra.index.sasi.utils.TypeUtil;
-import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.ImmediateFuture;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Maps;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
@@ -98,15 +100,24 @@
indexes.put(entry.getKey(), newIndex(entry.getValue()));
}
+ @Override
public void begin()
{}
- public void startPartition(DecoratedKey key, long curPosition)
+ @Override
+ public void startPartition(DecoratedKey key, long keyPosition, long KeyPositionForSASI)
{
currentKey = key;
- currentKeyPosition = curPosition;
+ currentKeyPosition = KeyPositionForSASI;
}
+ @Override
+ public void staticRow(Row staticRow)
+ {
+ nextUnfilteredCluster(staticRow);
+ }
+
+ @Override
public void nextUnfilteredCluster(Unfiltered unfiltered)
{
if (!unfiltered.isRow())
@@ -126,6 +137,7 @@
});
}
+ @Override
public void complete()
{
if (isComplete)
@@ -168,7 +180,7 @@
protected class Index
{
@VisibleForTesting
- protected final String outputFile;
+ protected final File outputFile;
private final ColumnIndex columnIndex;
private final AbstractAnalyzer analyzer;
@@ -183,7 +195,7 @@
public Index(ColumnIndex columnIndex)
{
this.columnIndex = columnIndex;
- this.outputFile = descriptor.filenameFor(columnIndex.getComponent());
+ this.outputFile = descriptor.fileFor(columnIndex.getComponent());
this.analyzer = columnIndex.getAnalyzer();
this.segments = new HashSet<>();
this.maxMemorySize = maxMemorySize(columnIndex);
@@ -244,15 +256,14 @@
final OnDiskIndexBuilder builder = currentBuilder;
currentBuilder = newIndexBuilder();
- final String segmentFile = filename(isFinal);
+ final File segmentFile = file(isFinal);
return () -> {
long start = nanoTime();
try
{
- File index = new File(segmentFile);
- return builder.finish(index) ? new OnDiskIndex(index, columnIndex.getValidator(), null) : null;
+ return builder.finish(segmentFile) ? new OnDiskIndex(segmentFile, columnIndex.getValidator(), null) : null;
}
catch (Exception | FSError e)
{
@@ -310,13 +321,13 @@
OnDiskIndexBuilder builder = newIndexBuilder();
builder.finish(Pair.create(combinedMin, combinedMax),
- new File(outputFile),
+ outputFile,
new CombinedTermIterator(parts));
}
catch (Exception | FSError e)
{
logger.error("Failed to flush index {}.", outputFile, e);
- FileUtils.delete(outputFile);
+ outputFile.tryDelete();
}
finally
{
@@ -330,7 +341,7 @@
if (part != null)
FileUtils.closeQuietly(part);
- FileUtils.delete(outputFile + "_" + segment);
+ outputFile.withSuffix("_" + segment).tryDelete();
}
latch.decrement();
@@ -348,9 +359,9 @@
return new OnDiskIndexBuilder(keyValidator, columnIndex.getValidator(), columnIndex.getMode().mode);
}
- public String filename(boolean isFinal)
+ public File file(boolean isFinal)
{
- return outputFile + (isFinal ? "" : "_" + segmentNumber++);
+ return isFinal ? outputFile : outputFile.withSuffix("_" + segmentNumber++);
}
}
diff --git a/src/java/org/apache/cassandra/io/IGenericSerializer.java b/src/java/org/apache/cassandra/io/IGenericSerializer.java
new file mode 100644
index 0000000..5be21cc
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/IGenericSerializer.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public interface IGenericSerializer<T, I extends DataInput, O extends DataOutput>
+{
+ void serialize(T t, O out) throws IOException;
+
+ T deserialize(I in) throws IOException;
+
+ long serializedSize(T t);
+}
diff --git a/src/java/org/apache/cassandra/io/ISerializer.java b/src/java/org/apache/cassandra/io/ISerializer.java
index 637a1c7..da9049d 100644
--- a/src/java/org/apache/cassandra/io/ISerializer.java
+++ b/src/java/org/apache/cassandra/io/ISerializer.java
@@ -22,7 +22,7 @@
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-public interface ISerializer<T>
+public interface ISerializer<T> extends IGenericSerializer<T, DataInputPlus, DataOutputPlus>
{
/**
* Serialize the specified type into the specified DataOutput instance.
@@ -32,7 +32,8 @@
* @param out DataOutput into which serialization needs to happen.
* @throws java.io.IOException
*/
- public void serialize(T t, DataOutputPlus out) throws IOException;
+ @Override
+ void serialize(T t, DataOutputPlus out) throws IOException;
/**
* Deserialize from the specified DataInput instance.
@@ -40,11 +41,13 @@
* @throws IOException
* @return the type that was deserialized
*/
- public T deserialize(DataInputPlus in) throws IOException;
+ @Override
+ T deserialize(DataInputPlus in) throws IOException;
- public long serializedSize(T t);
+ @Override
+ long serializedSize(T t);
- public default void skip(DataInputPlus in) throws IOException
+ default void skip(DataInputPlus in) throws IOException
{
deserialize(in);
}
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index 024e4ef..7d54e09 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -29,7 +29,12 @@
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.util.ChecksumWriter;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -66,14 +71,14 @@
* Create CompressedSequentialWriter without digest file.
*
* @param file File to write
- * @param offsetsPath File name to write compression metadata
+ * @param offsetsFile File to write compression metadata
* @param digestFile File to write digest
* @param option Write option (buffer size and type will be set the same as compression params)
* @param parameters Compression mparameters
* @param sstableMetadataCollector Metadata collector
*/
public CompressedSequentialWriter(File file,
- String offsetsPath,
+ File offsetsFile,
File digestFile,
SequentialWriterOption option,
CompressionParams parameters,
@@ -95,7 +100,7 @@
maxCompressedLength = parameters.maxCompressedLength();
/* Index File (-CompressionInfo.db component) and it's header */
- metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsPath);
+ metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsFile);
this.sstableMetadataCollector = sstableMetadataCollector;
crcMetadata = new ChecksumWriter(new DataOutputStream(Channels.newOutputStream(channel)));
@@ -397,4 +402,4 @@
this.nextChunkIndex = nextChunkIndex;
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
index 5af9c92..d2fab30 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
@@ -17,14 +17,11 @@
*/
package org.apache.cassandra.io.compress;
-import java.nio.file.NoSuchFileException;
-import java.io.DataInput;
import java.io.DataOutput;
import java.io.EOFException;
-
-import org.apache.cassandra.io.util.*;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -32,7 +29,6 @@
import java.util.TreeSet;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
import com.google.common.primitives.Longs;
import org.apache.cassandra.db.TypeSizes;
@@ -40,18 +36,25 @@
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.io.util.SafeMemory;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.utils.concurrent.Transactional;
import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.Transactional;
+import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable;
/**
* Holds metadata about compressed file
+ * TODO extract interface ICompressionMetadata which will just provide non-resource properties
*/
-public class CompressionMetadata
+public class CompressionMetadata extends WrappedSharedCloseable
{
// dataLength can represent either the true length of the file
// or some shorter value, in the case we want to impose a shorter limit on readers
@@ -60,42 +63,18 @@
public final long compressedFileLength;
private final Memory chunkOffsets;
private final long chunkOffsetsSize;
- public final String indexFilePath;
+ public final File chunksIndexFile;
public final CompressionParams parameters;
- /**
- * Create metadata about given compressed file including uncompressed data length, chunk size
- * and list of the chunk offsets of the compressed data.
- *
- * This is an expensive operation! Don't create more than one for each
- * sstable.
- *
- * @param dataFilePath Path to the compressed file
- *
- * @return metadata about given compressed file.
- */
- public static CompressionMetadata create(String dataFilePath)
- {
- return createWithLength(dataFilePath, new File(dataFilePath).length());
- }
-
- public static CompressionMetadata createWithLength(String dataFilePath, long compressedLength)
- {
- return new CompressionMetadata(Descriptor.fromFilename(dataFilePath), compressedLength);
- }
-
@VisibleForTesting
- public CompressionMetadata(Descriptor desc, long compressedLength)
+ @SuppressWarnings("resource")
+ public static CompressionMetadata open(File chunksIndexFile, long compressedLength, boolean hasMaxCompressedSize)
{
- this(desc.filenameFor(Component.COMPRESSION_INFO), compressedLength, desc.version.hasMaxCompressedLength());
- }
+ CompressionParams parameters;
+ long dataLength;
+ Memory chunkOffsets;
- @VisibleForTesting
- public CompressionMetadata(String indexFilePath, long compressedLength, boolean hasMaxCompressedSize)
- {
- this.indexFilePath = indexFilePath;
-
- try (FileInputStreamPlus stream = new File(indexFilePath).newInputStream())
+ try (FileInputStreamPlus stream = chunksIndexFile.newInputStream())
{
String compressorName = stream.readUTF();
int optionCount = stream.readInt();
@@ -120,7 +99,6 @@
}
dataLength = stream.readLong();
- compressedFileLength = compressedLength;
chunkOffsets = readChunkOffsets(stream);
}
catch (FileNotFoundException | NoSuchFileException e)
@@ -129,22 +107,39 @@
}
catch (IOException e)
{
- throw new CorruptSSTableException(e, indexFilePath);
+ throw new CorruptSSTableException(e, chunksIndexFile);
}
- this.chunkOffsetsSize = chunkOffsets.size();
+ return new CompressionMetadata(chunksIndexFile, parameters, chunkOffsets, chunkOffsets.size(), dataLength, compressedLength);
}
// do not call this constructor directly, unless used in testing
@VisibleForTesting
- public CompressionMetadata(String filePath, CompressionParams parameters, Memory offsets, long offsetsSize, long dataLength, long compressedLength)
+ public CompressionMetadata(File chunksIndexFile,
+ CompressionParams parameters,
+ Memory chunkOffsets,
+ long chunkOffsetsSize,
+ long dataLength,
+ long compressedFileLength)
{
- this.indexFilePath = filePath;
+ super(chunkOffsets);
+ this.chunksIndexFile = chunksIndexFile;
this.parameters = parameters;
this.dataLength = dataLength;
- this.compressedFileLength = compressedLength;
- this.chunkOffsets = offsets;
- this.chunkOffsetsSize = offsetsSize;
+ this.compressedFileLength = compressedFileLength;
+ this.chunkOffsets = chunkOffsets;
+ this.chunkOffsetsSize = chunkOffsetsSize;
+ }
+
+ private CompressionMetadata(CompressionMetadata copy)
+ {
+ super(copy);
+ this.chunksIndexFile = copy.chunksIndexFile;
+ this.parameters = copy.parameters;
+ this.dataLength = copy.dataLength;
+ this.compressedFileLength = copy.compressedFileLength;
+ this.chunkOffsets = copy.chunkOffsets;
+ this.chunkOffsetsSize = copy.chunkOffsetsSize;
}
public ICompressor compressor()
@@ -171,11 +166,19 @@
return chunkOffsets.size();
}
+ @Override
public void addTo(Ref.IdentityCollection identities)
{
+ super.addTo(identities);
identities.add(chunkOffsets);
}
+ @Override
+ public CompressionMetadata sharedCopy()
+ {
+ return new CompressionMetadata(this);
+ }
+
/**
* Read offsets of the individual chunks from the given input.
*
@@ -183,7 +186,7 @@
*
* @return collection of the chunk offsets.
*/
- private Memory readChunkOffsets(DataInput input)
+ private static Memory readChunkOffsets(FileInputStreamPlus input)
{
final int chunkCount;
try
@@ -194,7 +197,7 @@
}
catch (IOException e)
{
- throw new FSReadError(e, indexFilePath);
+ throw new FSReadError(e, input.file);
}
@SuppressWarnings("resource")
@@ -218,10 +221,10 @@
if (e instanceof EOFException)
{
String msg = String.format("Corrupted Index File %s: read %d but expected %d chunks.",
- indexFilePath, i, chunkCount);
- throw new CorruptSSTableException(new IOException(msg, e), indexFilePath);
+ input.file.path(), i, chunkCount);
+ throw new CorruptSSTableException(new IOException(msg, e), input.file);
}
- throw new FSReadError(e, indexFilePath);
+ throw new FSReadError(e, input.file);
}
}
@@ -237,11 +240,11 @@
long idx = 8 * (position / parameters.chunkLength());
if (idx >= chunkOffsetsSize)
- throw new CorruptSSTableException(new EOFException(), indexFilePath);
+ throw new CorruptSSTableException(new EOFException(), chunksIndexFile);
if (idx < 0)
throw new CorruptSSTableException(new IllegalArgumentException(String.format("Invalid negative chunk index %d with position %d", idx, position)),
- indexFilePath);
+ chunksIndexFile);
long chunkOffset = chunkOffsets.getLong(idx);
long nextChunkOffset = (idx + 8 == chunkOffsetsSize)
@@ -251,6 +254,28 @@
return new Chunk(chunkOffset, (int) (nextChunkOffset - chunkOffset - 4)); // "4" bytes reserved for checksum
}
+ public long getDataOffsetForChunkOffset(long chunkOffset)
+ {
+ long l = 0;
+ long h = (chunkOffsetsSize >> 3) - 1;
+ long idx, offset;
+
+ while (l <= h)
+ {
+ idx = (l + h) >>> 1;
+ offset = chunkOffsets.getLong(idx << 3);
+
+ if (offset < chunkOffset)
+ l = idx + 1;
+ else if (offset > chunkOffset)
+ h = idx - 1;
+ else
+ return idx * parameters.chunkLength();
+ }
+
+ throw new IllegalArgumentException("No chunk with offset " + chunkOffset);
+ }
+
/**
* @param sections Collection of sections in uncompressed file. Should not contain sections that overlap each other.
* @return Total chunk size in bytes for given sections including checksum.
@@ -315,16 +340,11 @@
return offsets.toArray(new Chunk[offsets.size()]);
}
- public void close()
- {
- chunkOffsets.close();
- }
-
public static class Writer extends Transactional.AbstractTransactional implements Transactional
{
// path to the file
private final CompressionParams parameters;
- private final String filePath;
+ private final File file;
private int maxCount = 100;
private SafeMemory offsets = new SafeMemory(maxCount * 8L);
private int count = 0;
@@ -332,15 +352,15 @@
// provided by user when setDescriptor
private long dataLength, chunkCount;
- private Writer(CompressionParams parameters, String path)
+ private Writer(CompressionParams parameters, File file)
{
this.parameters = parameters;
- filePath = path;
+ this.file = file;
}
- public static Writer open(CompressionParams parameters, String path)
+ public static Writer open(CompressionParams parameters, File file)
{
- return new Writer(parameters, path);
+ return new Writer(parameters, file);
}
public void addOffset(long offset)
@@ -375,7 +395,7 @@
}
catch (IOException e)
{
- throw new FSWriteError(e, filePath);
+ throw new FSWriteError(e, file);
}
}
@@ -387,6 +407,7 @@
return this;
}
+ @Override
public void doPrepare()
{
assert chunkCount == count;
@@ -401,7 +422,7 @@
}
// flush the data to disk
- try (FileOutputStreamPlus out = new FileOutputStreamPlus(filePath))
+ try (FileOutputStreamPlus out = file.newOutputStream(File.WriteMode.OVERWRITE))
{
writeHeader(out, dataLength, count);
for (int i = 0; i < count; i++)
@@ -412,11 +433,11 @@
}
catch (FileNotFoundException | NoSuchFileException fnfe)
{
- throw Throwables.propagate(fnfe);
+ throw new RuntimeException(fnfe);
}
catch (IOException e)
{
- throw new FSWriteError(e, filePath);
+ throw new FSWriteError(e, file);
}
}
@@ -435,7 +456,7 @@
if (tCount < this.count)
compressedLength = tOffsets.getLong(tCount * 8L);
- return new CompressionMetadata(filePath, parameters, tOffsets, tCount * 8L, dataLength, compressedLength);
+ return new CompressionMetadata(file, parameters, tOffsets, tCount * 8L, dataLength, compressedLength);
}
/**
@@ -461,16 +482,19 @@
count = chunkIndex;
}
+ @Override
protected Throwable doPostCleanup(Throwable failed)
{
return offsets.close(failed);
}
+ @Override
protected Throwable doCommit(Throwable accumulate)
{
return accumulate;
}
+ @Override
protected Throwable doAbort(Throwable accumulate)
{
return accumulate;
@@ -495,6 +519,7 @@
this.length = length;
}
+ @Override
public boolean equals(Object o)
{
if (this == o) return true;
@@ -504,6 +529,7 @@
return length == chunk.length && offset == chunk.offset;
}
+ @Override
public int hashCode()
{
int result = (int) (offset ^ (offset >>> 32));
@@ -511,6 +537,7 @@
return result;
}
+ @Override
public String toString()
{
return String.format("Chunk<offset: %d, length: %d>", offset, length);
@@ -519,17 +546,20 @@
static class ChunkSerializer implements IVersionedSerializer<Chunk>
{
+ @Override
public void serialize(Chunk chunk, DataOutputPlus out, int version) throws IOException
{
out.writeLong(chunk.offset);
out.writeInt(chunk.length);
}
+ @Override
public Chunk deserialize(DataInputPlus in, int version) throws IOException
{
return new Chunk(in.readLong(), in.readInt());
}
+ @Override
public long serializedSize(Chunk chunk, int version)
{
long size = TypeSizes.sizeof(chunk.offset);
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractMetricsProviders.java b/src/java/org/apache/cassandra/io/sstable/AbstractMetricsProviders.java
new file mode 100644
index 0000000..15b1161
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractMetricsProviders.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public abstract class AbstractMetricsProviders<R extends SSTableReader> implements MetricsProviders
+{
+ protected final <T extends Number> GaugeProvider<T> newGaugeProvider(String name, Function<Iterable<R>, T> combiner)
+ {
+ return new SimpleGaugeProvider<>(this::map, name, combiner);
+ }
+
+ protected final <T extends Number> GaugeProvider<T> newGaugeProvider(String name, T neutralValue, Function<R, T> extractor, BiFunction<T, T, T> combiner)
+ {
+ return new SimpleGaugeProvider<>(this::map, name, readers -> {
+ T total = neutralValue;
+ for (R reader : readers)
+ total = combiner.apply(total, extractor.apply(reader));
+ return total;
+ });
+ }
+
+ protected abstract R map(SSTableReader r);
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractRowIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/AbstractRowIndexEntry.java
new file mode 100644
index 0000000..7fcfbe2
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractRowIndexEntry.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+
+import org.apache.cassandra.cache.IMeasurableMemory;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * The base RowIndexEntry is not stored on disk, only specifies a position in the data file
+ */
+public abstract class AbstractRowIndexEntry implements IMeasurableMemory
+{
+ public final long position;
+
+ public AbstractRowIndexEntry(long position)
+ {
+ this.position = position;
+ }
+
+ /**
+ * Row position in a data file
+ */
+ public long getPosition()
+ {
+ return position;
+ }
+
+ /**
+ * @return true if this index entry contains the row-level tombstone and column summary. Otherwise,
+ * caller should fetch these from the row header.
+ */
+ public boolean isIndexed()
+ {
+ return blockCount() > 1;
+ }
+
+ public DeletionTime deletionTime()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public int blockCount()
+ {
+ return 0;
+ }
+
+ public abstract SSTableFormat<?, ?> getSSTableFormat();
+
+ /**
+ * Serialize this entry for key cache
+ *
+ * @param out the output stream for serialized entry
+ */
+ public abstract void serializeForCache(DataOutputPlus out) throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
new file mode 100644
index 0000000..46c2129
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
@@ -0,0 +1,585 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.UnfilteredDeserializer;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.utils.vint.VIntCoding.VIntOutOfRangeException;
+
+
+public abstract class AbstractSSTableIterator<RIE extends AbstractRowIndexEntry> implements UnfilteredRowIterator
+{
+ protected final SSTableReader sstable;
+ // We could use sstable.metadata(), but that can change during execution so it's good hygiene to grab an immutable instance
+ protected final TableMetadata metadata;
+
+ protected final DecoratedKey key;
+ protected final DeletionTime partitionLevelDeletion;
+ protected final ColumnFilter columns;
+ protected final DeserializationHelper helper;
+
+ protected final Row staticRow;
+ protected final Reader reader;
+
+ protected final FileHandle ifile;
+
+ private boolean isClosed;
+
+ protected final Slices slices;
+
+ @SuppressWarnings("resource") // We need this because the analysis is not able to determine that we do close
+ // file on every path where we created it.
+ protected AbstractSSTableIterator(SSTableReader sstable,
+ FileDataInput file,
+ DecoratedKey key,
+ RIE indexEntry,
+ Slices slices,
+ ColumnFilter columnFilter,
+ FileHandle ifile)
+ {
+ this.sstable = sstable;
+ this.metadata = sstable.metadata();
+ this.ifile = ifile;
+ this.key = key;
+ this.columns = columnFilter;
+ this.slices = slices;
+ this.helper = new DeserializationHelper(metadata, sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL, columnFilter);
+
+ if (indexEntry == null)
+ {
+ this.partitionLevelDeletion = DeletionTime.LIVE;
+ this.reader = null;
+ this.staticRow = Rows.EMPTY_STATIC_ROW;
+ }
+ else
+ {
+ boolean shouldCloseFile = file == null;
+ try
+ {
+ // We seek to the beginning to the partition if either:
+ // - the partition is not indexed; we then have a single block to read anyway
+ // (and we need to read the partition deletion time).
+ // - we're querying static columns.
+ boolean needSeekAtPartitionStart = !indexEntry.isIndexed() || !columns.fetchedColumns().statics.isEmpty();
+
+ if (needSeekAtPartitionStart)
+ {
+ // Not indexed (or is reading static), set to the beginning of the partition and read partition level deletion there
+ if (file == null)
+ file = sstable.getFileDataInput(indexEntry.position);
+ else
+ file.seek(indexEntry.position);
+
+ ByteBufferUtil.skipShortLength(file); // Skip partition key
+ this.partitionLevelDeletion = DeletionTime.serializer.deserialize(file);
+
+ // Note that this needs to be called after file != null and after the partitionDeletion has been set, but before readStaticRow
+ // (since it uses it) so we can't move that up (but we'll be able to simplify as soon as we drop support for the old file format).
+ this.reader = createReader(indexEntry, file, shouldCloseFile);
+ this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics);
+ }
+ else
+ {
+ this.partitionLevelDeletion = indexEntry.deletionTime();
+ this.staticRow = Rows.EMPTY_STATIC_ROW;
+ this.reader = createReader(indexEntry, file, shouldCloseFile);
+ }
+ if (!partitionLevelDeletion.validate())
+ UnfilteredValidation.handleInvalid(metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
+
+ if (reader != null && !slices.isEmpty())
+ reader.setForSlice(nextSlice());
+
+ if (reader == null && file != null && shouldCloseFile)
+ file.close();
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ String filePath = file.getPath();
+ if (shouldCloseFile)
+ {
+ try
+ {
+ file.close();
+ }
+ catch (IOException suppressed)
+ {
+ e.addSuppressed(suppressed);
+ }
+ }
+ throw new CorruptSSTableException(e, filePath);
+ }
+ }
+ }
+
+ private Slice nextSlice()
+ {
+ return slices.get(nextSliceIndex());
+ }
+
+ /**
+ * Returns the index of the next slice to process.
+ * @return the index of the next slice to process
+ */
+ protected abstract int nextSliceIndex();
+
+ /**
+ * Checks if there are more slice to process.
+ * @return {@code true} if there are more slice to process, {@code false} otherwise.
+ */
+ protected abstract boolean hasMoreSlices();
+
+ private static Row readStaticRow(SSTableReader sstable,
+ FileDataInput file,
+ DeserializationHelper helper,
+ Columns statics) throws IOException
+ {
+ if (!sstable.header.hasStatic())
+ return Rows.EMPTY_STATIC_ROW;
+
+ if (statics.isEmpty())
+ {
+ UnfilteredSerializer.serializer.skipStaticRow(file, sstable.header, helper);
+ return Rows.EMPTY_STATIC_ROW;
+ }
+ else
+ {
+ return UnfilteredSerializer.serializer.deserializeStaticRow(file, sstable.header, helper);
+ }
+ }
+
+ protected abstract Reader createReaderInternal(RIE indexEntry, FileDataInput file, boolean shouldCloseFile);
+
+ private Reader createReader(RIE indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ return slices.isEmpty() ? new NoRowsReader(file, shouldCloseFile)
+ : createReaderInternal(indexEntry, file, shouldCloseFile);
+ };
+
+ public TableMetadata metadata()
+ {
+ return metadata;
+ }
+
+ public RegularAndStaticColumns columns()
+ {
+ return columns.fetchedColumns();
+ }
+
+ public DecoratedKey partitionKey()
+ {
+ return key;
+ }
+
+ public DeletionTime partitionLevelDeletion()
+ {
+ return partitionLevelDeletion;
+ }
+
+ public Row staticRow()
+ {
+ return staticRow;
+ }
+
+ public EncodingStats stats()
+ {
+ return sstable.stats();
+ }
+
+ public boolean hasNext()
+ {
+ while (true)
+ {
+ if (reader == null)
+ return false;
+
+ if (reader.hasNext())
+ return true;
+
+ if (!hasMoreSlices())
+ return false;
+
+ slice(nextSlice());
+ }
+ }
+
+ public Unfiltered next()
+ {
+ assert reader != null;
+ return reader.next();
+ }
+
+ private void slice(Slice slice)
+ {
+ try
+ {
+ if (reader != null)
+ reader.setForSlice(slice);
+ }
+ catch (IOException e)
+ {
+ try
+ {
+ closeInternal();
+ }
+ catch (IOException suppressed)
+ {
+ e.addSuppressed(suppressed);
+ }
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, reader.toString());
+ }
+ }
+
+ public void remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ private void closeInternal() throws IOException
+ {
+ // It's important to make closing idempotent since it would bad to double-close 'file' as its a RandomAccessReader
+ // and its close is not idemptotent in the case where we recycle it.
+ if (isClosed)
+ return;
+
+ if (reader != null)
+ reader.close();
+
+ isClosed = true;
+ }
+
+ public void close()
+ {
+ try
+ {
+ closeInternal();
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, reader.toString());
+ }
+ }
+
+ public interface Reader extends Iterator<Unfiltered>, Closeable {
+
+ void setForSlice(Slice slice) throws IOException;
+
+ void seekToPosition(long columnOffset) throws IOException;
+ }
+
+ public abstract class AbstractReader implements Reader
+ {
+ private final boolean shouldCloseFile;
+ public FileDataInput file;
+
+ public UnfilteredDeserializer deserializer;
+
+ // Records the currently open range tombstone (if any)
+ public DeletionTime openMarker;
+
+ protected AbstractReader(FileDataInput file, boolean shouldCloseFile)
+ {
+ this.file = file;
+ this.shouldCloseFile = shouldCloseFile;
+
+ if (file != null)
+ createDeserializer();
+ }
+
+ private void createDeserializer()
+ {
+ assert file != null && deserializer == null;
+ deserializer = UnfilteredDeserializer.create(metadata, file, sstable.header, helper);
+ }
+
+ public void seekToPosition(long position) throws IOException
+ {
+ // This may be the first time we're actually looking into the file
+ if (file == null)
+ {
+ file = sstable.getFileDataInput(position);
+ createDeserializer();
+ }
+ else
+ {
+ file.seek(position);
+ deserializer.clearState();
+ }
+ }
+
+ protected void updateOpenMarker(RangeTombstoneMarker marker)
+ {
+ // Note that we always read index blocks in forward order so this method is always called in forward order
+ openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null;
+ }
+
+ public boolean hasNext()
+ {
+ try
+ {
+ return hasNextInternal();
+ }
+ catch (IOException | IndexOutOfBoundsException | VIntOutOfRangeException e)
+ {
+ try
+ {
+ closeInternal();
+ }
+ catch (IOException suppressed)
+ {
+ e.addSuppressed(suppressed);
+ }
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, toString());
+ }
+ }
+
+ public Unfiltered next()
+ {
+ try
+ {
+ return nextInternal();
+ }
+ catch (IOException e)
+ {
+ try
+ {
+ closeInternal();
+ }
+ catch (IOException suppressed)
+ {
+ e.addSuppressed(suppressed);
+ }
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, toString());
+ }
+ }
+
+ // Set the reader so its hasNext/next methods return values within the provided slice
+ public abstract void setForSlice(Slice slice) throws IOException;
+
+ protected abstract boolean hasNextInternal() throws IOException;
+ protected abstract Unfiltered nextInternal() throws IOException;
+
+ @Override
+ public void close() throws IOException
+ {
+ if (shouldCloseFile && file != null)
+ file.close();
+ }
+
+ @Override
+ public String toString()
+ {
+ return file != null ? file.toString() : "null";
+ }
+ }
+
+ protected class ForwardReader extends AbstractReader
+ {
+ // The start of the current slice. This will be null as soon as we know we've passed that bound.
+ protected ClusteringBound<?> start;
+ // The end of the current slice. Will never be null.
+ protected ClusteringBound<?> end = BufferClusteringBound.TOP;
+
+ protected Unfiltered next; // the next element to return: this is computed by hasNextInternal().
+
+ protected boolean sliceDone; // set to true once we know we have no more result for the slice. This is in particular
+ // used by the indexed reader when we know we can't have results based on the index.
+
+ public ForwardReader(FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ }
+
+ public void setForSlice(Slice slice) throws IOException
+ {
+ start = slice.start().isBottom() ? null : slice.start();
+ end = slice.end();
+
+ sliceDone = false;
+ next = null;
+ }
+
+ // Skip all data that comes before the currently set slice.
+ // Return what should be returned at the end of this, or null if nothing should.
+ private Unfiltered handlePreSliceData() throws IOException
+ {
+ assert deserializer != null;
+
+ // Note that the following comparison is not strict. The reason is that the only cases
+ // where it can be == is if the "next" is a RT start marker (either a '[' of a ')[' boundary),
+ // and if we had a strict inequality and an open RT marker before this, we would issue
+ // the open marker first, and then return then next later, which would send in the
+ // stream both '[' (or '(') and then ')[' for the same clustering value, which is wrong.
+ // By using a non-strict inequality, we avoid that problem (if we do get ')[' for the same
+ // clustering value than the slice, we'll simply record it in 'openMarker').
+ while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0)
+ {
+ if (deserializer.nextIsRow())
+ deserializer.skipNext();
+ else
+ updateOpenMarker((RangeTombstoneMarker)deserializer.readNext());
+ }
+
+ ClusteringBound<?> sliceStart = start;
+ start = null;
+
+ // We've reached the beginning of our queried slice. If we have an open marker
+ // we should return that first.
+ if (openMarker != null)
+ return new RangeTombstoneBoundMarker(sliceStart, openMarker);
+
+ return null;
+ }
+
+ // Compute the next element to return, assuming we're in the middle to the slice
+ // and the next element is either in the slice, or just after it. Returns null
+ // if we're done with the slice.
+ protected Unfiltered computeNext() throws IOException
+ {
+ assert deserializer != null;
+
+ while (true)
+ {
+ // We use a same reasoning as in handlePreSliceData regarding the strictness of the inequality below.
+ // We want to exclude deserialized unfiltered equal to end, because 1) we won't miss any rows since those
+ // woudn't be equal to a slice bound and 2) a end bound can be equal to a start bound
+ // (EXCL_END(x) == INCL_START(x) for instance) and in that case we don't want to return start bound because
+ // it's fundamentally excluded. And if the bound is a end (for a range tombstone), it means it's exactly
+ // our slice end, but in that case we will properly close the range tombstone anyway as part of our "close
+ // an open marker" code in hasNextInterna
+ if (!deserializer.hasNext() || deserializer.compareNextTo(end) >= 0)
+ return null;
+
+ Unfiltered next = deserializer.readNext();
+ UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable);
+ // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
+ if (next.isEmpty())
+ continue;
+
+ if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
+ updateOpenMarker((RangeTombstoneMarker) next);
+ return next;
+ }
+ }
+
+ protected boolean hasNextInternal() throws IOException
+ {
+ if (next != null)
+ return true;
+
+ if (sliceDone)
+ return false;
+
+ if (start != null)
+ {
+ Unfiltered unfiltered = handlePreSliceData();
+ if (unfiltered != null)
+ {
+ next = unfiltered;
+ return true;
+ }
+ }
+
+ next = computeNext();
+ if (next != null)
+ return true;
+
+ // for current slice, no data read from deserialization
+ sliceDone = true;
+ // If we have an open marker, we should not close it, there could be more slices
+ if (openMarker != null)
+ {
+ next = new RangeTombstoneBoundMarker(end, openMarker);
+ return true;
+ }
+ return false;
+ }
+
+ protected Unfiltered nextInternal() throws IOException
+ {
+ if (!hasNextInternal())
+ throw new NoSuchElementException();
+
+ Unfiltered toReturn = next;
+ next = null;
+ return toReturn;
+ }
+ }
+
+
+ // Reader for when we have Slices.NONE but need to read static row or partition level deletion
+ private class NoRowsReader extends AbstractReader
+ {
+ private NoRowsReader(FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ }
+
+ @Override
+ public void setForSlice(Slice slice)
+ {
+ // no-op
+ }
+
+ @Override
+ public boolean hasNextInternal()
+ {
+ return false;
+ }
+
+ @Override
+ protected Unfiltered nextInternal() throws IOException
+ {
+ throw new NoSuchElementException();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
index 49ceb76..429995c 100644
--- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
@@ -18,8 +18,8 @@
package org.apache.cassandra.io.sstable;
-import java.io.IOException;
import java.io.Closeable;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -27,7 +27,10 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
@@ -43,7 +46,7 @@
protected final File directory;
protected final TableMetadataRef metadata;
protected final RegularAndStaticColumns columns;
- protected SSTableFormat.Type formatType = SSTableFormat.Type.current();
+ protected SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
protected static final AtomicReference<SSTableId> id = new AtomicReference<>(SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
protected boolean makeRangeAware = false;
@@ -54,9 +57,9 @@
this.columns = columns;
}
- protected void setSSTableFormatType(SSTableFormat.Type type)
+ protected void setSSTableFormatType(SSTableFormat<?, ?> type)
{
- this.formatType = type;
+ this.format = type;
}
protected void setRangeAwareWriting(boolean makeRangeAware)
@@ -64,25 +67,26 @@
this.makeRangeAware = makeRangeAware;
}
- protected SSTableTxnWriter createWriter() throws IOException
+ protected SSTableTxnWriter createWriter(SSTable.Owner owner) throws IOException
{
SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS);
if (makeRangeAware)
- return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, formatType, 0, header);
+ return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, 0, header);
return SSTableTxnWriter.create(metadata,
- createDescriptor(directory, metadata.keyspace, metadata.name, formatType),
+ createDescriptor(directory, metadata.keyspace, metadata.name, format),
0,
ActiveRepairService.UNREPAIRED_SSTABLE,
ActiveRepairService.NO_PENDING_REPAIR,
false,
0,
header,
- Collections.emptySet());
+ Collections.emptySet(),
+ owner);
}
- private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt) throws IOException
+ private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat<?, ?> fmt) throws IOException
{
SSTableId nextGen = getNextId(directory, columnFamily);
return new Descriptor(directory, keyspace, columnFamily, nextGen, fmt);
@@ -95,7 +99,7 @@
try (Stream<Path> existingPaths = Files.list(directory.toPath()))
{
Stream<SSTableId> existingIds = existingPaths.map(File::new)
- .map(SSTable::tryDescriptorFromFilename)
+ .map(SSTable::tryDescriptorFromFile)
.filter(d -> d != null && d.cfname.equals(columnFamily))
.map(d -> d.id);
diff --git a/src/java/org/apache/cassandra/io/sstable/BloomFilterTracker.java b/src/java/org/apache/cassandra/io/sstable/BloomFilterTracker.java
deleted file mode 100644
index 07523a0..0000000
--- a/src/java/org/apache/cassandra/io/sstable/BloomFilterTracker.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class BloomFilterTracker
-{
- private final AtomicLong falsePositiveCount = new AtomicLong(0);
- private final AtomicLong truePositiveCount = new AtomicLong(0);
- private final AtomicLong trueNegativeCount = new AtomicLong(0);
- private long lastFalsePositiveCount = 0L;
- private long lastTruePositiveCount = 0L;
- private long lastTrueNegativeCount = 0L;
-
- public void addFalsePositive()
- {
- falsePositiveCount.incrementAndGet();
- }
-
- public void addTruePositive()
- {
- truePositiveCount.incrementAndGet();
- }
-
- public void addTrueNegative()
- {
- trueNegativeCount.incrementAndGet();
- }
-
- public long getFalsePositiveCount()
- {
- return falsePositiveCount.get();
- }
-
- public long getRecentFalsePositiveCount()
- {
- long fpc = getFalsePositiveCount();
- try
- {
- return (fpc - lastFalsePositiveCount);
- }
- finally
- {
- lastFalsePositiveCount = fpc;
- }
- }
-
- public long getTruePositiveCount()
- {
- return truePositiveCount.get();
- }
-
- public long getRecentTruePositiveCount()
- {
- long tpc = getTruePositiveCount();
- try
- {
- return (tpc - lastTruePositiveCount);
- }
- finally
- {
- lastTruePositiveCount = tpc;
- }
- }
-
- public long getTrueNegativeCount()
- {
- return trueNegativeCount.get();
- }
-
- public long getRecentTrueNegativeCount()
- {
- long tnc = getTrueNegativeCount();
- try
- {
- return (tnc - lastTrueNegativeCount);
- }
- finally
- {
- lastTrueNegativeCount = tnc;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
index 6d54561..d0bcafd 100644
--- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
@@ -34,8 +34,6 @@
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
-import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
@@ -44,6 +42,8 @@
import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.functions.types.UserType;
import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.Slices;
@@ -54,7 +54,17 @@
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaTransformations;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
+import org.apache.cassandra.schema.Views;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -366,7 +376,7 @@
{
private File directory;
- protected SSTableFormat.Type formatType = null;
+ protected SSTableFormat<?, ?> format = null;
private CreateTableStatement.Raw schemaStatement;
private final List<CreateTypeStatement.Raw> typeStatements;
@@ -563,11 +573,11 @@
String keyspaceName = schemaStatement.keyspace();
Schema.instance.transform(SchemaTransformations.addKeyspace(KeyspaceMetadata.create(keyspaceName,
- KeyspaceParams.simple(1),
- Tables.none(),
- Views.none(),
- Types.none(),
- Functions.none()), true));
+ KeyspaceParams.simple(1),
+ Tables.none(),
+ Views.none(),
+ Types.none(),
+ UserFunctions.none()), true));
KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName);
@@ -587,8 +597,8 @@
? new SSTableSimpleWriter(directory, ref, preparedModificationStatement.updatedColumns())
: new SSTableSimpleUnsortedWriter(directory, ref, preparedModificationStatement.updatedColumns(), bufferSizeInMiB);
- if (formatType != null)
- writer.setSSTableFormatType(formatType);
+ if (format != null)
+ writer.setSSTableFormatType(format);
return new CQLSSTableWriter(writer, preparedModificationStatement, preparedModificationStatement.getBindVariables());
}
diff --git a/src/java/org/apache/cassandra/io/sstable/Component.java b/src/java/org/apache/cassandra/io/sstable/Component.java
index d87d8b7..f2eea99 100644
--- a/src/java/org/apache/cassandra/io/sstable/Component.java
+++ b/src/java/org/apache/cassandra/io/sstable/Component.java
@@ -17,11 +17,18 @@
*/
package org.apache.cassandra.io.sstable;
-import java.util.EnumSet;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types;
/**
* SSTables are made up of multiple components in separate files. Components are
@@ -32,84 +39,149 @@
{
public static final char separator = '-';
- final static EnumSet<Type> TYPES = EnumSet.allOf(Type.class);
-
/**
* WARNING: Be careful while changing the names or string representation of the enum
* members. Streaming code depends on the names during streaming (Ref: CASSANDRA-14556).
*/
- public enum Type
+ public final static class Type
{
- // the base data for an sstable: the remaining components can be regenerated
- // based on the data component
- DATA("Data.db"),
- // index of the row keys with pointers to their positions in the data file
- PRIMARY_INDEX("Index.db"),
- // serialized bloom filter for the row keys in the sstable
- FILTER("Filter.db"),
- // file to hold information about uncompressed data length, chunk offsets etc.
- COMPRESSION_INFO("CompressionInfo.db"),
- // statistical metadata about the content of the sstable
- STATS("Statistics.db"),
- // holds CRC32 checksum of the data file
- DIGEST("Digest.crc32"),
- // holds the CRC32 for chunks in an a uncompressed file.
- CRC("CRC.db"),
- // holds SSTable Index Summary (sampling of Index component)
- SUMMARY("Summary.db"),
- // table of contents, stores the list of all components for the sstable
- TOC("TOC.txt"),
- // built-in secondary index (may be multiple per sstable)
- SECONDARY_INDEX("SI_.*.db"),
- // custom component, used by e.g. custom compaction strategy
- CUSTOM(null);
+ private final static CopyOnWriteArrayList<Type> typesCollector = new CopyOnWriteArrayList<>();
- final String repr;
+ public static final List<Type> all = Collections.unmodifiableList(typesCollector);
- Type(String repr)
+ public final int id;
+ public final String name;
+ public final String repr;
+ private final Component singleton;
+
+ @SuppressWarnings("rawtypes")
+ public final Class<? extends SSTableFormat> formatClass;
+
+ /**
+ * Creates a new non-singleton type and registers it a global type registry - see {@link #registerType(Type)}.
+ *
+ * @param name type name, must be unique for this and all parent formats
+ * @param repr the regular expression to be used to recognize a name represents this type
+ * @param formatClass format class for which this type is defined for
+ */
+ public static Type create(String name, String repr, Class<? extends SSTableFormat<?, ?>> formatClass)
{
+ return new Type(name, repr, false, formatClass);
+ }
+
+ /**
+ * Creates a new singleton type and registers it in a global type registry - see {@link #registerType(Type)}.
+ *
+ * @param name type name, must be unique for this and all parent formats
+ * @param repr the regular expression to be used to recognize a name represents this type
+ * @param formatClass format class for which this type is defined for
+ */
+ public static Type createSingleton(String name, String repr, Class<? extends SSTableFormat<?, ?>> formatClass)
+ {
+ return new Type(name, repr, true, formatClass);
+ }
+
+ private Type(String name, String repr, boolean isSingleton, Class<? extends SSTableFormat<?, ?>> formatClass)
+ {
+ this.name = Objects.requireNonNull(name);
this.repr = repr;
+ this.id = typesCollector.size();
+ this.formatClass = formatClass == null ? SSTableFormat.class : formatClass;
+ this.singleton = isSingleton ? new Component(this) : null;
+
+ registerType(this);
+ }
+
+ /**
+ * If you have two formats registered, they may both define a type say `INDEX`. It is allowed even though
+ * they have the same name because they are not in the same branch. Though, we cannot let a custom type
+ * define a type `TOC` which is declared on the top level.
+ * So, e.g. given we have `TOC@SSTableFormat`, and `BigFormat` tries to define `TOC@BigFormat`, we should
+ * forbid that; but, given we have `INDEX@BigFormat`, we should allow to define `INDEX@TrieFormat` as those
+ * types are be distinguishable via format type.
+ *
+ * @param type a type to be registered
+ */
+ private static void registerType(Type type)
+ {
+ synchronized (typesCollector)
+ {
+ if (typesCollector.stream().anyMatch(t -> (Objects.equals(t.name, type.name) || Objects.equals(t.repr, type.repr)) && (t.formatClass.isAssignableFrom(type.formatClass))))
+ throw new AssertionError("Type named " + type.name + " is already registered");
+
+ typesCollector.add(type);
+ }
}
@VisibleForTesting
- public static Type fromRepresentation(String repr)
+ public static Type fromRepresentation(String repr, SSTableFormat<?, ?> format)
{
- for (Type type : TYPES)
+ for (Type type : Type.all)
{
- if (type.repr != null && Pattern.matches(type.repr, repr))
+ if (type.repr != null && Pattern.matches(type.repr, repr) && type.formatClass.isAssignableFrom(format.getClass()))
return type;
}
- return CUSTOM;
+ return Types.CUSTOM;
+ }
+
+ public static Component createComponent(String repr, SSTableFormat<?, ?> format)
+ {
+ Type type = fromRepresentation(repr, format);
+ if (type.singleton != null)
+ return type.singleton;
+ else
+ return new Component(type, repr);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Type type = (Type) o;
+ return id == type.id;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return id;
+ }
+
+ @Override
+ public String toString()
+ {
+ return name;
+ }
+
+ public Component getSingleton()
+ {
+ return Objects.requireNonNull(singleton);
+ }
+
+ public Component createComponent(String repr)
+ {
+ Preconditions.checkArgument(singleton == null);
+ return new Component(this, repr);
}
}
- // singleton components for types that don't need ids
- public final static Component DATA = new Component(Type.DATA);
- public final static Component PRIMARY_INDEX = new Component(Type.PRIMARY_INDEX);
- public final static Component FILTER = new Component(Type.FILTER);
- public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO);
- public final static Component STATS = new Component(Type.STATS);
- public final static Component DIGEST = new Component(Type.DIGEST);
- public final static Component CRC = new Component(Type.CRC);
- public final static Component SUMMARY = new Component(Type.SUMMARY);
- public final static Component TOC = new Component(Type.TOC);
-
public final Type type;
public final String name;
public final int hashCode;
- public Component(Type type)
+ private Component(Type type)
{
this(type, type.repr);
- assert type != Type.CUSTOM;
}
- public Component(Type type, String name)
+ private Component(Type type, String name)
{
assert name != null : "Component name cannot be null";
+
this.type = type;
this.name = name;
- this.hashCode = Objects.hashCode(type, name);
+ this.hashCode = Objects.hash(type, name);
}
/**
@@ -127,26 +199,24 @@
* @return the component corresponding to {@code name}. Note that this always return a component as an unrecognized
* name is parsed into a CUSTOM component.
*/
- public static Component parse(String name)
+ public static Component parse(String name, SSTableFormat<?, ?> format)
{
- Type type = Type.fromRepresentation(name);
+ return Type.createComponent(name, format);
+ }
- // Build (or retrieve singleton for) the component object
- switch (type)
- {
- case DATA: return Component.DATA;
- case PRIMARY_INDEX: return Component.PRIMARY_INDEX;
- case FILTER: return Component.FILTER;
- case COMPRESSION_INFO: return Component.COMPRESSION_INFO;
- case STATS: return Component.STATS;
- case DIGEST: return Component.DIGEST;
- case CRC: return Component.CRC;
- case SUMMARY: return Component.SUMMARY;
- case TOC: return Component.TOC;
- case SECONDARY_INDEX: return new Component(Type.SECONDARY_INDEX, name);
- case CUSTOM: return new Component(Type.CUSTOM, name);
- default: throw new AssertionError();
- }
+ public static Iterable<Component> getSingletonsFor(SSTableFormat<?, ?> format)
+ {
+ return Iterables.transform(Iterables.filter(Type.all, t -> t.singleton != null && t.formatClass.isAssignableFrom(format.getClass())), t -> t.singleton);
+ }
+
+ public static Iterable<Component> getSingletonsFor(Class<? extends SSTableFormat<?, ?>> formatClass)
+ {
+ return Iterables.transform(Iterables.filter(Type.all, t -> t.singleton != null && t.formatClass.isAssignableFrom(formatClass)), t -> t.singleton);
+ }
+
+ public boolean isValidFor(Descriptor descriptor)
+ {
+ return type.formatClass.isAssignableFrom(descriptor.version.format.getClass());
}
@Override
@@ -162,8 +232,8 @@
return true;
if (!(o instanceof Component))
return false;
- Component that = (Component)o;
- return this.type == that.type && this.name.equals(that.name);
+ Component that = (Component) o;
+ return this.hashCode == that.hashCode && this.type == that.type && this.name.equals(that.name);
}
@Override
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index 75d5185..d77d1f1 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -17,13 +17,21 @@
*/
package org.apache.cassandra.io.sstable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.Version;
@@ -32,6 +40,7 @@
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.Pair;
+import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.cassandra.io.sstable.Component.separator;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -44,6 +53,24 @@
*/
public class Descriptor
{
+ private static final Logger logger = LoggerFactory.getLogger(Descriptor.class);
+
+ // Current SSTable directory format is {keyspace}/{tableName}-{tableId}[/backups|/snapshots/{tag}][/.{indexName}]/{component}.db
+ // * {var} are mandatory components
+ // * [var] are optional components
+ static final Pattern SSTABLE_DIR_PATTERN = Pattern.compile(".*/(?<keyspace>\\w+)/" +
+ "(?<tableName>\\w+)-(?<tableId>[0-9a-f]{32})/" +
+ "(backups/|snapshots/(?<tag>[\\w-]+)/)?" +
+ "(\\.(?<indexName>[\\w-]+)/)?" +
+ "(?<component>[\\w-]+)\\.(?<ext>[\\w]+)$");
+
+ // Pre 2.1 SSTable directory format is {keyspace}/{tableName}-{tableId}[/backups|/snapshots/{tag}][/.{indexName}]/{component}.db
+ static final Pattern LEGACY_SSTABLE_DIR_PATTERN = Pattern.compile(".*/(?<keyspace>\\w+)/" +
+ "(?<tableName>\\w+)/" +
+ "(backups/|snapshots/(?<tag>[\\w-]+)/)?" +
+ "(\\.(?<indexName>[\\w-]+)/)?" +
+ "(?<component>[\\w-]+)\\.(?<ext>[\\w]+)$");
+
private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$";
private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);
@@ -60,8 +87,9 @@
public final String ksname;
public final String cfname;
public final SSTableId id;
- public final SSTableFormat.Type formatType;
private final int hashCode;
+ private final String prefix;
+ private final File baseFile;
/**
* A descriptor that assumes CURRENT_VERSION.
@@ -69,75 +97,90 @@
@VisibleForTesting
public Descriptor(File directory, String ksname, String cfname, SSTableId id)
{
- this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, id, SSTableFormat.Type.current());
+ this(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion(), directory, ksname, cfname, id);
}
/**
* Constructor for sstable writers only.
*/
- public Descriptor(File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType)
+ public Descriptor(File directory, String ksname, String cfname, SSTableId id, SSTableFormat<?, ?> format)
{
- this(formatType.info.getLatestVersion(), directory, ksname, cfname, id, formatType);
+ this(format.getLatestVersion(), directory, ksname, cfname, id);
}
@VisibleForTesting
- public Descriptor(String version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType)
+ public Descriptor(String version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat<?, ?> format)
{
- this(formatType.info.getVersion(version), directory, ksname, cfname, id, formatType);
+ this(format.getVersion(version), directory, ksname, cfname, id);
}
- public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType)
+ public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id)
{
- assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass());
+ checkNotNull(version);
+ checkNotNull(directory);
+ checkNotNull(ksname);
+ checkNotNull(cfname);
+
this.version = version;
this.directory = directory.toCanonical();
this.ksname = ksname;
this.cfname = cfname;
this.id = id;
- this.formatType = formatType;
+
+ StringBuilder buf = new StringBuilder();
+ appendFileName(buf);
+ this.prefix = buf.toString();
+ this.baseFile = new File(directory.toPath().resolve(prefix));
// directory is unnecessary for hashCode, and for simulator consistency we do not include it
- hashCode = Objects.hashCode(version, id, ksname, cfname, formatType);
+ hashCode = Objects.hashCode(version, id, ksname, cfname);
}
- public String tmpFilenameFor(Component component)
+ private String tmpFilenameFor(Component component)
{
- return filenameFor(component) + TMP_EXT;
+ return fileFor(component) + TMP_EXT;
}
- /**
- * @return a unique temporary file name for given component during entire-sstable-streaming.
- */
- public String tmpFilenameForStreaming(Component component)
+ public File tmpFileFor(Component component)
+ {
+ return new File(directory.toPath().resolve(tmpFilenameFor(component)));
+ }
+
+ private String tmpFilenameForStreaming(Component component)
{
// Use UUID to handle concurrent streamings on the same sstable.
// TMP_EXT allows temp file to be removed by {@link ColumnFamilyStore#scrubDataDirectories}
return String.format("%s.%s%s", filenameFor(component), nextTimeUUID(), TMP_EXT);
}
- public String filenameFor(Component component)
+ /**
+ * @return a unique temporary file name for given component during entire-sstable-streaming.
+ */
+ public File tmpFileForStreaming(Component component)
{
- return baseFilename() + separator + component.name();
+ return new File(directory.toPath().resolve(tmpFilenameForStreaming(component)));
+ }
+
+ private String filenameFor(Component component)
+ {
+ return prefix + separator + component.name();
}
public File fileFor(Component component)
{
- return new File(filenameFor(component));
+ return new File(directory.toPath().resolve(filenameFor(component)));
}
- public String baseFilename()
+ public File baseFile()
{
- StringBuilder buff = new StringBuilder();
- buff.append(directory).append(File.pathSeparator());
- appendFileName(buff);
- return buff.toString();
+ return baseFile;
}
private void appendFileName(StringBuilder buff)
{
buff.append(version).append(separator);
buff.append(id.toString());
- buff.append(separator).append(formatType.name);
+ buff.append(separator).append(version.format.name());
}
public String relativeFilenameFor(Component component)
@@ -153,9 +196,9 @@
return buff.toString();
}
- public SSTableFormat getFormat()
+ public SSTableFormat<?, ?> getFormat()
{
- return formatType.info;
+ return version.format;
}
/** Return any temporary files found in the directory */
@@ -171,6 +214,22 @@
return ret;
}
+ /**
+ * Returns the set of components consisting of the provided mandatory components and those optional components
+ * for which the corresponding file exists.
+ */
+ public Set<Component> getComponents(Set<Component> mandatory, Set<Component> optional)
+ {
+ ImmutableSet.Builder<Component> builder = ImmutableSet.builder();
+ builder.addAll(mandatory);
+ for (Component component : optional)
+ {
+ if (fileFor(component).exists())
+ builder.add(component);
+ }
+ return builder.build();
+ }
+
public static boolean isValidFile(File file)
{
String filename = file.name();
@@ -180,23 +239,6 @@
/**
* Parse a sstable filename into a Descriptor.
* <p>
- * This is a shortcut for {@code fromFilename(new File(filename))}.
- *
- * @param filename the filename to a sstable component.
- * @return the descriptor for the parsed file.
- *
- * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
- * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
- * versions.
- */
- public static Descriptor fromFilename(String filename)
- {
- return fromFilename(new File(filename));
- }
-
- /**
- * Parse a sstable filename into a Descriptor.
- * <p>
* SSTables files are all located within subdirectories of the form {@code <keyspace>/<table>/}. Normal sstables are
* are directly within that subdirectory structure while 2ndary index, backups and snapshot are each inside an
* additional subdirectory. The file themselves have the form:
@@ -211,13 +253,31 @@
* mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
* versions.
*/
- public static Descriptor fromFilename(File file)
+ public static Descriptor fromFile(File file)
{
- return fromFilenameWithComponent(file).left;
+ return fromFileWithComponent(file).left;
+ }
+
+ public static Component componentFromFile(File file)
+ {
+ String name = file.name();
+ List<String> tokens = filenameTokens(name);
+
+ return Component.parse(tokens.get(3), formatFromName(name, tokens));
+ }
+
+ private static SSTableFormat<?, ?> formatFromName(String fileName, List<String> tokens)
+ {
+ String formatString = tokens.get(2);
+ SSTableFormat<?, ?> format = DatabaseDescriptor.getSSTableFormats().get(formatString);
+ if (format == null)
+ throw invalidSSTable(fileName, "unknown 'format' part (%s)", formatString);
+ return format;
}
/**
* Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part.
+ * The keyspace/table name will be extracted from the directory path.
*
* @param file the {@code File} object for the filename to parse.
* @return a pair of the descriptor and component corresponding to the provided {@code file}.
@@ -226,14 +286,77 @@
* mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
* versions.
*/
- public static Pair<Descriptor, Component> fromFilenameWithComponent(File file)
+ public static Pair<Descriptor, Component> fromFileWithComponent(File file)
+ {
+ return fromFileWithComponent(file, true);
+ }
+
+ public static Pair<Descriptor, Component> fromFileWithComponent(File file, boolean validateDirs)
{
// We need to extract the keyspace and table names from the parent directories, so make sure we deal with the
// absolute path.
if (!file.isAbsolute())
file = file.toAbsolute();
+ SSTableInfo info = validateAndExtractInfo(file);
String name = file.name();
+
+ String keyspaceName = "";
+ String tableName = "";
+
+ Matcher sstableDirMatcher = SSTABLE_DIR_PATTERN.matcher(file.toString());
+
+ // Use pre-2.1 SSTable format if current one does not match it
+ if (!sstableDirMatcher.find(0))
+ {
+ sstableDirMatcher = LEGACY_SSTABLE_DIR_PATTERN.matcher(file.toString());
+ }
+
+ if (sstableDirMatcher.find(0))
+ {
+ keyspaceName = sstableDirMatcher.group("keyspace");
+ tableName = sstableDirMatcher.group("tableName");
+ String indexName = sstableDirMatcher.group("indexName");
+ if (indexName != null)
+ {
+ tableName = String.format("%s.%s", tableName, indexName);
+ }
+ }
+ else if (validateDirs)
+ {
+ logger.debug("Could not extract keyspace/table info from sstable directory {}", file.toString());
+ throw invalidSSTable(name, String.format("cannot extract keyspace and table name from %s; make sure the sstable is in the proper sub-directories", file));
+ }
+
+ return Pair.create(new Descriptor(info.version, parentOf(name, file), keyspaceName, tableName, info.id), info.component);
+ }
+
+ /**
+ * Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part.
+ *
+ * @param file the {@code File} object for the filename to parse.
+ * @param keyspace The keyspace name of the file. If <code>null</code>, then the keyspace name will be extracted
+ * from the directory path.
+ * @param table The table name of the file. If <code>null</code>, then the table name will be extracted from the
+ * directory path.
+ * @return a pair of the descriptor and component corresponding to the provided {@code file}.
+ * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
+ * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
+ * versions.
+ */
+ public static Pair<Descriptor, Component> fromFileWithComponent(File file, String keyspace, String table)
+ {
+ if (null == keyspace || null == table)
+ {
+ return fromFileWithComponent(file);
+ }
+
+ SSTableInfo info = validateAndExtractInfo(file);
+ return Pair.create(new Descriptor(info.version, parentOf(file.name(), file), keyspace, table, info.id), info.component);
+ }
+
+ private static List<String> filenameTokens(String name)
+ {
List<String> tokens = filenameSplitter.splitToList(name);
int size = tokens.size();
@@ -245,11 +368,16 @@
// Note that we assume it's an old format sstable if it has the right number of tokens: this is not perfect
// but we're just trying to be helpful, not perfect.
if (size == 5 || size == 6)
- throw new IllegalArgumentException(String.format("%s is of version %s which is now unsupported and cannot be read.",
- name,
- tokens.get(size - 3)));
+ throw new IllegalArgumentException(String.format("%s is of version %s which is now unsupported and cannot be read.", name, tokens.get(size - 3)));
throw new IllegalArgumentException(String.format("Invalid sstable file %s: the name doesn't look like a supported sstable file name", name));
}
+ return tokens;
+ }
+
+ private static SSTableInfo validateAndExtractInfo(File file)
+ {
+ String name = file.name();
+ List<String> tokens = filenameTokens(name);
String versionString = tokens.get(0);
if (!Version.validate(versionString))
@@ -265,49 +393,28 @@
throw invalidSSTable(name, "the 'id' part (%s) of the name doesn't parse as a valid unique identifier", tokens.get(1));
}
- String formatString = tokens.get(2);
- SSTableFormat.Type format;
- try
- {
- format = SSTableFormat.Type.validate(formatString);
- }
- catch (RuntimeException e)
- {
- throw invalidSSTable(name, "unknown 'format' part (%s)", formatString);
- }
+ SSTableFormat<?, ?> format = formatFromName(name, tokens);
+ Component component = Component.parse(tokens.get(3), format);
- Component component = Component.parse(tokens.get(3));
-
- Version version = format.info.getVersion(versionString);
+ Version version = format.getVersion(versionString);
if (!version.isCompatible())
throw invalidSSTable(name, "incompatible sstable version (%s); you should have run upgradesstables before upgrading", versionString);
- File directory = parentOf(name, file);
- File tableDir = directory;
+ return new SSTableInfo(version, id, component);
+ }
- // Check if it's a 2ndary index directory (not that it doesn't exclude it to be also a backup or snapshot)
- String indexName = "";
- if (Directories.isSecondaryIndexFolder(tableDir))
+ private static class SSTableInfo
+ {
+ final Version version;
+ final SSTableId id;
+ final Component component;
+
+ SSTableInfo(Version version, SSTableId id, Component component)
{
- indexName = tableDir.name();
- tableDir = parentOf(name, tableDir);
+ this.version = version;
+ this.id = id;
+ this.component = component;
}
-
- // Then it can be a backup or a snapshot
- if (tableDir.name().equals(Directories.BACKUPS_SUBDIR) && tableDir.parent().name().contains("-"))
- tableDir = tableDir.parent();
- else
- {
- File keyspaceOrSnapshotDir = parentOf(name, tableDir);
- if (keyspaceOrSnapshotDir.name().equals(Directories.SNAPSHOT_SUBDIR)
- && parentOf(name, keyspaceOrSnapshotDir).name().contains("-"))
- tableDir = parentOf(name, keyspaceOrSnapshotDir);
- }
-
- String table = tableDir.name().split("-")[0] + indexName;
- String keyspace = parentOf(name, tableDir).name();
-
- return Pair.create(new Descriptor(version, directory, keyspace, table, id, format), component);
}
private static File parentOf(String name, File file)
@@ -336,10 +443,21 @@
return version.isCompatible();
}
+ public Set<Component> discoverComponents()
+ {
+ Set<Component> components = Sets.newHashSetWithExpectedSize(Component.Type.all.size());
+ for (Component component : Component.getSingletonsFor(version.format))
+ {
+ if (fileFor(component).exists())
+ components.add(component);
+ }
+ return components;
+ }
+
@Override
public String toString()
{
- return baseFilename();
+ return baseFile().absolutePath();
}
@Override
@@ -350,12 +468,13 @@
if (!(o instanceof Descriptor))
return false;
Descriptor that = (Descriptor)o;
+ if (this.hashCode != that.hashCode)
+ return false;
return that.directory.equals(this.directory)
&& that.id.equals(this.id)
&& that.ksname.equals(this.ksname)
&& that.cfname.equals(this.cfname)
- && that.version.equals(this.version)
- && that.formatType == this.formatType;
+ && that.version.equals(this.version);
}
@Override
diff --git a/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
new file mode 100644
index 0000000..8976ed4
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator implements ISSTableScanner
+{
+ private final SSTableReader sstable;
+
+ public EmptySSTableScanner(SSTableReader sstable)
+ {
+ this.sstable = sstable;
+ }
+
+ public long getBytesScanned()
+ {
+ return 0;
+ }
+
+ public long getLengthInBytes()
+ {
+ return 0;
+ }
+
+ public long getCompressedLengthInBytes()
+ {
+ return 0;
+ }
+
+ public Set<SSTableReader> getBackingSSTables()
+ {
+ return ImmutableSet.of(sstable);
+ }
+
+ public long getCurrentPosition()
+ {
+ return 0;
+ }
+
+ public TableMetadata metadata()
+ {
+ return sstable.metadata();
+ }
+
+ public boolean hasNext()
+ {
+ return false;
+ }
+
+ public UnfilteredRowIterator next()
+ {
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/GaugeProvider.java b/src/java/org/apache/cassandra/io/sstable/GaugeProvider.java
new file mode 100644
index 0000000..d0680be
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/GaugeProvider.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+
+public abstract class GaugeProvider<T extends Number>
+{
+ public final String name;
+
+ public GaugeProvider(String name)
+ {
+ this.name = name;
+ }
+
+ public abstract Gauge<T> getTableGauge(ColumnFamilyStore cfs);
+
+ public abstract Gauge<T> getKeyspaceGauge(Keyspace keyspace);
+
+ public abstract Gauge<T> getGlobalGauge();
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/IOOptions.java b/src/java/org/apache/cassandra/io/sstable/IOOptions.java
new file mode 100644
index 0000000..1f395a7
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/IOOptions.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DiskOptimizationStrategy;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+
+public class IOOptions
+{
+ public static IOOptions fromDatabaseDescriptor()
+ {
+ return new IOOptions(DatabaseDescriptor.getDiskOptimizationStrategy(),
+ DatabaseDescriptor.getDiskAccessMode(),
+ DatabaseDescriptor.getIndexAccessMode(),
+ DatabaseDescriptor.getDiskOptimizationEstimatePercentile(),
+ SequentialWriterOption.newBuilder()
+ .trickleFsync(DatabaseDescriptor.getTrickleFsync())
+ .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024)
+ .build(),
+ DatabaseDescriptor.getFlushCompression());
+ }
+
+ public final DiskOptimizationStrategy diskOptimizationStrategy;
+ public final Config.DiskAccessMode defaultDiskAccessMode;
+ public final Config.DiskAccessMode indexDiskAccessMode;
+ public final double diskOptimizationEstimatePercentile;
+ public final SequentialWriterOption writerOptions;
+ public final Config.FlushCompression flushCompression;
+
+ public IOOptions(DiskOptimizationStrategy diskOptimizationStrategy,
+ Config.DiskAccessMode defaultDiskAccessMode,
+ Config.DiskAccessMode indexDiskAccessMode,
+ double diskOptimizationEstimatePercentile,
+ SequentialWriterOption writerOptions,
+ Config.FlushCompression flushCompression)
+ {
+ this.diskOptimizationStrategy = diskOptimizationStrategy;
+ this.defaultDiskAccessMode = defaultDiskAccessMode;
+ this.indexDiskAccessMode = indexDiskAccessMode;
+ this.diskOptimizationEstimatePercentile = diskOptimizationEstimatePercentile;
+ this.writerOptions = writerOptions;
+ this.flushCompression = flushCompression;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
index af661b7..671bccb 100644
--- a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
@@ -64,7 +64,8 @@
if (throwable != null)
{
- Throwables.propagate(throwable);
+ Throwables.throwIfUnchecked(throwable);
+ throw new RuntimeException(throwable);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/IScrubber.java b/src/java/org/apache/cassandra/io/sstable/IScrubber.java
new file mode 100644
index 0000000..50c6eb3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/IScrubber.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.StringJoiner;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.utils.Closeable;
+
+public interface IScrubber extends Closeable
+{
+ void scrub();
+
+ void close();
+
+ CompactionInfo.Holder getScrubInfo();
+
+ @VisibleForTesting
+ ScrubResult scrubWithResult();
+
+ static Options.Builder options()
+ {
+ return new Options.Builder();
+ }
+
+ final class ScrubResult
+ {
+ public final int goodPartitions;
+ public final int badPartitions;
+ public final int emptyPartitions;
+
+ public ScrubResult(int goodPartitions, int badPartitions, int emptyPartitions)
+ {
+ this.goodPartitions = goodPartitions;
+ this.badPartitions = badPartitions;
+ this.emptyPartitions = emptyPartitions;
+ }
+ }
+
+ class Options
+ {
+ public final boolean checkData;
+ public final boolean reinsertOverflowedTTLRows;
+ public final boolean skipCorrupted;
+
+ private Options(boolean checkData, boolean reinsertOverflowedTTLRows, boolean skipCorrupted)
+ {
+ this.checkData = checkData;
+ this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows;
+ this.skipCorrupted = skipCorrupted;
+ }
+
+ @Override
+ public String toString()
+ {
+ return new StringJoiner(", ", Options.class.getSimpleName() + "[", "]")
+ .add("checkData=" + checkData)
+ .add("reinsertOverflowedTTLRows=" + reinsertOverflowedTTLRows)
+ .add("skipCorrupted=" + skipCorrupted)
+ .toString();
+ }
+
+ public static class Builder
+ {
+ private boolean checkData = false;
+ private boolean reinsertOverflowedTTLRows = false;
+ private boolean skipCorrupted = false;
+
+ public Builder checkData()
+ {
+ this.checkData = true;
+ return this;
+ }
+
+ public Builder checkData(boolean checkData)
+ {
+ this.checkData = checkData;
+ return this;
+ }
+
+ public Builder reinsertOverflowedTTLRows()
+ {
+ this.reinsertOverflowedTTLRows = true;
+ return this;
+ }
+
+ public Builder reinsertOverflowedTTLRows(boolean reinsertOverflowedTTLRows)
+ {
+ this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows;
+ return this;
+ }
+
+ public Builder skipCorrupted()
+ {
+ this.skipCorrupted = true;
+ return this;
+ }
+
+ public Builder skipCorrupted(boolean skipCorrupted)
+ {
+ this.skipCorrupted = skipCorrupted;
+ return this;
+ }
+
+ public Options build()
+ {
+ return new Options(checkData, reinsertOverflowedTTLRows, skipCorrupted);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/IVerifier.java b/src/java/org/apache/cassandra/io/sstable/IVerifier.java
new file mode 100644
index 0000000..62ec065
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/IVerifier.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.Closeable;
+import java.util.Collection;
+import java.util.function.Function;
+
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.service.StorageService;
+
+public interface IVerifier extends Closeable
+{
+ static Options.Builder options()
+ {
+ return new Options.Builder();
+ }
+
+ void verify();
+
+ @Override
+ void close();
+
+ CompactionInfo.Holder getVerifyInfo();
+
+ class Options
+ {
+ public final boolean invokeDiskFailurePolicy;
+
+ /**
+ * Force extended verification - unless it is enabled, extended verificiation will be done only
+ * if there is no digest present. Setting it along with quick makes no sense.
+ */
+ public final boolean extendedVerification;
+
+ public final boolean checkVersion;
+ public final boolean mutateRepairStatus;
+ public final boolean checkOwnsTokens;
+
+ /**
+ * Quick check which does not include sstable data verification.
+ */
+ public final boolean quick;
+
+ public final Function<String, ? extends Collection<Range<Token>>> tokenLookup;
+
+ private Options(boolean invokeDiskFailurePolicy,
+ boolean extendedVerification,
+ boolean checkVersion,
+ boolean mutateRepairStatus,
+ boolean checkOwnsTokens,
+ boolean quick,
+ Function<String, ? extends Collection<Range<Token>>> tokenLookup)
+ {
+ this.invokeDiskFailurePolicy = invokeDiskFailurePolicy;
+ this.extendedVerification = extendedVerification;
+ this.checkVersion = checkVersion;
+ this.mutateRepairStatus = mutateRepairStatus;
+ this.checkOwnsTokens = checkOwnsTokens;
+ this.quick = quick;
+ this.tokenLookup = tokenLookup;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Options{" +
+ "invokeDiskFailurePolicy=" + invokeDiskFailurePolicy +
+ ", extendedVerification=" + extendedVerification +
+ ", checkVersion=" + checkVersion +
+ ", mutateRepairStatus=" + mutateRepairStatus +
+ ", checkOwnsTokens=" + checkOwnsTokens +
+ ", quick=" + quick +
+ '}';
+ }
+
+ public static class Builder
+ {
+ private boolean invokeDiskFailurePolicy = false; // invoking disk failure policy can stop the node if we find a corrupt stable
+ private boolean extendedVerification = false;
+ private boolean checkVersion = false;
+ private boolean mutateRepairStatus = false; // mutating repair status can be dangerous
+ private boolean checkOwnsTokens = false;
+ private boolean quick = false;
+ private Function<String, ? extends Collection<Range<Token>>> tokenLookup = StorageService.instance::getLocalAndPendingRanges;
+
+ public Builder invokeDiskFailurePolicy(boolean param)
+ {
+ this.invokeDiskFailurePolicy = param;
+ return this;
+ }
+
+ public Builder extendedVerification(boolean param)
+ {
+ this.extendedVerification = param;
+ return this;
+ }
+
+ public Builder checkVersion(boolean param)
+ {
+ this.checkVersion = param;
+ return this;
+ }
+
+ public Builder mutateRepairStatus(boolean param)
+ {
+ this.mutateRepairStatus = param;
+ return this;
+ }
+
+ public Builder checkOwnsTokens(boolean param)
+ {
+ this.checkOwnsTokens = param;
+ return this;
+ }
+
+ public Builder quick(boolean param)
+ {
+ this.quick = param;
+ return this;
+ }
+
+ public Builder tokenLookup(Function<String, ? extends Collection<Range<Token>>> tokenLookup)
+ {
+ this.tokenLookup = tokenLookup;
+ return this;
+ }
+
+ public Options build()
+ {
+ return new Options(invokeDiskFailurePolicy, extendedVerification, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
index fa0fb2c..dbce416 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
@@ -23,12 +23,12 @@
import org.apache.cassandra.db.ClusteringPrefix;
import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ObjectSizes;
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
deleted file mode 100644
index 303adfd..0000000
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable;
-import org.apache.cassandra.utils.memory.MemoryUtil;
-
-import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-
-/*
- * Layout of Memory for index summaries:
- *
- * There are two sections:
- * 1. A "header" containing the offset into `bytes` of entries in the summary summary data, consisting of
- * one four byte position for each entry in the summary. This allows us do simple math in getIndex()
- * to find the position in the Memory to start reading the actual index summary entry.
- * (This is necessary because keys can have different lengths.)
- * 2. A sequence of (DecoratedKey, position) pairs, where position is the offset into the actual index file.
- */
-public class IndexSummary extends WrappedSharedCloseable
-{
- private static final Logger logger = LoggerFactory.getLogger(IndexSummary.class);
- public static final IndexSummarySerializer serializer = new IndexSummarySerializer();
-
- /**
- * A lower bound for the average number of partitions in between each index summary entry. A lower value means
- * that more partitions will have an entry in the index summary when at the full sampling level.
- */
- private final int minIndexInterval;
-
- private final IPartitioner partitioner;
- private final int sizeAtFullSampling;
- // we permit the memory to span a range larger than we use,
- // so we have an accompanying count and length for each part
- // we split our data into two ranges: offsets (indexing into entries),
- // and entries containing the summary data
- private final Memory offsets;
- private final int offsetCount;
- // entries is a list of (partition key, index file offset) pairs
- private final Memory entries;
- private final long entriesLength;
-
- /**
- * A value between 1 and BASE_SAMPLING_LEVEL that represents how many of the original
- * index summary entries ((1 / indexInterval) * numKeys) have been retained.
- *
- * Thus, this summary contains (samplingLevel / BASE_SAMPLING_LEVEL) * ((1 / indexInterval) * numKeys)) entries.
- */
- private final int samplingLevel;
-
- public IndexSummary(IPartitioner partitioner, Memory offsets, int offsetCount, Memory entries, long entriesLength,
- int sizeAtFullSampling, int minIndexInterval, int samplingLevel)
- {
- super(new Memory[] { offsets, entries });
- assert offsets.getInt(0) == 0;
- this.partitioner = partitioner;
- this.minIndexInterval = minIndexInterval;
- this.offsetCount = offsetCount;
- this.entriesLength = entriesLength;
- this.sizeAtFullSampling = sizeAtFullSampling;
- this.offsets = offsets;
- this.entries = entries;
- this.samplingLevel = samplingLevel;
- assert samplingLevel > 0;
- }
-
- private IndexSummary(IndexSummary copy)
- {
- super(copy);
- this.partitioner = copy.partitioner;
- this.minIndexInterval = copy.minIndexInterval;
- this.offsetCount = copy.offsetCount;
- this.entriesLength = copy.entriesLength;
- this.sizeAtFullSampling = copy.sizeAtFullSampling;
- this.offsets = copy.offsets;
- this.entries = copy.entries;
- this.samplingLevel = copy.samplingLevel;
- }
-
- // binary search is notoriously more difficult to get right than it looks; this is lifted from
- // Harmony's Collections implementation
- public int binarySearch(PartitionPosition key)
- {
- // We will be comparing non-native Keys, so use a buffer with appropriate byte order
- ByteBuffer hollow = MemoryUtil.getHollowDirectByteBuffer().order(ByteOrder.BIG_ENDIAN);
- int low = 0, mid = offsetCount, high = mid - 1, result = -1;
- while (low <= high)
- {
- mid = (low + high) >> 1;
- fillTemporaryKey(mid, hollow);
- result = -DecoratedKey.compareTo(partitioner, hollow, key);
- if (result > 0)
- {
- low = mid + 1;
- }
- else if (result == 0)
- {
- return mid;
- }
- else
- {
- high = mid - 1;
- }
- }
-
- return -mid - (result < 0 ? 1 : 2);
- }
-
- /**
- * Gets the position of the actual index summary entry in our Memory attribute, 'bytes'.
- * @param index The index of the entry or key to get the position for
- * @return an offset into our Memory attribute where the actual entry resides
- */
- public int getPositionInSummary(int index)
- {
- // The first section of bytes holds a four-byte position for each entry in the summary, so just multiply by 4.
- return offsets.getInt(index << 2);
- }
-
- public byte[] getKey(int index)
- {
- long start = getPositionInSummary(index);
- int keySize = (int) (calculateEnd(index) - start - 8L);
- byte[] key = new byte[keySize];
- entries.getBytes(start, key, 0, keySize);
- return key;
- }
-
- private void fillTemporaryKey(int index, ByteBuffer buffer)
- {
- long start = getPositionInSummary(index);
- int keySize = (int) (calculateEnd(index) - start - 8L);
- entries.setByteBuffer(buffer, start, keySize);
- }
-
- public void addTo(Ref.IdentityCollection identities)
- {
- super.addTo(identities);
- identities.add(offsets);
- identities.add(entries);
- }
-
- public long getPosition(int index)
- {
- return entries.getLong(calculateEnd(index) - 8);
- }
-
- public long getEndInSummary(int index)
- {
- return calculateEnd(index);
- }
-
- private long calculateEnd(int index)
- {
- return index == (offsetCount - 1) ? entriesLength : getPositionInSummary(index + 1);
- }
-
- public int getMinIndexInterval()
- {
- return minIndexInterval;
- }
-
- public double getEffectiveIndexInterval()
- {
- return (BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval;
- }
-
- /**
- * Returns an estimate of the total number of keys in the SSTable.
- */
- public long getEstimatedKeyCount()
- {
- return ((long) getMaxNumberOfEntries() + 1) * minIndexInterval;
- }
-
- public int size()
- {
- return offsetCount;
- }
-
- public int getSamplingLevel()
- {
- return samplingLevel;
- }
-
- /**
- * Returns the number of entries this summary would have if it were at the full sampling level, which is equal
- * to the number of entries in the primary on-disk index divided by the min index interval.
- */
- public int getMaxNumberOfEntries()
- {
- return sizeAtFullSampling;
- }
-
- /**
- * Returns the amount of off-heap memory used for the entries portion of this summary.
- * @return size in bytes
- */
- long getEntriesLength()
- {
- return entriesLength;
- }
-
- Memory getOffsets()
- {
- return offsets;
- }
-
- Memory getEntries()
- {
- return entries;
- }
-
- public long getOffHeapSize()
- {
- return offsetCount * 4 + entriesLength;
- }
-
- /**
- * Returns the number of primary (on-disk) index entries between the index summary entry at `index` and the next
- * index summary entry (assuming there is one). Without any downsampling, this will always be equivalent to
- * the index interval.
- *
- * @param index the index of an index summary entry (between zero and the index entry size)
- *
- * @return the number of partitions after `index` until the next partition with a summary entry
- */
- public int getEffectiveIndexIntervalAfterIndex(int index)
- {
- return Downsampling.getEffectiveIndexIntervalAfterIndex(index, samplingLevel, minIndexInterval);
- }
-
- public IndexSummary sharedCopy()
- {
- return new IndexSummary(this);
- }
-
- public static class IndexSummarySerializer
- {
- public void serialize(IndexSummary t, DataOutputPlus out) throws IOException
- {
- out.writeInt(t.minIndexInterval);
- out.writeInt(t.offsetCount);
- out.writeLong(t.getOffHeapSize());
- out.writeInt(t.samplingLevel);
- out.writeInt(t.sizeAtFullSampling);
- // our on-disk representation treats the offsets and the summary data as one contiguous structure,
- // in which the offsets are based from the start of the structure. i.e., if the offsets occupy
- // X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
- // the summary values are indexed from zero, so we apply a correction to the offsets when de/serializing.
- // In this case adding X to each of the offsets.
- int baseOffset = t.offsetCount * 4;
- for (int i = 0 ; i < t.offsetCount ; i++)
- {
- int offset = t.offsets.getInt(i * 4) + baseOffset;
- // our serialization format for this file uses native byte order, so if this is different to the
- // default Java serialization order (BIG_ENDIAN) we have to reverse our bytes
- if (ByteOrder.nativeOrder() != ByteOrder.BIG_ENDIAN)
- offset = Integer.reverseBytes(offset);
- out.writeInt(offset);
- }
- out.write(t.entries, 0, t.entriesLength);
- }
-
- @SuppressWarnings("resource")
- public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
- {
- int minIndexInterval = in.readInt();
- if (minIndexInterval != expectedMinIndexInterval)
- {
- throw new IOException(String.format("Cannot read index summary because min_index_interval changed from %d to %d.",
- minIndexInterval, expectedMinIndexInterval));
- }
-
- int offsetCount = in.readInt();
- long offheapSize = in.readLong();
- int samplingLevel = in.readInt();
- int fullSamplingSummarySize = in.readInt();
-
- int effectiveIndexInterval = (int) Math.ceil((BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval);
- if (effectiveIndexInterval > maxIndexInterval)
- {
- throw new IOException(String.format("Rebuilding index summary because the effective index interval (%d) is higher than" +
- " the current max index interval (%d)", effectiveIndexInterval, maxIndexInterval));
- }
-
- Memory offsets = Memory.allocate(offsetCount * 4);
- Memory entries = Memory.allocate(offheapSize - offsets.size());
- try
- {
- FBUtilities.copy(in, new MemoryOutputStream(offsets), offsets.size());
- FBUtilities.copy(in, new MemoryOutputStream(entries), entries.size());
- }
- catch (IOException ioe)
- {
- offsets.free();
- entries.free();
- throw ioe;
- }
- // our on-disk representation treats the offsets and the summary data as one contiguous structure,
- // in which the offsets are based from the start of the structure. i.e., if the offsets occupy
- // X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
- // the summary values are indexed from zero, so we apply a correction to the offsets when de/serializing.
- // In this case subtracting X from each of the offsets.
- for (int i = 0 ; i < offsets.size() ; i += 4)
- offsets.setInt(i, (int) (offsets.getInt(i) - offsets.size()));
- return new IndexSummary(partitioner, offsets, offsetCount, entries, entries.size(), fullSamplingSummarySize, minIndexInterval, samplingLevel);
- }
-
- /**
- * Deserializes the first and last key stored in the summary
- *
- * Only for use by offline tools like SSTableMetadataViewer, otherwise SSTable.first/last should be used.
- */
- public Pair<DecoratedKey, DecoratedKey> deserializeFirstLastKey(DataInputStream in, IPartitioner partitioner) throws IOException
- {
- in.skipBytes(4); // minIndexInterval
- int offsetCount = in.readInt();
- long offheapSize = in.readLong();
- in.skipBytes(8); // samplingLevel, fullSamplingSummarySize
-
- in.skip(offsetCount * 4);
- in.skip(offheapSize - offsetCount * 4);
-
- DecoratedKey first = partitioner.decorateKey(ByteBufferUtil.readWithLength(in));
- DecoratedKey last = partitioner.decorateKey(ByteBufferUtil.readWithLength(in));
- return Pair.create(first, last);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java
deleted file mode 100644
index 75cca84..0000000
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.nio.ByteOrder;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.io.util.Memory;
-import org.apache.cassandra.io.util.SafeMemoryWriter;
-
-import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-
-public class IndexSummaryBuilder implements AutoCloseable
-{
- private static final Logger logger = LoggerFactory.getLogger(IndexSummaryBuilder.class);
-
- static final String defaultExpectedKeySizeName = Config.PROPERTY_PREFIX + "index_summary_expected_key_size";
- static long defaultExpectedKeySize = Long.valueOf(System.getProperty(defaultExpectedKeySizeName, "64"));
-
- // the offset in the keys memory region to look for a given summary boundary
- private final SafeMemoryWriter offsets;
- private final SafeMemoryWriter entries;
-
- private final int minIndexInterval;
- private final int samplingLevel;
- private final int[] startPoints;
- private long keysWritten = 0;
- private long indexIntervalMatches = 0;
- private long nextSamplePosition;
-
- // for each ReadableBoundary, we map its dataLength property to itself, permitting us to lookup the
- // last readable boundary from the perspective of the data file
- // [data file position limit] => [ReadableBoundary]
- private TreeMap<Long, ReadableBoundary> lastReadableByData = new TreeMap<>();
- // for each ReadableBoundary, we map its indexLength property to itself, permitting us to lookup the
- // last readable boundary from the perspective of the index file
- // [index file position limit] => [ReadableBoundary]
- private TreeMap<Long, ReadableBoundary> lastReadableByIndex = new TreeMap<>();
- // the last synced data file position
- private long dataSyncPosition;
- // the last synced index file position
- private long indexSyncPosition;
-
- // the last summary interval boundary that is fully readable in both data and index files
- private ReadableBoundary lastReadableBoundary;
-
- /**
- * Represents a boundary that is guaranteed fully readable in the summary, index file and data file.
- * The key contained is the last key readable if the index and data files have been flushed to the
- * stored lengths.
- */
- public static class ReadableBoundary
- {
- public final DecoratedKey lastKey;
- public final long indexLength;
- public final long dataLength;
- public final int summaryCount;
- public final long entriesLength;
- public ReadableBoundary(DecoratedKey lastKey, long indexLength, long dataLength, int summaryCount, long entriesLength)
- {
- this.lastKey = lastKey;
- this.indexLength = indexLength;
- this.dataLength = dataLength;
- this.summaryCount = summaryCount;
- this.entriesLength = entriesLength;
- }
- }
-
- /**
- * Build an index summary builder.
- *
- * @param expectedKeys - the number of keys we expect in the sstable
- * @param minIndexInterval - the minimum interval between entries selected for sampling
- * @param samplingLevel - the level at which entries are sampled
- */
- public IndexSummaryBuilder(long expectedKeys, int minIndexInterval, int samplingLevel)
- {
- this.samplingLevel = samplingLevel;
- this.startPoints = Downsampling.getStartPoints(BASE_SAMPLING_LEVEL, samplingLevel);
-
- long expectedEntrySize = getEntrySize(defaultExpectedKeySize);
- long maxExpectedEntries = expectedKeys / minIndexInterval;
- long maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
- if (maxExpectedEntriesSize > Integer.MAX_VALUE)
- {
- // that's a _lot_ of keys, and a very low min index interval
- int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys * expectedEntrySize) / Integer.MAX_VALUE);
- maxExpectedEntries = expectedKeys / effectiveMinInterval;
- maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
- assert maxExpectedEntriesSize <= Integer.MAX_VALUE : maxExpectedEntriesSize;
- logger.warn("min_index_interval of {} is too low for {} expected keys of avg size {}; using interval of {} instead",
- minIndexInterval, expectedKeys, defaultExpectedKeySize, effectiveMinInterval);
- this.minIndexInterval = effectiveMinInterval;
- }
- else
- {
- this.minIndexInterval = minIndexInterval;
- }
-
- // for initializing data structures, adjust our estimates based on the sampling level
- maxExpectedEntries = Math.max(1, (maxExpectedEntries * samplingLevel) / BASE_SAMPLING_LEVEL);
- offsets = new SafeMemoryWriter(4 * maxExpectedEntries).order(ByteOrder.nativeOrder());
- entries = new SafeMemoryWriter(expectedEntrySize * maxExpectedEntries).order(ByteOrder.nativeOrder());
-
- // the summary will always contain the first index entry (downsampling will never remove it)
- nextSamplePosition = 0;
- indexIntervalMatches++;
- }
-
- /**
- * Given a key, return how long the serialized index summary entry will be.
- */
- private static long getEntrySize(DecoratedKey key)
- {
- return getEntrySize(key.getKey().remaining());
- }
-
- /**
- * Given a key size, return how long the serialized index summary entry will be, that is add 8 bytes to
- * accomodate for the size of the position.
- */
- private static long getEntrySize(long keySize)
- {
- return keySize + TypeSizes.sizeof(0L);
- }
-
- // the index file has been flushed to the provided position; stash it and use that to recalculate our max readable boundary
- public void markIndexSynced(long upToPosition)
- {
- indexSyncPosition = upToPosition;
- refreshReadableBoundary();
- }
-
- // the data file has been flushed to the provided position; stash it and use that to recalculate our max readable boundary
- public void markDataSynced(long upToPosition)
- {
- dataSyncPosition = upToPosition;
- refreshReadableBoundary();
- }
-
- private void refreshReadableBoundary()
- {
- // grab the readable boundary prior to the given position in either the data or index file
- Map.Entry<?, ReadableBoundary> byData = lastReadableByData.floorEntry(dataSyncPosition);
- Map.Entry<?, ReadableBoundary> byIndex = lastReadableByIndex.floorEntry(indexSyncPosition);
- if (byData == null || byIndex == null)
- return;
-
- // take the lowest of the two, and stash it
- lastReadableBoundary = byIndex.getValue().indexLength < byData.getValue().indexLength
- ? byIndex.getValue() : byData.getValue();
-
- // clear our data prior to this, since we no longer need it
- lastReadableByData.headMap(lastReadableBoundary.dataLength, false).clear();
- lastReadableByIndex.headMap(lastReadableBoundary.indexLength, false).clear();
- }
-
- public ReadableBoundary getLastReadableBoundary()
- {
- return lastReadableBoundary;
- }
-
- public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart) throws IOException
- {
- return maybeAddEntry(decoratedKey, indexStart, 0, 0);
- }
-
- /**
- *
- * @param decoratedKey the key for this record
- * @param indexStart the position in the index file this record begins
- * @param indexEnd the position in the index file we need to be able to read to (exclusive) to read this record
- * @param dataEnd the position in the data file we need to be able to read to (exclusive) to read this record
- * a value of 0 indicates we are not tracking readable boundaries
- */
- public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart, long indexEnd, long dataEnd) throws IOException
- {
- if (keysWritten == nextSamplePosition)
- {
- if ((entries.length() + getEntrySize(decoratedKey)) <= Integer.MAX_VALUE)
- {
- offsets.writeInt((int) entries.length());
- entries.write(decoratedKey.getKey());
- entries.writeLong(indexStart);
- setNextSamplePosition(keysWritten);
- }
- else
- {
- // we cannot fully sample this sstable due to too much memory in the index summary, so let's tell the user
- logger.error("Memory capacity of index summary exceeded (2GiB), index summary will not cover full sstable, " +
- "you should increase min_sampling_level");
- }
- }
- else if (dataEnd != 0 && keysWritten + 1 == nextSamplePosition)
- {
- // this is the last key in this summary interval, so stash it
- ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int) (offsets.length() / 4), entries.length());
- lastReadableByData.put(dataEnd, boundary);
- lastReadableByIndex.put(indexEnd, boundary);
- }
-
- keysWritten++;
- return this;
- }
-
- // calculate the next key we will store to our summary
- private void setNextSamplePosition(long position)
- {
- tryAgain: while (true)
- {
- position += minIndexInterval;
- long test = indexIntervalMatches++;
- for (int start : startPoints)
- if ((test - start) % BASE_SAMPLING_LEVEL == 0)
- continue tryAgain;
-
- nextSamplePosition = position;
- return;
- }
- }
-
- public void prepareToCommit()
- {
- // this method should only be called when we've finished appending records, so we truncate the
- // memory we're using to the exact amount required to represent it before building our summary
- entries.trim();
- offsets.trim();
- }
-
- public IndexSummary build(IPartitioner partitioner)
- {
- return build(partitioner, null);
- }
-
- // build the summary up to the provided boundary; this is backed by shared memory between
- // multiple invocations of this build method
- public IndexSummary build(IPartitioner partitioner, ReadableBoundary boundary)
- {
- assert entries.length() > 0;
-
- int count = (int) (offsets.length() / 4);
- long entriesLength = entries.length();
- if (boundary != null)
- {
- count = boundary.summaryCount;
- entriesLength = boundary.entriesLength;
- }
-
- int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) minIndexInterval);
- assert count > 0;
- return new IndexSummary(partitioner, offsets.currentBuffer().sharedCopy(),
- count, entries.currentBuffer().sharedCopy(), entriesLength,
- sizeAtFullSampling, minIndexInterval, samplingLevel);
- }
-
- // close the builder and release any associated memory
- public void close()
- {
- entries.close();
- offsets.close();
- }
-
- public Throwable close(Throwable accumulate)
- {
- accumulate = entries.close(accumulate);
- accumulate = offsets.close(accumulate);
- return accumulate;
- }
-
- static int entriesAtSamplingLevel(int samplingLevel, int maxSummarySize)
- {
- return (int) Math.ceil((samplingLevel * maxSummarySize) / (double) BASE_SAMPLING_LEVEL);
- }
-
- static int calculateSamplingLevel(int currentSamplingLevel, int currentNumEntries, long targetNumEntries, int minIndexInterval, int maxIndexInterval)
- {
- // effective index interval == (BASE_SAMPLING_LEVEL / samplingLevel) * minIndexInterval
- // so we can just solve for minSamplingLevel here:
- // maxIndexInterval == (BASE_SAMPLING_LEVEL / minSamplingLevel) * minIndexInterval
- int effectiveMinSamplingLevel = Math.max(1, (int) Math.ceil((BASE_SAMPLING_LEVEL * minIndexInterval) / (double) maxIndexInterval));
-
- // Algebraic explanation for calculating the new sampling level (solve for newSamplingLevel):
- // originalNumEntries = (baseSamplingLevel / currentSamplingLevel) * currentNumEntries
- // newSpaceUsed = (newSamplingLevel / baseSamplingLevel) * originalNumEntries
- // newSpaceUsed = (newSamplingLevel / baseSamplingLevel) * (baseSamplingLevel / currentSamplingLevel) * currentNumEntries
- // newSpaceUsed = (newSamplingLevel / currentSamplingLevel) * currentNumEntries
- // (newSpaceUsed * currentSamplingLevel) / currentNumEntries = newSamplingLevel
- int newSamplingLevel = (int) (targetNumEntries * currentSamplingLevel) / currentNumEntries;
- return Math.min(BASE_SAMPLING_LEVEL, Math.max(effectiveMinSamplingLevel, newSamplingLevel));
- }
-
- /**
- * Downsamples an existing index summary to a new sampling level.
- * @param existing an existing IndexSummary
- * @param newSamplingLevel the target level for the new IndexSummary. This must be less than the current sampling
- * level for `existing`.
- * @param partitioner the partitioner used for the index summary
- * @return a new IndexSummary
- */
- @SuppressWarnings("resource")
- public static IndexSummary downsample(IndexSummary existing, int newSamplingLevel, int minIndexInterval, IPartitioner partitioner)
- {
- // To downsample the old index summary, we'll go through (potentially) several rounds of downsampling.
- // Conceptually, each round starts at position X and then removes every Nth item. The value of X follows
- // a particular pattern to evenly space out the items that we remove. The value of N decreases by one each
- // round.
-
- int currentSamplingLevel = existing.getSamplingLevel();
- assert currentSamplingLevel > newSamplingLevel;
- assert minIndexInterval == existing.getMinIndexInterval();
-
- // calculate starting indexes for downsampling rounds
- int[] startPoints = Downsampling.getStartPoints(currentSamplingLevel, newSamplingLevel);
-
- // calculate new off-heap size
- int newKeyCount = existing.size();
- long newEntriesLength = existing.getEntriesLength();
- for (int start : startPoints)
- {
- for (int j = start; j < existing.size(); j += currentSamplingLevel)
- {
- newKeyCount--;
- long length = existing.getEndInSummary(j) - existing.getPositionInSummary(j);
- newEntriesLength -= length;
- }
- }
-
- Memory oldEntries = existing.getEntries();
- Memory newOffsets = Memory.allocate(newKeyCount * 4);
- Memory newEntries = Memory.allocate(newEntriesLength);
-
- // Copy old entries to our new Memory.
- int i = 0;
- int newEntriesOffset = 0;
- outer:
- for (int oldSummaryIndex = 0; oldSummaryIndex < existing.size(); oldSummaryIndex++)
- {
- // to determine if we can skip this entry, go through the starting points for our downsampling rounds
- // and see if the entry's index is covered by that round
- for (int start : startPoints)
- {
- if ((oldSummaryIndex - start) % currentSamplingLevel == 0)
- continue outer;
- }
-
- // write the position of the actual entry in the index summary (4 bytes)
- newOffsets.setInt(i * 4, newEntriesOffset);
- i++;
- long start = existing.getPositionInSummary(oldSummaryIndex);
- long length = existing.getEndInSummary(oldSummaryIndex) - start;
- newEntries.put(newEntriesOffset, oldEntries, start, length);
- newEntriesOffset += length;
- }
- assert newEntriesOffset == newEntriesLength;
- return new IndexSummary(partitioner, newOffsets, newKeyCount, newEntries, newEntriesLength,
- existing.getMaxNumberOfEntries(), minIndexInterval, newSamplingLevel);
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
deleted file mode 100644
index b11ad2b..0000000
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
-
-import com.codahale.metrics.Timer;
-import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.ExecutorUtils;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.MBeanWrapper;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-/**
- * Manages the fixed-size memory pool for index summaries, periodically resizing them
- * in order to give more memory to hot sstables and less memory to cold sstables.
- */
-public class IndexSummaryManager implements IndexSummaryManagerMBean
-{
- private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManager.class);
- public static final String MBEAN_NAME = "org.apache.cassandra.db:type=IndexSummaries";
- public static final IndexSummaryManager instance;
-
- private long memoryPoolBytes;
-
- private final ScheduledExecutorPlus executor;
-
- // our next scheduled resizing run
- private ScheduledFuture future;
-
- static
- {
- instance = new IndexSummaryManager();
- MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME);
- }
-
- private IndexSummaryManager()
- {
- executor = executorFactory().scheduled(false, "IndexSummaryManager", Thread.MIN_PRIORITY);
-
- long indexSummarySizeInMB = DatabaseDescriptor.getIndexSummaryCapacityInMiB();
- int interval = DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes();
- logger.info("Initializing index summary manager with a memory pool size of {} MB and a resize interval of {} minutes",
- indexSummarySizeInMB, interval);
-
- setMemoryPoolCapacityInMB(DatabaseDescriptor.getIndexSummaryCapacityInMiB());
- setResizeIntervalInMinutes(DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
- }
-
- public int getResizeIntervalInMinutes()
- {
- return DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes();
- }
-
- public void setResizeIntervalInMinutes(int resizeIntervalInMinutes)
- {
- int oldInterval = getResizeIntervalInMinutes();
- DatabaseDescriptor.setIndexSummaryResizeIntervalInMinutes(resizeIntervalInMinutes);
-
- long initialDelay;
- if (future != null)
- {
- initialDelay = oldInterval < 0
- ? resizeIntervalInMinutes
- : Math.max(0, resizeIntervalInMinutes - (oldInterval - future.getDelay(TimeUnit.MINUTES)));
- future.cancel(false);
- }
- else
- {
- initialDelay = resizeIntervalInMinutes;
- }
-
- if (resizeIntervalInMinutes < 0)
- {
- future = null;
- return;
- }
-
- future = executor.scheduleWithFixedDelay(new WrappedRunnable()
- {
- protected void runMayThrow() throws Exception
- {
- redistributeSummaries();
- }
- }, initialDelay, resizeIntervalInMinutes, TimeUnit.MINUTES);
- }
-
- // for testing only
- @VisibleForTesting
- Long getTimeToNextResize(TimeUnit timeUnit)
- {
- if (future == null)
- return null;
-
- return future.getDelay(timeUnit);
- }
-
- public long getMemoryPoolCapacityInMB()
- {
- return memoryPoolBytes / 1024L / 1024L;
- }
-
- public Map<String, Integer> getIndexIntervals()
- {
- List<SSTableReader> sstables = getAllSSTables();
- Map<String, Integer> intervals = new HashMap<>(sstables.size());
- for (SSTableReader sstable : sstables)
- intervals.put(sstable.getFilename(), (int) Math.round(sstable.getEffectiveIndexInterval()));
-
- return intervals;
- }
-
- public double getAverageIndexInterval()
- {
- List<SSTableReader> sstables = getAllSSTables();
- double total = 0.0;
- for (SSTableReader sstable : sstables)
- total += sstable.getEffectiveIndexInterval();
- return total / sstables.size();
- }
-
- public void setMemoryPoolCapacityInMB(long memoryPoolCapacityInMB)
- {
- this.memoryPoolBytes = memoryPoolCapacityInMB * 1024L * 1024L;
- }
-
- /**
- * Returns the actual space consumed by index summaries for all sstables.
- * @return space currently used in MB
- */
- public double getMemoryPoolSizeInMB()
- {
- long total = 0;
- for (SSTableReader sstable : getAllSSTables())
- total += sstable.getIndexSummaryOffHeapSize();
- return total / 1024.0 / 1024.0;
- }
-
- private List<SSTableReader> getAllSSTables()
- {
- List<SSTableReader> result = new ArrayList<>();
- for (Keyspace ks : Keyspace.all())
- {
- for (ColumnFamilyStore cfStore: ks.getColumnFamilyStores())
- result.addAll(cfStore.getLiveSSTables());
- }
-
- return result;
- }
-
- /**
- * Marks the non-compacting sstables as compacting for index summary redistribution for all keyspaces/tables.
- *
- * @return Pair containing:
- * left: total size of the off heap index summaries for the sstables we were unable to mark compacting (they were involved in other compactions)
- * right: the transactions, keyed by table id.
- */
- @SuppressWarnings("resource")
- private Pair<Long, Map<TableId, LifecycleTransaction>> getRestributionTransactions()
- {
- List<SSTableReader> allCompacting = new ArrayList<>();
- Map<TableId, LifecycleTransaction> allNonCompacting = new HashMap<>();
- for (Keyspace ks : Keyspace.all())
- {
- for (ColumnFamilyStore cfStore: ks.getColumnFamilyStores())
- {
- Set<SSTableReader> nonCompacting, allSSTables;
- LifecycleTransaction txn;
- do
- {
- View view = cfStore.getTracker().getView();
- allSSTables = ImmutableSet.copyOf(view.select(SSTableSet.CANONICAL));
- nonCompacting = ImmutableSet.copyOf(view.getUncompacting(allSSTables));
- }
- while (null == (txn = cfStore.getTracker().tryModify(nonCompacting, OperationType.INDEX_SUMMARY)));
-
- allNonCompacting.put(cfStore.metadata.id, txn);
- allCompacting.addAll(Sets.difference(allSSTables, nonCompacting));
- }
- }
- long nonRedistributingOffHeapSize = allCompacting.stream().mapToLong(SSTableReader::getIndexSummaryOffHeapSize).sum();
- return Pair.create(nonRedistributingOffHeapSize, allNonCompacting);
- }
-
- public void redistributeSummaries() throws IOException
- {
- if (CompactionManager.instance.isGlobalCompactionPaused())
- return;
- Pair<Long, Map<TableId, LifecycleTransaction>> redistributionTransactionInfo = getRestributionTransactions();
- Map<TableId, LifecycleTransaction> transactions = redistributionTransactionInfo.right;
- long nonRedistributingOffHeapSize = redistributionTransactionInfo.left;
- try (Timer.Context ctx = CompactionManager.instance.getMetrics().indexSummaryRedistributionTime.time())
- {
- redistributeSummaries(new IndexSummaryRedistribution(transactions,
- nonRedistributingOffHeapSize,
- this.memoryPoolBytes));
- }
- catch (Exception e)
- {
- if (e instanceof CompactionInterruptedException)
- {
- logger.info("Index summary interrupted: {}", e.getMessage());
- }
- else
- {
- logger.error("Got exception during index summary redistribution", e);
- throw e;
- }
- }
- finally
- {
- try
- {
- FBUtilities.closeAll(transactions.values());
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
- * Attempts to fairly distribute a fixed pool of memory for index summaries across a set of SSTables based on
- * their recent read rates.
- * @param redistribution encapsulating the transactions containing the sstables we are to redistribute the
- * memory pool across and a size (in bytes) that the total index summary space usage
- * should stay close to or under, if possible
- * @return a list of new SSTableReader instances
- */
- @VisibleForTesting
- public static List<SSTableReader> redistributeSummaries(IndexSummaryRedistribution redistribution) throws IOException
- {
- return CompactionManager.instance.runIndexSummaryRedistribution(redistribution);
- }
-
- @VisibleForTesting
- public void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
- {
- if (future != null)
- {
- future.cancel(false);
- future = null;
- }
- ExecutorUtils.shutdownAndWait(timeout, unit, executor);
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java
deleted file mode 100644
index 9ba3d40..0000000
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.util.Map;
-
-public interface IndexSummaryManagerMBean
-{
- public long getMemoryPoolCapacityInMB();
- public void setMemoryPoolCapacityInMB(long memoryPoolCapacityInMB);
-
- /**
- * Returns the current actual off-heap memory usage of the index summaries for all non-compacting sstables.
- * @return The amount of memory used in MiB.
- */
- public double getMemoryPoolSizeInMB();
-
- /**
- * Returns a map of SSTable filenames to their current effective index interval.
- */
- public Map<String, Integer> getIndexIntervals();
-
- public double getAverageIndexInterval();
-
- public void redistributeSummaries() throws IOException;
-
- public int getResizeIntervalInMinutes();
-
- /**
- * Set resizeIntervalInMinutes = -1 for disabled; This is the equivalent of index_summary_resize_interval being
- * set to null in cassandra.yaml
- */
- public void setResizeIntervalInMinutes(int resizeIntervalInMinutes);
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
deleted file mode 100644
index 8bbe709..0000000
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.concurrent.Refs;
-
-import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-
-public class IndexSummaryRedistribution extends CompactionInfo.Holder
-{
- private static final Logger logger = LoggerFactory.getLogger(IndexSummaryRedistribution.class);
-
- // The target (or ideal) number of index summary entries must differ from the actual number of
- // entries by this ratio in order to trigger an upsample or downsample of the summary. Because
- // upsampling requires reading the primary index in order to rebuild the summary, the threshold
- // for upsampling is is higher.
- static final double UPSAMPLE_THRESHOLD = 1.5;
- static final double DOWNSAMPLE_THESHOLD = 0.75;
-
- private final Map<TableId, LifecycleTransaction> transactions;
- private final long nonRedistributingOffHeapSize;
- private final long memoryPoolBytes;
- private final TimeUUID compactionId;
- private volatile long remainingSpace;
-
- /**
- *
- * @param transactions the transactions for the different keyspaces/tables we are to redistribute
- * @param nonRedistributingOffHeapSize the total index summary off heap size for all sstables we were not able to mark compacting (due to them being involved in other compactions)
- * @param memoryPoolBytes size of the memory pool
- */
- public IndexSummaryRedistribution(Map<TableId, LifecycleTransaction> transactions, long nonRedistributingOffHeapSize, long memoryPoolBytes)
- {
- this.transactions = transactions;
- this.nonRedistributingOffHeapSize = nonRedistributingOffHeapSize;
- this.memoryPoolBytes = memoryPoolBytes;
- this.compactionId = nextTimeUUID();
- }
-
- public List<SSTableReader> redistributeSummaries() throws IOException
- {
- long start = nanoTime();
- logger.info("Redistributing index summaries");
- List<SSTableReader> redistribute = new ArrayList<>();
- for (LifecycleTransaction txn : transactions.values())
- {
- redistribute.addAll(txn.originals());
- }
-
- long total = nonRedistributingOffHeapSize;
- for (SSTableReader sstable : redistribute)
- total += sstable.getIndexSummaryOffHeapSize();
-
- logger.info("Beginning redistribution of index summaries for {} sstables with memory pool size {} MiB; current spaced used is {} MiB",
- redistribute.size(), memoryPoolBytes / 1024L / 1024L, total / 1024.0 / 1024.0);
-
- final Map<SSTableReader, Double> readRates = new HashMap<>(redistribute.size());
- double totalReadsPerSec = 0.0;
- for (SSTableReader sstable : redistribute)
- {
- if (isStopRequested())
- throw new CompactionInterruptedException(getCompactionInfo());
-
- if (sstable.getReadMeter() != null)
- {
- Double readRate = sstable.getReadMeter().fifteenMinuteRate();
- totalReadsPerSec += readRate;
- readRates.put(sstable, readRate);
- }
- }
- logger.trace("Total reads/sec across all sstables in index summary resize process: {}", totalReadsPerSec);
-
- // copy and sort by read rates (ascending)
- List<SSTableReader> sstablesByHotness = new ArrayList<>(redistribute);
- Collections.sort(sstablesByHotness, new ReadRateComparator(readRates));
-
- long remainingBytes = memoryPoolBytes - nonRedistributingOffHeapSize;
-
- logger.trace("Index summaries for compacting SSTables are using {} MiB of space",
- (memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
- List<SSTableReader> newSSTables;
- try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
- {
- newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
-
- for (LifecycleTransaction txn : transactions.values())
- txn.finish();
- }
- total = nonRedistributingOffHeapSize;
- for (SSTableReader sstable : newSSTables)
- total += sstable.getIndexSummaryOffHeapSize();
-
- logger.info("Completed resizing of index summaries; current approximate memory used: {} MiB, time spent: {}ms",
- total / 1024.0 / 1024.0, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
-
- return newSSTables;
- }
-
- private List<SSTableReader> adjustSamplingLevels(List<SSTableReader> sstables,
- Map<TableId, LifecycleTransaction> transactions,
- double totalReadsPerSec, long memoryPoolCapacity) throws IOException
- {
- List<ResampleEntry> toDownsample = new ArrayList<>(sstables.size() / 4);
- List<ResampleEntry> toUpsample = new ArrayList<>(sstables.size() / 4);
- List<ResampleEntry> forceResample = new ArrayList<>();
- List<ResampleEntry> forceUpsample = new ArrayList<>();
- List<SSTableReader> newSSTables = new ArrayList<>(sstables.size());
-
- // Going from the coldest to the hottest sstables, try to give each sstable an amount of space proportional
- // to the number of total reads/sec it handles.
- remainingSpace = memoryPoolCapacity;
- for (SSTableReader sstable : sstables)
- {
- if (isStopRequested())
- throw new CompactionInterruptedException(getCompactionInfo());
-
- int minIndexInterval = sstable.metadata().params.minIndexInterval;
- int maxIndexInterval = sstable.metadata().params.maxIndexInterval;
-
- double readsPerSec = sstable.getReadMeter() == null ? 0.0 : sstable.getReadMeter().fifteenMinuteRate();
- long idealSpace = Math.round(remainingSpace * (readsPerSec / totalReadsPerSec));
-
- // figure out how many entries our idealSpace would buy us, and pick a new sampling level based on that
- int currentNumEntries = sstable.getIndexSummarySize();
- double avgEntrySize = sstable.getIndexSummaryOffHeapSize() / (double) currentNumEntries;
- long targetNumEntries = Math.max(1, Math.round(idealSpace / avgEntrySize));
- int currentSamplingLevel = sstable.getIndexSummarySamplingLevel();
- int maxSummarySize = sstable.getMaxIndexSummarySize();
-
- // if the min_index_interval changed, calculate what our current sampling level would be under the new min
- if (sstable.getMinIndexInterval() != minIndexInterval)
- {
- int effectiveSamplingLevel = (int) Math.round(currentSamplingLevel * (minIndexInterval / (double) sstable.getMinIndexInterval()));
- maxSummarySize = (int) Math.round(maxSummarySize * (sstable.getMinIndexInterval() / (double) minIndexInterval));
- logger.trace("min_index_interval changed from {} to {}, so the current sampling level for {} is effectively now {} (was {})",
- sstable.getMinIndexInterval(), minIndexInterval, sstable, effectiveSamplingLevel, currentSamplingLevel);
- currentSamplingLevel = effectiveSamplingLevel;
- }
-
- int newSamplingLevel = IndexSummaryBuilder.calculateSamplingLevel(currentSamplingLevel, currentNumEntries, targetNumEntries,
- minIndexInterval, maxIndexInterval);
- int numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, maxSummarySize);
- double effectiveIndexInterval = sstable.getEffectiveIndexInterval();
-
- if (logger.isTraceEnabled())
- logger.trace("{} has {} reads/sec; ideal space for index summary: {} ({} entries); considering moving " +
- "from level {} ({} entries, {}) " +
- "to level {} ({} entries, {})",
- sstable.getFilename(), readsPerSec, FBUtilities.prettyPrintMemory(idealSpace), targetNumEntries,
- currentSamplingLevel, currentNumEntries, FBUtilities.prettyPrintMemory((long) (currentNumEntries * avgEntrySize)),
- newSamplingLevel, numEntriesAtNewSamplingLevel, FBUtilities.prettyPrintMemory((long) (numEntriesAtNewSamplingLevel * avgEntrySize)));
-
- if (effectiveIndexInterval < minIndexInterval)
- {
- // The min_index_interval was changed; re-sample to match it.
- logger.trace("Forcing resample of {} because the current index interval ({}) is below min_index_interval ({})",
- sstable, effectiveIndexInterval, minIndexInterval);
- long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
- forceResample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
- remainingSpace -= spaceUsed;
- }
- else if (effectiveIndexInterval > maxIndexInterval)
- {
- // The max_index_interval was lowered; force an upsample to the effective minimum sampling level
- logger.trace("Forcing upsample of {} because the current index interval ({}) is above max_index_interval ({})",
- sstable, effectiveIndexInterval, maxIndexInterval);
- newSamplingLevel = Math.max(1, (BASE_SAMPLING_LEVEL * minIndexInterval) / maxIndexInterval);
- numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, sstable.getMaxIndexSummarySize());
- long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
- forceUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
- remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel;
- }
- else if (targetNumEntries >= currentNumEntries * UPSAMPLE_THRESHOLD && newSamplingLevel > currentSamplingLevel)
- {
- long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
- toUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
- remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel;
- }
- else if (targetNumEntries < currentNumEntries * DOWNSAMPLE_THESHOLD && newSamplingLevel < currentSamplingLevel)
- {
- long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
- toDownsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
- remainingSpace -= spaceUsed;
- }
- else
- {
- // keep the same sampling level
- logger.trace("SSTable {} is within thresholds of ideal sampling", sstable);
- remainingSpace -= sstable.getIndexSummaryOffHeapSize();
- newSSTables.add(sstable);
- transactions.get(sstable.metadata().id).cancel(sstable);
- }
- totalReadsPerSec -= readsPerSec;
- }
-
- if (remainingSpace > 0)
- {
- Pair<List<SSTableReader>, List<ResampleEntry>> result = distributeRemainingSpace(toDownsample, remainingSpace);
- toDownsample = result.right;
- newSSTables.addAll(result.left);
- for (SSTableReader sstable : result.left)
- transactions.get(sstable.metadata().id).cancel(sstable);
- }
-
- // downsample first, then upsample
- logger.info("index summaries: downsample: {}, force resample: {}, upsample: {}, force upsample: {}", toDownsample.size(), forceResample.size(), toUpsample.size(), forceUpsample.size());
- toDownsample.addAll(forceResample);
- toDownsample.addAll(toUpsample);
- toDownsample.addAll(forceUpsample);
- for (ResampleEntry entry : toDownsample)
- {
- if (isStopRequested())
- throw new CompactionInterruptedException(getCompactionInfo());
-
- SSTableReader sstable = entry.sstable;
- logger.trace("Re-sampling index summary for {} from {}/{} to {}/{} of the original number of entries",
- sstable, sstable.getIndexSummarySamplingLevel(), Downsampling.BASE_SAMPLING_LEVEL,
- entry.newSamplingLevel, Downsampling.BASE_SAMPLING_LEVEL);
- ColumnFamilyStore cfs = Keyspace.open(sstable.metadata().keyspace).getColumnFamilyStore(sstable.metadata().id);
- long oldSize = sstable.bytesOnDisk();
- SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(cfs, entry.newSamplingLevel);
- long newSize = replacement.bytesOnDisk();
- newSSTables.add(replacement);
- transactions.get(sstable.metadata().id).update(replacement, true);
- addHooks(cfs, transactions, oldSize, newSize);
- }
-
- return newSSTables;
- }
-
- /**
- * Add hooks to correctly update the storage load metrics once the transaction is closed/aborted
- */
- @SuppressWarnings("resource") // Transactions are closed in finally outside of this method
- private void addHooks(ColumnFamilyStore cfs, Map<TableId, LifecycleTransaction> transactions, long oldSize, long newSize)
- {
- LifecycleTransaction txn = transactions.get(cfs.metadata.id);
- txn.runOnCommit(() -> {
- // The new size will be added in Transactional.commit() as an updated SSTable, more details: CASSANDRA-13738
- StorageMetrics.load.dec(oldSize);
- cfs.metric.liveDiskSpaceUsed.dec(oldSize);
- cfs.metric.totalDiskSpaceUsed.dec(oldSize);
- });
- txn.runOnAbort(() -> {
- // the local disk was modified but book keeping couldn't be commited, apply the delta
- long delta = oldSize - newSize; // if new is larger this will be negative, so dec will become a inc
- StorageMetrics.load.dec(delta);
- cfs.metric.liveDiskSpaceUsed.dec(delta);
- cfs.metric.totalDiskSpaceUsed.dec(delta);
- });
- }
-
- @VisibleForTesting
- static Pair<List<SSTableReader>, List<ResampleEntry>> distributeRemainingSpace(List<ResampleEntry> toDownsample, long remainingSpace)
- {
- // sort by the amount of space regained by doing the downsample operation; we want to try to avoid operations
- // that will make little difference.
- Collections.sort(toDownsample, new Comparator<ResampleEntry>()
- {
- public int compare(ResampleEntry o1, ResampleEntry o2)
- {
- return Double.compare(o1.sstable.getIndexSummaryOffHeapSize() - o1.newSpaceUsed,
- o2.sstable.getIndexSummaryOffHeapSize() - o2.newSpaceUsed);
- }
- });
-
- int noDownsampleCutoff = 0;
- List<SSTableReader> willNotDownsample = new ArrayList<>();
- while (remainingSpace > 0 && noDownsampleCutoff < toDownsample.size())
- {
- ResampleEntry entry = toDownsample.get(noDownsampleCutoff);
-
- long extraSpaceRequired = entry.sstable.getIndexSummaryOffHeapSize() - entry.newSpaceUsed;
- // see if we have enough leftover space to keep the current sampling level
- if (extraSpaceRequired <= remainingSpace)
- {
- logger.trace("Using leftover space to keep {} at the current sampling level ({})",
- entry.sstable, entry.sstable.getIndexSummarySamplingLevel());
- willNotDownsample.add(entry.sstable);
- remainingSpace -= extraSpaceRequired;
- }
- else
- {
- break;
- }
-
- noDownsampleCutoff++;
- }
- return Pair.create(willNotDownsample, toDownsample.subList(noDownsampleCutoff, toDownsample.size()));
- }
-
- public CompactionInfo getCompactionInfo()
- {
- return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
- }
-
- public boolean isGlobal()
- {
- return true;
- }
-
- /** Utility class for sorting sstables by their read rates. */
- private static class ReadRateComparator implements Comparator<SSTableReader>
- {
- private final Map<SSTableReader, Double> readRates;
-
- ReadRateComparator(Map<SSTableReader, Double> readRates)
- {
- this.readRates = readRates;
- }
-
- @Override
- public int compare(SSTableReader o1, SSTableReader o2)
- {
- Double readRate1 = readRates.get(o1);
- Double readRate2 = readRates.get(o2);
- if (readRate1 == null && readRate2 == null)
- return 0;
- else if (readRate1 == null)
- return -1;
- else if (readRate2 == null)
- return 1;
- else
- return Double.compare(readRate1, readRate2);
- }
- }
-
- private static class ResampleEntry
- {
- public final SSTableReader sstable;
- public final long newSpaceUsed;
- public final int newSamplingLevel;
-
- ResampleEntry(SSTableReader sstable, long newSpaceUsed, int newSamplingLevel)
- {
- this.sstable = sstable;
- this.newSpaceUsed = newSpaceUsed;
- this.newSamplingLevel = newSamplingLevel;
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
index ceacf87..dbe501f 100644
--- a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java
@@ -19,103 +19,48 @@
import java.io.IOException;
import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CloseableIterator;
public class KeyIterator extends AbstractIterator<DecoratedKey> implements CloseableIterator<DecoratedKey>
{
- private final static class In
- {
- private final File path;
- private volatile RandomAccessReader in;
-
- public In(File path)
- {
- this.path = path;
- }
-
- private void maybeInit()
- {
- if (in != null)
- return;
-
- synchronized (this)
- {
- if (in == null)
- {
- in = RandomAccessReader.open(path);
- }
- }
- }
-
- public DataInputPlus get()
- {
- maybeInit();
- return in;
- }
-
- public boolean isEOF()
- {
- maybeInit();
- return in.isEOF();
- }
-
- public void close()
- {
- if (in != null)
- in.close();
- }
-
- public long getFilePointer()
- {
- maybeInit();
- return in.getFilePointer();
- }
-
- public long length()
- {
- maybeInit();
- return in.length();
- }
- }
-
- private final Descriptor desc;
- private final In in;
private final IPartitioner partitioner;
+ private final KeyReader it;
private final ReadWriteLock fileAccessLock;
+ private final long totalBytes;
- private long keyPosition;
+ private boolean initialized = false;
- public KeyIterator(Descriptor desc, TableMetadata metadata)
+ public KeyIterator(KeyReader it, IPartitioner partitioner, long totalBytes, ReadWriteLock fileAccessLock)
{
- this.desc = desc;
- in = new In(new File(desc.filenameFor(Component.PRIMARY_INDEX)));
- partitioner = metadata.partitioner;
- fileAccessLock = new ReentrantReadWriteLock();
+ this.it = it;
+ this.partitioner = partitioner;
+ this.totalBytes = totalBytes;
+ this.fileAccessLock = fileAccessLock;
}
protected DecoratedKey computeNext()
{
- fileAccessLock.readLock().lock();
+ if (fileAccessLock != null)
+ fileAccessLock.readLock().lock();
try
{
- if (in.isEOF())
- return endOfData();
-
- keyPosition = in.getFilePointer();
- DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in.get()));
- RowIndexEntry.Serializer.skip(in.get(), desc.version); // skip remainder of the entry
- return key;
+ if (!initialized)
+ {
+ initialized = true;
+ return it.isExhausted()
+ ? endOfData()
+ : partitioner.decorateKey(it.key());
+ }
+ else
+ {
+ return it.advance()
+ ? partitioner.decorateKey(it.key())
+ : endOfData();
+ }
}
catch (IOException e)
{
@@ -123,45 +68,44 @@
}
finally
{
- fileAccessLock.readLock().unlock();
+ if (fileAccessLock != null)
+ fileAccessLock.readLock().unlock();
}
}
public void close()
{
- fileAccessLock.writeLock().lock();
+ if (fileAccessLock != null)
+ fileAccessLock.writeLock().lock();
try
{
- in.close();
+ it.close();
}
finally
{
- fileAccessLock.writeLock().unlock();
+ if (fileAccessLock != null)
+ fileAccessLock.writeLock().unlock();
}
}
public long getBytesRead()
{
- fileAccessLock.readLock().lock();
+ if (fileAccessLock != null)
+ fileAccessLock.readLock().lock();
try
{
- return in.getFilePointer();
+ return it.isExhausted() ? totalBytes : it.dataPosition();
}
finally
{
- fileAccessLock.readLock().unlock();
+ if (fileAccessLock != null)
+ fileAccessLock.readLock().unlock();
}
}
public long getTotalBytes()
{
- // length is final in the referenced object.
- // no need to acquire the lock
- return in.length();
+ return totalBytes;
}
- public long getKeyPosition()
- {
- return keyPosition;
- }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/KeyReader.java b/src/java/org/apache/cassandra/io/sstable/KeyReader.java
new file mode 100644
index 0000000..88ee145
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/KeyReader.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Reads keys from an SSTable.
+ * <p/>
+ * It is specific to SSTable format how the keys are read but in general the assumption is that it will read all the
+ * keys in the order as they are placed in data file.
+ * <p/>
+ * After creating it, it should be at the first key. Unless the SSTable is empty, {@link #key()},
+ * {@link #keyPositionForSecondaryIndex()} and {@link #dataPosition()} should return approriate values. If there is
+ * no data, {@link #isExhausted()} returns {@code true}. In order to move to the next key, {@link #advance()} should be
+ * called. It returns {@code true} if the reader moved to the next key; otherwise, there is no more data to read and
+ * the reader is exhausted. When the reader is exhausted, return values of {@link #key()},
+ * {@link #keyPositionForSecondaryIndex()} and {@link #dataPosition()} are undefined.
+ */
+public interface KeyReader extends Closeable
+{
+ /**
+ * Current key
+ */
+ ByteBuffer key();
+
+ /**
+ * Position in the component preferred for reading keys. This is specific to SSTable implementation
+ */
+ long keyPositionForSecondaryIndex();
+
+ /**
+ * Position in the data file where the associated content resides
+ */
+ long dataPosition();
+
+ /**
+ * Moves the iterator forward. Returns false if we reach EOF and there nothing more to read
+ */
+ boolean advance() throws IOException;
+
+ /**
+ * Returns true if we reach EOF
+ */
+ boolean isExhausted();
+
+ /**
+ * Resets the iterator to the initial position
+ */
+ void reset() throws IOException;
+
+ /**
+ * Closes the iterator quietly
+ */
+ @Override
+ void close();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/MetricsProviders.java b/src/java/org/apache/cassandra/io/sstable/MetricsProviders.java
new file mode 100644
index 0000000..cc52b63
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/MetricsProviders.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+public interface MetricsProviders
+{
+ Iterable<GaugeProvider<?>> getGaugeProviders();
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java
new file mode 100644
index 0000000..9caef0b
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+public class RangeAwareSSTableWriter implements SSTableMultiWriter
+{
+ private final List<PartitionPosition> boundaries;
+ private final List<Directories.DataDirectory> directories;
+ private final int sstableLevel;
+ private final long estimatedKeys;
+ private final long repairedAt;
+ private final TimeUUID pendingRepair;
+ private final boolean isTransient;
+ private final SSTableFormat<?, ?> format;
+ private final SerializationHeader header;
+ private final LifecycleNewTracker lifecycleNewTracker;
+ private int currentIndex = -1;
+ public final ColumnFamilyStore cfs;
+ private final List<SSTableMultiWriter> finishedWriters = new ArrayList<>();
+ private final List<SSTableReader> finishedReaders = new ArrayList<>();
+ private SSTableMultiWriter currentWriter = null;
+
+ public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat<?, ?> format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException
+ {
+ DiskBoundaries db = cfs.getDiskBoundaries();
+ directories = db.directories;
+ this.sstableLevel = sstableLevel;
+ this.cfs = cfs;
+ this.estimatedKeys = estimatedKeys / directories.size();
+ this.repairedAt = repairedAt;
+ this.pendingRepair = pendingRepair;
+ this.isTransient = isTransient;
+ this.format = format;
+ this.lifecycleNewTracker = lifecycleNewTracker;
+ this.header = header;
+ boundaries = db.positions;
+ if (boundaries == null)
+ {
+ Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
+ if (localDir == null)
+ throw new IOException(String.format("Insufficient disk space to store %s",
+ FBUtilities.prettyPrintMemory(totalSize)));
+ Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
+ currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
+ }
+ }
+
+ private void maybeSwitchWriter(DecoratedKey key)
+ {
+ if (boundaries == null)
+ return;
+
+ boolean switched = false;
+ while (currentIndex < 0 || key.compareTo(boundaries.get(currentIndex)) > 0)
+ {
+ switched = true;
+ currentIndex++;
+ }
+
+ if (switched)
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+
+ Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format);
+ currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
+ }
+ }
+
+ public boolean append(UnfilteredRowIterator partition)
+ {
+ maybeSwitchWriter(partition.partitionKey());
+ return currentWriter.append(partition);
+ }
+
+ @Override
+ public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+ currentWriter = null;
+ for (SSTableMultiWriter writer : finishedWriters)
+ {
+ if (writer.getFilePointer() > 0)
+ finishedReaders.addAll(writer.finish(repairedAt, maxDataAge, openResult));
+ else
+ SSTableMultiWriter.abortOrDie(writer);
+ }
+ return finishedReaders;
+ }
+
+ @Override
+ public Collection<SSTableReader> finish(boolean openResult)
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+ currentWriter = null;
+ for (SSTableMultiWriter writer : finishedWriters)
+ {
+ if (writer.getFilePointer() > 0)
+ finishedReaders.addAll(writer.finish(openResult));
+ else
+ SSTableMultiWriter.abortOrDie(writer);
+ }
+ return finishedReaders;
+ }
+
+ @Override
+ public Collection<SSTableReader> finished()
+ {
+ return finishedReaders;
+ }
+
+ @Override
+ public SSTableMultiWriter setOpenResult(boolean openResult)
+ {
+ finishedWriters.forEach((w) -> w.setOpenResult(openResult));
+ currentWriter.setOpenResult(openResult);
+ return this;
+ }
+
+ public String getFilename()
+ {
+ return String.join("/", cfs.keyspace.getName(), cfs.getTableName());
+ }
+
+ @Override
+ public long getFilePointer()
+ {
+ return currentWriter != null ? currentWriter.getFilePointer() : 0L;
+ }
+
+ @Override
+ public TableId getTableId()
+ {
+ return cfs.metadata.id;
+ }
+
+ @Override
+ public Throwable commit(Throwable accumulate)
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+ currentWriter = null;
+ for (SSTableMultiWriter writer : finishedWriters)
+ accumulate = writer.commit(accumulate);
+ return accumulate;
+ }
+
+ @Override
+ public Throwable abort(Throwable accumulate)
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+ currentWriter = null;
+ for (SSTableMultiWriter finishedWriter : finishedWriters)
+ accumulate = finishedWriter.abort(accumulate);
+
+ return accumulate;
+ }
+
+ @Override
+ public void prepareToCommit()
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+ currentWriter = null;
+ finishedWriters.forEach(SSTableMultiWriter::prepareToCommit);
+ }
+
+ @Override
+ public void close()
+ {
+ if (currentWriter != null)
+ finishedWriters.add(currentWriter);
+ currentWriter = null;
+ finishedWriters.forEach(SSTableMultiWriter::close);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
index 826b91d..1cd780e 100644
--- a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
@@ -17,15 +17,18 @@
*/
package org.apache.cassandra.io.sstable;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.IMergeIterator;
import org.apache.cassandra.utils.MergeIterator;
+import org.apache.cassandra.utils.Throwables;
/**
* Caller must acquire and release references to the sstables used here.
@@ -39,7 +42,17 @@
{
iters = new ArrayList<>(sstables.size());
for (SSTableReader sstable : sstables)
- iters.add(new KeyIterator(sstable.descriptor, sstable.metadata()));
+ {
+ try
+ {
+ iters.add(sstable.keyIterator());
+ }
+ catch (IOException ex)
+ {
+ iters.forEach(FileUtils::closeQuietly);
+ throw new RuntimeException("Failed to create a key iterator for sstable " + sstable.getFilename());
+ }
+ }
}
private void maybeInit()
@@ -78,7 +91,17 @@
public void close()
{
if (mi != null)
+ {
mi.close();
+ }
+ else
+ {
+ // if merging iterator was not initialized before this reducing iterator is closed, we need to close the
+ // underlying iterators manually
+ Throwable err = Throwables.close(null, iters);
+ if (err != null)
+ throw Throwables.unchecked(err);
+ }
}
public long getTotalBytes()
@@ -121,4 +144,4 @@
{
throw new UnsupportedOperationException();
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java
index 81030c2..5156adc 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTable.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java
@@ -17,16 +17,15 @@
*/
package org.apache.cassandra.io.sstable;
-
-import java.io.FileNotFoundException;
-import java.io.IOError;
-import java.io.IOException;
-import java.io.PrintWriter;
+import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
+import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -34,43 +33,35 @@
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
-import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.DiskOptimizationStrategy;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.metrics.TableMetrics;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.memory.HeapCloner;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.concurrent.SharedCloseable;
-import static org.apache.cassandra.io.util.File.WriteMode.APPEND;
+import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
/**
- * This class is built on top of the SequenceFile. It stores
- * data on disk in sorted fashion. However the sorting is upto
- * the application. This class expects keys to be handed to it
- * in sorted order.
- *
- * A separate index file is maintained as well, containing the
- * SSTable keys and the offset into the SSTable at which they are found.
- * Every 1/indexInterval key is read into memory when the SSTable is opened.
- *
- * Finally, a bloom filter file is also kept for the keys in each SSTable.
+ * This class represents an abstract sstable on disk whose keys and corresponding partitions are stored in
+ * a {@link SSTableFormat.Components#DATA} file in order as imposed by {@link DecoratedKey#comparator}.
*/
public abstract class SSTable
{
@@ -78,70 +69,101 @@
public static final int TOMBSTONE_HISTOGRAM_BIN_SIZE = 100;
public static final int TOMBSTONE_HISTOGRAM_SPOOL_SIZE = 100000;
- public static final int TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS = Integer.valueOf(System.getProperty("cassandra.streaminghistogram.roundseconds", "60"));
+ public static final int TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS = CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.getInt();
public final Descriptor descriptor;
protected final Set<Component> components;
public final boolean compression;
- public DecoratedKey first;
- public DecoratedKey last;
-
- protected final DiskOptimizationStrategy optimizationStrategy;
protected final TableMetadataRef metadata;
- protected SSTable(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata, DiskOptimizationStrategy optimizationStrategy)
- {
- // In almost all cases, metadata shouldn't be null, but allowing null allows to create a mostly functional SSTable without
- // full schema definition. SSTableLoader use that ability
- assert descriptor != null;
- assert components != null;
+ public final ChunkCache chunkCache;
+ public final IOOptions ioOptions;
- this.descriptor = descriptor;
- Set<Component> dataComponents = new HashSet<>(components);
- this.compression = dataComponents.contains(Component.COMPRESSION_INFO);
- this.components = new CopyOnWriteArraySet<>(dataComponents);
- this.metadata = metadata;
- this.optimizationStrategy = Objects.requireNonNull(optimizationStrategy);
+ @Nullable
+ private final WeakReference<Owner> owner;
+
+ public SSTable(Builder<?, ?> builder, Owner owner)
+ {
+ this.owner = new WeakReference<>(owner);
+ checkNotNull(builder.descriptor);
+ checkNotNull(builder.getComponents());
+
+ this.descriptor = builder.descriptor;
+ this.ioOptions = builder.getIOOptions();
+ this.components = new CopyOnWriteArraySet<>(builder.getComponents());
+ this.compression = components.contains(Components.COMPRESSION_INFO);
+ this.metadata = builder.getTableMetadataRef();
+ this.chunkCache = builder.getChunkCache();
}
+ public final Optional<Owner> owner()
+ {
+ if (owner == null)
+ return Optional.empty();
+ return Optional.ofNullable(owner.get());
+ }
+
+ public static void rename(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
+ {
+ components.stream()
+ .filter(c -> !newdesc.getFormat().generatedOnLoadComponents().contains(c))
+ .filter(c -> !c.equals(Components.DATA))
+ .forEach(c -> tmpdesc.fileFor(c).move(newdesc.fileFor(c)));
+
+ // do -Data last because -Data present should mean the sstable was completely renamed before crash
+ tmpdesc.fileFor(Components.DATA).move(newdesc.fileFor(Components.DATA));
+
+ // rename it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
+ components.stream()
+ .filter(c -> newdesc.getFormat().generatedOnLoadComponents().contains(c))
+ .forEach(c -> tmpdesc.fileFor(c).tryMove(newdesc.fileFor(c)));
+ }
+
+ public static void copy(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
+ {
+ components.stream()
+ .filter(c -> !newdesc.getFormat().generatedOnLoadComponents().contains(c))
+ .filter(c -> !c.equals(Components.DATA))
+ .forEach(c -> FileUtils.copyWithConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c)));
+
+ // do -Data last because -Data present should mean the sstable was completely copied before crash
+ FileUtils.copyWithConfirm(tmpdesc.fileFor(Components.DATA), newdesc.fileFor(Components.DATA));
+
+ // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
+ components.stream()
+ .filter(c -> newdesc.getFormat().generatedOnLoadComponents().contains(c))
+ .forEach(c -> FileUtils.copyWithOutConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c)));
+ }
+
+ public static void hardlink(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
+ {
+ components.stream()
+ .filter(c -> !newdesc.getFormat().generatedOnLoadComponents().contains(c))
+ .filter(c -> !c.equals(Components.DATA))
+ .forEach(c -> FileUtils.createHardLinkWithConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c)));
+
+ // do -Data last because -Data present should mean the sstable was completely copied before crash
+ FileUtils.createHardLinkWithConfirm(tmpdesc.fileFor(Components.DATA), newdesc.fileFor(Components.DATA));
+
+ // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
+ components.stream()
+ .filter(c -> newdesc.getFormat().generatedOnLoadComponents().contains(c))
+ .forEach(c -> FileUtils.createHardLinkWithoutConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c)));
+ }
+
+ public abstract DecoratedKey getFirst();
+
+ public abstract DecoratedKey getLast();
+
+ public abstract AbstractBounds<Token> getBounds();
+
@VisibleForTesting
public Set<Component> getComponents()
{
return ImmutableSet.copyOf(components);
}
- /**
- * We use a ReferenceQueue to manage deleting files that have been compacted
- * and for which no more SSTable references exist. But this is not guaranteed
- * to run for each such file because of the semantics of the JVM gc. So,
- * we write a marker to `compactedFilename` when a file is compacted;
- * if such a marker exists on startup, the file should be removed.
- *
- * This method will also remove SSTables that are marked as temporary.
- *
- * @return true if the file was deleted
- */
- public static boolean delete(Descriptor desc, Set<Component> components)
- {
- logger.info("Deleting sstable: {}", desc);
- // remove the DATA component first if it exists
- if (components.contains(Component.DATA))
- FileUtils.deleteWithConfirm(desc.filenameFor(Component.DATA));
- for (Component component : components)
- {
- if (component.equals(Component.DATA) || component.equals(Component.SUMMARY))
- continue;
-
- FileUtils.deleteWithConfirm(desc.filenameFor(component));
- }
-
- if (components.contains(Component.SUMMARY))
- FileUtils.delete(desc.filenameFor(Component.SUMMARY));
-
- return true;
- }
-
public TableMetadata metadata()
{
return metadata.get();
@@ -157,25 +179,9 @@
return getPartitioner().decorateKey(key);
}
- /**
- * If the given @param key occupies only part of a larger buffer, allocate a new buffer that is only
- * as large as necessary.
- */
- public static DecoratedKey getMinimalKey(DecoratedKey key)
- {
- return key.getKey().position() > 0 || key.getKey().hasRemaining() || !key.getKey().hasArray()
- ? new BufferDecoratedKey(key.getToken(), HeapCloner.instance.clone(key.getKey()))
- : key;
- }
-
public String getFilename()
{
- return descriptor.filenameFor(Component.DATA);
- }
-
- public String getIndexFilename()
- {
- return descriptor.filenameFor(Component.PRIMARY_INDEX);
+ return descriptor.fileFor(Components.DATA).absolutePath();
}
public String getColumnFamilyName()
@@ -192,11 +198,32 @@
{
List<String> ret = new ArrayList<>(components.size());
for (Component component : components)
- ret.add(descriptor.filenameFor(component));
+ ret.add(descriptor.fileFor(component).absolutePath());
return ret;
}
/**
+ * The method sets fields for this sstable representation on the provided {@link Builder}. The method is intended
+ * to be called from the overloaded {@code unbuildTo} method in subclasses.
+ *
+ * @param builder the builder on which the fields should be set
+ * @param sharedCopy whether the {@link SharedCloseable} resources should be passed as shared copies or directly;
+ * note that the method will overwrite the fields representing {@link SharedCloseable} only if
+ * they are not set in the builder yet (the relevant fields in the builder are {@code null}).
+ * Although {@link SSTable} does not keep any references to resources, the parameters is added
+ * for the possible future fields and for consistency with the overloaded implementations in
+ * subclasses
+ * @return the same instance of builder as provided
+ */
+ protected final <B extends Builder<?, B>> B unbuildTo(B builder, boolean sharedCopy)
+ {
+ return builder.setTableMetadataRef(metadata)
+ .setComponents(components)
+ .setChunkCache(chunkCache)
+ .setIOOptions(ioOptions);
+ }
+
+ /**
* Parse a sstable filename into both a {@link Descriptor} and {@code Component} object.
*
* @param file the filename to parse.
@@ -208,7 +235,29 @@
{
try
{
- return Descriptor.fromFilenameWithComponent(file);
+ return Descriptor.fromFileWithComponent(file);
+ }
+ catch (Throwable e)
+ {
+ return null;
+ }
+ }
+
+ /**
+ * Parse a sstable filename into both a {@link Descriptor} and {@code Component} object.
+ *
+ * @param file the filename to parse.
+ * @param keyspace The keyspace name of the file.
+ * @param table The table name of the file.
+ * @return a pair of the {@code Descriptor} and {@code Component} corresponding to {@code file} if it corresponds to
+ * a valid and supported sstable filename, {@code null} otherwise. Note that components of an unknown type will be
+ * returned as CUSTOM ones.
+ */
+ public static Pair<Descriptor, Component> tryComponentFromFilename(File file, String keyspace, String table)
+ {
+ try
+ {
+ return Descriptor.fromFileWithComponent(file, keyspace, table);
}
catch (Throwable e)
{
@@ -226,11 +275,11 @@
* @return the {@code Descriptor} corresponding to {@code file} if it corresponds to a valid and supported sstable
* filename, {@code null} otherwise.
*/
- public static Descriptor tryDescriptorFromFilename(File file)
+ public static Descriptor tryDescriptorFromFile(File file)
{
try
{
- return Descriptor.fromFilename(file);
+ return Descriptor.fromFile(file);
}
catch (Throwable e)
{
@@ -238,150 +287,10 @@
}
}
- /**
- * Discovers existing components for the descriptor. Slow: only intended for use outside the critical path.
- */
- public static Set<Component> componentsFor(final Descriptor desc)
- {
- try
- {
- try
- {
- return readTOC(desc);
- }
- catch (FileNotFoundException | NoSuchFileException e)
- {
- Set<Component> components = discoverComponentsFor(desc);
- if (components.isEmpty())
- return components; // sstable doesn't exist yet
-
- if (!components.contains(Component.TOC))
- components.add(Component.TOC);
- appendTOC(desc, components);
- return components;
- }
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
-
- public static Set<Component> discoverComponentsFor(Descriptor desc)
- {
- Set<Component.Type> knownTypes = Sets.difference(Component.TYPES, Collections.singleton(Component.Type.CUSTOM));
- Set<Component> components = Sets.newHashSetWithExpectedSize(knownTypes.size());
- for (Component.Type componentType : knownTypes)
- {
- Component component = new Component(componentType);
- if (new File(desc.filenameFor(component)).exists())
- components.add(component);
- }
- return components;
- }
-
- /** @return An estimate of the number of keys contained in the given index file. */
- public static long estimateRowsFromIndex(RandomAccessReader ifile, Descriptor descriptor) throws IOException
- {
- // collect sizes for the first 10000 keys, or first 10 mebibytes of data
- final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length());
- int keys = 0;
- while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP)
- {
- ByteBufferUtil.skipShortLength(ifile);
- RowIndexEntry.Serializer.skip(ifile, descriptor.version);
- keys++;
- }
- assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile;
- long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys);
- ifile.seek(0);
- return estimatedRows;
- }
-
- public long bytesOnDisk()
- {
- long bytes = 0;
- for (Component component : components)
- {
- bytes += new File(descriptor.filenameFor(component)).length();
- }
- return bytes;
- }
-
@Override
public String toString()
{
- return getClass().getSimpleName() + "(" +
- "path='" + getFilename() + '\'' +
- ')';
- }
-
- /**
- * Reads the list of components from the TOC component.
- * @return set of components found in the TOC
- */
- protected static Set<Component> readTOC(Descriptor descriptor) throws IOException
- {
- return readTOC(descriptor, true);
- }
-
- /**
- * Reads the list of components from the TOC component.
- * @param skipMissing, skip adding the component to the returned set if the corresponding file is missing.
- * @return set of components found in the TOC
- */
- protected static Set<Component> readTOC(Descriptor descriptor, boolean skipMissing) throws IOException
- {
- File tocFile = new File(descriptor.filenameFor(Component.TOC));
- List<String> componentNames = Files.readAllLines(tocFile.toPath());
- Set<Component> components = Sets.newHashSetWithExpectedSize(componentNames.size());
- for (String componentName : componentNames)
- {
- Component component = new Component(Component.Type.fromRepresentation(componentName), componentName);
- if (skipMissing && !new File(descriptor.filenameFor(component)).exists())
- logger.error("Missing component: {}", descriptor.filenameFor(component));
- else
- components.add(component);
- }
- return components;
- }
-
- /**
- * Appends new component names to the TOC component.
- */
- protected static void appendTOC(Descriptor descriptor, Collection<Component> components)
- {
- File tocFile = new File(descriptor.filenameFor(Component.TOC));
- try (FileOutputStreamPlus out = tocFile.newOutputStream(APPEND);
- PrintWriter w = new PrintWriter(out))
- {
- for (Component component : components)
- w.println(component.name);
- w.flush();
- out.sync();
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, tocFile);
- }
- }
-
- /**
- * Registers new custom components. Used by custom compaction strategies.
- * Adding a component for the second time is a no-op.
- * Don't remove this - this method is a part of the public API, intended for use by custom compaction strategies.
- * @param newComponents collection of components to be added
- */
- public synchronized void addComponents(Collection<Component> newComponents)
- {
- Collection<Component> componentsToAdd = Collections2.filter(newComponents, Predicates.not(Predicates.in(components)));
- appendTOC(descriptor, componentsToAdd);
- components.addAll(componentsToAdd);
- }
-
- public AbstractBounds<Token> getBounds()
- {
- return AbstractBounds.bounds(first.getToken(), true, last.getToken(), true);
+ return String.format("%s:%s(path='%s')", getClass().getSimpleName(), descriptor.version.format.name(), getFilename());
}
public static void validateRepairedMetadata(long repairedAt, TimeUUID pendingRepair, boolean isTransient)
@@ -390,6 +299,118 @@
"pendingRepair cannot be set on a repaired sstable");
Preconditions.checkArgument(!isTransient || (pendingRepair != NO_PENDING_REPAIR),
"isTransient can only be true for sstables pending repair");
+ }
+ /**
+ * Registers new custom components. Used by custom compaction strategies.
+ * Adding a component for the second time is a no-op.
+ * Don't remove this - this method is a part of the public API, intended for use by custom compaction strategies.
+ *
+ * @param newComponents collection of components to be added
+ */
+ public synchronized void addComponents(Collection<Component> newComponents)
+ {
+ Collection<Component> componentsToAdd = Collections2.filter(newComponents, Predicates.not(Predicates.in(components)));
+ TOCComponent.appendTOC(descriptor, componentsToAdd);
+ components.addAll(componentsToAdd);
+ }
+
+ public interface Owner
+ {
+ Double getCrcCheckChance();
+
+ OpOrder.Barrier newReadOrderingBarrier();
+
+ TableMetrics getMetrics();
+ }
+
+ /**
+ * A builder of this sstable representation. It should be extended for each implementation with the specific fields.
+ *
+ * @param <S> type of the sstable representation to be build with this builder
+ * @param <B> type of this builder
+ */
+ public static class Builder<S extends SSTable, B extends Builder<S, B>>
+ {
+ public final Descriptor descriptor;
+
+ private Set<Component> components;
+ private TableMetadataRef tableMetadataRef;
+ private ChunkCache chunkCache = ChunkCache.instance;
+ private IOOptions ioOptions = IOOptions.fromDatabaseDescriptor();
+
+ public Builder(Descriptor descriptor)
+ {
+ checkNotNull(descriptor);
+ this.descriptor = descriptor;
+ }
+
+ public B setComponents(Collection<Component> components)
+ {
+ if (components != null)
+ {
+ components.forEach(c -> Preconditions.checkState(c.isValidFor(descriptor), "Invalid component type for sstable format " + descriptor.version.format.name()));
+ this.components = ImmutableSet.copyOf(components);
+ }
+ else
+ {
+ this.components = null;
+ }
+ return (B) this;
+ }
+
+ public B addComponents(Collection<Component> components)
+ {
+ if (components == null || components.isEmpty())
+ return (B) this;
+
+ if (this.components == null)
+ return setComponents(components);
+
+ return setComponents(Sets.union(this.components, ImmutableSet.copyOf(components)));
+ }
+
+ public B setTableMetadataRef(TableMetadataRef ref)
+ {
+ this.tableMetadataRef = ref;
+ return (B) this;
+ }
+
+ public B setChunkCache(ChunkCache chunkCache)
+ {
+ this.chunkCache = chunkCache;
+ return (B) this;
+ }
+
+ public B setIOOptions(IOOptions ioOptions)
+ {
+ this.ioOptions = ioOptions;
+ return (B) this;
+ }
+
+ public Descriptor getDescriptor()
+ {
+ return descriptor;
+ }
+
+ public Set<Component> getComponents()
+ {
+ return components;
+ }
+
+ public TableMetadataRef getTableMetadataRef()
+ {
+ return tableMetadataRef;
+ }
+
+ public ChunkCache getChunkCache()
+ {
+ return chunkCache;
+ }
+
+ public IOOptions getIOOptions()
+ {
+ return ioOptions;
+ }
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
new file mode 100644
index 0000000..025d8cb
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+/**
+ * Observer for events in the lifecycle of writing out an sstable.
+ */
+public interface SSTableFlushObserver
+{
+ /**
+ * Called before writing any data to the sstable.
+ */
+ void begin();
+
+ /**
+ * Called when a new partition in being written to the sstable,
+ * but before any cells are processed (see {@link #nextUnfilteredCluster(Unfiltered)}).
+ *
+ * @param key the key being appended to SSTable.
+ * @param keyPosition the position of the key in the SSTable data file
+ * @param KeyPositionForSASI SSTable format specific key position for storage attached indexes, it can be
+ * in data file or in some index file. It is the same position as returned by
+ * {@link KeyReader#keyPositionForSecondaryIndex()} for the same format, and the same
+ * position as expected by {@link SSTableReader#keyAtPositionFromSecondaryIndex(long)}.
+ */
+ void startPartition(DecoratedKey key, long keyPosition, long KeyPositionForSASI);
+
+ /**
+ * Called when a static row is being written to the sstable. If static columns are present in the table, it is called
+ * after {@link #startPartition(DecoratedKey, long, long)} and before any calls to {@link #nextUnfilteredCluster(Unfiltered)}.
+ *
+ * @param staticRow static row appended to the sstable, can be empty, may not be {@code null}
+ */
+ void staticRow(Row staticRow);
+
+ /**
+ * Called after the unfiltered cluster is written to the sstable.
+ * Will be preceded by a call to {@code startPartition(DecoratedKey, long)},
+ * and the cluster should be assumed to belong to that partition.
+ *
+ * @param unfilteredCluster The unfiltered cluster being added to SSTable.
+ */
+ void nextUnfilteredCluster(Unfiltered unfilteredCluster);
+
+ /**
+ * Called when all data is written to the file and it's ready to be finished up.
+ */
+ void complete();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
index f78200a..e785196 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java
@@ -35,7 +35,6 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,8 +53,11 @@
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Schema;
@@ -66,15 +68,15 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_SKIP_AUTOMATIC_UDT_FIX;
+
/**
* Validates and fixes type issues in the serialization-header of sstables.
*/
public abstract class SSTableHeaderFix
{
// C* 3.0 upgrade code
-
- private static final String SKIPAUTOMATICUDTFIX = "cassandra.skipautomaticudtfix";
- private static final boolean SKIP_AUTOMATIC_FIX_ON_UPGRADE = Boolean.getBoolean(SKIPAUTOMATICUDTFIX);
+ private static final boolean SKIP_AUTOMATIC_FIX_ON_UPGRADE = CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getBoolean();
public static void fixNonFrozenUDTIfUpgradeFrom30()
{
@@ -94,7 +96,7 @@
"sstable metadata serialization-headers",
previousVersionString,
FBUtilities.getReleaseVersionString(),
- SKIPAUTOMATICUDTFIX);
+ CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getKey());
return;
}
@@ -301,7 +303,7 @@
.filter(p -> {
try
{
- return Descriptor.fromFilenameWithComponent(new File(p)).right.type == Component.Type.DATA;
+ return Descriptor.fromFileWithComponent(new File(p)).right.type == Types.DATA;
}
catch (IllegalArgumentException t)
{
@@ -309,8 +311,8 @@
return false;
}
})
- .map(Path::toString)
- .map(Descriptor::fromFilename)
+ .map(File::new)
+ .map(file -> Descriptor.fromFileWithComponent(file, false).left)
.forEach(descriptors::add);
}
@@ -342,8 +344,8 @@
return;
}
- Set<Component> components = SSTable.discoverComponentsFor(desc);
- if (components.stream().noneMatch(c -> c.type == Component.Type.STATS))
+ Set<Component> components = desc.discoverComponents();
+ if (components.stream().noneMatch(c -> c.type == Types.STATS))
{
error("sstable %s has no -Statistics.db component.", desc);
return;
@@ -845,7 +847,7 @@
private void writeNewMetadata(Descriptor desc, Map<MetadataType, MetadataComponent> newMetadata)
{
- String file = desc.filenameFor(Component.STATS);
+ File file = desc.fileFor(Components.STATS);
info.accept(String.format(" Writing new metadata file %s", file));
try
{
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableId.java b/src/java/org/apache/cassandra/io/sstable/SSTableId.java
index a3d95dd..7a2235b 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableId.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableId.java
@@ -33,7 +33,7 @@
* A new implementation must adhere to the following invariants:
* - Must be locally sortable - that is, the comparison must reflect the comparison of generation times of identifiers
* generated on the same node
- * - String representation must *not* include the {@link Descriptor#FILENAME_SEPARATOR} character, see {@link Descriptor#fromFilenameWithComponent(File)}
+ * - String representation must *not* include the {@link Descriptor#FILENAME_SEPARATOR} character, see {@link Descriptor#fromFileWithComponent(File)}
* - must be case-insensitive because the sstables can be stored on case-insensitive file system
* <p>
*/
@@ -50,7 +50,7 @@
* {@link Builder#fromString(String)}
* <p>
* Must not contain any {@link Descriptor#FILENAME_SEPARATOR} character as it is used in the Descriptor
- * see {@link Descriptor#fromFilenameWithComponent(File)}
+ * see {@link Descriptor#fromFileWithComponent(File)}
*/
@Override
String toString();
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
index 76e12c8..b05b3c3 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
@@ -17,16 +17,26 @@
*/
package org.apache.cassandra.io.sstable;
-import java.io.*;
+import java.io.IOError;
+import java.io.IOException;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.utils.vint.VIntCoding.VIntOutOfRangeException;
+
public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterator>, UnfilteredRowIterator
{
private final SSTableReader sstable;
@@ -69,13 +79,16 @@
}
@SuppressWarnings("resource")
- public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, RowIndexEntry<?> indexEntry, DecoratedKey key, boolean tombstoneOnly)
+ public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, long dataPosition, DecoratedKey key, boolean tombstoneOnly)
{
try
{
- dfile.seek(indexEntry.position);
+ dfile.seek(dataPosition);
ByteBufferUtil.skipShortLength(dfile); // Skip partition key
DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(dfile);
+ if (!partitionLevelDeletion.validate())
+ UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
+
DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
SSTableSimpleIterator iterator = tombstoneOnly
? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion)
@@ -125,7 +138,7 @@
{
return iterator.hasNext();
}
- catch (IndexOutOfBoundsException e)
+ catch (IndexOutOfBoundsException | VIntOutOfRangeException e)
{
sstable.markSuspect();
throw new CorruptSSTableException(e, filename);
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index 7ddbe72..187d6ce 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@ -17,25 +17,43 @@
*/
package org.apache.cassandra.io.sstable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
-import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.io.FSError;
-import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.streaming.*;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.streaming.OutgoingStream;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamEvent;
+import org.apache.cassandra.streaming.StreamEventHandler;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.streaming.StreamingChannel;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Pair;
-
import org.apache.cassandra.utils.concurrent.Ref;
import static org.apache.cassandra.streaming.StreamingChannel.Factory.Global.streamingFactory;
@@ -48,13 +66,13 @@
{
private final File directory;
private final String keyspace;
+ private final String table;
private final Client client;
private final int connectionsPerHost;
private final OutputHandler outputHandler;
private final Set<InetAddressAndPort> failedHosts = new HashSet<>();
private final List<SSTableReader> sstables = new ArrayList<>();
- private final Multimap<InetAddressAndPort, OutgoingStream> streamingDetails = HashMultimap.create();
public SSTableLoader(File directory, Client client, OutputHandler outputHandler)
{
@@ -63,18 +81,25 @@
public SSTableLoader(File directory, Client client, OutputHandler outputHandler, int connectionsPerHost, String targetKeyspace)
{
+ this(directory, client, outputHandler, connectionsPerHost, targetKeyspace, null);
+ }
+
+ public SSTableLoader(File directory, Client client, OutputHandler outputHandler, int connectionsPerHost, String targetKeyspace, String targetTable)
+ {
this.directory = directory;
this.keyspace = targetKeyspace != null ? targetKeyspace : directory.parent().name();
+ this.table = targetTable;
this.client = client;
this.outputHandler = outputHandler;
this.connectionsPerHost = connectionsPerHost;
}
@SuppressWarnings("resource")
- protected Collection<SSTableReader> openSSTables(final Map<InetAddressAndPort, Collection<Range<Token>>> ranges)
+ private Multimap<InetAddressAndPort, CassandraOutgoingFile> openSSTables(final Map<InetAddressAndPort, Collection<Range<Token>>> ranges)
{
outputHandler.output("Opening sstables and calculating sections to stream");
+ Multimap<InetAddressAndPort, CassandraOutgoingFile> streamingDetails = HashMultimap.create();
LifecycleTransaction.getFiles(directory.toPath(),
(file, type) ->
{
@@ -87,15 +112,27 @@
return false;
}
- Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(file);
+ Pair<Descriptor, Component> p;
+ if (null != keyspace && null != table)
+ {
+ p = SSTable.tryComponentFromFilename(file, keyspace, table);
+ }
+ else
+ {
+ p = SSTable.tryComponentFromFilename(file);
+ }
+
Descriptor desc = p == null ? null : p.left;
- if (p == null || !p.right.equals(Component.DATA))
+ if (p == null || !p.right.equals(Components.DATA))
return false;
- if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists())
+ for (Component c : desc.getFormat().primaryComponents())
{
- outputHandler.output(String.format("Skipping file %s because index is missing", name));
- return false;
+ if (!desc.fileFor(c).exists())
+ {
+ outputHandler.output(String.format("Skipping file %s because %s is missing", name, c.name));
+ return false;
+ }
}
TableMetadataRef metadata = client.getTableMetadata(desc.cfname);
@@ -105,22 +142,14 @@
return false;
}
- Set<Component> components = new HashSet<>();
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- if (new File(desc.filenameFor(Component.SUMMARY)).exists())
- components.add(Component.SUMMARY);
- if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
- components.add(Component.COMPRESSION_INFO);
- if (new File(desc.filenameFor(Component.STATS)).exists())
- components.add(Component.STATS);
+ Set<Component> components = desc.getComponents(desc.getFormat().primaryComponents(), desc.getFormat().uploadComponents());
try
{
// To conserve memory, open SSTableReaders without bloom filters and discard
// the index summary after calculating the file sections to stream and the estimated
// number of keys for each endpoint. See CASSANDRA-5555 for details.
- SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata);
+ SSTableReader sstable = SSTableReader.openForBatch(null, desc, components, metadata);
sstables.add(sstable);
// calculate the sstable sections to stream as well as the estimated number of
@@ -138,12 +167,12 @@
long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
Ref<SSTableReader> ref = sstable.ref();
- OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.BULK_LOAD, ref, sstableSections, tokenRanges, estimatedKeys);
+ CassandraOutgoingFile stream = new CassandraOutgoingFile(StreamOperation.BULK_LOAD, ref, sstableSections, tokenRanges, estimatedKeys);
streamingDetails.put(endpoint, stream);
}
// to conserve heap space when bulk loading
- sstable.releaseSummary();
+ sstable.releaseInMemoryComponents();
}
catch (FSError e)
{
@@ -154,7 +183,7 @@
},
Directories.OnTxnErr.IGNORE);
- return sstables;
+ return streamingDetails;
}
public StreamResultFuture stream()
@@ -170,14 +199,14 @@
StreamPlan plan = new StreamPlan(StreamOperation.BULK_LOAD, connectionsPerHost, false, null, PreviewKind.NONE).connectionFactory(client.getConnectionFactory());
Map<InetAddressAndPort, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap();
- openSSTables(endpointToRanges);
- if (sstables.isEmpty())
+ Multimap<InetAddressAndPort, CassandraOutgoingFile> streamingDetails = openSSTables(endpointToRanges);
+ if (streamingDetails.isEmpty())
{
// return empty result
return plan.execute();
}
- outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet()));
+ outputHandler.output(String.format("Streaming relevant part of %s to %s", names(streamingDetails.values()), endpointToRanges.keySet()));
for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : endpointToRanges.entrySet())
{
@@ -185,13 +214,8 @@
if (toIgnore.contains(remote))
continue;
- List<OutgoingStream> streams = new LinkedList<>();
-
// references are acquired when constructing the SSTableStreamingSections above
- for (OutgoingStream stream : streamingDetails.get(remote))
- {
- streams.add(stream);
- }
+ List<OutgoingStream> streams = new LinkedList<>(streamingDetails.get(remote));
plan.transferStreams(remote, streams);
}
@@ -213,10 +237,13 @@
*/
private void releaseReferences()
{
- for (SSTableReader sstable : sstables)
+ Iterator<SSTableReader> it = sstables.iterator();
+ while (it.hasNext())
{
+ SSTableReader sstable = it.next();
sstable.selfRef().release();
assert sstable.selfRef().globalCount() == 0 : String.format("for sstable = %s, ref count = %d", sstable, sstable.selfRef().globalCount());
+ it.remove();
}
}
@@ -230,12 +257,9 @@
}
}
- private String names(Collection<SSTableReader> sstables)
+ private String names(Collection<CassandraOutgoingFile> sstables)
{
- StringBuilder builder = new StringBuilder();
- for (SSTableReader sstable : sstables)
- builder.append(sstable.descriptor.filenameFor(Component.DATA)).append(" ");
- return builder.toString();
+ return sstables.stream().map(CassandraOutgoingFile::getName).distinct().collect(Collectors.joining(" "));
}
public Set<InetAddressAndPort> getFailedHosts()
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java b/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java
new file mode 100644
index 0000000..6b494d6
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.tracing.Tracing;
+
+/**
+ * Listener for receiving notifications associated with reading SSTables.
+ */
+public interface SSTableReadsListener
+{
+ /**
+ * The reasons for skipping an SSTable
+ */
+ enum SkippingReason
+ {
+ BLOOM_FILTER("Bloom filter allows skipping sstable {}"),
+ MIN_MAX_KEYS("Check against min and max keys allows skipping sstable {}"),
+ PARTITION_INDEX_LOOKUP("Partition index lookup allows skipping sstable {}"),
+ INDEX_ENTRY_NOT_FOUND("Partition index lookup complete (bloom filter false positive) for sstable {}");
+
+ private final String message;
+
+ SkippingReason(String message)
+ {
+ this.message = message;
+ }
+
+ public void trace(Descriptor descriptor)
+ {
+ Tracing.trace(message, descriptor.id);
+ }
+ }
+
+ /**
+ * The reasons for selecting an SSTable
+ */
+ enum SelectionReason
+ {
+ KEY_CACHE_HIT("Key cache hit for sstable {}, size = {}"),
+ INDEX_ENTRY_FOUND("Partition index found for sstable {}, size = {}");
+
+ private final String message;
+
+ SelectionReason(String message)
+ {
+ this.message = message;
+ }
+
+ public void trace(Descriptor descriptor, AbstractRowIndexEntry entry)
+ {
+ Tracing.trace(message, descriptor.id, entry.blockCount());
+ }
+ }
+
+ /**
+ * Listener that does nothing.
+ */
+ static final SSTableReadsListener NOOP_LISTENER = new SSTableReadsListener() {};
+
+ /**
+ * Handles notification that the specified SSTable has been skipped during a single partition query.
+ *
+ * @param sstable the SSTable reader
+ * @param reason the reason for which the SSTable has been skipped
+ */
+ default void onSSTableSkipped(SSTableReader sstable, SkippingReason reason)
+ {
+ }
+
+ /**
+ * Handles notification that the specified SSTable has been selected during a single partition query.
+ *
+ * @param sstable the SSTable reader
+ * @param indexEntry the index entry
+ * @param reason the reason for which the SSTable has been selected
+ */
+ default void onSSTableSelected(SSTableReader sstable, SelectionReason reason)
+ {
+ }
+
+ /**
+ * Handles notification that the specified SSTable is being scanned during a partition range query.
+ *
+ * @param sstable the SSTable reader of the SSTable being scanned.
+ */
+ default void onScanningStarted(SSTableReader sstable)
+ {
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
index e394bbd..82ae4dc 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
@@ -17,22 +17,18 @@
*/
package org.apache.cassandra.io.sstable;
-import java.lang.ref.WeakReference;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.cache.InstrumentingCache;
-import org.apache.cassandra.cache.KeyCacheKey;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.concurrent.Transactional;
/**
@@ -43,12 +39,12 @@
* for on-close (i.e. when all references expire) that drops the page cache prior to that key position
*
* hard-links are created for each partially written sstable so that readers opened against them continue to work past
- * the rename of the temporary file, which is deleted once all readers against the hard-link have been closed.
+ * renaming of the temporary file, which is deleted once all readers against the hard-link have been closed.
* If for any reason the writer is rolled over, we immediately rename and fully expose the completed file in the Tracker.
*
- * On abort we restore the original lower bounds to the existing readers and delete any temporary files we had in progress,
- * but leave any hard-links in place for the readers we opened to cleanup when they're finished as we would had we finished
- * successfully.
+ * On abort, we restore the original lower bounds to the existing readers and delete any temporary files we had in progress,
+ * but leave any hard-links in place for the readers we opened, and clean-up when the readers finish as we would do
+ * if we had finished successfully.
*/
public class SSTableRewriter extends Transactional.AbstractTransactional implements Transactional
{
@@ -69,7 +65,6 @@
private final boolean eagerWriterMetaRelease; // true if the writer metadata should be released when switch is called
private SSTableWriter writer;
- private Map<DecoratedKey, RowIndexEntry> cachedKeys = new HashMap<>();
// for testing (TODO: remove when have byteman setup)
private boolean throwEarly, throwLate;
@@ -117,31 +112,16 @@
return writer;
}
- public RowIndexEntry append(UnfilteredRowIterator partition)
+ public AbstractRowIndexEntry append(UnfilteredRowIterator partition)
{
- // we do this before appending to ensure we can resetAndTruncate() safely if the append fails
+ // we do this before appending to ensure we can resetAndTruncate() safely if appending fails
DecoratedKey key = partition.partitionKey();
maybeReopenEarly(key);
- RowIndexEntry index = writer.append(partition);
- if (DatabaseDescriptor.shouldMigrateKeycacheOnCompaction())
- {
- if (!transaction.isOffline() && index != null)
- {
- for (SSTableReader reader : transaction.originals())
- {
- if (reader.getCachedPosition(key, false) != null)
- {
- cachedKeys.put(key, index);
- break;
- }
- }
- }
- }
- return index;
+ return writer.append(partition);
}
// attempts to append the row, if fails resets the writer position
- public RowIndexEntry tryAppend(UnfilteredRowIterator partition)
+ public AbstractRowIndexEntry tryAppend(UnfilteredRowIterator partition)
{
writer.mark();
try
@@ -163,20 +143,18 @@
{
for (SSTableReader reader : transaction.originals())
{
- RowIndexEntry index = reader.getPosition(key, SSTableReader.Operator.GE);
- NativeLibrary.trySkipCache(reader.getFilename(), 0, index == null ? 0 : index.position);
+ reader.trySkipFileCacheBefore(key);
}
}
else
{
- SSTableReader reader = writer.setMaxDataAge(maxAge).openEarly();
- if (reader != null)
- {
+ writer.setMaxDataAge(maxAge);
+ writer.openEarly(reader -> {
transaction.update(reader, false);
currentlyOpenedEarlyAt = writer.getFilePointer();
- moveStarts(reader, reader.last);
+ moveStarts(reader.last);
transaction.checkpoint();
- }
+ });
}
}
}
@@ -210,27 +188,13 @@
* one, the old *instance* will have reference count == 0 and if we were to start a new compaction with that old
* instance, we would get exceptions.
*
- * @param newReader the rewritten reader that replaces them for this region
* @param lowerbound if !reset, must be non-null, and marks the exclusive lowerbound of the start for each sstable
*/
- private void moveStarts(SSTableReader newReader, DecoratedKey lowerbound)
+ private void moveStarts(DecoratedKey lowerbound)
{
if (transaction.isOffline() || preemptiveOpenInterval == Long.MAX_VALUE)
return;
- newReader.setupOnline();
- List<DecoratedKey> invalidateKeys = null;
- if (!cachedKeys.isEmpty())
- {
- invalidateKeys = new ArrayList<>(cachedKeys.size());
- for (Map.Entry<DecoratedKey, RowIndexEntry> cacheKey : cachedKeys.entrySet())
- {
- invalidateKeys.add(cacheKey.getKey());
- newReader.cacheKey(cacheKey.getKey(), cacheKey.getValue());
- }
- }
-
- cachedKeys.clear();
for (SSTableReader sstable : transaction.originals())
{
// we call getCurrentReplacement() to support multiple rewriters operating over the same source readers at once.
@@ -241,49 +205,19 @@
if (latest.first.compareTo(lowerbound) > 0)
continue;
- Runnable runOnClose = invalidateKeys != null ? new InvalidateKeys(latest, invalidateKeys) : null;
if (lowerbound.compareTo(latest.last) >= 0)
{
if (!transaction.isObsolete(latest))
- {
- if (runOnClose != null)
- {
- latest.runOnClose(runOnClose);
- }
transaction.obsolete(latest);
- }
continue;
}
- DecoratedKey newStart = latest.firstKeyBeyond(lowerbound);
- assert newStart != null;
- SSTableReader replacement = latest.cloneWithNewStart(newStart, runOnClose);
- transaction.update(replacement, true);
- }
- }
-
- private static final class InvalidateKeys implements Runnable
- {
- final List<KeyCacheKey> cacheKeys = new ArrayList<>();
- final WeakReference<InstrumentingCache<KeyCacheKey, ?>> cacheRef;
-
- private InvalidateKeys(SSTableReader reader, Collection<DecoratedKey> invalidate)
- {
- this.cacheRef = new WeakReference<>(reader.getKeyCache());
- if (cacheRef.get() != null)
+ if (!transaction.isObsolete(latest))
{
- for (DecoratedKey key : invalidate)
- cacheKeys.add(reader.getCacheKey(key));
- }
- }
-
- public void run()
- {
- for (KeyCacheKey key : cacheKeys)
- {
- InstrumentingCache<KeyCacheKey, ?> cache = cacheRef.get();
- if (cache != null)
- cache.remove(key);
+ DecoratedKey newStart = latest.firstKeyBeyond(lowerbound);
+ assert newStart != null;
+ SSTableReader replacement = latest.cloneWithNewStart(newStart);
+ transaction.update(replacement, true);
}
}
}
@@ -291,7 +225,10 @@
public void switchWriter(SSTableWriter newWriter)
{
if (newWriter != null)
- writers.add(newWriter.setMaxDataAge(maxAge));
+ {
+ newWriter.setMaxDataAge(maxAge);
+ writers.add(newWriter);
+ }
if (eagerWriterMetaRelease && writer != null)
writer.releaseMetadataOverhead();
@@ -310,12 +247,15 @@
return;
}
+ // Open fully completed sstables early. This is also required for the final sstable in a set (where newWriter
+ // is null) to permit the compilation of a canonical set of sstables (see View.select).
if (preemptiveOpenInterval != Long.MAX_VALUE)
{
// we leave it as a tmp file, but we open it and add it to the Tracker
- SSTableReader reader = writer.setMaxDataAge(maxAge).openFinalEarly();
+ writer.setMaxDataAge(maxAge);
+ SSTableReader reader = writer.openFinalEarly();
transaction.update(reader, false);
- moveStarts(reader, reader.last);
+ moveStarts(reader.last);
transaction.checkpoint();
}
@@ -370,7 +310,9 @@
for (SSTableWriter writer : writers)
{
assert writer.getFilePointer() > 0;
- writer.setRepairedAt(repairedAt).setOpenResult(true).prepareToCommit();
+ writer.setRepairedAt(repairedAt);
+ writer.setOpenResult(true);
+ writer.prepareToCommit();
SSTableReader reader = writer.finished();
transaction.update(reader, false);
preparedForCommit.add(reader);
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
index 3b1ed1b..0851574 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
@@ -83,7 +83,7 @@
{
// todo: inefficient - we create and serialize a PU just to get its size, then recreate it
// todo: either allow PartitionUpdateBuilder to have .build() called several times or pre-calculate the size
- currentSize += PartitionUpdate.serializer.serializedSize(createPartitionUpdateBuilder(key).build(), formatType.info.getLatestVersion().correspondingMessagingVersion());
+ currentSize += PartitionUpdate.serializer.serializedSize(createPartitionUpdateBuilder(key).build(), format.getLatestVersion().correspondingMessagingVersion());
previous = createPartitionUpdateBuilder(key);
buffer.put(key, previous);
}
@@ -97,7 +97,7 @@
// improve that. In particular, what we count is closer to the serialized value, but it's debatable that it's the right thing
// to count since it will take a lot more space in memory and the bufferSize if first and foremost used to avoid OOM when
// using this writer.
- currentSize += UnfilteredSerializer.serializer.serializedSize(row, helper, 0, formatType.info.getLatestVersion().correspondingMessagingVersion());
+ currentSize += UnfilteredSerializer.serializer.serializedSize(row, helper, 0, format.getLatestVersion().correspondingMessagingVersion());
}
private void maybeSync() throws SyncException
@@ -182,7 +182,10 @@
if (diskWriter.exception instanceof IOException)
throw (IOException) diskWriter.exception;
else
- throw Throwables.propagate(diskWriter.exception);
+ {
+ Throwables.throwIfUnchecked(diskWriter.exception);
+ throw new RuntimeException(diskWriter.exception);
+ }
}
}
@@ -211,7 +214,7 @@
if (b == SENTINEL)
return;
- try (SSTableTxnWriter writer = createWriter())
+ try (SSTableTxnWriter writer = createWriter(null))
{
for (Map.Entry<DecoratedKey, PartitionUpdate.Builder> entry : b.entrySet())
writer.append(entry.getValue().build().unfilteredIterator());
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
index c9356a3..744f35a 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
@@ -21,7 +21,8 @@
import com.google.common.base.Throwables;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.TableMetadataRef;
@@ -51,7 +52,7 @@
private SSTableTxnWriter getOrCreateWriter() throws IOException
{
if (writer == null)
- writer = createWriter();
+ writer = createWriter(null);
return writer;
}
@@ -85,7 +86,9 @@
}
catch (Throwable t)
{
- throw Throwables.propagate(writer == null ? t : writer.abort(t));
+ Throwable e = writer == null ? t : writer.abort(t);
+ Throwables.throwIfUnchecked(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
index 83c5542..fad7d54 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
@@ -28,7 +28,6 @@
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.index.Index;
-import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@@ -113,7 +112,7 @@
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
- SSTableFormat.Type type,
+ SSTableFormat<?, ?> type,
int sstableLevel,
SerializationHeader header)
{
@@ -144,12 +143,13 @@
boolean isTransient,
int sstableLevel,
SerializationHeader header,
- Collection<Index> indexes)
+ Collection<Index> indexes,
+ SSTable.Owner owner)
{
// if the column family store does not exist, we create a new default SSTableMultiWriter to use:
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel);
- SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn);
+ SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn, owner);
return new SSTableTxnWriter(txn, writer);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java
new file mode 100644
index 0000000..91e6490
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.schema.TableId;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+
+public class SSTableZeroCopyWriter extends SSTable implements SSTableMultiWriter
+{
+ private static final Logger logger = LoggerFactory.getLogger(SSTableZeroCopyWriter.class);
+
+ private volatile SSTableReader finalReader;
+ private final Map<Component.Type, SequentialWriter> componentWriters;
+
+ public SSTableZeroCopyWriter(Builder<?, ?> builder,
+ LifecycleNewTracker lifecycleNewTracker,
+ SSTable.Owner owner)
+ {
+ super(builder, owner);
+
+ lifecycleNewTracker.trackNew(this);
+ this.componentWriters = new HashMap<>();
+
+ if (!descriptor.getFormat().streamingComponents().containsAll(components))
+ throw new AssertionError(format("Unsupported streaming component detected %s",
+ Sets.difference(ImmutableSet.copyOf(components), descriptor.getFormat().streamingComponents())));
+
+ for (Component c : components)
+ componentWriters.put(c.type, makeWriter(descriptor, c));
+ }
+
+ @Override
+ public DecoratedKey getFirst()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DecoratedKey getLast()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AbstractBounds<Token> getBounds()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ private SequentialWriter makeWriter(Descriptor descriptor, Component component)
+ {
+ return new SequentialWriter(descriptor.fileFor(component), ioOptions.writerOptions, false);
+ }
+
+ private void write(DataInputPlus in, long size, SequentialWriter out) throws FSWriteError
+ {
+ final int BUFFER_SIZE = 1 << 20;
+ long bytesRead = 0;
+ byte[] buff = new byte[BUFFER_SIZE];
+ try
+ {
+ while (bytesRead < size)
+ {
+ int toRead = (int) Math.min(size - bytesRead, BUFFER_SIZE);
+ in.readFully(buff, 0, toRead);
+ int count = Math.min(toRead, BUFFER_SIZE);
+ out.write(buff, 0, count);
+ bytesRead += count;
+ }
+ out.sync(); // finish will also call sync(). Leaving here to get stuff flushed as early as possible
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, out.getFile());
+ }
+ }
+
+ @Override
+ public boolean append(UnfilteredRowIterator partition)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
+ {
+ return finish(openResult);
+ }
+
+ @Override
+ public Collection<SSTableReader> finish(boolean openResult)
+ {
+ setOpenResult(openResult);
+
+ for (SequentialWriter writer : componentWriters.values())
+ writer.finish();
+
+ return finished();
+ }
+
+ @Override
+ public Collection<SSTableReader> finished()
+ {
+ if (finalReader == null)
+ finalReader = SSTableReader.open(owner().orElse(null), descriptor, components, metadata);
+
+ return ImmutableList.of(finalReader);
+ }
+
+ @Override
+ public SSTableMultiWriter setOpenResult(boolean openResult)
+ {
+ return null;
+ }
+
+ @Override
+ public long getFilePointer()
+ {
+ return 0;
+ }
+
+ @Override
+ public TableId getTableId()
+ {
+ return metadata.id;
+ }
+
+ @Override
+ public Throwable commit(Throwable accumulate)
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ accumulate = writer.commit(accumulate);
+ return accumulate;
+ }
+
+ @Override
+ public Throwable abort(Throwable accumulate)
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ accumulate = writer.abort(accumulate);
+ return accumulate;
+ }
+
+ @Override
+ public void prepareToCommit()
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ writer.prepareToCommit();
+ }
+
+ @Override
+ public void close()
+ {
+ for (SequentialWriter writer : componentWriters.values())
+ writer.close();
+ }
+
+ public void writeComponent(Component.Type type, DataInputPlus in, long size) throws ClosedChannelException
+ {
+ logger.info("Writing component {} to {} length {}", type, componentWriters.get(type).getPath(), prettyPrintMemory(size));
+
+ if (in instanceof AsyncStreamingInputPlus)
+ write((AsyncStreamingInputPlus) in, size, componentWriters.get(type));
+ else
+ write(in, size, componentWriters.get(type));
+ }
+
+ private void write(AsyncStreamingInputPlus in, long size, SequentialWriter writer) throws ClosedChannelException
+ {
+ logger.info("Block Writing component to {} length {}", writer.getPath(), prettyPrintMemory(size));
+
+ try
+ {
+ in.consume(writer::writeDirectlyToChannel, size);
+ writer.sync();
+ }
+ catch (EOFException e)
+ {
+ in.close();
+ }
+ catch (ClosedChannelException e)
+ {
+ // FSWriteError triggers disk failure policy, but if we get a connection issue we do not want to do that
+ // so rethrow so the error handling logic higher up is able to deal with this
+ // see CASSANDRA-17116
+ throw e;
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, writer.getPath());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable_API.md b/src/java/org/apache/cassandra/io/sstable/SSTable_API.md
new file mode 100644
index 0000000..6a04406
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTable_API.md
@@ -0,0 +1,262 @@
+# SSTable API
+
+[CEP-17](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-17%3A+SSTable+format+API)
+/ [CASSANDRA-17056](https://issues.apache.org/jira/browse/CASSANDRA-17056)
+
+## SSTable format
+
+SSTable format is an implementation of the `SSTableFormat` interface. It is responsible for creating readers, writers,
+scrubbers, verifiers, and other components for processing the sstables. An SSTable format implementation comes with
+a factory class implementing the `SSTableFormat.Factory` interface. The factory is required to provide a unique name
+of the implementation and a method for creating the format instance.
+
+## Configuration specification
+
+SSTable format factories are discovered using
+[Java Service Loader](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html) mechanism. The loaded
+format implementations can be used to read the existing sstables. The write format is chosen based on the configuration.
+If it is not specified, `BigFormat` implementation is assumed.
+
+Optional SSTable formats configuration can be supplied in the _cassandra.yaml_ file under the `sstable` key.
+
+```yaml
+sstable:
+ selected_format: ⟨name of the default SSTableFormat implementation⟩
+ format:
+ ⟨format1 name⟩:
+ param1: ⟨format specific parameter 1⟩
+ param2: ⟨format specific parameter 2⟩
+ # ...
+ ⟨format2 name⟩:
+ param1: ⟨format specific parameter 1⟩
+ param2: ⟨format specific parameter 2⟩
+ # ...
+ # ...
+```
+
+Each implementation must have a unique name that is used to unanonimously identify the format. The name must be
+consistently returned by `name()` methods in the `SSTableFormat` and `SSTableFormat.Factory` implementations. It must
+include only lowercase ASCII letters.
+
+Parameters specified under the key named after the format name are passed to the factory method of the corresponding
+implementation. All of those parameters are optional and depend on the implementation.
+
+The assumed default configuration - which is equivalent to empty configuration:
+```yaml
+sstable:
+ selected_format: big
+```
+
+Example configuration which uses `bti` as the default:
+```yaml
+sstable:
+ selected_format: bti
+ format:
+ big:
+ param1: value1
+ param2: value2
+ bti:
+ param1: value1
+ param2: value2
+
+```
+
+## Components
+
+Each sstable consists of a set of components - required and optional. A component constitutes an identifier required
+to obtain the exact file with an sstable descriptor. Components are grouped by type. A type may define either
+a singleton component (for example, _stats_ component) or a non-singleton component (for example, _secondary index_
+component).
+
+A set of generic types of components that are thought of as common to all the sstable implementations is defined in the
+[`SSTableFormat.Components`](format/SSTableFormat.java) class. They include singleton types like `DATA`,
+`COMPRESSION_INFO`, `STATS`, `FILTER`, `DIGEST`, `CRC`, and `TOC`, which comes with predefined singleton component
+instances, as well as non-singleton types like `SECONDARY_INDEX` and `CUSTOM`.
+
+Apart from the generic components, each sstable format implementation may describe its specific component types.
+For example, the _big table_ format describes additionally `PRIMARY_INDEX` and `SUMMARY` singleton types and
+the corresponding singleton components (see [`BigFormat.Components`](format/big/BigFormat.java)).
+
+Custom types can be created with one of the `Component.Type.create(name, repr, formatClass)`,
+`Component.Type.createSingleton(name, repr, formatClass)` methods. Each created type is registered in a global types'
+registry. Types registry is hierarchical which means that an sstable implementation may use types defined for its
+format class and for all parent format classes (for example, the types defined for the `BigFormat` class extend the set
+of types defined for the `SSTableFormat` interface).
+
+For example, types defined for `BigFormat`:
+
+```java
+public static class Types extends SSTableFormat.Components.Types
+{
+ public static final Component.Type PRIMARY_INDEX = Component.Type.createSingleton("PRIMARY_INDEX", "Index.db", BigFormat.class);
+ public static final Component.Type SUMMARY = Component.Type.createSingleton("SUMMARY", "Summary.db", BigFormat.class);
+}
+```
+
+Singleton components are immediately associated with the singleton types and retrieved with the `<type>.getSingleton()`
+method:
+
+```java
+public static class Components extends AbstractSSTableFormat.Components
+{
+ public final static Component PRIMARY_INDEX = Types.PRIMARY_INDEX.getSingleton();
+ public final static Component SUMMARY = Types.SUMMARY.getSingleton();
+}
+```
+
+Non-singleton components are created explicitly as follows:
+
+```java
+Component idx1 = Types.SECONDARY_INDEX.createComponent("SI_idx1.db");
+```
+
+## Implementation
+
+We strongly suggest the main format class to extend [`AbstractSSTableFormat`](format/AbstractSSTableFormat.java) because
+it includes the expected implementation of a couple of methods that should not be reimplemented differently.
+
+### Initialization
+
+Cassandra either initializes the sstable format class as a singleton by calling its constructor or obtains the instance
+by accessing a static field called `instance` in the class. As a part of the initialization, Cassandra calls the `setup`
+method and provides the configuration parameters. Right after initialization, Cassandra calls the `allComponents` method
+to confirm all the components defined for the format are initialized and usable.
+
+### Predefined sets of components
+
+SSTable format defines a couple of collections of components. You should declare those collections as constant and
+immutable sets.
+
+### Reader
+
+#### Construction
+
+An sstable reader ([`SSTableReader`](format/SSTableReader.java)) is responsible for reading the data from an sstable.
+It is created by a _simple builder_ ([`SSTableReader.Builder`](format/SSTableReader.java)) or a _loading
+builder_ ([`SSTableReaderLoadingBuilder`](format/SSTableReaderLoadingBuilder.java)). The builders are supplied by
+a _reader factory_ ([`SSTableFormat.SSTableReaderFactory`](format/SSTableFormat.java)).
+
+The constructor of a particular `SSTableReader` implementation should accept two parameters - one is the format-specific
+_simple builder_ and the other one is an sstable owner (usually a `ColumnFamilyStore` instance, but it can be null
+either). The constructor should be simple and not do anything but assign internal fields with values from the builder.
+
+A simple builder does not perform any logic except basic validation - it only stores the provided values the reader
+constructor can access. A new reader implementation should include a public static simple builder inner class that
+extends the `SSTableReader.Builder` generic reader builder (or `SSTableReaderWithFilter.Builder`, see [below](#filter)).
+
+In contrast to the simple builder, a loading builder can perform additional operations like more complex validation,
+opening resources, loading caches, indexes, filters, etc. It internally creates a simple builder and eventually
+instantiates a reader.
+
+#### General notes
+
+Note that if the builder carries some closeable resources to the reader, they should be returned by the `setupInstance`
+method.
+
+You will find some `cloneXXX` methods to implement - remember to create a reader clone in a lambda passed to
+the `runWithLock()` method.
+
+#### Unbuilding
+It is convenient to implement the `unbuildTo` method, which takes a _simple builder_ and initializes it so that
+the builder can produce the same reader. The method should also take the `sharedCopy` boolean argument denoting whether
+it should copy the fields referencing closeable resources to the builder directly or as (shared) copies. The convention
+also requires copying the resources only if they are unset in the builder (the field is null). The method should call
+the `super.unbuildTo` method as a first step so that all the fields managed by the parent class are copied and in
+the actual implementation only the fields specific to this format have to be assigned.
+
+For example, the implementation of that method in a reader for the _big table_ format is as follows:
+
+```java
+protected final Builder unbuildTo(Builder builder, boolean sharedCopy)
+{
+ Builder b = super.unbuildTo(builder, sharedCopy);
+ if (builder.getIndexFile() == null)
+ b.setIndexFile(sharedCopy ? sharedCopyOrNull(ifile) : ifile);
+ if (builder.getIndexSummary() == null)
+ b.setIndexSummary(sharedCopy ? sharedCopyOrNull(indexSummary) : indexSummary);
+
+ b.setKeyCache(keyCache);
+
+ return b;
+}
+```
+
+#### Filter
+
+If the sstable includes a _filter_, the reader class should extend
+the [`SSTableReaderWithFilter`](format/SSTableReaderWithFilter.java) abstract reader (and its _simple builder_ should
+extend the `SSTableReaderWithFilter.SSTableReaderWithFilterBuilder` builder).
+
+The reader with filter provides the `isPresentInFilter` method for extending implementation. It also implements other
+filter-specific methods the system relies on if the implemented reader extends that class.
+
+Note that if the implemented reader extends the `SSTableReaderWithFilter` class, it should include the `FILTER`
+component in the appropriate component sets.
+
+The reader with filter implementation comes with additional [metrics](filter/BloomFilterMetrics.java) - read more about custom
+metrics support [here](#metrics).
+
+#### Index summary
+
+Some sstable format implementations, such as _big table_ format, may use _index summaries_. If a reader uses _index
+summaries_ it should implement the [`IndexSummarySupport`](indexsummary/IndexSummarySupport.java) interface.
+
+The support for _index summaries_ comes with additional [metrics](indexsummary/IndexSummaryMetrics.java) - read more
+about custom metrics support [here](#metrics).
+
+#### Key cache
+
+If an sstable format implementation uses row key cache, it should implement
+the[`KeyCacheSupport`](keycache/KeyCacheSupport.java) interface. In particular, it should store a `KeyCache` instance
+and return it with the `getKeyCache()` method. The interface has the default implementations of several methods
+the system relies on if the reader implements the `KeyCacheSupport` interface.
+
+The interface comes with additional [metrics](keycache/KeyCacheMetrics.java) - read more about custom metrics support
+[here](#metrics).
+
+#### Metrics
+
+A custom sstable format implementation may provide additional metrics on a table, keyspace, and global level. Those
+metrics are accessible via JMX. The `SSTableFormat` implementation exposes the additional metrics by implementing the
+`SSTableFormat.getFormatSpecificMetricsProviders` method. The method should return a singleton object implementing the
+[`MetricsProviders`](MetricsProviders.java) interface. Currently, there is only support for custom gauges, but it can be
+extended when needed.
+
+Each custom metric (gauge) is an implementation of the [GaugeProvider](GaugeProvider.java) abstract class. Although the
+class expects the implementation to provide a gauge for each level of aggregation, there is a helper class -
+[SimpleGaugeProvider](SimpleGaugeProvider.java) - which does that automatically with a supplied reduction lambda. There
+is [`AbstractMetricsProviders`](AbstractMetricsProviders.java) class which is a partial implementation of the
+`MetricsProviders` interface and leverages `SimpleGaugeProvider` in the offered methods.
+
+Example - additional metrics for sstables supporting index summaries (see
+[`IndexSummaryMetrics`](indexsummary/IndexSummaryMetrics.java) for a full example):
+```java
+private final GaugeProvider<Long> indexSummaryOffHeapMemoryUsed = newGaugeProvider("IndexSummaryOffHeapMemoryUsed",
+ 0L,
+ r -> r.getIndexSummary().getOffHeapSize(),
+ Long::sum);
+```
+
+### Writer
+
+#### Construction
+
+An sstable writer ([`SSTableWriter`](format/SSTableWriter.java)) is responsible for writing the data to sstable files.
+It is created by a _builder_ ([`SSTableWriter.Builder`](format/SSTableWriter.java)). The builder is supplied by
+a _writer factory_ ([`SSTableFormat.SSTableWriterFactory`](format/SSTableFormat.java)).
+
+#### SortedTableWriter
+
+There are not many methods to be implemented in the writer. The most notable one is `append` which should write
+the provided partition to disk. However, there is a generic default implementation
+[`SortedTableWriter`](format/SortedTableWriter.java) which handles things like writing to the data file using
+the default serializers, generic support for a partition index, notifications, metadata collection, and building
+a filter. The writer triggers fine-grained events when data are added and those methods can be overridden in
+the subclasses to apply specific behaviours (for example, `onPartitionStart`, `onRow`, `onStaticRow`, etc.).
+Eventually it calls an abstract `createRowIndexEntry` method which should be implemented in the subclass.
+
+### Scrubber and verifier
+
+A custom sstable format should also come with its own verifier and scrubber implementing [`IVerifier`](IVerifier.java)
+and [`IScrubber`](IScrubber.java) interfaces correspondingly. A generic partial implementation is provided in
+[`SortedTableVerifier`](format/SortedTableVerifier.java) and [`SortedTableScrubber`](format/SortedTableScrubber.java).
diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleGaugeProvider.java b/src/java/org/apache/cassandra/io/sstable/SimpleGaugeProvider.java
new file mode 100644
index 0000000..1929090
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SimpleGaugeProvider.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+import com.google.common.collect.Iterables;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class SimpleGaugeProvider<T extends Number, R extends SSTableReader> extends GaugeProvider<T>
+{
+ private final Function<SSTableReader, R> mapper;
+ private final Function<Iterable<R>, T> combiner;
+
+ public SimpleGaugeProvider(Function<SSTableReader, R> mapper, String name, Function<Iterable<R>, T> combiner)
+ {
+ super(name);
+ this.mapper = mapper;
+ this.combiner = combiner;
+ }
+
+ @Override
+ public Gauge<T> getTableGauge(ColumnFamilyStore cfs)
+ {
+ return () -> combine(cfs.getLiveSSTables());
+ }
+
+ @Override
+ public Gauge<T> getKeyspaceGauge(Keyspace keyspace)
+ {
+ return () -> combine(getAllReaders(keyspace));
+ }
+
+ @Override
+ public Gauge<T> getGlobalGauge()
+ {
+ return () -> combine(Iterables.concat(Iterables.transform(Keyspace.all(), SimpleGaugeProvider::getAllReaders)));
+ }
+
+ private T combine(Iterable<SSTableReader> allReaders)
+ {
+ Iterable<R> readers = Iterables.filter(Iterables.transform(allReaders, mapper::apply), Objects::nonNull);
+ return combiner.apply(readers);
+ }
+
+ private static Iterable<SSTableReader> getAllReaders(Keyspace keyspace)
+ {
+ return Iterables.concat(Iterables.transform(keyspace.getColumnFamilyStores(), ColumnFamilyStore::getLiveSSTables));
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
index ec28528..381c5eb 100644
--- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
@@ -20,7 +20,6 @@
import java.util.Collection;
import java.util.Collections;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -45,13 +44,15 @@
public boolean append(UnfilteredRowIterator partition)
{
- RowIndexEntry<?> indexEntry = writer.append(partition);
+ AbstractRowIndexEntry indexEntry = writer.append(partition);
return indexEntry != null;
}
public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
{
- return Collections.singleton(writer.finish(repairedAt, maxDataAge, openResult));
+ writer.setRepairedAt(repairedAt);
+ writer.setMaxDataAge(maxDataAge);
+ return Collections.singleton(writer.finish(openResult));
}
public Collection<SSTableReader> finish(boolean openResult)
@@ -116,9 +117,20 @@
MetadataCollector metadataCollector,
SerializationHeader header,
Collection<Index> indexes,
- LifecycleNewTracker lifecycleNewTracker)
+ LifecycleNewTracker lifecycleNewTracker,
+ SSTable.Owner owner)
{
- SSTableWriter writer = SSTableWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, indexes, lifecycleNewTracker);
+ SSTableWriter writer = descriptor.getFormat().getWriterFactory().builder(descriptor)
+ .setKeyCount(keyCount)
+ .setRepairedAt(repairedAt)
+ .setPendingRepair(pendingRepair)
+ .setTransientSSTable(isTransient)
+ .setTableMetadataRef(metadata)
+ .setMetadataCollector(metadataCollector)
+ .setSerializationHeader(header)
+ .addDefaultComponents()
+ .addFlushObserversForSecondaryIndexes(indexes, lifecycleNewTracker.opType())
+ .build(lifecycleNewTracker, owner);
return new SimpleSSTableMultiWriter(writer, lifecycleNewTracker);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterMetrics.java b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterMetrics.java
new file mode 100644
index 0000000..ee5fc3c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterMetrics.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.filter;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.cassandra.io.sstable.AbstractMetricsProviders;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+
+public class BloomFilterMetrics<R extends SSTableReaderWithFilter> extends AbstractMetricsProviders<R>
+{
+ public final static BloomFilterMetrics<?> instance = new BloomFilterMetrics<>();
+
+ @Override
+ protected R map(SSTableReader r)
+ {
+ if (r instanceof SSTableReaderWithFilter)
+ return (R) r;
+ return null;
+ }
+
+ /**
+ * Number of false positives in bloom filter
+ */
+ public final GaugeProvider<Long> bloomFilterFalsePositives = newGaugeProvider("BloomFilterFalsePositives",
+ 0L,
+ r -> r.getFilterTracker().getFalsePositiveCount(),
+ Long::sum);
+
+ /**
+ * Number of false positives in bloom filter from last read
+ */
+ public final GaugeProvider<Long> recentBloomFilterFalsePositives = newGaugeProvider("RecentBloomFilterFalsePositives",
+ 0L,
+ r -> r.getFilterTracker().getRecentFalsePositiveCount(),
+ Long::sum);
+
+ /**
+ * Disk space used by bloom filter
+ */
+ public final GaugeProvider<Long> bloomFilterDiskSpaceUsed = newGaugeProvider("BloomFilterDiskSpaceUsed",
+ 0L,
+ SSTableReaderWithFilter::getFilterSerializedSize,
+ Long::sum);
+
+ /**
+ * Off heap memory used by bloom filter
+ */
+ public final GaugeProvider<Long> bloomFilterOffHeapMemoryUsed = newGaugeProvider("BloomFilterOffHeapMemoryUsed",
+ 0L,
+ SSTableReaderWithFilter::getFilterOffHeapSize,
+ Long::sum);
+
+ /**
+ * False positive ratio of bloom filter
+ */
+ public final GaugeProvider<Double> bloomFilterFalseRatio = newGaugeProvider("BloomFilterFalseRatio", readers -> {
+ long falsePositiveCount = 0L;
+ long truePositiveCount = 0L;
+ long trueNegativeCount = 0L;
+ for (SSTableReaderWithFilter sstable : readers)
+ {
+ falsePositiveCount += sstable.getFilterTracker().getFalsePositiveCount();
+ truePositiveCount += sstable.getFilterTracker().getTruePositiveCount();
+ trueNegativeCount += sstable.getFilterTracker().getTrueNegativeCount();
+ }
+ if (falsePositiveCount == 0L && truePositiveCount == 0L)
+ return 0d;
+ return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
+ });
+
+ /**
+ * False positive ratio of bloom filter from last read
+ */
+ public final GaugeProvider<Double> recentBloomFilterFalseRatio = newGaugeProvider("RecentBloomFilterFalseRatio", readers -> {
+ long falsePositiveCount = 0L;
+ long truePositiveCount = 0L;
+ long trueNegativeCount = 0L;
+ for (SSTableReaderWithFilter sstable : readers)
+ {
+ falsePositiveCount += sstable.getFilterTracker().getRecentFalsePositiveCount();
+ truePositiveCount += sstable.getFilterTracker().getRecentTruePositiveCount();
+ trueNegativeCount += sstable.getFilterTracker().getRecentTrueNegativeCount();
+ }
+ if (falsePositiveCount == 0L && truePositiveCount == 0L)
+ return 0d;
+ return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
+ });
+
+ private final List<GaugeProvider<?>> gaugeProviders = Arrays.asList(bloomFilterFalsePositives,
+ recentBloomFilterFalsePositives,
+ bloomFilterDiskSpaceUsed,
+ bloomFilterOffHeapMemoryUsed,
+ bloomFilterFalseRatio,
+ recentBloomFilterFalseRatio);
+
+ public List<GaugeProvider<?>> getGaugeProviders()
+ {
+ return gaugeProviders;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java
new file mode 100644
index 0000000..3629263
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.filter;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+
+public class BloomFilterTracker
+{
+ private final LongAdder falsePositiveCount = new LongAdder();
+ private final LongAdder truePositiveCount = new LongAdder();
+ private final LongAdder trueNegativeCount = new LongAdder();
+ private final AtomicLong lastFalsePositiveCount = new AtomicLong();
+ private final AtomicLong lastTruePositiveCount = new AtomicLong();
+ private final AtomicLong lastTrueNegativeCount = new AtomicLong();
+
+ public void addFalsePositive()
+ {
+ falsePositiveCount.increment();
+ }
+
+ public void addTruePositive()
+ {
+ truePositiveCount.increment();
+ }
+
+ public void addTrueNegative()
+ {
+ trueNegativeCount.increment();
+ }
+
+ public long getFalsePositiveCount()
+ {
+ return falsePositiveCount.sum();
+ }
+
+ public long getRecentFalsePositiveCount()
+ {
+ long fpc = getFalsePositiveCount();
+ long last = lastFalsePositiveCount.getAndSet(fpc);
+ return fpc - last;
+ }
+
+ public long getTruePositiveCount()
+ {
+ return truePositiveCount.sum();
+ }
+
+ public long getRecentTruePositiveCount()
+ {
+ long tpc = getTruePositiveCount();
+ long last = lastTruePositiveCount.getAndSet(tpc);
+ return tpc - last;
+ }
+
+ public long getTrueNegativeCount()
+ {
+ return trueNegativeCount.sum();
+ }
+
+ public long getRecentTrueNegativeCount()
+ {
+ long tnc = getTrueNegativeCount();
+ long last = lastTrueNegativeCount.getAndSet(tnc);
+ return tnc - last;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java
new file mode 100644
index 0000000..7fc1985
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class AbstractSSTableFormat<R extends SSTableReader, W extends SSTableWriter> implements SSTableFormat<R, W>
+{
+ public final String name;
+ protected final Map<String, String> options;
+
+ protected AbstractSSTableFormat(String name, Map<String, String> options)
+ {
+ this.name = Objects.requireNonNull(name);
+ this.options = options;
+ }
+
+ @Override
+ public final String name()
+ {
+ return name;
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AbstractSSTableFormat<?, ?> that = (AbstractSSTableFormat<?, ?>) o;
+ return Objects.equals(name, that.name);
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return Objects.hash(name);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java b/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java
new file mode 100644
index 0000000..82c10bf
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.nio.file.NoSuchFileException;
+import java.util.Set;
+
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.util.File;
+
+public class CompressionInfoComponent
+{
+ public static CompressionMetadata maybeLoad(Descriptor descriptor, Set<Component> components)
+ {
+ if (components.contains(Components.COMPRESSION_INFO))
+ return load(descriptor);
+
+ return null;
+ }
+
+ public static CompressionMetadata loadIfExists(Descriptor descriptor)
+ {
+ if (descriptor.fileFor(Components.COMPRESSION_INFO).exists())
+ return load(descriptor);
+
+ return null;
+ }
+
+ public static CompressionMetadata load(Descriptor descriptor)
+ {
+ return CompressionMetadata.open(descriptor.fileFor(Components.COMPRESSION_INFO),
+ descriptor.fileFor(Components.DATA).length(),
+ descriptor.version.hasMaxCompressedLength());
+ }
+
+ /**
+ * Best-effort checking to verify the expected compression info component exists, according to the TOC file.
+ * The verification depends on the existence of TOC file. If absent, the verification is skipped.
+ *
+ * @param descriptor
+ * @param actualComponents, actual components listed from the file system.
+ * @throws CorruptSSTableException, if TOC expects compression info but not found from disk.
+ * @throws FSReadError, if unable to read from TOC file.
+ */
+ public static void verifyCompressionInfoExistenceIfApplicable(Descriptor descriptor, Set<Component> actualComponents) throws CorruptSSTableException, FSReadError
+ {
+ File tocFile = descriptor.fileFor(Components.TOC);
+ if (tocFile.exists())
+ {
+ try
+ {
+ Set<Component> expectedComponents = TOCComponent.loadTOC(descriptor, false);
+ if (expectedComponents.contains(Components.COMPRESSION_INFO) && !actualComponents.contains(Components.COMPRESSION_INFO))
+ {
+ File compressionInfoFile = descriptor.fileFor(Components.COMPRESSION_INFO);
+ throw new CorruptSSTableException(new NoSuchFileException(compressionInfoFile.absolutePath()), compressionInfoFile);
+ }
+ }
+ catch (IOException e)
+ {
+ throw new FSReadError(e, tocFile);
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java b/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java
new file mode 100644
index 0000000..9367cb4
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import org.apache.cassandra.config.Config.FlushCompression;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.ChecksummedSequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class DataComponent
+{
+ public static SequentialWriter buildWriter(Descriptor descriptor,
+ TableMetadata metadata,
+ SequentialWriterOption options,
+ MetadataCollector metadataCollector,
+ OperationType operationType,
+ FlushCompression flushCompression)
+ {
+ if (metadata.params.compression.isEnabled())
+ {
+ final CompressionParams compressionParams = buildCompressionParams(metadata, operationType, flushCompression);
+
+ return new CompressedSequentialWriter(descriptor.fileFor(Components.DATA),
+ descriptor.fileFor(Components.COMPRESSION_INFO),
+ descriptor.fileFor(Components.DIGEST),
+ options,
+ compressionParams,
+ metadataCollector);
+ }
+ else
+ {
+ return new ChecksummedSequentialWriter(descriptor.fileFor(Components.DATA),
+ descriptor.fileFor(Components.CRC),
+ descriptor.fileFor(Components.DIGEST),
+ options);
+ }
+ }
+
+ /**
+ * Given an OpType, determine the correct Compression Parameters
+ *
+ * @return {@link CompressionParams}
+ */
+ private static CompressionParams buildCompressionParams(TableMetadata metadata, OperationType operationType, FlushCompression flushCompression)
+ {
+ CompressionParams compressionParams = metadata.params.compression;
+ final ICompressor compressor = compressionParams.getSstableCompressor();
+
+ if (null != compressor && operationType == OperationType.FLUSH)
+ {
+ // When we are flushing out of the memtable throughput of the compressor is critical as flushes,
+ // especially of large tables, can queue up and potentially block writes.
+ // This optimization allows us to fall back to a faster compressor if a particular
+ // compression algorithm indicates we should. See CASSANDRA-15379 for more details.
+ switch (flushCompression)
+ {
+ // It is relatively easier to insert a Noop compressor than to disable compressed writing
+ // entirely as the "compression" member field is provided outside the scope of this class.
+ // It may make sense in the future to refactor the ownership of the compression flag so that
+ // We can bypass the CompressedSequentialWriter in this case entirely.
+ case none:
+ compressionParams = CompressionParams.NOOP;
+ break;
+ case fast:
+ if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION))
+ {
+ // The default compressor is generally fast (LZ4 with 16KiB block size)
+ compressionParams = CompressionParams.DEFAULT;
+ break;
+ }
+ // else fall through
+ case table:
+ default:
+ break;
+ }
+ }
+ return compressionParams;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java b/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java
new file mode 100644
index 0000000..9f99d7d
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.BloomFilterSerializer;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+
+public class FilterComponent
+{
+ private static final Logger logger = LoggerFactory.getLogger(FilterComponent.class);
+
+ final static boolean rebuildFilterOnFPChanceChange = false;
+ final static double filterFPChanceTolerance = 0d;
+
+ private FilterComponent()
+ {
+ }
+
+ /**
+ * Load bloom filter from Filter.db file.
+ */
+ public static IFilter load(Descriptor descriptor) throws IOException
+ {
+ File filterFile = descriptor.fileFor(Components.FILTER);
+
+ if (!filterFile.exists())
+ return null;
+
+ if (filterFile.length() == 0)
+ return FilterFactory.AlwaysPresent;
+
+ try (FileInputStreamPlus stream = descriptor.fileFor(Components.FILTER).newInputStream())
+ {
+ return BloomFilterSerializer.forVersion(descriptor.version.hasOldBfFormat()).deserialize(stream);
+ }
+ catch (IOException ex)
+ {
+ throw new IOException("Failed to load Bloom filter for SSTable: " + descriptor.baseFile(), ex);
+ }
+ }
+
+ public static void save(IFilter filter, Descriptor descriptor, boolean deleteOnFailure) throws IOException
+ {
+ File filterFile = descriptor.fileFor(Components.FILTER);
+ try (FileOutputStreamPlus stream = filterFile.newOutputStream(File.WriteMode.OVERWRITE))
+ {
+ filter.serialize(stream, descriptor.version.hasOldBfFormat());
+ stream.flush();
+ stream.sync();
+ }
+ catch (IOException ex)
+ {
+ if (deleteOnFailure)
+ descriptor.fileFor(Components.FILTER).deleteIfExists();
+ throw new IOException("Failed to save Bloom filter for SSTable: " + descriptor.baseFile(), ex);
+ }
+ }
+
+ /**
+ * Optionally loads a Bloom filter.
+ * If the filter is not needed (FP chance is neglectable), it returns {@link FilterFactory#AlwaysPresent}.
+ * If the filter is expected to be recreated for various reasons the method returns {@code null}.
+ * Otherwise, an attempt to load the filter is made and if it succeeds, the loaded filter is returned.
+ * If loading fails, the method returns {@code null}.
+ *
+ * @return {@link FilterFactory#AlwaysPresent}, loaded filter or {@code null} (which means that the filter should be rebuilt)
+ */
+ public static IFilter maybeLoadBloomFilter(Descriptor descriptor, Set<Component> components, TableMetadata metadata, ValidationMetadata validationMetadata)
+ {
+ double currentFPChance = validationMetadata != null ? validationMetadata.bloomFilterFPChance : Double.NaN;
+ double desiredFPChance = metadata.params.bloomFilterFpChance;
+
+ IFilter filter = null;
+ if (!shouldUseBloomFilter(desiredFPChance))
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("Bloom filter for {} will not be loaded because fpChance={} is negligible", descriptor, desiredFPChance);
+
+ return FilterFactory.AlwaysPresent;
+ }
+ else if (!components.contains(Components.FILTER) || Double.isNaN(currentFPChance))
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("Bloom filter for {} will not be loaded because the filter component is missing or sstable lacks validation metadata", descriptor);
+
+ return null;
+ }
+ else if (!isFPChanceDiffNegligible(desiredFPChance, currentFPChance) && rebuildFilterOnFPChanceChange)
+ {
+ if (logger.isTraceEnabled())
+ logger.trace("Bloom filter for {} will not be loaded because fpChance has changed from {} to {} and the filter should be recreated", descriptor, currentFPChance, desiredFPChance);
+
+ return null;
+ }
+
+ try
+ {
+ filter = load(descriptor);
+ if (filter == null || !filter.isInformative())
+ logger.info("Bloom filter for {} is missing or invalid", descriptor);
+ }
+ catch (IOException ex)
+ {
+ logger.info("Bloom filter for " + descriptor + " could not be deserialized", ex);
+ }
+
+ return filter;
+ }
+
+ static boolean shouldUseBloomFilter(double fpChance)
+ {
+ return !(Math.abs(1 - fpChance) <= filterFPChanceTolerance);
+ }
+
+ static boolean isFPChanceDiffNegligible(double fpChance1, double fpChance2)
+ {
+ return Math.abs(fpChance1 - fpChance2) <= filterFPChanceTolerance;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java b/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java
new file mode 100644
index 0000000..45dfc62
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.IOOptions;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+
+public class IndexComponent
+{
+ public static FileHandle.Builder fileBuilder(File file, IOOptions ioOptions, ChunkCache chunkCache)
+ {
+ return new FileHandle.Builder(file).withChunkCache(chunkCache)
+ .mmapped(ioOptions.indexDiskAccessMode);
+ }
+
+ public static FileHandle.Builder fileBuilder(Component component, SSTable ssTable)
+ {
+ return fileBuilder(ssTable.descriptor.fileFor(component), ssTable.ioOptions, ssTable.chunkCache);
+ }
+
+ public static FileHandle.Builder fileBuilder(Component component, SSTable.Builder<?, ?> builder)
+ {
+ return fileBuilder(builder.descriptor.fileFor(component), builder.getIOOptions(), builder.getChunkCache());
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java
deleted file mode 100644
index e9b15b1..0000000
--- a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.DiskBoundaries;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.TimeUUID;
-
-public class RangeAwareSSTableWriter implements SSTableMultiWriter
-{
- private final List<PartitionPosition> boundaries;
- private final List<Directories.DataDirectory> directories;
- private final int sstableLevel;
- private final long estimatedKeys;
- private final long repairedAt;
- private final TimeUUID pendingRepair;
- private final boolean isTransient;
- private final SSTableFormat.Type format;
- private final SerializationHeader header;
- private final LifecycleNewTracker lifecycleNewTracker;
- private int currentIndex = -1;
- public final ColumnFamilyStore cfs;
- private final List<SSTableMultiWriter> finishedWriters = new ArrayList<>();
- private final List<SSTableReader> finishedReaders = new ArrayList<>();
- private SSTableMultiWriter currentWriter = null;
-
- public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat.Type format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException
- {
- DiskBoundaries db = cfs.getDiskBoundaries();
- directories = db.directories;
- this.sstableLevel = sstableLevel;
- this.cfs = cfs;
- this.estimatedKeys = estimatedKeys / directories.size();
- this.repairedAt = repairedAt;
- this.pendingRepair = pendingRepair;
- this.isTransient = isTransient;
- this.format = format;
- this.lifecycleNewTracker = lifecycleNewTracker;
- this.header = header;
- boundaries = db.positions;
- if (boundaries == null)
- {
- Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
- if (localDir == null)
- throw new IOException(String.format("Insufficient disk space to store %s",
- FBUtilities.prettyPrintMemory(totalSize)));
- Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
- currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
- }
- }
-
- private void maybeSwitchWriter(DecoratedKey key)
- {
- if (boundaries == null)
- return;
-
- boolean switched = false;
- while (currentIndex < 0 || key.compareTo(boundaries.get(currentIndex)) > 0)
- {
- switched = true;
- currentIndex++;
- }
-
- if (switched)
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
-
- Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format);
- currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
- }
- }
-
- public boolean append(UnfilteredRowIterator partition)
- {
- maybeSwitchWriter(partition.partitionKey());
- return currentWriter.append(partition);
- }
-
- @Override
- public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
- currentWriter = null;
- for (SSTableMultiWriter writer : finishedWriters)
- {
- if (writer.getFilePointer() > 0)
- finishedReaders.addAll(writer.finish(repairedAt, maxDataAge, openResult));
- else
- SSTableMultiWriter.abortOrDie(writer);
- }
- return finishedReaders;
- }
-
- @Override
- public Collection<SSTableReader> finish(boolean openResult)
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
- currentWriter = null;
- for (SSTableMultiWriter writer : finishedWriters)
- {
- if (writer.getFilePointer() > 0)
- finishedReaders.addAll(writer.finish(openResult));
- else
- SSTableMultiWriter.abortOrDie(writer);
- }
- return finishedReaders;
- }
-
- @Override
- public Collection<SSTableReader> finished()
- {
- return finishedReaders;
- }
-
- @Override
- public SSTableMultiWriter setOpenResult(boolean openResult)
- {
- finishedWriters.forEach((w) -> w.setOpenResult(openResult));
- currentWriter.setOpenResult(openResult);
- return this;
- }
-
- public String getFilename()
- {
- return String.join("/", cfs.keyspace.getName(), cfs.getTableName());
- }
-
- @Override
- public long getFilePointer()
- {
- return currentWriter != null ? currentWriter.getFilePointer() : 0L;
- }
-
- @Override
- public TableId getTableId()
- {
- return cfs.metadata.id;
- }
-
- @Override
- public Throwable commit(Throwable accumulate)
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
- currentWriter = null;
- for (SSTableMultiWriter writer : finishedWriters)
- accumulate = writer.commit(accumulate);
- return accumulate;
- }
-
- @Override
- public Throwable abort(Throwable accumulate)
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
- currentWriter = null;
- for (SSTableMultiWriter finishedWriter : finishedWriters)
- accumulate = finishedWriter.abort(accumulate);
-
- return accumulate;
- }
-
- @Override
- public void prepareToCommit()
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
- currentWriter = null;
- finishedWriters.forEach(SSTableMultiWriter::prepareToCommit);
- }
-
- @Override
- public void close()
- {
- if (currentWriter != null)
- finishedWriters.add(currentWriter);
- currentWriter = null;
- finishedWriters.forEach(SSTableMultiWriter::close);
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFlushObserver.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFlushObserver.java
deleted file mode 100644
index 569925e..0000000
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFlushObserver.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format;
-
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.rows.Unfiltered;
-
-/**
- * Observer for events in the lifecycle of writing out an sstable.
- */
-public interface SSTableFlushObserver
-{
- /**
- * Called before writing any data to the sstable.
- */
- void begin();
-
- /**
- * Called when a new partition in being written to the sstable,
- * but before any cells are processed (see {@link #nextUnfilteredCluster(Unfiltered)}).
- *
- * @param key The key being appended to SSTable.
- * @param indexPosition The position of the key in the SSTable PRIMARY_INDEX file.
- */
- void startPartition(DecoratedKey key, long indexPosition);
-
- /**
- * Called after the unfiltered cluster is written to the sstable.
- * Will be preceded by a call to {@code startPartition(DecoratedKey, long)},
- * and the cluster should be assumed to belong to that partition.
- *
- * @param unfilteredCluster The unfiltered cluster being added to SSTable.
- */
- void nextUnfilteredCluster(Unfiltered unfilteredCluster);
-
- /**
- * Called when all data is written to the file and it's ready to be finished up.
- */
- void complete();
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
index 14f6602..1caa87b 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
@@ -17,61 +17,198 @@
*/
package org.apache.cassandra.io.sstable.format;
-import com.google.common.base.CharMatcher;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nonnull;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.MetricsProviders;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Pair;
/**
* Provides the accessors to data on disk.
*/
-public interface SSTableFormat
+public interface SSTableFormat<R extends SSTableReader, W extends SSTableWriter>
{
- static boolean enableSSTableDevelopmentTestMode = Boolean.getBoolean("cassandra.test.sstableformatdevelopment");
-
+ String name();
Version getLatestVersion();
Version getVersion(String version);
- SSTableWriter.Factory getWriterFactory();
- SSTableReader.Factory getReaderFactory();
+ SSTableWriterFactory<W, ?> getWriterFactory();
- RowIndexEntry.IndexSerializer<?> getIndexSerializer(TableMetadata metadata, Version version, SerializationHeader header);
+ SSTableReaderFactory<R, ?> getReaderFactory();
- public static enum Type
+ /**
+ * All the components that the writter can produce when saving an sstable, as well as all the components
+ * that the reader can read.
+ */
+ Set<Component> allComponents();
+
+ Set<Component> streamingComponents();
+
+ Set<Component> primaryComponents();
+
+ /**
+ * Returns components required by offline compaction tasks - like splitting sstables.
+ */
+ Set<Component> batchComponents();
+
+ /**
+ * Returns the components which should be selected for upload by the sstable loader.
+ */
+ Set<Component> uploadComponents();
+
+ /**
+ * Returns a set of the components that can be changed after an sstable was written.
+ */
+ Set<Component> mutableComponents();
+
+ /**
+ * Returns a set of components that can be automatically generated when loading sstable and thus are not mandatory.
+ */
+ Set<Component> generatedOnLoadComponents();
+
+ KeyCacheValueSerializer<R, ?> getKeyCacheValueSerializer();
+
+ /**
+ * Returns a new scrubber for an sstable. Note that the transaction must contain only one reader
+ * and the reader must match the provided cfs.
+ */
+ IScrubber getScrubber(ColumnFamilyStore cfs,
+ LifecycleTransaction transaction,
+ OutputHandler outputHandler,
+ IScrubber.Options options);
+
+ MetricsProviders getFormatSpecificMetricsProviders();
+
+ void deleteOrphanedComponents(Descriptor descriptor, Set<Component> components);
+
+ /**
+ * Deletes the existing components of the sstables represented by the provided descriptor.
+ * The method is also responsible for cleaning up the in-memory resources occupied by the stuff related to that
+ * sstables, such as row key cache entries.
+ */
+ void delete(Descriptor descriptor);
+
+ interface SSTableReaderFactory<R extends SSTableReader, B extends SSTableReader.Builder<R, B>>
{
- //The original sstable format
- BIG("big", BigFormat.instance);
+ /**
+ * A simple builder which creates an instnace of {@link SSTableReader} with the provided parameters.
+ * It expects that all the required resources to be opened/loaded externally by the caller.
+ * <p>
+ * The builder is expected to perform basic validation of the provided parameters.
+ */
+ SSTableReader.Builder<R, B> builder(Descriptor descriptor);
- public final SSTableFormat info;
- public final String name;
+ /**
+ * A builder which opens/loads all the required resources upon execution of
+ * {@link SSTableReaderLoadingBuilder#build(SSTable.Owner, boolean, boolean)} and passed them to the created
+ * reader instance. If the creation of {@link SSTableReader} fails, no resources should be left opened.
+ * <p>
+ * The builder is expected to perform basic validation of the provided parameters.
+ */
+ SSTableReaderLoadingBuilder<R, B> loadingBuilder(Descriptor descriptor, TableMetadataRef tableMetadataRef, Set<Component> components);
- public static Type current()
+ /**
+ * Retrieves a key range for the given sstable at the lowest cost - that is, without opening all sstables files
+ * if possible.
+ */
+ Pair<DecoratedKey, DecoratedKey> readKeyRange(Descriptor descriptor, IPartitioner partitioner) throws IOException;
+
+ Class<R> getReaderClass();
+ }
+
+ interface SSTableWriterFactory<W extends SSTableWriter, B extends SSTableWriter.Builder<W, B>>
+ {
+ /**
+ * Returns a new builder which can create instance of {@link SSTableWriter} with the provided parameters.
+ * Similarly to the loading builder, it should open the required resources when
+ * the {@link SSTableWriter.Builder#build(LifecycleNewTracker, SSTable.Owner)} method is called.
+ * It should not let the caller passing any closeable resources directly, that is, via setters.
+ * If building fails, all the opened resources should be released.
+ */
+ B builder(Descriptor descriptor);
+
+ /**
+ * Tries to estimate the size of all the sstable files from the provided parameters.
+ */
+ long estimateSize(SSTableWriter.SSTableSizeParameters parameters);
+ }
+
+ class Components
+ {
+ public static class Types
{
- return BIG;
+ // the base data for an sstable: the remaining components can be regenerated
+ // based on the data component
+ public static final Component.Type DATA = Component.Type.createSingleton("DATA", "Data.db", null);
+ // file to hold information about uncompressed data length, chunk offsets etc.
+ public static final Component.Type COMPRESSION_INFO = Component.Type.createSingleton("COMPRESSION_INFO", "CompressionInfo.db", null);
+ // statistical metadata about the content of the sstable
+ public static final Component.Type STATS = Component.Type.createSingleton("STATS", "Statistics.db", null);
+ // serialized bloom filter for the row keys in the sstable
+ public static final Component.Type FILTER = Component.Type.createSingleton("FILTER", "Filter.db", null);
+ // holds CRC32 checksum of the data file
+ public static final Component.Type DIGEST = Component.Type.createSingleton("DIGEST", "Digest.crc32", null);
+ // holds the CRC32 for chunks in an uncompressed file.
+ public static final Component.Type CRC = Component.Type.createSingleton("CRC", "CRC.db", null);
+ // table of contents, stores the list of all components for the sstable
+ public static final Component.Type TOC = Component.Type.createSingleton("TOC", "TOC.txt", null);
+ // built-in secondary index (may exist multiple per sstable)
+ public static final Component.Type SECONDARY_INDEX = Component.Type.create("SECONDARY_INDEX", "SI_.*.db", null);
+ // custom component, used by e.g. custom compaction strategy
+ public static final Component.Type CUSTOM = Component.Type.create("CUSTOM", null, null);
}
- private Type(String name, SSTableFormat info)
- {
- //Since format comes right after generation
- //we disallow formats with numeric names
- assert !CharMatcher.digit().matchesAllOf(name);
+ // singleton components for types that don't need ids
+ public final static Component DATA = Types.DATA.getSingleton();
+ public final static Component COMPRESSION_INFO = Types.COMPRESSION_INFO.getSingleton();
+ public final static Component STATS = Types.STATS.getSingleton();
+ public final static Component FILTER = Types.FILTER.getSingleton();
+ public final static Component DIGEST = Types.DIGEST.getSingleton();
+ public final static Component CRC = Types.CRC.getSingleton();
+ public final static Component TOC = Types.TOC.getSingleton();
+ }
- this.name = name;
- this.info = info;
- }
+ interface KeyCacheValueSerializer<R extends SSTableReader, T extends AbstractRowIndexEntry>
+ {
+ void skip(DataInputPlus input) throws IOException;
- public static Type validate(String name)
- {
- for (Type valid : Type.values())
- {
- if (valid.name.equalsIgnoreCase(name))
- return valid;
- }
+ T deserialize(R reader, DataInputPlus input) throws IOException;
- throw new IllegalArgumentException("No Type constant " + name);
- }
+ void serialize(T entry, DataOutputPlus output) throws IOException;
+ }
+
+ interface Factory
+ {
+ /**
+ * Returns a name of the format. Format name must not be empty, must be unique and must consist only of lowercase letters.
+ */
+ String name();
+
+ /**
+ * Returns an instance of the sstable format configured with the provided options.
+ * <p/>
+ * The method is expected to validate the options, and throw
+ * {@link org.apache.cassandra.exceptions.ConfigurationException} if the validation fails.
+ *
+ * @param options overrides for the default options, can be empty, cannot be null
+ */
+ SSTableFormat<?, ?> getInstance(@Nonnull Map<String, String> options);
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index f26cf65..fd34c02 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -20,123 +20,133 @@
import java.io.IOException;
import java.lang.ref.WeakReference;
-import java.nio.ByteBuffer;
-import java.nio.file.NoSuchFileException;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import com.google.common.primitives.Longs;
import com.google.common.util.concurrent.RateLimiter;
-
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.db.rows.UnfilteredSource;
-import org.apache.cassandra.concurrent.ExecutorPlus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
-import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
-
-import org.apache.cassandra.cache.InstrumentingCache;
-import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredSource;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.FSError;
-import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.metadata.*;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.CheckedFunction;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.metrics.RestorableMeter;
-import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.*;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.SelfRefCounted;
+import org.apache.cassandra.utils.concurrent.SharedCloseable;
+import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQueue;
+import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull;
/**
- * An SSTableReader can be constructed in a number of places, but typically is either
- * read from disk at startup, or constructed from a flushed memtable, or after compaction
- * to replace some existing sstables. However once created, an sstablereader may also be modified.
- *
- * A reader's OpenReason describes its current stage in its lifecycle, as follows:
- *
- *
- * <pre> {@code
- * NORMAL
- * From: None => Reader has been read from disk, either at startup or from a flushed memtable
- * EARLY => Reader is the final result of a compaction
- * MOVED_START => Reader WAS being compacted, but this failed and it has been restored to NORMAL status
- *
- * EARLY
- * From: None => Reader is a compaction replacement that is either incomplete and has been opened
- * to represent its partial result status, or has been finished but the compaction
- * it is a part of has not yet completed fully
- * EARLY => Same as from None, only it is not the first time it has been
- *
- * MOVED_START
- * From: NORMAL => Reader is being compacted. This compaction has not finished, but the compaction result
- * is either partially or fully opened, to either partially or fully replace this reader.
- * This reader's start key has been updated to represent this, so that reads only hit
- * one or the other reader.
- *
- * METADATA_CHANGE
- * From: NORMAL => Reader has seen low traffic and the amount of memory available for index summaries is
- * constrained, so its index summary has been downsampled.
- * METADATA_CHANGE => Same
- * } </pre>
- *
- * Note that in parallel to this, there are two different Descriptor types; TMPLINK and FINAL; the latter corresponds
- * to NORMAL state readers and all readers that replace a NORMAL one. TMPLINK is used for EARLY state readers and
- * no others.
- *
- * When a reader is being compacted, if the result is large its replacement may be opened as EARLY before compaction
- * completes in order to present the result to consumers earlier. In this case the reader will itself be changed to
- * a MOVED_START state, where its start no longer represents its on-disk minimum key. This is to permit reads to be
- * directed to only one reader when the two represent the same data. The EARLY file can represent a compaction result
- * that is either partially complete and still in-progress, or a complete and immutable sstable that is part of a larger
- * macro compaction action that has not yet fully completed.
- *
- * Currently ALL compaction results at least briefly go through an EARLY open state prior to completion, regardless
- * of if early opening is enabled.
- *
- * Since a reader can be created multiple times over the same shared underlying resources, and the exact resources
- * it shares between each instance differ subtly, we track the lifetime of any underlying resource with its own
- * reference count, which each instance takes a Ref to. Each instance then tracks references to itself, and once these
- * all expire it releases its Refs to these underlying resources.
- *
- * There is some shared cleanup behaviour needed only once all sstablereaders in a certain stage of their lifecycle
- * (i.e. EARLY or NORMAL opening), and some that must only occur once all readers of any kind over a single logical
- * sstable have expired. These are managed by the TypeTidy and GlobalTidy classes at the bottom, and are effectively
- * managed as another resource each instance tracks its own Ref instance to, to ensure all of these resources are
- * cleaned up safely and can be debugged otherwise.
- *
+ * An SSTableReader can be constructed in a number of places, but typically is either read from disk at startup, or
+ * constructed from a flushed memtable, or after compaction to replace some existing sstables. However once created,
+ * an sstablereader may also be modified.
+ * <p>
+ * A reader's {@link OpenReason} describes its current stage in its lifecycle. Note that in parallel to this, there are
+ * two different Descriptor types; TMPLINK and FINAL; the latter corresponds to {@link OpenReason#NORMAL} state readers
+ * and all readers that replace a {@link OpenReason#NORMAL} one. TMPLINK is used for {@link OpenReason#EARLY} state
+ * readers and no others.
+ * <p>
+ * When a reader is being compacted, if the result is large its replacement may be opened as {@link OpenReason#EARLY}
+ * before compaction completes in order to present the result to consumers earlier. In this case the reader will itself
+ * be changed to a {@link OpenReason#MOVED_START} state, where its start no longer represents its on-disk minimum key.
+ * This is to permit reads to be directed to only one reader when the two represent the same data.
+ * The {@link OpenReason#EARLY} file can represent a compaction result that is either partially complete and still
+ * in-progress, or a complete and immutable sstable that is part of a larger macro compaction action that has not yet
+ * fully completed.
+ * <p>
+ * Currently ALL compaction results at least briefly go through an {@link OpenReason#EARLY} open state prior to completion,
+ * regardless of if early opening is enabled.
+ * <p>
+ * Since a reader can be created multiple times over the same shared underlying resources, and the exact resources it
+ * shares between each instance differ subtly, we track the lifetime of any underlying resource with its own reference
+ * count, which each instance takes a {@link Ref} to. Each instance then tracks references to itself, and once these
+ * all expire it releases all its {@link Ref} to these underlying resources.
+ * <p>
+ * There is some shared cleanup behaviour needed only once all readers in a certain stage of their lifecycle
+ * (i.e. {@link OpenReason#EARLY} or {@link OpenReason#NORMAL} opening), and some that must only occur once all readers
+ * of any kind over a single logical sstable have expired. These are managed by the {@link InstanceTidier} and
+ * {@link GlobalTidy} classes at the bottom, and are effectively managed as another resource each instance tracks its
+ * own {@link Ref} instance to, to ensure all of these resources are cleaned up safely and can be debugged otherwise.
+ * <p>
* TODO: fill in details about Tracker and lifecycle interactions for tools, and for compaction strategies
*/
public abstract class SSTableReader extends SSTable implements UnfilteredSource, SelfRefCounted<SSTableReader>
@@ -146,6 +156,7 @@
private static final boolean TRACK_ACTIVITY = CassandraRelevantProperties.DISABLE_SSTABLE_ACTIVITY_TRACKING.getBoolean();
private static final ScheduledExecutorPlus syncExecutor = initSyncExecutor();
+
private static ScheduledExecutorPlus initSyncExecutor()
{
if (DatabaseDescriptor.isClientOrToolInitialized())
@@ -157,81 +168,98 @@
// Immediately remove readMeter sync task when cancelled.
// TODO: should we set this by default on all scheduled executors?
if (syncExecutor instanceof ScheduledThreadPoolExecutor)
- ((ScheduledThreadPoolExecutor)syncExecutor).setRemoveOnCancelPolicy(true);
+ ((ScheduledThreadPoolExecutor) syncExecutor).setRemoveOnCancelPolicy(true);
return syncExecutor;
}
+
private static final RateLimiter meterSyncThrottle = RateLimiter.create(100.0);
- public static final Comparator<SSTableReader> maxTimestampDescending = (o1, o2) -> Long.compare(o2.getMaxTimestamp(), o1.getMaxTimestamp());
- public static final Comparator<SSTableReader> maxTimestampAscending = (o1, o2) -> Long.compare(o1.getMaxTimestamp(), o2.getMaxTimestamp());
+ public static final Comparator<SSTableReader> maxTimestampAscending = Comparator.comparingLong(SSTableReader::getMaxTimestamp);
+ public static final Comparator<SSTableReader> maxTimestampDescending = maxTimestampAscending.reversed();
// it's just an object, which we use regular Object equality on; we introduce a special class just for easy recognition
- public static final class UniqueIdentifier {}
+ public static final class UniqueIdentifier
+ {
+ }
+ public final UniqueIdentifier instanceId = new UniqueIdentifier();
- public static final Comparator<SSTableReader> sstableComparator = (o1, o2) -> o1.first.compareTo(o2.first);
+ public static final Comparator<SSTableReader> sstableComparator = Comparator.comparing(o -> o.first);
+ public static final Ordering<SSTableReader> sstableOrdering = Ordering.from(sstableComparator);
public static final Comparator<SSTableReader> idComparator = Comparator.comparing(t -> t.descriptor.id, SSTableIdFactory.COMPARATOR);
public static final Comparator<SSTableReader> idReverseComparator = idComparator.reversed();
- public static final Ordering<SSTableReader> sstableOrdering = Ordering.from(sstableComparator);
-
- public static final Comparator<SSTableReader> sizeComparator = new Comparator<SSTableReader>()
- {
- public int compare(SSTableReader o1, SSTableReader o2)
- {
- return Longs.compare(o1.onDiskLength(), o2.onDiskLength());
- }
- };
+ public static final Comparator<SSTableReader> sizeComparator = (o1, o2) -> Longs.compare(o1.onDiskLength(), o2.onDiskLength());
/**
* maxDataAge is a timestamp in local server time (e.g. Global.currentTimeMilli) which represents an upper bound
* to the newest piece of data stored in the sstable. In other words, this sstable does not contain items created
* later than maxDataAge.
- *
+ * <p>
* The field is not serialized to disk, so relying on it for more than what truncate does is not advised.
- *
+ * <p>
* When a new sstable is flushed, maxDataAge is set to the time of creation.
* When a sstable is created from compaction, maxDataAge is set to max of all merged sstables.
- *
+ * <p>
* The age is in milliseconds since epoc and is local to this host.
*/
public final long maxDataAge;
public enum OpenReason
{
+ /**
+ * <ul>
+ * <li>From {@code None} - Reader has been read from disk, either at startup or from a flushed memtable</li>
+ * <li>From {@link #EARLY} - Reader is the final result of a compaction</li>
+ * <li>From {@link #MOVED_START} - Reader WAS being compacted, but this failed and it has been restored
+ * to {code NORMAL}status</li>
+ * </ul>
+ */
NORMAL,
+
+ /**
+ * <ul>
+ * <li>From {@code None} - Reader is a compaction replacement that is either incomplete and has been opened
+ * to represent its partial result status, or has been finished but the compaction it is a part of has not
+ * yet completed fully</li>
+ * <li>From {@link #EARLY} - Same as from {@code None}, only it is not the first time it has been
+ * </ul>
+ */
EARLY,
+
+ /**
+ * From:
+ * <ul>
+ * <li>From {@link #NORMAL} - Reader has seen low traffic and the amount of memory available for index summaries
+ * is constrained, so its index summary has been downsampled</li>
+ * <li>From {@link #METADATA_CHANGE} - Same
+ * </ul>
+ */
METADATA_CHANGE,
+
+ /**
+ * <ul>
+ * <li>From {@link #NORMAL} - Reader is being compacted. This compaction has not finished, but the compaction
+ * result is either partially or fully opened, to either partially or fully replace this reader. This reader's
+ * start key has been updated to represent this, so that reads only hit one or the other reader.</li>
+ * </ul>
+ */
MOVED_START
}
public final OpenReason openReason;
- public final UniqueIdentifier instanceId = new UniqueIdentifier();
- // indexfile and datafile: might be null before a call to load()
- protected final FileHandle ifile;
protected final FileHandle dfile;
- protected final IFilter bf;
- public final IndexSummary indexSummary;
-
- protected final RowIndexEntry.IndexSerializer<?> rowIndexEntrySerializer;
-
- protected InstrumentingCache<KeyCacheKey, RowIndexEntry> keyCache;
-
- protected final BloomFilterTracker bloomFilterTracker = new BloomFilterTracker();
// technically isCompacted is not necessary since it should never be unreferenced unless it is also compacted,
// but it seems like a good extra layer of protection against reference counting bugs to not delete data based on that alone
- protected final AtomicBoolean isSuspect = new AtomicBoolean(false);
+ public final AtomicBoolean isSuspect = new AtomicBoolean(false);
// not final since we need to be able to change level on a file.
protected volatile StatsMetadata sstableMetadata;
public final SerializationHeader header;
- protected final AtomicLong keyCacheHit = new AtomicLong(0);
- protected final AtomicLong keyCacheRequest = new AtomicLong(0);
-
private final InstanceTidier tidy;
private final Ref<SSTableReader> selfRef;
@@ -239,6 +267,10 @@
private volatile double crcCheckChance;
+ public final DecoratedKey first;
+ public final DecoratedKey last;
+ public final AbstractBounds<Token> bounds;
+
/**
* Calculate approximate key count.
* If cardinality estimator is available on all given sstables, then this method use them to estimate
@@ -264,7 +296,7 @@
try
{
- CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
+ CompactionMetadata metadata = StatsComponent.load(sstable.descriptor).compactionMetadata();
// If we can't load the CompactionMetadata, we are forced to estimate the keys using the index
// summary. (CASSANDRA-10676)
if (metadata == null)
@@ -305,233 +337,66 @@
return count;
}
- /**
- * Estimates how much of the keys we would keep if the sstables were compacted together
- */
- public static double estimateCompactionGain(Set<SSTableReader> overlapping)
+ public static SSTableReader open(SSTable.Owner owner, Descriptor descriptor)
{
- Set<ICardinality> cardinalities = new HashSet<>(overlapping.size());
- for (SSTableReader sstable : overlapping)
- {
- try
- {
- ICardinality cardinality = ((CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION)).cardinalityEstimator;
- if (cardinality != null)
- cardinalities.add(cardinality);
- else
- logger.trace("Got a null cardinality estimator in: {}", sstable.getFilename());
- }
- catch (IOException e)
- {
- logger.warn("Could not read up compaction metadata for {}", sstable, e);
- }
- }
- long totalKeyCountBefore = 0;
- for (ICardinality cardinality : cardinalities)
- {
- totalKeyCountBefore += cardinality.cardinality();
- }
- if (totalKeyCountBefore == 0)
- return 1;
-
- long totalKeyCountAfter = mergeCardinalities(cardinalities).cardinality();
- logger.trace("Estimated compaction gain: {}/{}={}", totalKeyCountAfter, totalKeyCountBefore, ((double)totalKeyCountAfter)/totalKeyCountBefore);
- return ((double)totalKeyCountAfter)/totalKeyCountBefore;
+ return open(owner, descriptor, null);
}
- private static ICardinality mergeCardinalities(Collection<ICardinality> cardinalities)
+ public static SSTableReader open(SSTable.Owner owner, Descriptor desc, TableMetadataRef metadata)
{
- ICardinality base = new HyperLogLogPlus(13, 25); // see MetadataCollector.cardinality
- try
- {
- base = base.merge(cardinalities.toArray(new ICardinality[cardinalities.size()]));
- }
- catch (CardinalityMergeException e)
- {
- logger.warn("Could not merge cardinalities", e);
- }
- return base;
+ return open(owner, desc, null, metadata);
}
- public static SSTableReader open(Descriptor descriptor)
+ public static SSTableReader open(SSTable.Owner owner, Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
- TableMetadataRef metadata;
- if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
- {
- int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
- String indexName = descriptor.cfname.substring(i + 1);
- metadata = Schema.instance.getIndexTableMetadataRef(descriptor.ksname, indexName);
- if (metadata == null)
- throw new AssertionError("Could not find index metadata for index cf " + i);
- }
- else
- {
- metadata = Schema.instance.getTableMetadataRef(descriptor.ksname, descriptor.cfname);
- }
- return open(descriptor, metadata);
- }
-
- public static SSTableReader open(Descriptor desc, TableMetadataRef metadata)
- {
- return open(desc, componentsFor(desc), metadata);
- }
-
- public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
- {
- return open(descriptor, components, metadata, true, false);
+ return open(owner, descriptor, components, metadata, true, false);
}
// use only for offline or "Standalone" operations
public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs)
{
- return open(descriptor, components, cfs.metadata, false, true);
+ return open(cfs, descriptor, components, cfs.metadata, false, true);
}
// use only for offline or "Standalone" operations
- public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata)
+ public static SSTableReader openNoValidation(SSTable.Owner owner, Descriptor descriptor, TableMetadataRef metadata)
{
- return open(descriptor, componentsFor(descriptor), metadata, false, true);
+ return open(owner, descriptor, null, metadata, false, true);
}
/**
* Open SSTable reader to be used in batch mode(such as sstableloader).
- *
- * @param descriptor
- * @param components
- * @param metadata
- * @return opened SSTableReader
- * @throws IOException
*/
- public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
+ public static SSTableReader openForBatch(SSTable.Owner owner, Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
- // Minimum components without which we can't do anything
- assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
- assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
- verifyCompressionInfoExistenceIfApplicable(descriptor, components);
-
- EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata;
- try
- {
- sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
- }
- catch (IOException e)
- {
- throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
- }
-
- ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
- StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
- SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
-
- // Check if sstable is created using same partitioner.
- // Partitioner can be null, which indicates older version of sstable or no stats available.
- // In that case, we skip the check.
- String partitionerName = metadata.get().partitioner.getClass().getCanonicalName();
- if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
- {
- logger.error("Cannot open {}; partitioner {} does not match system partitioner {}. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
- descriptor, validationMetadata.partitioner, partitionerName);
- System.exit(1);
- }
-
- try
- {
- return new SSTableReaderBuilder.ForBatch(descriptor, metadata, components, statsMetadata, header.toHeader(metadata.get())).build();
- }
- catch (UnknownColumnException e)
- {
- throw new IllegalStateException(e);
- }
+ return open(owner, descriptor, components, metadata, true, true);
}
/**
* Open an SSTable for reading
+ *
+ * @param owner owning entity
* @param descriptor SSTable to open
* @param components Components included with this SSTable
- * @param metadata for this SSTables CF
- * @param validate Check SSTable for corruption (limited)
- * @param isOffline Whether we are opening this SSTable "offline", for example from an external tool or not for inclusion in queries (validations)
- * This stops regenerating BF + Summaries and also disables tracking of hotness for the SSTable.
+ * @param metadata for this SSTables CF
+ * @param validate Check SSTable for corruption (limited)
+ * @param isOffline Whether we are opening this SSTable "offline", for example from an external tool or not for inclusion in queries (validations)
+ * This stops regenerating BF + Summaries and also disables tracking of hotness for the SSTable.
* @return {@link SSTableReader}
- * @throws IOException
*/
- public static SSTableReader open(Descriptor descriptor,
+ public static SSTableReader open(Owner owner,
+ Descriptor descriptor,
Set<Component> components,
TableMetadataRef metadata,
boolean validate,
boolean isOffline)
{
- // Minimum components without which we can't do anything
- assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
- assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
+ SSTableReaderLoadingBuilder<?, ?> builder = descriptor.getFormat().getReaderFactory().loadingBuilder(descriptor, metadata, components);
- // For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content
- assert components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
-
- verifyCompressionInfoExistenceIfApplicable(descriptor, components);
-
- EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
-
- Map<MetadataType, MetadataComponent> sstableMetadata;
- try
- {
- sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
- }
- catch (Throwable t)
- {
- throw new CorruptSSTableException(t, descriptor.filenameFor(Component.STATS));
- }
- ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
- StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
- SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
- assert header != null;
-
- // Check if sstable is created using same partitioner.
- // Partitioner can be null, which indicates older version of sstable or no stats available.
- // In that case, we skip the check.
- String partitionerName = metadata.get().partitioner.getClass().getCanonicalName();
- if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
- {
- logger.error("Cannot open {}; partitioner {} does not match system partitioner {}. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
- descriptor, validationMetadata.partitioner, partitionerName);
- System.exit(1);
- }
-
- SSTableReader sstable;
- try
- {
- sstable = new SSTableReaderBuilder.ForRead(descriptor,
- metadata,
- validationMetadata,
- isOffline,
- components,
- statsMetadata,
- header.toHeader(metadata.get())).build();
- }
- catch (UnknownColumnException e)
- {
- throw new IllegalStateException(e);
- }
-
- try
- {
- if (validate)
- sstable.validate();
-
- if (sstable.getKeyCache() != null)
- logger.trace("key cache contains {}/{} keys", sstable.getKeyCache().size(), sstable.getKeyCache().getCapacity());
-
- return sstable;
- }
- catch (Throwable t)
- {
- sstable.selfRef().release();
- throw new CorruptSSTableException(t, sstable.getFilename());
- }
+ return builder.build(owner, validate, !isOffline);
}
- public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor, Set<Component>>> entries,
+ public static Collection<SSTableReader> openAll(SSTable.Owner owner, Set<Map.Entry<Descriptor, Set<Component>>> entries,
final TableMetadataRef metadata)
{
final Collection<SSTableReader> sstables = newBlockingQueue();
@@ -541,29 +406,25 @@
{
for (final Map.Entry<Descriptor, Set<Component>> entry : entries)
{
- Runnable runnable = new Runnable()
- {
- public void run()
+ Runnable runnable = () -> {
+ SSTableReader sstable;
+ try
{
- SSTableReader sstable;
- try
- {
- sstable = open(entry.getKey(), entry.getValue(), metadata);
- }
- catch (CorruptSSTableException ex)
- {
- JVMStabilityInspector.inspectThrowable(ex);
- logger.error("Corrupt sstable {}; skipping table", entry, ex);
- return;
- }
- catch (FSError ex)
- {
- JVMStabilityInspector.inspectThrowable(ex);
- logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
- return;
- }
- sstables.add(sstable);
+ sstable = open(owner, entry.getKey(), entry.getValue(), metadata);
}
+ catch (CorruptSSTableException ex)
+ {
+ JVMStabilityInspector.inspectThrowable(ex);
+ logger.error("Corrupt sstable {}; skipping table", entry, ex);
+ return;
+ }
+ catch (FSError ex)
+ {
+ JVMStabilityInspector.inspectThrowable(ex);
+ logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
+ return;
+ }
+ sstables.add(sstable);
};
executor.submit(runnable);
}
@@ -583,102 +444,62 @@
}
return sstables;
-
}
- /**
- * Open a RowIndexedReader which already has its state initialized (by SSTableWriter).
- */
- public static SSTableReader internalOpen(Descriptor desc,
- Set<Component> components,
- TableMetadataRef metadata,
- FileHandle ifile,
- FileHandle dfile,
- IndexSummary summary,
- IFilter bf,
- long maxDataAge,
- StatsMetadata sstableMetadata,
- OpenReason openReason,
- SerializationHeader header)
+ protected SSTableReader(Builder<?, ?> builder, Owner owner)
{
- assert desc != null && ifile != null && dfile != null && summary != null && bf != null && sstableMetadata != null;
+ super(builder, owner);
- return new SSTableReaderBuilder.ForWriter(desc, metadata, maxDataAge, components, sstableMetadata, openReason, header)
- .bf(bf).ifile(ifile).dfile(dfile).summary(summary).build();
- }
+ this.sstableMetadata = builder.getStatsMetadata();
+ this.header = builder.getSerializationHeader();
+ this.dfile = builder.getDataFile();
+ this.maxDataAge = builder.getMaxDataAge();
+ this.openReason = builder.getOpenReason();
+ this.first = builder.getFirst();
+ this.last = builder.getLast();
+ this.bounds = AbstractBounds.strictlyWrapsAround(first.getToken(), last.getToken())
+ ? null // this will cause the validation to fail, but the reader is opened with no validation,
+ // e.g. for scrubbing, we should accept screwed bounds
+ : AbstractBounds.bounds(first.getToken(), true, last.getToken(), true);
- /**
- * Best-effort checking to verify the expected compression info component exists, according to the TOC file.
- * The verification depends on the existence of TOC file. If absent, the verification is skipped.
- * @param descriptor
- * @param actualComponents, actual components listed from the file system.
- * @throws CorruptSSTableException, if TOC expects compression info but not found from disk.
- * @throws FSReadError, if unable to read from TOC file.
- */
- public static void verifyCompressionInfoExistenceIfApplicable(Descriptor descriptor,
- Set<Component> actualComponents)
- throws CorruptSSTableException, FSReadError
- {
- File tocFile = new File(descriptor.filenameFor(Component.TOC));
- if (tocFile.exists())
- {
- try
- {
- Set<Component> expectedComponents = readTOC(descriptor, false);
- if (expectedComponents.contains(Component.COMPRESSION_INFO) && !actualComponents.contains(Component.COMPRESSION_INFO))
- {
- String compressionInfoFileName = descriptor.filenameFor(Component.COMPRESSION_INFO);
- throw new CorruptSSTableException(new NoSuchFileException(compressionInfoFileName), compressionInfoFileName);
- }
- }
- catch (IOException e)
- {
- throw new FSReadError(e, tocFile);
- }
- }
- }
-
- protected SSTableReader(SSTableReaderBuilder builder)
- {
- this(builder.descriptor,
- builder.components,
- builder.metadataRef,
- builder.maxDataAge,
- builder.statsMetadata,
- builder.openReason,
- builder.header,
- builder.summary,
- builder.dfile,
- builder.ifile,
- builder.bf);
- }
-
- protected SSTableReader(final Descriptor desc,
- Set<Component> components,
- TableMetadataRef metadata,
- long maxDataAge,
- StatsMetadata sstableMetadata,
- OpenReason openReason,
- SerializationHeader header,
- IndexSummary summary,
- FileHandle dfile,
- FileHandle ifile,
- IFilter bf)
- {
- super(desc, components, metadata, DatabaseDescriptor.getDiskOptimizationStrategy());
- this.sstableMetadata = sstableMetadata;
- this.header = header;
- this.indexSummary = summary;
- this.dfile = dfile;
- this.ifile = ifile;
- this.bf = bf;
- this.maxDataAge = maxDataAge;
- this.openReason = openReason;
- this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), desc.version, header);
- tidy = new InstanceTidier(descriptor, metadata.id);
+ tidy = new InstanceTidier(descriptor, owner);
selfRef = new Ref<>(this, tidy);
}
+ @Override
+ public DecoratedKey getFirst()
+ {
+ return first;
+ }
+
+ @Override
+ public DecoratedKey getLast()
+ {
+ return last;
+ }
+
+ @Override
+ public AbstractBounds<Token> getBounds()
+ {
+ return Objects.requireNonNull(bounds, "Bounds were not created because the sstable is out of order");
+ }
+
+ public DataIntegrityMetadata.ChecksumValidator maybeGetChecksumValidator() throws IOException
+ {
+ if (descriptor.fileFor(Components.CRC).exists())
+ return new DataIntegrityMetadata.ChecksumValidator(descriptor.fileFor(Components.DATA), descriptor.fileFor(Components.CRC));
+ else
+ return null;
+ }
+
+ public DataIntegrityMetadata.FileDigestValidator maybeGetDigestValidator() throws IOException
+ {
+ if (descriptor.fileFor(Components.DIGEST).exists())
+ return new DataIntegrityMetadata.FileDigestValidator(descriptor.fileFor(Components.DATA), descriptor.fileFor(Components.DIGEST));
+ else
+ return null;
+ }
+
public static long getTotalBytes(Iterable<SSTableReader> sstables)
{
long sum = 0;
@@ -713,60 +534,7 @@
public void setupOnline()
{
- // under normal operation we can do this at any time, but SSTR is also used outside C* proper,
- // e.g. by BulkLoader, which does not initialize the cache. As a kludge, we set up the cache
- // here when we know we're being wired into the rest of the server infrastructure.
- InstrumentingCache<KeyCacheKey, RowIndexEntry> maybeKeyCache = CacheService.instance.keyCache;
- if (maybeKeyCache.getCapacity() > 0)
- keyCache = maybeKeyCache;
-
- final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata().id);
- if (cfs != null)
- setCrcCheckChance(cfs.getCrcCheckChance());
- }
-
- /**
- * Save index summary to Summary.db file.
- */
- public static void saveSummary(Descriptor descriptor, DecoratedKey first, DecoratedKey last, IndexSummary summary)
- {
- File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
- if (summariesFile.exists())
- FileUtils.deleteWithConfirm(summariesFile);
-
- try (DataOutputStreamPlus oStream = new FileOutputStreamPlus(summariesFile))
- {
- IndexSummary.serializer.serialize(summary, oStream);
- ByteBufferUtil.writeWithLength(first.getKey(), oStream);
- ByteBufferUtil.writeWithLength(last.getKey(), oStream);
- }
- catch (IOException e)
- {
- logger.error("Cannot save SSTable Summary: ", e);
-
- // corrupted hence delete it and let it load it now.
- if (summariesFile.exists())
- FileUtils.deleteWithConfirm(summariesFile);
- }
- }
-
- public static void saveBloomFilter(Descriptor descriptor, IFilter filter)
- {
- File filterFile = new File(descriptor.filenameFor(Component.FILTER));
- try (DataOutputStreamPlus stream = new FileOutputStreamPlus(filterFile))
- {
- BloomFilterSerializer.serialize((BloomFilter) filter, stream);
- stream.flush();
- }
- catch (IOException e)
- {
- logger.trace("Cannot save SSTable bloomfilter: ", e);
-
- // corrupted hence delete it and let it load it now.
- if (filterFile.exists())
- FileUtils.deleteWithConfirm(filterFile);
- }
-
+ owner().ifPresent(o -> setCrcCheckChance(o.getCrcCheckChance()));
}
/**
@@ -774,7 +542,7 @@
*
* @param task to be guarded by sstable lock
*/
- public <R> R runWithLock(CheckedFunction<Descriptor, R, IOException> task) throws IOException
+ public <R, E extends Exception> R runWithLock(CheckedFunction<Descriptor, R, E> task) throws E
{
synchronized (tidy.global)
{
@@ -799,317 +567,111 @@
}
}
- // These runnables must NOT be an anonymous or non-static inner class, nor must it retain a reference chain to this reader
+ /**
+ * The runnable passed to this method must not be an anonymous or non-static inner class. It can be a lambda or a
+ * method reference provided that it does not retain a reference chain to this reader.
+ */
public void runOnClose(final Runnable runOnClose)
{
+ if (runOnClose == null)
+ return;
+
synchronized (tidy.global)
{
final Runnable existing = tidy.runOnClose;
- tidy.runOnClose = AndThen.get(existing, runOnClose);
- }
- }
-
- private static class AndThen implements Runnable
- {
- final Runnable runFirst;
- final Runnable runSecond;
-
- private AndThen(Runnable runFirst, Runnable runSecond)
- {
- this.runFirst = runFirst;
- this.runSecond = runSecond;
- }
-
- public void run()
- {
- runFirst.run();
- runSecond.run();
- }
-
- static Runnable get(Runnable runFirst, Runnable runSecond)
- {
- if (runFirst == null)
- return runSecond;
- return new AndThen(runFirst, runSecond);
+ if (existing == null)
+ tidy.runOnClose = runOnClose;
+ else
+ tidy.runOnClose = () -> {
+ existing.run();
+ runOnClose.run();
+ };
}
}
/**
- * Clone this reader with the provided start and open reason, and set the clone as replacement.
+ * The method sets fields specific to this {@link SSTableReader} and the parent {@link SSTable} on the provided
+ * {@link Builder}. The method is intended to be called from the overloaded {@code unbuildTo} method in subclasses.
*
- * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive
- * opening of compaction results).
- * @param reason the {@code OpenReason} for the replacement.
- *
- * @return the cloned reader. That reader is set as a replacement by the method.
+ * @param builder the builder on which the fields should be set
+ * @param sharedCopy whether the {@link SharedCloseable} resources should be passed as shared copies or directly;
+ * note that the method will overwrite the fields representing {@link SharedCloseable} only if
+ * they are not set in the builder yet (the relevant fields in the builder are {@code null}).
+ * @return the same instance of builder as provided
*/
- private SSTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason)
+ protected final <B extends Builder<?, B>> B unbuildTo(B builder, boolean sharedCopy)
{
- return cloneAndReplace(newFirst, reason, indexSummary.sharedCopy());
+ B b = super.unbuildTo(builder, sharedCopy);
+ if (builder.getDataFile() == null)
+ b.setDataFile(sharedCopy ? sharedCopyOrNull(dfile) : dfile);
+
+ b.setStatsMetadata(sstableMetadata);
+ b.setSerializationHeader(header);
+ b.setMaxDataAge(maxDataAge);
+ b.setOpenReason(openReason);
+ b.setFirst(first);
+ b.setLast(last);
+ b.setSuspected(isSuspect.get());
+ return b;
}
- /**
- * Clone this reader with the new values and set the clone as replacement.
- *
- * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive
- * opening of compaction results).
- * @param reason the {@code OpenReason} for the replacement.
- * @param newSummary the index summary for the replacement.
- *
- * @return the cloned reader. That reader is set as a replacement by the method.
- */
- private SSTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason, IndexSummary newSummary)
- {
- SSTableReader replacement = internalOpen(descriptor,
- components,
- metadata,
- ifile != null ? ifile.sharedCopy() : null,
- dfile.sharedCopy(),
- newSummary,
- bf.sharedCopy(),
- maxDataAge,
- sstableMetadata,
- reason,
- header);
+ public abstract SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart);
- replacement.first = newFirst;
- replacement.last = last;
- replacement.isSuspect.set(isSuspect.get());
- return replacement;
- }
-
- /**
- * Clone this reader with the new values and set the clone as replacement.
- *
- * @param newBloomFilter for the replacement
- *
- * @return the cloned reader. That reader is set as a replacement by the method.
- */
- @VisibleForTesting
- public SSTableReader cloneAndReplace(IFilter newBloomFilter)
- {
- SSTableReader replacement = internalOpen(descriptor,
- components,
- metadata,
- ifile.sharedCopy(),
- dfile.sharedCopy(),
- indexSummary,
- newBloomFilter,
- maxDataAge,
- sstableMetadata,
- openReason,
- header);
-
- replacement.first = first;
- replacement.last = last;
- replacement.isSuspect.set(isSuspect.get());
- return replacement;
- }
-
- public SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart)
- {
- synchronized (tidy.global)
- {
- return cloneAndReplace(restoredStart, OpenReason.NORMAL);
- }
- }
-
- // runOnClose must NOT be an anonymous or non-static inner class, nor must it retain a reference chain to this reader
- public SSTableReader cloneWithNewStart(DecoratedKey newStart, final Runnable runOnClose)
- {
- synchronized (tidy.global)
- {
- assert openReason != OpenReason.EARLY;
- // TODO: merge with caller's firstKeyBeyond() work,to save time
- if (newStart.compareTo(first) > 0)
- {
- final long dataStart = getPosition(newStart, Operator.EQ).position;
- final long indexStart = getIndexScanPosition(newStart);
- this.tidy.runOnClose = new DropPageCache(dfile, dataStart, ifile, indexStart, runOnClose);
- }
-
- return cloneAndReplace(newStart, OpenReason.MOVED_START);
- }
- }
-
- private static class DropPageCache implements Runnable
- {
- final FileHandle dfile;
- final long dfilePosition;
- final FileHandle ifile;
- final long ifilePosition;
- final Runnable andThen;
-
- private DropPageCache(FileHandle dfile, long dfilePosition, FileHandle ifile, long ifilePosition, Runnable andThen)
- {
- this.dfile = dfile;
- this.dfilePosition = dfilePosition;
- this.ifile = ifile;
- this.ifilePosition = ifilePosition;
- this.andThen = andThen;
- }
-
- public void run()
- {
- dfile.dropPageCache(dfilePosition);
-
- if (ifile != null)
- ifile.dropPageCache(ifilePosition);
- if (andThen != null)
- andThen.run();
- }
- }
-
- /**
- * Returns a new SSTableReader with the same properties as this SSTableReader except that a new IndexSummary will
- * be built at the target samplingLevel. This (original) SSTableReader instance will be marked as replaced, have
- * its DeletingTask removed, and have its periodic read-meter sync task cancelled.
- * @param samplingLevel the desired sampling level for the index summary on the new SSTableReader
- * @return a new SSTableReader
- * @throws IOException
- */
- @SuppressWarnings("resource")
- public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
- {
- assert openReason != OpenReason.EARLY;
-
- int minIndexInterval = metadata().params.minIndexInterval;
- int maxIndexInterval = metadata().params.maxIndexInterval;
- double effectiveInterval = indexSummary.getEffectiveIndexInterval();
-
- IndexSummary newSummary;
-
- // We have to rebuild the summary from the on-disk primary index in three cases:
- // 1. The sampling level went up, so we need to read more entries off disk
- // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary
- // at full sampling (and consequently at any other sampling level)
- // 3. The max_index_interval was lowered, forcing us to raise the sampling level
- if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval)
- {
- newSummary = buildSummaryAtLevel(samplingLevel);
- }
- else if (samplingLevel < indexSummary.getSamplingLevel())
- {
- // we can use the existing index summary to make a smaller one
- newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, getPartitioner());
- }
- else
- {
- throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " +
- "no adjustments to min/max_index_interval");
- }
-
- // Always save the resampled index with lock to avoid racing with entire-sstable streaming
- synchronized (tidy.global)
- {
- saveSummary(descriptor, first, last, newSummary);
- return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary);
- }
- }
-
- private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
- {
- // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
- RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
- try
- {
- long indexSize = primaryIndex.length();
- try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata().params.minIndexInterval, newSamplingLevel))
- {
- long indexPosition;
- while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
- {
- summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
- RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version);
- }
-
- return summaryBuilder.build(getPartitioner());
- }
- }
- finally
- {
- FileUtils.closeQuietly(primaryIndex);
- }
- }
+ public abstract SSTableReader cloneWithNewStart(DecoratedKey newStart);
public RestorableMeter getReadMeter()
{
return readMeter;
}
- public int getIndexSummarySamplingLevel()
+ /**
+ * All the resources which should be released upon closing this sstable reader are registered with in
+ * {@link GlobalTidy}. This method lets close a provided resource explicitly any time and unregister it from
+ * {@link GlobalTidy} so that it is not tried to be released twice.
+ *
+ * @param closeable a resource to be closed
+ */
+ protected void closeInternalComponent(AutoCloseable closeable)
{
- return indexSummary.getSamplingLevel();
+ synchronized (tidy.global)
+ {
+ boolean removed = tidy.closeables.remove(closeable);
+ Preconditions.checkState(removed);
+ try
+ {
+ closeable.close();
+ }
+ catch (Exception ex)
+ {
+ throw new RuntimeException("Failed to close " + closeable, ex);
+ }
+ }
}
- public long getIndexSummaryOffHeapSize()
- {
- return indexSummary.getOffHeapSize();
- }
+ /**
+ * This method is expected to close the components which occupy memory but are not needed when we just want to
+ * stream the components (for example, when SSTable is opened with SSTableLoader). The method should call
+ * {@link #closeInternalComponent(AutoCloseable)} for each such component. Leaving the implementation empty is
+ * valid given there are not such resources to release.
+ */
+ public abstract void releaseInMemoryComponents();
- public int getMinIndexInterval()
+ /**
+ * Perform any validation needed for the reader upon creation before returning it from the {@link Builder}.
+ */
+ public void validate()
{
- return indexSummary.getMinIndexInterval();
- }
-
- public double getEffectiveIndexInterval()
- {
- return indexSummary.getEffectiveIndexInterval();
- }
-
- public void releaseSummary()
- {
- tidy.releaseSummary();
- }
-
- private void validate()
- {
- if (this.first.compareTo(this.last) > 0)
+ if (this.first.compareTo(this.last) > 0 || bounds == null)
{
throw new CorruptSSTableException(new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)), getFilename());
}
}
/**
- * Gets the position in the index file to start scanning to find the given key (at most indexInterval keys away,
- * modulo downsampling of the index summary). Always returns a {@code value >= 0}
- */
- public long getIndexScanPosition(PartitionPosition key)
- {
- if (openReason == OpenReason.MOVED_START && key.compareTo(first) < 0)
- key = first;
-
- return getIndexScanPositionFromBinarySearchResult(indexSummary.binarySearch(key), indexSummary);
- }
-
- @VisibleForTesting
- public static long getIndexScanPositionFromBinarySearchResult(int binarySearchResult, IndexSummary referencedIndexSummary)
- {
- if (binarySearchResult == -1)
- return 0;
- else
- return referencedIndexSummary.getPosition(getIndexSummaryIndexFromBinarySearchResult(binarySearchResult));
- }
-
- public static int getIndexSummaryIndexFromBinarySearchResult(int binarySearchResult)
- {
- if (binarySearchResult < 0)
- {
- // binary search gives us the first index _greater_ than the key searched for,
- // i.e., its insertion position
- int greaterThan = (binarySearchResult + 1) * -1;
- if (greaterThan == 0)
- return -1;
- return greaterThan - 1;
- }
- else
- {
- return binarySearchResult;
- }
- }
-
- /**
- * Returns the compression metadata for this sstable.
+ * Returns the compression metadata for this sstable. Note that the compression metdata is a resource and should not
+ * be closed by the caller.
+ * TODO do not return a closeable resource or return a shared copy
+ *
* @throws IllegalStateException if the sstable is not compressed
*/
public CompressionMetadata getCompressionMetadata()
@@ -1122,6 +684,7 @@
/**
* Returns the amount of memory in bytes used off heap by the compression meta-data.
+ *
* @return the amount of memory in bytes used off heap by the compression meta-data
*/
public long getCompressionMetadataOffHeapSize()
@@ -1132,166 +695,30 @@
return getCompressionMetadata().offHeapSize();
}
- public IFilter getBloomFilter()
- {
- return bf;
- }
-
- public long getBloomFilterSerializedSize()
- {
- return bf.serializedSize();
- }
+ /**
+ * Calculates an estimate of the number of keys in the sstable represented by this reader.
+ */
+ public abstract long estimatedKeys();
/**
- * Returns the amount of memory in bytes used off heap by the bloom filter.
- * @return the amount of memory in bytes used off heap by the bloom filter
+ * Calculates an estimate of the number of keys for the given ranges in the sstable represented by this reader.
*/
- public long getBloomFilterOffHeapSize()
- {
- return bf.offHeapSize();
- }
+ public abstract long estimatedKeysForRanges(Collection<Range<Token>> ranges);
/**
- * @return An estimate of the number of keys in this SSTable based on the index summary.
+ * Returns whether methods like {@link #estimatedKeys()} or {@link #estimatedKeysForRanges(Collection)} can return
+ * sensible estimations.
*/
- public long estimatedKeys()
- {
- return indexSummary.getEstimatedKeyCount();
- }
+ public abstract boolean isEstimationInformative();
/**
- * @param ranges
- * @return An estimate of the number of keys for given ranges in this SSTable.
+ * Returns sample keys for the provided token range.
*/
- public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
- {
- long sampleKeyCount = 0;
- List<IndexesBounds> sampleIndexes = getSampleIndexesForRanges(indexSummary, ranges);
- for (IndexesBounds sampleIndexRange : sampleIndexes)
- sampleKeyCount += (sampleIndexRange.upperPosition - sampleIndexRange.lowerPosition + 1);
-
- // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling
- long estimatedKeys = sampleKeyCount * ((long) Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel();
- return Math.max(1, estimatedKeys);
- }
-
- /**
- * Returns the number of entries in the IndexSummary. At full sampling, this is approximately 1/INDEX_INTERVALth of
- * the keys in this SSTable.
- */
- public int getIndexSummarySize()
- {
- return indexSummary.size();
- }
-
- /**
- * Returns the approximate number of entries the IndexSummary would contain if it were at full sampling.
- */
- public int getMaxIndexSummarySize()
- {
- return indexSummary.getMaxNumberOfEntries();
- }
-
- /**
- * Returns the key for the index summary entry at `index`.
- */
- public byte[] getIndexSummaryKey(int index)
- {
- return indexSummary.getKey(index);
- }
-
- private static List<IndexesBounds> getSampleIndexesForRanges(IndexSummary summary, Collection<Range<Token>> ranges)
- {
- // use the index to determine a minimal section for each range
- List<IndexesBounds> positions = new ArrayList<>();
-
- for (Range<Token> range : Range.normalize(ranges))
- {
- PartitionPosition leftPosition = range.left.maxKeyBound();
- PartitionPosition rightPosition = range.right.maxKeyBound();
-
- int left = summary.binarySearch(leftPosition);
- if (left < 0)
- left = (left + 1) * -1;
- else
- // left range are start exclusive
- left = left + 1;
- if (left == summary.size())
- // left is past the end of the sampling
- continue;
-
- int right = Range.isWrapAround(range.left, range.right)
- ? summary.size() - 1
- : summary.binarySearch(rightPosition);
- if (right < 0)
- {
- // range are end inclusive so we use the previous index from what binarySearch give us
- // since that will be the last index we will return
- right = (right + 1) * -1;
- if (right == 0)
- // Means the first key is already stricly greater that the right bound
- continue;
- right--;
- }
-
- if (left > right)
- // empty range
- continue;
- positions.add(new IndexesBounds(left, right));
- }
- return positions;
- }
-
- public Iterable<DecoratedKey> getKeySamples(final Range<Token> range)
- {
- final List<IndexesBounds> indexRanges = getSampleIndexesForRanges(indexSummary, Collections.singletonList(range));
-
- if (indexRanges.isEmpty())
- return Collections.emptyList();
-
- return new Iterable<DecoratedKey>()
- {
- public Iterator<DecoratedKey> iterator()
- {
- return new Iterator<DecoratedKey>()
- {
- private Iterator<IndexesBounds> rangeIter = indexRanges.iterator();
- private IndexesBounds current;
- private int idx;
-
- public boolean hasNext()
- {
- if (current == null || idx > current.upperPosition)
- {
- if (rangeIter.hasNext())
- {
- current = rangeIter.next();
- idx = current.lowerPosition;
- return true;
- }
- return false;
- }
-
- return true;
- }
-
- public DecoratedKey next()
- {
- byte[] bytes = indexSummary.getKey(idx++);
- return decorateKey(ByteBuffer.wrap(bytes));
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
- };
- }
- };
- }
+ public abstract Iterable<DecoratedKey> getKeySamples(final Range<Token> range);
/**
* Determine the minimal set of sections that can be extracted from this SSTable to cover the given ranges.
+ *
* @return A sorted list of (offset,end) pairs that cover the given ranges in the datafile for this SSTable.
*/
public List<PartitionPositionBounds> getPositionsForRanges(Collection<Range<Token>> ranges)
@@ -1309,10 +736,10 @@
if (leftBound.compareTo(last) > 0 || rightBound.compareTo(first) < 0)
continue;
- long left = getPosition(leftBound, Operator.GT).position;
+ long left = getPosition(leftBound, Operator.GT);
long right = (rightBound.compareTo(last) > 0)
? uncompressedLength()
- : getPosition(rightBound, Operator.GT).position;
+ : getPosition(rightBound, Operator.GT);
if (left == right)
// empty range
@@ -1324,146 +751,91 @@
return positions;
}
- public KeyCacheKey getCacheKey(DecoratedKey key)
- {
- return new KeyCacheKey(metadata(), descriptor, key.getKey());
- }
-
- public void cacheKey(DecoratedKey key, RowIndexEntry info)
- {
- CachingParams caching = metadata().params.caching;
-
- if (!caching.cacheKeys() || keyCache == null || keyCache.getCapacity() == 0)
- return;
-
- KeyCacheKey cacheKey = new KeyCacheKey(metadata(), descriptor, key.getKey());
- logger.trace("Adding cache entry for {} -> {}", cacheKey, info);
- keyCache.put(cacheKey, info);
- }
-
- public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
- {
- if (isKeyCacheEnabled())
- return getCachedPosition(new KeyCacheKey(metadata(), descriptor, key.getKey()), updateStats);
- return null;
- }
-
- protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
- {
- if (isKeyCacheEnabled())
- {
- if (updateStats)
- {
- RowIndexEntry cachedEntry = keyCache.get(unifiedKey);
- keyCacheRequest.incrementAndGet();
- if (cachedEntry != null)
- {
- keyCacheHit.incrementAndGet();
- bloomFilterTracker.addTruePositive();
- }
- return cachedEntry;
- }
- else
- {
- return keyCache.getInternal(unifiedKey);
- }
- }
- return null;
- }
-
- public boolean isKeyCacheEnabled()
- {
- return keyCache != null && metadata().params.caching.cacheKeys();
- }
-
/**
* Retrieves the position while updating the key cache and the stats.
+ *
* @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
- * allow key selection by token bounds but only if op != * EQ
- * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
*/
- public final RowIndexEntry getPosition(PartitionPosition key, Operator op)
+ public final long getPosition(PartitionPosition key, Operator op)
{
return getPosition(key, op, SSTableReadsListener.NOOP_LISTENER);
}
- /**
- * Retrieves the position while updating the key cache and the stats.
- * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
- * allow key selection by token bounds but only if op != * EQ
- * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
- * @param listener the {@code SSTableReaderListener} that must handle the notifications.
- */
- public final RowIndexEntry getPosition(PartitionPosition key, Operator op, SSTableReadsListener listener)
+ public final long getPosition(PartitionPosition key, Operator op, SSTableReadsListener listener)
{
- return getPosition(key, op, true, false, listener);
+ return getPosition(key, op, true, listener);
}
- public final RowIndexEntry getPosition(PartitionPosition key,
- Operator op,
- boolean updateCacheAndStats)
+ public final long getPosition(PartitionPosition key,
+ Operator op,
+ boolean updateStats)
{
- return getPosition(key, op, updateCacheAndStats, false, SSTableReadsListener.NOOP_LISTENER);
+ return getPosition(key, op, updateStats, SSTableReadsListener.NOOP_LISTENER);
}
/**
- * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
- * allow key selection by token bounds but only if op != * EQ
- * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
- * @param updateCacheAndStats true if updating stats and cache
- * @param listener a listener used to handle internal events
+ * Retrieve a position in data file according to the provided key and operator.
+ *
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateStats true if updating stats and cache
+ * @param listener a listener used to handle internal events
* @return The index entry corresponding to the key, or null if the key is not present
*/
- protected abstract RowIndexEntry getPosition(PartitionPosition key,
- Operator op,
- boolean updateCacheAndStats,
- boolean permitMatchPastLast,
- SSTableReadsListener listener);
+ protected long getPosition(PartitionPosition key,
+ Operator op,
+ boolean updateStats,
+ SSTableReadsListener listener)
+ {
+ AbstractRowIndexEntry rie = getRowIndexEntry(key, op, updateStats, listener);
+ return rie != null ? rie.position : -1;
+ }
- public abstract UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed);
+ /**
+ * Retrieve an index entry for the partition found according to the provided key and operator.
+ *
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateStats true if updating stats and cache
+ * @param listener a listener used to handle internal events
+ * @return The index entry corresponding to the key, or null if the key is not present
+ */
+ @VisibleForTesting
+ protected abstract AbstractRowIndexEntry getRowIndexEntry(PartitionPosition key,
+ Operator op,
+ boolean updateStats,
+ SSTableReadsListener listener);
- public abstract UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly);
+ public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, long dataPosition, boolean tombstoneOnly)
+ {
+ return SSTableIdentityIterator.create(this, file, dataPosition, key, tombstoneOnly);
+ }
+
+ /**
+ * Returns a {@link KeyReader} over all keys in the sstable.
+ */
+ public abstract KeyReader keyReader() throws IOException;
+
+ /**
+ * Returns a {@link KeyIterator} over all keys in the sstable.
+ */
+ public KeyIterator keyIterator() throws IOException
+ {
+ return new KeyIterator(keyReader(), getPartitioner(), uncompressedLength(), new ReentrantReadWriteLock());
+ }
/**
* Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
*/
- public DecoratedKey firstKeyBeyond(PartitionPosition token)
- {
- if (token.compareTo(first) < 0)
- return first;
-
- long sampledPosition = getIndexScanPosition(token);
-
- if (ifile == null)
- return null;
-
- String path = null;
- try (FileDataInput in = ifile.createReader(sampledPosition))
- {
- path = in.getPath();
- while (!in.isEOF())
- {
- ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
- DecoratedKey indexDecoratedKey = decorateKey(indexKey);
- if (indexDecoratedKey.compareTo(token) > 0)
- return indexDecoratedKey;
-
- RowIndexEntry.Serializer.skip(in, descriptor.version);
- }
- }
- catch (IOException e)
- {
- markSuspect();
- throw new CorruptSSTableException(e, path);
- }
-
- return null;
- }
+ public abstract DecoratedKey firstKeyBeyond(PartitionPosition token);
/**
- * @return The length in bytes of the data for this SSTable. For
- * compressed files, this is not the same thing as the on disk size (see
- * onDiskLength())
+ * Returns the length in bytes of the (uncompressed) data for this SSTable. For compressed files, this is not
+ * the same thing as the on disk size (see {@link #onDiskLength()}).
*/
public long uncompressedLength()
{
@@ -1471,9 +843,8 @@
}
/**
- * @return The length in bytes of the on disk size for this SSTable. For
- * compressed files, this is not the same thing as the data length (see
- * length())
+ * The length in bytes of the on disk size for this SSTable. For compressed files, this is not the same thing
+ * as the data length (see {@link #uncompressedLength()}).
*/
public long onDiskLength()
{
@@ -1487,10 +858,8 @@
}
/**
- * Set the value of CRC check chance. The argument supplied is obtained
- * from the the property of the owning CFS. Called when either the SSTR
- * is initialized, or the CFS's property is updated via JMX
- * @param crcCheckChance
+ * Set the value of CRC check chance. The argument supplied is obtained from the property of the owning CFS.
+ * Called when either the SSTR is initialized, or the CFS's property is updated via JMX
*/
public void setCrcCheckChance(double crcCheckChance)
{
@@ -1500,11 +869,11 @@
/**
* Mark the sstable as obsolete, i.e., compacted into newer sstables.
- *
- * When calling this function, the caller must ensure that the SSTableReader is not referenced anywhere
- * except for threads holding a reference.
- *
- * multiple times is usually buggy (see exceptions in Tracker.unmarkCompacting and removeOldSSTablesSize).
+ * <p>
+ * When calling this function, the caller must ensure that the SSTableReader is not referenced anywhere except for
+ * threads holding a reference.
+ * <p>
+ * Calling it multiple times is usually buggy.
*/
public void markObsolete(Runnable tidier)
{
@@ -1514,7 +883,7 @@
synchronized (tidy.global)
{
assert !tidy.isReplaced;
- assert tidy.global.obsoletion == null: this + " was already marked compacted";
+ assert tidy.global.obsoletion == null : this + " was already marked compacted";
tidy.global.obsoletion = tidier;
tidy.global.stopReadMeterPersistence();
@@ -1581,20 +950,27 @@
*/
public abstract ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator);
+ /**
+ * Create a {@link FileDataInput} for the data file of the sstable represented by this reader. This method returns
+ * a newly opened resource which must be closed by the caller.
+ *
+ * @param position the data input will be opened and seek to this position
+ */
public FileDataInput getFileDataInput(long position)
{
return dfile.createReader(position);
}
/**
- * Tests if the sstable contains data newer than the given age param (in localhost currentMilli time).
- * This works in conjunction with maxDataAge which is an upper bound on the create of data in this sstable.
- * @param age The age to compare the maxDataAre of this sstable. Measured in millisec since epoc on this host
- * @return True iff this sstable contains data that's newer than the given age parameter.
+ * Tests if the sstable contains data newer than the given age param (in localhost currentMillis time).
+ * This works in conjunction with maxDataAge which is an upper bound on the data in the sstable represented
+ * by this reader.
+ *
+ * @return {@code true} iff this sstable contains data that's newer than the given timestamp
*/
- public boolean newSince(long age)
+ public boolean newSince(long timestampMillis)
{
- return maxDataAge > age;
+ return maxDataAge > timestampMillis;
}
public void createLinks(String snapshotDirectoryPath)
@@ -1616,7 +992,7 @@
{
for (Component component : components)
{
- File sourceFile = new File(descriptor.filenameFor(component));
+ File sourceFile = descriptor.fileFor(component);
if (!sourceFile.exists())
continue;
if (null != limiter)
@@ -1631,25 +1007,17 @@
return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE;
}
- public DecoratedKey keyAt(long indexPosition) throws IOException
- {
- DecoratedKey key;
- try (FileDataInput in = ifile.createReader(indexPosition))
- {
- if (in.isEOF())
- return null;
-
- key = decorateKey(ByteBufferUtil.readWithShortLength(in));
-
- // hint read path about key location if caching is enabled
- // this saves index summary lookup and index file iteration which whould be pretty costly
- // especially in presence of promoted column indexes
- if (isKeyCacheEnabled())
- cacheKey(key, rowIndexEntrySerializer.deserialize(in));
- }
-
- return key;
- }
+ /**
+ * Reads the key stored at the position saved in SASI.
+ * <p>
+ * When SASI is created, it uses key locations retrieved from {@link KeyReader#keyPositionForSecondaryIndex()}.
+ * This method is to read the key stored at such position. It is up to the concrete SSTable format implementation
+ * what that position means and which file it refers. The only requirement is that it is consistent with what
+ * {@link KeyReader#keyPositionForSecondaryIndex()} returns.
+ *
+ * @return key if found, {@code null} otherwise
+ */
+ public abstract DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException;
public boolean isPendingRepair()
{
@@ -1694,55 +1062,29 @@
final static class Equals extends Operator
{
- public int apply(int comparison) { return -comparison; }
+ public int apply(int comparison)
+ {
+ return -comparison;
+ }
}
final static class GreaterThanOrEqualTo extends Operator
{
- public int apply(int comparison) { return comparison >= 0 ? 0 : 1; }
+ public int apply(int comparison)
+ {
+ return comparison >= 0 ? 0 : 1;
+ }
}
final static class GreaterThan extends Operator
{
- public int apply(int comparison) { return comparison > 0 ? 0 : 1; }
+ public int apply(int comparison)
+ {
+ return comparison > 0 ? 0 : 1;
+ }
}
}
- public long getBloomFilterFalsePositiveCount()
- {
- return bloomFilterTracker.getFalsePositiveCount();
- }
-
- public long getRecentBloomFilterFalsePositiveCount()
- {
- return bloomFilterTracker.getRecentFalsePositiveCount();
- }
-
- public long getBloomFilterTruePositiveCount()
- {
- return bloomFilterTracker.getTruePositiveCount();
- }
-
- public long getRecentBloomFilterTruePositiveCount()
- {
- return bloomFilterTracker.getRecentTruePositiveCount();
- }
-
- public long getBloomFilterTrueNegativeCount()
- {
- return bloomFilterTracker.getTrueNegativeCount();
- }
-
- public long getRecentBloomFilterTrueNegativeCount()
- {
- return bloomFilterTracker.getRecentTrueNegativeCount();
- }
-
- public InstrumentingCache<KeyCacheKey, RowIndexEntry> getKeyCache()
- {
- return keyCache;
- }
-
public EstimatedHistogram getEstimatedPartitionSize()
{
return sstableMetadata.estimatedPartitionSize;
@@ -1825,8 +1167,8 @@
public int getAvgColumnSetPerRow()
{
return sstableMetadata.totalRows < 0
- ? -1
- : (sstableMetadata.totalRows == 0 ? 0 : (int)(sstableMetadata.totalColumnsSet / sstableMetadata.totalRows));
+ ? -1
+ : (sstableMetadata.totalRows == 0 ? 0 : (int) (sstableMetadata.totalColumnsSet / sstableMetadata.totalRows));
}
public int getSSTableLevel()
@@ -1860,16 +1202,16 @@
/**
* Reloads the sstable metadata from disk.
- *
+ * <p>
* Called after level is changed on sstable, for example if the sstable is dropped to L0
- *
+ * <p>
* Might be possible to remove in future versions
*
* @throws IOException
*/
public void reloadSSTableMetadata() throws IOException
{
- this.sstableMetadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS);
+ this.sstableMetadata = StatsComponent.load(descriptor).statsMetadata();
}
public StatsMetadata getSSTableMetadata()
@@ -1888,11 +1230,10 @@
return dfile.createReader();
}
- public RandomAccessReader openIndexReader()
+ public void trySkipFileCacheBefore(DecoratedKey key)
{
- if (ifile != null)
- return ifile.createReader();
- return null;
+ long position = getPosition(key, SSTableReader.Operator.GE);
+ NativeLibrary.trySkipCache(descriptor.fileFor(Components.DATA).absolutePath(), 0, position < 0 ? 0 : position);
}
public ChannelProxy getDataChannel()
@@ -1900,39 +1241,12 @@
return dfile.channel;
}
- public ChannelProxy getIndexChannel()
- {
- return ifile.channel;
- }
-
- public FileHandle getIndexFile()
- {
- return ifile;
- }
-
/**
- * @param component component to get timestamp.
- * @return last modified time for given component. 0 if given component does not exist or IO error occurs.
+ * @return last modified time for data component. 0 if given component does not exist or IO error occurs.
*/
- public long getCreationTimeFor(Component component)
+ public long getDataCreationTime()
{
- return new File(descriptor.filenameFor(component)).lastModified();
- }
-
- /**
- * @return Number of key cache hit
- */
- public long getKeyCacheHit()
- {
- return keyCacheHit.get();
- }
-
- /**
- * @return Number of key cache request
- */
- public long getKeyCacheRequest()
- {
- return keyCacheRequest.get();
+ return descriptor.fileFor(Components.DATA).lastModified();
}
/**
@@ -1967,9 +1281,16 @@
return selfRef.ref();
}
- void setup(boolean trackHotness)
+ protected List<AutoCloseable> setupInstance(boolean trackHotness)
{
- tidy.setup(this, TRACK_ACTIVITY && trackHotness);
+ return Collections.singletonList(dfile);
+ }
+
+ public void setup(boolean trackHotness)
+ {
+ assert tidy.closeables == null;
+ trackHotness &= TRACK_ACTIVITY;
+ tidy.setup(this, trackHotness, setupInstance(trackHotness));
this.readMeter = tidy.global.readMeter;
}
@@ -1983,39 +1304,35 @@
{
identities.add(this);
identities.add(tidy.globalRef);
- dfile.addTo(identities);
- ifile.addTo(identities);
- bf.addTo(identities);
- indexSummary.addTo(identities);
-
- }
-
- public boolean maybePresent(DecoratedKey key)
- {
- // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing),
- // we check index file instead.
- return bf instanceof AlwaysPresentFilter && getPosition(key, Operator.EQ, false) != null || bf.isPresent(key);
+ tidy.closeables.forEach(c -> {
+ if (c instanceof SharedCloseable)
+ ((SharedCloseable) c).addTo(identities);
+ });
}
/**
+ * The method verifies whether the sstable may contain the provided key. The method does approximation using
+ * Bloom filter if it is present and if it is not, performs accurate check in the index.
+ */
+ public abstract boolean mayContainAssumingKeyIsInRange(DecoratedKey key);
+
+ /**
* One instance per SSTableReader we create.
- *
+ * <p>
* We can create many InstanceTidiers (one for every time we reopen an sstable with MOVED_START for example),
* but there can only be one GlobalTidy for one single logical sstable.
- *
+ * <p>
* When the InstanceTidier cleansup, it releases its reference to its GlobalTidy; when all InstanceTidiers
* for that type have run, the GlobalTidy cleans up.
*/
- private static final class InstanceTidier implements Tidy
+ protected static final class InstanceTidier implements Tidy
{
private final Descriptor descriptor;
- private final TableId tableId;
- private IFilter bf;
- private IndexSummary summary;
+ private final WeakReference<Owner> owner;
- private FileHandle dfile;
- private FileHandle ifile;
+ private List<? extends AutoCloseable> closeables;
private Runnable runOnClose;
+
private boolean isReplaced = false;
// a reference to our shared tidy instance, that
@@ -2025,26 +1342,25 @@
private volatile boolean setup;
- void setup(SSTableReader reader, boolean trackHotness)
+ public void setup(SSTableReader reader, boolean trackHotness, Collection<? extends AutoCloseable> closeables)
{
- this.setup = true;
- this.bf = reader.bf;
- this.summary = reader.indexSummary;
- this.dfile = reader.dfile;
- this.ifile = reader.ifile;
// get a new reference to the shared descriptor-type tidy
this.globalRef = GlobalTidy.get(reader);
this.global = globalRef.get();
if (trackHotness)
global.ensureReadMeter();
+ this.closeables = new ArrayList<>(closeables);
+ // to avoid tidy seeing partial state, set setup=true at the end
+ this.setup = true;
}
- InstanceTidier(Descriptor descriptor, TableId tableId)
+ private InstanceTidier(Descriptor descriptor, Owner owner)
{
this.descriptor = descriptor;
- this.tableId = tableId;
+ this.owner = new WeakReference<>(owner);
}
+ @Override
public void tidy()
{
if (logger.isTraceEnabled())
@@ -2054,15 +1370,17 @@
if (!setup)
return;
- final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
final OpOrder.Barrier barrier;
- if (cfs != null)
+ Owner owner = this.owner.get();
+ if (owner != null)
{
- barrier = cfs.readOrdering.newBarrier();
+ barrier = owner.newReadOrderingBarrier();
barrier.issue();
}
else
+ {
barrier = null;
+ }
ScheduledExecutors.nonPeriodicTasks.execute(new Runnable()
{
@@ -2077,17 +1395,36 @@
if (logger.isTraceEnabled())
logger.trace("Async instance tidier for {}, after barrier", descriptor);
- if (bf != null)
- bf.close();
- if (summary != null)
- summary.close();
- if (runOnClose != null)
+ Throwable exceptions = null;
+ if (runOnClose != null) try
+ {
runOnClose.run();
- if (dfile != null)
- dfile.close();
- if (ifile != null)
- ifile.close();
- globalRef.release();
+ }
+ catch (RuntimeException | Error ex)
+ {
+ logger.error("Failed to run on-close listeners for sstable " + descriptor.baseFile(), ex);
+ exceptions = ex;
+ }
+
+ Throwable closeExceptions = Throwables.close(null, Iterables.filter(closeables, Objects::nonNull));
+ if (closeExceptions != null)
+ {
+ logger.error("Failed to close some sstable components of " + descriptor.baseFile(), closeExceptions);
+ exceptions = Throwables.merge(exceptions, closeExceptions);
+ }
+
+ try
+ {
+ globalRef.release();
+ }
+ catch (RuntimeException | Error ex)
+ {
+ logger.error("Failed to release the global ref of " + descriptor.baseFile(), ex);
+ exceptions = Throwables.merge(exceptions, ex);
+ }
+
+ if (exceptions != null)
+ JVMStabilityInspector.inspectThrowable(exceptions);
if (logger.isTraceEnabled())
logger.trace("Async instance tidier for {}, completed", descriptor);
@@ -2101,23 +1438,17 @@
});
}
+ @Override
public String name()
{
return descriptor.toString();
}
-
- void releaseSummary()
- {
- summary.close();
- assert summary.isCleanedUp();
- summary = null;
- }
}
/**
* One instance per logical sstable. This both tracks shared cleanup and some shared state related
* to the sstable's lifecycle.
- *
+ * <p>
* All InstanceTidiers, on setup(), ask the static get() method for their shared state,
* and stash a reference to it to be released when they are. Once all such references are
* released, this shared tidy will be performed.
@@ -2160,17 +1491,16 @@
readMeter = SystemKeyspace.getSSTableReadMeter(desc.ksname, desc.cfname, desc.id);
// sync the average read rate to system.sstable_activity every five minutes, starting one minute from now
- readMeterSyncFuture = new WeakReference<>(syncExecutor.scheduleAtFixedRate(new Runnable()
+ readMeterSyncFuture = new WeakReference<>(syncExecutor.scheduleAtFixedRate(this::maybePersistSSTableReadMeter, 1, 5, TimeUnit.MINUTES));
+ }
+
+ void maybePersistSSTableReadMeter()
+ {
+ if (obsoletion == null && DatabaseDescriptor.getSStableReadRatePersistenceEnabled())
{
- public void run()
- {
- if (obsoletion == null)
- {
- meterSyncThrottle.acquire();
- SystemKeyspace.persistSSTableReadMeter(desc.ksname, desc.cfname, desc.id, readMeter);
- }
- }
- }, 1, 5, TimeUnit.MINUTES));
+ meterSyncThrottle.acquire();
+ SystemKeyspace.persistSSTableReadMeter(desc.ksname, desc.cfname, desc.id, readMeter);
+ }
}
private void stopReadMeterPersistence()
@@ -2191,8 +1521,8 @@
obsoletion.run();
// don't ideally want to dropPageCache for the file until all instances have been released
- NativeLibrary.trySkipCache(desc.filenameFor(Component.DATA), 0, 0);
- NativeLibrary.trySkipCache(desc.filenameFor(Component.PRIMARY_INDEX), 0, 0);
+ for (Component c : desc.discoverComponents())
+ NativeLibrary.trySkipCache(desc.fileFor(c).absolutePath(), 0, 0);
}
public String name()
@@ -2235,11 +1565,6 @@
GlobalTidy.lookup.clear();
}
- public static abstract class Factory
- {
- public abstract SSTableReader open(SSTableReaderBuilder builder);
- }
-
public static class PartitionPositionBounds
{
public final long lowerPosition;
@@ -2255,15 +1580,15 @@
public final int hashCode()
{
int hashCode = (int) lowerPosition ^ (int) (lowerPosition >>> 32);
- return 31 * (hashCode ^ (int) ((int) upperPosition ^ (upperPosition >>> 32)));
+ return 31 * (hashCode ^ (int) ((int) upperPosition ^ (upperPosition >>> 32)));
}
@Override
public final boolean equals(Object o)
{
- if(!(o instanceof PartitionPositionBounds))
+ if (!(o instanceof PartitionPositionBounds))
return false;
- PartitionPositionBounds that = (PartitionPositionBounds)o;
+ PartitionPositionBounds that = (PartitionPositionBounds) o;
return lowerPosition == that.lowerPosition && upperPosition == that.upperPosition;
}
}
@@ -2297,7 +1622,7 @@
/**
* Moves the sstable in oldDescriptor to a new place (with generation etc) in newDescriptor.
- *
+ * <p>
* All components given will be moved/renamed
*/
public static SSTableReader moveAndOpenSSTable(ColumnFamilyStore cfs, Descriptor oldDescriptor, Descriptor newDescriptor, Set<Component> components, boolean copyData)
@@ -2315,9 +1640,9 @@
logger.error(message);
throw new RuntimeException(message);
}
- if (new File(newDescriptor.filenameFor(Component.DATA)).exists())
+ if (newDescriptor.fileFor(Components.DATA).exists())
{
- String msg = String.format("File %s already exists, can't move the file there", newDescriptor.filenameFor(Component.DATA));
+ String msg = String.format("File %s already exists, can't move the file there", newDescriptor.fileFor(Components.DATA));
logger.error(msg);
throw new RuntimeException(msg);
}
@@ -2327,24 +1652,24 @@
try
{
logger.info("Hardlinking new SSTable {} to {}", oldDescriptor, newDescriptor);
- SSTableWriter.hardlink(oldDescriptor, newDescriptor, components);
+ hardlink(oldDescriptor, newDescriptor, components);
}
catch (FSWriteError ex)
{
logger.warn("Unable to hardlink new SSTable {} to {}, falling back to copying", oldDescriptor, newDescriptor, ex);
- SSTableWriter.copy(oldDescriptor, newDescriptor, components);
+ copy(oldDescriptor, newDescriptor, components);
}
}
else
{
logger.info("Moving new SSTable {} to {}", oldDescriptor, newDescriptor);
- SSTableWriter.rename(oldDescriptor, newDescriptor, components);
+ rename(oldDescriptor, newDescriptor, components);
}
SSTableReader reader;
try
{
- reader = SSTableReader.open(newDescriptor, components, cfs.metadata);
+ reader = open(cfs, newDescriptor, components, cfs.metadata);
}
catch (Throwable t)
{
@@ -2360,4 +1685,214 @@
ExecutorUtils.shutdownNowAndWait(timeout, unit, syncExecutor);
resetTidying();
}
+
+ /**
+ * @return the physical size on disk of all components for this SSTable in bytes
+ */
+ public long bytesOnDisk()
+ {
+ return bytesOnDisk(false);
+ }
+
+ /**
+ * @return the total logical/uncompressed size in bytes of all components for this SSTable
+ */
+ public long logicalBytesOnDisk()
+ {
+ return bytesOnDisk(true);
+ }
+
+ private long bytesOnDisk(boolean logical)
+ {
+ long bytes = 0;
+ for (Component component : components)
+ {
+ // Only the data file is compressable.
+ bytes += logical && component == Components.DATA && compression
+ ? getCompressionMetadata().dataLength
+ : descriptor.fileFor(component).length();
+ }
+ return bytes;
+ }
+
+ @VisibleForTesting
+ public void maybePersistSSTableReadMeter()
+ {
+ tidy.global.maybePersistSSTableReadMeter();
+ }
+
+ /**
+ * Returns a new verifier for this sstable. Note that the reader must match the provided cfs.
+ */
+ public abstract IVerifier getVerifier(ColumnFamilyStore cfs,
+ OutputHandler outputHandler,
+ boolean isOffline,
+ IVerifier.Options options);
+
+ /**
+ * A method to be called by {@link #getPosition(PartitionPosition, Operator, boolean, SSTableReadsListener)}
+ * and {@link #getRowIndexEntry(PartitionPosition, Operator, boolean, SSTableReadsListener)} methods when
+ * a searched key is found. It adds a trace message and notify the provided listener.
+ */
+ protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry)
+ {
+ reason.trace(descriptor, entry);
+
+ if (localListener != null)
+ localListener.onSSTableSelected(this, reason);
+ }
+
+ /**
+ * A method to be called by {@link #getPosition(PartitionPosition, Operator, boolean, SSTableReadsListener)}
+ * and {@link #getRowIndexEntry(PartitionPosition, Operator, boolean, SSTableReadsListener)} methods when
+ * a searched key is not found. It adds a trace message and notify the provided listener.
+ */
+ protected void notifySkipped(SSTableReadsListener.SkippingReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats)
+ {
+ reason.trace(descriptor);
+
+ if (localListener != null)
+ localListener.onSSTableSkipped(this, reason);
+ }
+
+ /**
+ * A builder of this sstable reader. It should be extended for each implementation of {@link SSTableReader} with
+ * the implementation specific fields.
+ *
+ * @param <R> type of the reader the builder creates
+ * @param <B> type of this builder
+ */
+ public abstract static class Builder<R extends SSTableReader, B extends Builder<R, B>> extends SSTable.Builder<R, B>
+ {
+ private long maxDataAge;
+ private StatsMetadata statsMetadata;
+ private OpenReason openReason;
+ private SerializationHeader serializationHeader;
+ private FileHandle dataFile;
+ private DecoratedKey first;
+ private DecoratedKey last;
+ private boolean suspected;
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ public B setMaxDataAge(long maxDataAge)
+ {
+ Preconditions.checkArgument(maxDataAge >= 0);
+ this.maxDataAge = maxDataAge;
+ return (B) this;
+ }
+
+ public B setStatsMetadata(StatsMetadata statsMetadata)
+ {
+ Preconditions.checkNotNull(statsMetadata);
+ this.statsMetadata = statsMetadata;
+ return (B) this;
+ }
+
+ public B setOpenReason(OpenReason openReason)
+ {
+ Preconditions.checkNotNull(openReason);
+ this.openReason = openReason;
+ return (B) this;
+ }
+
+ public B setSerializationHeader(SerializationHeader serializationHeader)
+ {
+ this.serializationHeader = serializationHeader;
+ return (B) this;
+ }
+
+ public B setDataFile(FileHandle dataFile)
+ {
+ this.dataFile = dataFile;
+ return (B) this;
+ }
+
+ public B setFirst(DecoratedKey first)
+ {
+ this.first = first != null ? first.retainable() : null;
+ return (B) this;
+ }
+
+ public B setLast(DecoratedKey last)
+ {
+ this.last = last != null ? last.retainable() : null;
+ return (B) this;
+ }
+
+ public B setSuspected(boolean suspected)
+ {
+ this.suspected = suspected;
+ return (B) this;
+ }
+
+ public long getMaxDataAge()
+ {
+ return maxDataAge;
+ }
+
+ public StatsMetadata getStatsMetadata()
+ {
+ return statsMetadata;
+ }
+
+ public OpenReason getOpenReason()
+ {
+ return openReason;
+ }
+
+ public SerializationHeader getSerializationHeader()
+ {
+ return serializationHeader;
+ }
+
+ public FileHandle getDataFile()
+ {
+ return dataFile;
+ }
+
+ public DecoratedKey getFirst()
+ {
+ return first;
+ }
+
+ public DecoratedKey getLast()
+ {
+ return last;
+ }
+
+ public boolean isSuspected()
+ {
+ return suspected;
+ }
+
+ protected abstract R buildInternal(Owner owner);
+
+ public R build(Owner owner, boolean validate, boolean online)
+ {
+ R reader = buildInternal(owner);
+
+ try
+ {
+ if (isSuspected())
+ reader.markSuspect();
+
+ reader.setup(online);
+
+ if (validate)
+ reader.validate();
+ }
+ catch (RuntimeException | Error ex)
+ {
+ JVMStabilityInspector.inspectThrowable(ex);
+ reader.selfRef().release();
+ throw ex;
+ }
+
+ return reader;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java
deleted file mode 100644
index 6ca74f0..0000000
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.io.sstable.format;
-
-import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
-import org.apache.cassandra.io.util.DiskOptimizationStrategy;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.cassandra.io.sstable.format.SSTableReader.OpenReason.NORMAL;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
-public abstract class SSTableReaderBuilder
-{
- private static final Logger logger = LoggerFactory.getLogger(SSTableReaderBuilder.class);
-
- protected final SSTableReader.Factory readerFactory;
- protected final Descriptor descriptor;
- protected final TableMetadataRef metadataRef;
- protected final TableMetadata metadata;
- protected final long maxDataAge;
- protected final Set<Component> components;
- protected final StatsMetadata statsMetadata;
- protected final SSTableReader.OpenReason openReason;
- protected final SerializationHeader header;
-
- protected IndexSummary summary;
- protected DecoratedKey first;
- protected DecoratedKey last;
- protected IFilter bf;
- protected FileHandle ifile;
- protected FileHandle dfile;
-
- public SSTableReaderBuilder(Descriptor descriptor,
- TableMetadataRef metadataRef,
- long maxDataAge,
- Set<Component> components,
- StatsMetadata statsMetadata,
- SSTableReader.OpenReason openReason,
- SerializationHeader header)
- {
- this.descriptor = descriptor;
- this.metadataRef = metadataRef;
- this.metadata = metadataRef.get();
- this.maxDataAge = maxDataAge;
- this.components = components;
- this.statsMetadata = statsMetadata;
- this.openReason = openReason;
- this.header = header;
- this.readerFactory = descriptor.getFormat().getReaderFactory();
- }
-
- public abstract SSTableReader build();
-
- public SSTableReaderBuilder dfile(FileHandle dfile)
- {
- this.dfile = dfile;
- return this;
- }
-
- public SSTableReaderBuilder ifile(FileHandle ifile)
- {
- this.ifile = ifile;
- return this;
- }
-
- public SSTableReaderBuilder bf(IFilter bf)
- {
- this.bf = bf;
- return this;
- }
-
- public SSTableReaderBuilder summary(IndexSummary summary)
- {
- this.summary = summary;
- return this;
- }
-
- /**
- * Load index summary, first key and last key from Summary.db file if it exists.
- *
- * if loaded index summary has different index interval from current value stored in schema,
- * then Summary.db file will be deleted and need to be rebuilt.
- */
- void loadSummary()
- {
- File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
- if (!summariesFile.exists())
- {
- if (logger.isDebugEnabled())
- logger.debug("SSTable Summary File {} does not exist", summariesFile.absolutePath());
- return;
- }
-
- DataInputStream iStream = null;
- try
- {
- iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath()));
- summary = IndexSummary.serializer.deserialize(iStream,
- metadata.partitioner,
- metadata.params.minIndexInterval,
- metadata.params.maxIndexInterval);
- first = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
- last = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
- }
- catch (IOException e)
- {
- if (summary != null)
- summary.close();
- logger.trace("Cannot deserialize SSTable Summary File {}: {}", summariesFile.path(), e.getMessage());
- // corrupted; delete it and fall back to creating a new summary
- FileUtils.closeQuietly(iStream);
- // delete it and fall back to creating a new summary
- FileUtils.deleteWithConfirm(summariesFile);
- }
- finally
- {
- FileUtils.closeQuietly(iStream);
- }
- }
-
- /**
- * Build index summary, first key, last key if {@code summaryLoaded} is false and recreate bloom filter if
- * {@code recreteBloomFilter} is true by reading through Index.db file.
- *
- * @param recreateBloomFilter true if recreate bloom filter
- * @param summaryLoaded true if index summary, first key and last key are already loaded and not need to build again
- */
- void buildSummaryAndBloomFilter(boolean recreateBloomFilter,
- boolean summaryLoaded,
- Set<Component> components,
- StatsMetadata statsMetadata) throws IOException
- {
- if (!components.contains(Component.PRIMARY_INDEX))
- return;
-
- if (logger.isDebugEnabled())
- logger.debug("Attempting to build summary for {}", descriptor);
-
-
- // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
- try (RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))))
- {
- long indexSize = primaryIndex.length();
- long histogramCount = statsMetadata.estimatedPartitionSize.count();
- long estimatedKeys = histogramCount > 0 && !statsMetadata.estimatedPartitionSize.isOverflowed()
- ? histogramCount
- : SSTable.estimateRowsFromIndex(primaryIndex, descriptor); // statistics is supposed to be optional
-
- if (recreateBloomFilter)
- bf = FilterFactory.getFilter(estimatedKeys, metadata.params.bloomFilterFpChance);
-
- try (IndexSummaryBuilder summaryBuilder = summaryLoaded ? null : new IndexSummaryBuilder(estimatedKeys, metadata.params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL))
- {
- long indexPosition;
-
- while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex);
- RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version);
- DecoratedKey decoratedKey = metadata.partitioner.decorateKey(key);
-
- if (!summaryLoaded)
- {
- if (first == null)
- first = decoratedKey;
- last = decoratedKey;
- }
-
- if (recreateBloomFilter)
- bf.add(decoratedKey);
-
- // if summary was already read from disk we don't want to re-populate it using primary index
- if (!summaryLoaded)
- {
- summaryBuilder.maybeAddEntry(decoratedKey, indexPosition);
- }
- }
-
- if (!summaryLoaded)
- summary = summaryBuilder.build(metadata.partitioner);
- }
- }
-
- if (!summaryLoaded)
- {
- first = SSTable.getMinimalKey(first);
- last = SSTable.getMinimalKey(last);
- }
- }
-
- /**
- * Load bloom filter from Filter.db file.
- *
- * @throws IOException
- */
- IFilter loadBloomFilter() throws IOException
- {
- try (FileInputStreamPlus stream = new File(descriptor.filenameFor(Component.FILTER)).newInputStream())
- {
- return BloomFilterSerializer.deserialize(stream, descriptor.version.hasOldBfFormat());
- }
- }
-
- public static class ForWriter extends SSTableReaderBuilder
- {
- public ForWriter(Descriptor descriptor,
- TableMetadataRef metadataRef,
- long maxDataAge,
- Set<Component> components,
- StatsMetadata statsMetadata,
- SSTableReader.OpenReason openReason,
- SerializationHeader header)
- {
- super(descriptor, metadataRef, maxDataAge, components, statsMetadata, openReason, header);
- }
-
- @Override
- public SSTableReader build()
- {
- SSTableReader reader = readerFactory.open(this);
-
- reader.setup(true);
- return reader;
- }
- }
-
- public static class ForBatch extends SSTableReaderBuilder
- {
- public ForBatch(Descriptor descriptor,
- TableMetadataRef metadataRef,
- Set<Component> components,
- StatsMetadata statsMetadata,
- SerializationHeader header)
- {
- super(descriptor, metadataRef, currentTimeMillis(), components, statsMetadata, NORMAL, header);
- }
-
- @Override
- public SSTableReader build()
- {
- String dataFilePath = descriptor.filenameFor(Component.DATA);
- long fileLength = new File(dataFilePath).length();
- logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength));
-
- initSummary(dataFilePath, components, statsMetadata);
-
- boolean compression = components.contains(Component.COMPRESSION_INFO);
- try (FileHandle.Builder ibuilder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX))
- .mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
- .withChunkCache(ChunkCache.instance);
- FileHandle.Builder dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(compression)
- .mmapped(DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap)
- .withChunkCache(ChunkCache.instance))
- {
- long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
- DiskOptimizationStrategy optimizationStrategy = DatabaseDescriptor.getDiskOptimizationStrategy();
- int dataBufferSize = optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
- int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / summary.size());
- ifile = ibuilder.bufferSize(indexBufferSize).complete();
- dfile = dbuilder.bufferSize(dataBufferSize).complete();
- bf = FilterFactory.AlwaysPresent;
-
- SSTableReader sstable = readerFactory.open(this);
-
- sstable.first = first;
- sstable.last = last;
-
- sstable.setup(false);
- return sstable;
- }
- }
-
- void initSummary(String dataFilePath, Set<Component> components, StatsMetadata statsMetadata)
- {
- loadSummary();
- if (summary == null)
- {
- try
- {
- buildSummaryAndBloomFilter(false, false, components, statsMetadata);
- }
- catch (IOException e)
- {
- throw new CorruptSSTableException(e, dataFilePath);
- }
- }
- }
- }
-
- public static class ForRead extends SSTableReaderBuilder
- {
- private final ValidationMetadata validationMetadata;
- private final boolean isOffline;
-
- public ForRead(Descriptor descriptor,
- TableMetadataRef metadataRef,
- ValidationMetadata validationMetadata,
- boolean isOffline,
- Set<Component> components,
- StatsMetadata statsMetadata,
- SerializationHeader header)
- {
- super(descriptor, metadataRef, currentTimeMillis(), components, statsMetadata, NORMAL, header);
- this.validationMetadata = validationMetadata;
- this.isOffline = isOffline;
- }
-
- @Override
- public SSTableReader build()
- {
- String dataFilePath = descriptor.filenameFor(Component.DATA);
- long fileLength = new File(dataFilePath).length();
- logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength));
-
- try
- {
- // load index and filter
- long start = nanoTime();
- load(validationMetadata, isOffline, components, DatabaseDescriptor.getDiskOptimizationStrategy(), statsMetadata);
- logger.trace("INDEX LOAD TIME for {}: {} ms.", descriptor, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
- }
- catch (IOException t)
- {
- throw new CorruptSSTableException(t, dataFilePath);
- }
-
- SSTableReader sstable = readerFactory.open(this);
-
- sstable.first = first;
- sstable.last = last;
-
- sstable.setup(!isOffline); // Don't track hotness if we're offline.
- return sstable;
- }
-
- /**
- * @param validation Metadata for SSTable being loaded
- * @param isOffline Whether the SSTable is being loaded by an offline tool (sstabledump, scrub, etc)
- */
- private void load(ValidationMetadata validation,
- boolean isOffline,
- Set<Component> components,
- DiskOptimizationStrategy optimizationStrategy,
- StatsMetadata statsMetadata) throws IOException
- {
- if (metadata.params.bloomFilterFpChance == 1.0)
- {
- // bf is disabled.
- load(false, !isOffline, optimizationStrategy, statsMetadata, components);
- bf = FilterFactory.AlwaysPresent;
- }
- else if (!components.contains(Component.PRIMARY_INDEX)) // What happens if filter component and primary index is missing?
- {
- // avoid any reading of the missing primary index component.
- // this should only happen during StandaloneScrubber
- load(false, !isOffline, optimizationStrategy, statsMetadata, components);
- }
- else if (!components.contains(Component.FILTER) || validation == null)
- {
- // bf is enabled, but filter component is missing.
- load(!isOffline, !isOffline, optimizationStrategy, statsMetadata, components);
- if (isOffline)
- bf = FilterFactory.AlwaysPresent;
- }
- else
- {
- // bf is enabled and fp chance matches the currently configured value.
- load(false, !isOffline, optimizationStrategy, statsMetadata, components);
- bf = loadBloomFilter();
- }
- }
-
- /**
- * Loads ifile, dfile and indexSummary, and optionally recreates and persists the bloom filter.
- * @param recreateBloomFilter Recreate the bloomfilter.
- * @param saveSummaryIfCreated for bulk loading purposes, if the summary was absent and needed to be built, you can
- * avoid persisting it to disk by setting this to false
- */
- void load(boolean recreateBloomFilter,
- boolean saveSummaryIfCreated,
- DiskOptimizationStrategy optimizationStrategy,
- StatsMetadata statsMetadata,
- Set<Component> components) throws IOException
- {
- try(FileHandle.Builder ibuilder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX))
- .mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
- .withChunkCache(ChunkCache.instance);
- FileHandle.Builder dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(components.contains(Component.COMPRESSION_INFO))
- .mmapped(DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap)
- .withChunkCache(ChunkCache.instance))
- {
- loadSummary();
- boolean buildSummary = summary == null || recreateBloomFilter;
- if (buildSummary)
- buildSummaryAndBloomFilter(recreateBloomFilter, summary != null, components, statsMetadata);
-
- int dataBufferSize = optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
-
- if (components.contains(Component.PRIMARY_INDEX))
- {
- long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
- int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / summary.size());
- ifile = ibuilder.bufferSize(indexBufferSize).complete();
- }
-
- dfile = dbuilder.bufferSize(dataBufferSize).complete();
-
- if (buildSummary)
- {
- if (saveSummaryIfCreated)
- SSTableReader.saveSummary(descriptor, first, last, summary);
- if (recreateBloomFilter)
- SSTableReader.saveBloomFilter(descriptor, bf);
- }
- }
- catch (Throwable t)
- { // Because the tidier has not been set-up yet in SSTableReader.open(), we must release the files in case of error
- if (ifile != null)
- {
- ifile.close();
- }
-
- if (dfile != null)
- {
- dfile.close();
- }
-
- if (summary != null)
- {
- summary.close();
- }
-
- throw t;
- }
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java
new file mode 100644
index 0000000..aedd860
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IOOptions;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
+import static org.apache.cassandra.io.sstable.format.SSTableReader.OpenReason.NORMAL;
+
+public abstract class SSTableReaderLoadingBuilder<R extends SSTableReader, B extends SSTableReader.Builder<R, B>>
+{
+ private final static Logger logger = LoggerFactory.getLogger(SSTableReaderLoadingBuilder.class);
+
+ protected final Descriptor descriptor;
+ protected final Set<Component> components;
+ protected final TableMetadataRef tableMetadataRef;
+ protected final IOOptions ioOptions;
+ protected final ChunkCache chunkCache;
+
+ public SSTableReaderLoadingBuilder(SSTable.Builder<?, ?> builder)
+ {
+ this.descriptor = builder.descriptor;
+ this.components = builder.getComponents() != null ? ImmutableSet.copyOf(builder.getComponents()) : TOCComponent.loadOrCreate(this.descriptor);
+ this.tableMetadataRef = builder.getTableMetadataRef() != null ? builder.getTableMetadataRef() : resolveTableMetadataRef();
+ this.ioOptions = builder.getIOOptions() != null ? builder.getIOOptions() : IOOptions.fromDatabaseDescriptor();
+ this.chunkCache = builder.getChunkCache() != null ? builder.getChunkCache() : ChunkCache.instance;
+
+ checkNotNull(this.components);
+ checkNotNull(this.tableMetadataRef);
+ }
+
+ public R build(SSTable.Owner owner, boolean validate, boolean online)
+ {
+ checkArgument(components.contains(Components.DATA), "Data component is missing for sstable %s", descriptor);
+ if (validate)
+ checkArgument(this.components.containsAll(descriptor.getFormat().primaryComponents()), "Some required components (%s) are missing for sstable %s", Sets.difference(descriptor.getFormat().primaryComponents(), this.components), descriptor);
+
+ B builder = (B) descriptor.getFormat().getReaderFactory().builder(descriptor);
+ builder.setOpenReason(NORMAL);
+ builder.setMaxDataAge(Clock.Global.currentTimeMillis());
+ builder.setTableMetadataRef(tableMetadataRef);
+ builder.setComponents(components);
+
+ R reader = null;
+
+ try
+ {
+ CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(descriptor, builder.getComponents());
+
+ long t0 = Clock.Global.currentTimeMillis();
+
+ openComponents(builder, owner, validate, online);
+
+ if (logger.isTraceEnabled())
+ logger.trace("SSTable {} loaded in {}ms", descriptor, Clock.Global.currentTimeMillis() - t0);
+
+ reader = builder.build(owner, validate, online);
+
+ return reader;
+ }
+ catch (RuntimeException | IOException | Error ex)
+ {
+ if (reader != null)
+ reader.selfRef().release();
+
+ JVMStabilityInspector.inspectThrowable(ex);
+
+ if (ex instanceof CorruptSSTableException)
+ throw (CorruptSSTableException) ex;
+
+ throw new CorruptSSTableException(ex, descriptor.baseFile());
+ }
+ }
+
+ public abstract KeyReader buildKeyReader(TableMetrics tableMetrics) throws IOException;
+
+ protected abstract void openComponents(B builder, SSTable.Owner owner, boolean validate, boolean online) throws IOException;
+
+ /**
+ * Check if sstable is created using same partitioner.
+ * Partitioner can be null, which indicates older version of sstable or no stats available.
+ * In that case, we skip the check.
+ */
+ protected void validatePartitioner(TableMetadata metadata, ValidationMetadata validationMetadata)
+ {
+ String partitionerName = metadata.partitioner.getClass().getCanonicalName();
+ if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
+ {
+ throw new CorruptSSTableException(new IOException(String.format("Cannot open %s; partitioner %s does not match system partitioner %s. " +
+ "Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, " +
+ "so you will need to edit that to match your old partitioner if upgrading.",
+ descriptor, validationMetadata.partitioner, partitionerName)),
+ descriptor.fileFor(Components.STATS));
+ }
+ }
+
+ private TableMetadataRef resolveTableMetadataRef()
+ {
+ TableMetadataRef metadata;
+ if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
+ {
+ int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
+ String indexName = descriptor.cfname.substring(i + 1);
+ metadata = Schema.instance.getIndexTableMetadataRef(descriptor.ksname, indexName);
+ if (metadata == null)
+ throw new AssertionError("Could not find index metadata for index cf " + i);
+ }
+ else
+ {
+ metadata = Schema.instance.getTableMetadataRef(descriptor.ksname, descriptor.cfname);
+ }
+
+ return metadata;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java
new file mode 100644
index 0000000..5aac1d6
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.filter.BloomFilterTracker;
+import org.apache.cassandra.utils.IFilter;
+
+import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull;
+
+public abstract class SSTableReaderWithFilter extends SSTableReader
+{
+ private final IFilter filter;
+ private final BloomFilterTracker filterTracker;
+
+ protected SSTableReaderWithFilter(Builder<?, ?> builder, Owner owner)
+ {
+ super(builder, owner);
+ this.filter = Objects.requireNonNull(builder.getFilter());
+ this.filterTracker = new BloomFilterTracker();
+ }
+
+ @Override
+ protected List<AutoCloseable> setupInstance(boolean trackHotness)
+ {
+ ArrayList<AutoCloseable> closeables = Lists.newArrayList(filter);
+ closeables.addAll(super.setupInstance(trackHotness));
+ return closeables;
+ }
+
+ protected final <B extends Builder<?, B>> B unbuildTo(B builder, boolean sharedCopy)
+ {
+ B b = super.unbuildTo(builder, sharedCopy);
+ if (builder.getFilter() == null)
+ b.setFilter(sharedCopy ? sharedCopyOrNull(filter) : filter);
+ return b;
+ }
+
+ protected boolean isPresentInFilter(IFilter.FilterKey key)
+ {
+ return filter.isPresent(key);
+ }
+
+ @Override
+ public boolean mayContainAssumingKeyIsInRange(DecoratedKey key)
+ {
+ // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing),
+ // we check index file instead.
+ return !filter.isInformative() && getPosition(key, Operator.EQ, false) >= 0 || filter.isPresent(key);
+ }
+
+ @Override
+ protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry)
+ {
+ super.notifySelected(reason, localListener, op, updateStats, entry);
+
+ if (!(updateStats && op == SSTableReader.Operator.EQ))
+ return;
+
+ filterTracker.addTruePositive();
+ }
+
+ @Override
+ protected void notifySkipped(SSTableReadsListener.SkippingReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats)
+ {
+ super.notifySkipped(reason, localListener, op, updateStats);
+
+ if (!updateStats)
+ return;
+
+ switch (reason)
+ {
+ case BLOOM_FILTER:
+ filterTracker.addTrueNegative();
+ break;
+ case MIN_MAX_KEYS:
+ // checking bloom filter against keys outside the sstable range make no sense so collecting
+ // statistics on that makes no sense either
+ break;
+ default:
+ if (op == SSTableReader.Operator.EQ)
+ filterTracker.addFalsePositive();
+ }
+ }
+
+ public BloomFilterTracker getFilterTracker()
+ {
+ return filterTracker;
+ }
+
+ public long getFilterSerializedSize()
+ {
+ return filter.serializedSize(descriptor.version.hasOldBfFormat());
+ }
+
+ public long getFilterOffHeapSize()
+ {
+ return filter.offHeapSize();
+ }
+
+ public abstract SSTableReaderWithFilter cloneAndReplace(IFilter filter);
+
+ public abstract static class Builder<R extends SSTableReaderWithFilter, B extends Builder<R, B>> extends SSTableReader.Builder<R, B>
+ {
+ private IFilter filter;
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ public B setFilter(IFilter filter)
+ {
+ this.filter = filter;
+ return (B) this;
+ }
+
+ public IFilter getFilter()
+ {
+ return this.filter;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java
deleted file mode 100644
index 6d384bf..0000000
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format;
-
-import org.apache.cassandra.db.RowIndexEntry;
-
-/**
- * Listener for receiving notifications associated with reading SSTables.
- */
-public interface SSTableReadsListener
-{
- /**
- * The reasons for skipping an SSTable
- */
- enum SkippingReason
- {
- BLOOM_FILTER,
- MIN_MAX_KEYS,
- PARTITION_INDEX_LOOKUP,
- INDEX_ENTRY_NOT_FOUND;
- }
-
- /**
- * The reasons for selecting an SSTable
- */
- enum SelectionReason
- {
- KEY_CACHE_HIT,
- INDEX_ENTRY_FOUND;
- }
-
- /**
- * Listener that does nothing.
- */
- static final SSTableReadsListener NOOP_LISTENER = new SSTableReadsListener() {};
-
- /**
- * Handles notification that the specified SSTable has been skipped during a single partition query.
- *
- * @param sstable the SSTable reader
- * @param reason the reason for which the SSTable has been skipped
- */
- default void onSSTableSkipped(SSTableReader sstable, SkippingReason reason)
- {
- }
-
- /**
- * Handles notification that the specified SSTable has been selected during a single partition query.
- *
- * @param sstable the SSTable reader
- * @param indexEntry the index entry
- * @param reason the reason for which the SSTable has been selected
- */
- default void onSSTableSelected(SSTableReader sstable, RowIndexEntry<?> indexEntry, SelectionReason reason)
- {
- }
-
- /**
- * Handles notification that the specified SSTable is being scanned during a partition range query.
- *
- * @param sstable the SSTable reader of the SSTable being scanned.
- */
- default void onScanningStarted(SSTableReader sstable)
- {
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
new file mode 100644
index 0000000..b15b2a5
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.AbstractBounds.Boundary;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
+
+import static org.apache.cassandra.dht.AbstractBounds.isEmpty;
+import static org.apache.cassandra.dht.AbstractBounds.maxLeft;
+import static org.apache.cassandra.dht.AbstractBounds.minRight;
+
+public abstract class SSTableScanner<S extends SSTableReader,
+ E extends AbstractRowIndexEntry,
+ I extends SSTableScanner<S, E, I>.BaseKeyScanningIterator>
+implements ISSTableScanner
+{
+ protected final AtomicBoolean isClosed = new AtomicBoolean(false);
+ protected final RandomAccessReader dfile;
+ protected final S sstable;
+
+ protected final Iterator<AbstractBounds<PartitionPosition>> rangeIterator;
+
+ protected final ColumnFilter columns;
+ protected final DataRange dataRange;
+ private final SSTableReadsListener listener;
+
+ protected I iterator;
+
+ protected long startScan = -1;
+ protected long bytesScanned = 0;
+
+ protected SSTableScanner(S sstable,
+ ColumnFilter columns,
+ DataRange dataRange,
+ Iterator<AbstractBounds<PartitionPosition>> rangeIterator,
+ SSTableReadsListener listener)
+ {
+ assert sstable != null;
+
+ this.dfile = sstable.openDataReader();
+ this.sstable = sstable;
+ this.columns = columns;
+ this.dataRange = dataRange;
+ this.rangeIterator = rangeIterator;
+ this.listener = listener;
+ }
+
+ protected static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReader sstable, Collection<Range<Token>> tokenRanges)
+ {
+ List<AbstractBounds<PartitionPosition>> boundsList = new ArrayList<>(tokenRanges.size());
+ for (Range<Token> range : Range.normalize(tokenRanges))
+ addRange(sstable, Range.makeRowRange(range), boundsList);
+ return boundsList;
+ }
+
+ protected static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReader sstable, DataRange dataRange)
+ {
+ List<AbstractBounds<PartitionPosition>> boundsList = new ArrayList<>(2);
+ addRange(sstable, dataRange.keyRange(), boundsList);
+ return boundsList;
+ }
+
+ protected static AbstractBounds<PartitionPosition> fullRange(SSTableReader sstable)
+ {
+ return new Bounds<>(sstable.first, sstable.last);
+ }
+
+ private static void addRange(SSTableReader sstable, AbstractBounds<PartitionPosition> requested, List<AbstractBounds<PartitionPosition>> boundsList)
+ {
+ if (requested instanceof Range && ((Range<?>) requested).isWrapAround())
+ {
+ if (requested.right.compareTo(sstable.first) >= 0)
+ {
+ // since we wrap, we must contain the whole sstable prior to stopKey()
+ Boundary<PartitionPosition> left = new Boundary<>(sstable.first, true);
+ Boundary<PartitionPosition> right;
+ right = requested.rightBoundary();
+ right = minRight(right, sstable.last, true);
+ if (!isEmpty(left, right))
+ boundsList.add(AbstractBounds.bounds(left, right));
+ }
+ if (requested.left.compareTo(sstable.last) <= 0)
+ {
+ // since we wrap, we must contain the whole sstable after dataRange.startKey()
+ Boundary<PartitionPosition> right = new Boundary<>(sstable.last, true);
+ Boundary<PartitionPosition> left;
+ left = requested.leftBoundary();
+ left = maxLeft(left, sstable.first, true);
+ if (!isEmpty(left, right))
+ boundsList.add(AbstractBounds.bounds(left, right));
+ }
+ }
+ else
+ {
+ assert !AbstractBounds.strictlyWrapsAround(requested.left, requested.right);
+ Boundary<PartitionPosition> left, right;
+ left = requested.leftBoundary();
+ right = requested.rightBoundary();
+ left = maxLeft(left, sstable.first, true);
+ // apparently isWrapAround() doesn't count Bounds that extend to the limit (min) as wrapping
+ right = requested.right.isMinimum() ? new Boundary<>(sstable.last, true)
+ : minRight(right, sstable.last, true);
+ if (!isEmpty(left, right))
+ boundsList.add(AbstractBounds.bounds(left, right));
+ }
+ }
+
+ public void close()
+ {
+ try
+ {
+ if (isClosed.compareAndSet(false, true))
+ {
+ markScanned();
+ doClose();
+ }
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+
+ protected abstract void doClose() throws IOException;
+
+ @Override
+ public long getLengthInBytes()
+ {
+ return sstable.uncompressedLength();
+ }
+
+
+ public long getCompressedLengthInBytes()
+ {
+ return sstable.onDiskLength();
+ }
+
+ @Override
+ public long getCurrentPosition()
+ {
+ return dfile.getFilePointer();
+ }
+
+ public long getBytesScanned()
+ {
+ return bytesScanned;
+ }
+
+ @Override
+ public Set<SSTableReader> getBackingSSTables()
+ {
+ return ImmutableSet.of(sstable);
+ }
+
+ public TableMetadata metadata()
+ {
+ return sstable.metadata();
+ }
+
+ public boolean hasNext()
+ {
+ if (iterator == null)
+ iterator = createIterator();
+ return iterator.hasNext();
+ }
+
+ public UnfilteredRowIterator next()
+ {
+ if (iterator == null)
+ iterator = createIterator();
+ return iterator.next();
+ }
+
+ public void remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ private I createIterator()
+ {
+ this.listener.onScanningStarted(sstable);
+ return doCreateIterator();
+ }
+
+ protected abstract I doCreateIterator();
+
+ private void markScanned()
+ {
+ if (startScan != -1)
+ {
+ bytesScanned += dfile.getFilePointer() - startScan;
+ startScan = -1;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s(dfile=%s sstable=%s)", getClass().getSimpleName(), dfile, sstable);
+ }
+
+ public abstract class BaseKeyScanningIterator extends AbstractIterator<UnfilteredRowIterator>
+ {
+ protected DecoratedKey currentKey;
+ protected E currentEntry;
+ private LazilyInitializedUnfilteredRowIterator currentRowIterator;
+
+ protected abstract boolean prepareToIterateRow() throws IOException;
+
+ protected abstract UnfilteredRowIterator getRowIterator(E indexEntry, DecoratedKey key) throws IOException;
+
+ protected UnfilteredRowIterator computeNext()
+ {
+ if (currentRowIterator != null && currentRowIterator.isOpen() && currentRowIterator.hasNext())
+ throw new IllegalStateException("The UnfilteredRowIterator returned by the last call to next() was initialized: " +
+ "it must be closed before calling hasNext() or next() again.");
+
+ try
+ {
+ markScanned();
+
+ if (!prepareToIterateRow())
+ return endOfData();
+
+ /*
+ * For a given partition key, we want to avoid hitting the data file unless we're explicitly asked.
+ * This is important for PartitionRangeReadCommand#checkCacheFilter.
+ */
+ return currentRowIterator = new LazilyInitializedUnfilteredRowIterator(currentKey)
+ {
+ // Store currentEntry reference during object instantiation as later (during initialize) the
+ // reference may point to a different entry.
+ private final E rowIndexEntry = currentEntry;
+
+ protected UnfilteredRowIterator initializeIterator()
+ {
+ try
+ {
+ startScan = rowIndexEntry.position;
+ return getRowIterator(rowIndexEntry, partitionKey());
+ }
+ catch (CorruptSSTableException | IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+ };
+ }
+ catch (CorruptSSTableException | IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index f82a7c2..255e1cd 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -18,48 +18,47 @@
package org.apache.cassandra.io.sstable.format;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Sets;
+import com.google.common.collect.ImmutableSet;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.DeletionPurger;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.db.rows.ComplexColumnData;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.io.util.MmappedRegionsCache;
+import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.Transactional;
+import static com.google.common.base.Preconditions.checkNotNull;
+
/**
- * This is the API all table writers must implement.
- *
- * TableWriter.create() is the primary way to create a writer for a particular format.
- * The format information is part of the Descriptor.
+ * A root class for a writer implementation. A writer must be created by passing an implementation-specific
+ * {@link Builder}, a {@link LifecycleNewTracker} and {@link SSTable.Owner} instances. Implementing classes should
+ * not extend that list and all the additional properties should be included in the builder.
*/
public abstract class SSTableWriter extends SSTable implements Transactional
{
@@ -69,143 +68,57 @@
protected long maxDataAge = -1;
protected final long keyCount;
protected final MetadataCollector metadataCollector;
- protected final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
protected final SerializationHeader header;
- protected final TransactionalProxy txnProxy = txnProxy();
protected final Collection<SSTableFlushObserver> observers;
+ protected final MmappedRegionsCache mmappedRegionsCache;
+ protected final TransactionalProxy txnProxy = txnProxy();
+ protected final LifecycleNewTracker lifecycleNewTracker;
+ protected DecoratedKey first;
+ protected DecoratedKey last;
+ /**
+ * The implementing method should return an instance of {@link TransactionalProxy} initialized with a list of all
+ * transactional resources included in this writer.
+ */
protected abstract TransactionalProxy txnProxy();
- // due to lack of multiple inheritance, we use an inner class to proxy our Transactional implementation details
- protected abstract class TransactionalProxy extends AbstractTransactional
+ protected SSTableWriter(Builder<?, ?> builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner)
{
- // should be set during doPrepare()
- protected SSTableReader finalReader;
- protected boolean openResult;
+ super(builder, owner);
+ checkNotNull(builder.getFlushObservers());
+ checkNotNull(builder.getMetadataCollector());
+ checkNotNull(builder.getSerializationHeader());
+
+ this.keyCount = builder.getKeyCount();
+ this.repairedAt = builder.getRepairedAt();
+ this.pendingRepair = builder.getPendingRepair();
+ this.isTransient = builder.isTransientSSTable();
+ this.metadataCollector = builder.getMetadataCollector();
+ this.header = builder.getSerializationHeader();
+ this.observers = builder.getFlushObservers();
+ this.mmappedRegionsCache = builder.getMmappedRegionsCache();
+ this.lifecycleNewTracker = lifecycleNewTracker;
+
+ lifecycleNewTracker.trackNew(this);
}
- protected SSTableWriter(Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- TableMetadataRef metadata,
- MetadataCollector metadataCollector,
- SerializationHeader header,
- Collection<SSTableFlushObserver> observers)
+ @Override
+ public DecoratedKey getFirst()
{
- super(descriptor, components(metadata.getLocal()), metadata, DatabaseDescriptor.getDiskOptimizationStrategy());
- this.keyCount = keyCount;
- this.repairedAt = repairedAt;
- this.pendingRepair = pendingRepair;
- this.isTransient = isTransient;
- this.metadataCollector = metadataCollector;
- this.header = header;
- this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), descriptor.version, header);
- this.observers = observers == null ? Collections.emptySet() : observers;
+ return first;
}
- public static SSTableWriter create(Descriptor descriptor,
- Long keyCount,
- Long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- TableMetadataRef metadata,
- MetadataCollector metadataCollector,
- SerializationHeader header,
- Collection<Index> indexes,
- LifecycleNewTracker lifecycleNewTracker)
+ @Override
+ public DecoratedKey getLast()
{
- Factory writerFactory = descriptor.getFormat().getWriterFactory();
- return writerFactory.open(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers(descriptor, indexes, lifecycleNewTracker.opType()), lifecycleNewTracker);
+ return last;
}
- public static SSTableWriter create(Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- int sstableLevel,
- SerializationHeader header,
- Collection<Index> indexes,
- LifecycleNewTracker lifecycleNewTracker)
+ @Override
+ public AbstractBounds<Token> getBounds()
{
- TableMetadataRef metadata = Schema.instance.getTableMetadataRef(descriptor);
- return create(metadata, descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, indexes, lifecycleNewTracker);
- }
-
- public static SSTableWriter create(TableMetadataRef metadata,
- Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- int sstableLevel,
- SerializationHeader header,
- Collection<Index> indexes,
- LifecycleNewTracker lifecycleNewTracker)
- {
- MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel);
- return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, lifecycleNewTracker);
- }
-
- @VisibleForTesting
- public static SSTableWriter create(Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- SerializationHeader header,
- Collection<Index> indexes,
- LifecycleNewTracker lifecycleNewTracker)
- {
- return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, 0, header, indexes, lifecycleNewTracker);
- }
-
- private static Set<Component> components(TableMetadata metadata)
- {
- Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA,
- Component.PRIMARY_INDEX,
- Component.STATS,
- Component.SUMMARY,
- Component.TOC,
- Component.DIGEST));
-
- if (metadata.params.bloomFilterFpChance < 1.0)
- components.add(Component.FILTER);
-
- if (metadata.params.compression.isEnabled())
- {
- components.add(Component.COMPRESSION_INFO);
- }
- else
- {
- // it would feel safer to actually add this component later in maybeWriteDigest(),
- // but the components are unmodifiable after construction
- components.add(Component.CRC);
- }
- return components;
- }
-
- private static Collection<SSTableFlushObserver> observers(Descriptor descriptor,
- Collection<Index> indexes,
- OperationType operationType)
- {
- if (indexes == null)
- return Collections.emptyList();
-
- List<SSTableFlushObserver> observers = new ArrayList<>(indexes.size());
- for (Index index : indexes)
- {
- SSTableFlushObserver observer = index.getFlushObserver(descriptor, operationType);
- if (observer != null)
- {
- observer.begin();
- observers.add(observer);
- }
- }
-
- return ImmutableList.copyOf(observers);
+ return (first != null && last != null) ? AbstractBounds.bounds(first.getToken(), true, last.getToken(), true)
+ : null;
}
public abstract void mark();
@@ -217,44 +130,62 @@
* @return the created index entry if something was written, that is if {@code iterator}
* wasn't empty, {@code null} otherwise.
*
- * @throws FSWriteError if a write to the dataFile fails
+ * @throws FSWriteError if writing to the dataFile fails
*/
- public abstract RowIndexEntry append(UnfilteredRowIterator iterator);
+ public abstract AbstractRowIndexEntry append(UnfilteredRowIterator iterator);
+ /**
+ * Returns a position in the uncompressed data - for uncompressed files it is the same as {@link #getOnDiskFilePointer()}
+ * but for compressed files it returns a position in the data rather than a position in the file on disk.
+ */
public abstract long getFilePointer();
+ /**
+ * Returns a position in the (compressed) data file on disk. See {@link #getFilePointer()}
+ */
public abstract long getOnDiskFilePointer();
+ /**
+ * Returns the amount of data already written to disk that may not be accurate (for example, the position after
+ * the recently flushed chunk).
+ */
public long getEstimatedOnDiskBytesWritten()
{
return getOnDiskFilePointer();
}
+ /**
+ * Reset the data file to the marked position (see {@link #mark()}) and truncate the rest of the file.
+ */
public abstract void resetAndTruncate();
- public SSTableWriter setRepairedAt(long repairedAt)
+ public void setRepairedAt(long repairedAt)
{
if (repairedAt > 0)
this.repairedAt = repairedAt;
- return this;
}
- public SSTableWriter setMaxDataAge(long maxDataAge)
+ public void setMaxDataAge(long maxDataAge)
{
this.maxDataAge = maxDataAge;
- return this;
}
- public SSTableWriter setOpenResult(boolean openResult)
+ public void setOpenResult(boolean openResult)
{
txnProxy.openResult = openResult;
- return this;
}
/**
- * Open the resultant SSTableReader before it has been fully written
+ * Open the resultant SSTableReader before it has been fully written.
+ * <p>
+ * The passed consumer will be called when the necessary data has been flushed to disk/cache. This may never happen
+ * (e.g. if the table was finished before the flushes materialized, or if this call returns false e.g. if a table
+ * was already prepared but hasn't reached readiness yet).
+ * <p>
+ * Uses callback instead of future because preparation and callback happen on the same thread.
*/
- public abstract SSTableReader openEarly();
+
+ public abstract void openEarly(Consumer<SSTableReader> doWhenReady);
/**
* Open the resultant SSTableReader once it has been fully written, but before the
@@ -262,17 +193,11 @@
*/
public abstract SSTableReader openFinalEarly();
- public SSTableReader finish(long repairedAt, long maxDataAge, boolean openResult)
- {
- if (repairedAt > 0)
- this.repairedAt = repairedAt;
- this.maxDataAge = maxDataAge;
- return finish(openResult);
- }
+ protected abstract SSTableReader openFinal(SSTableReader.OpenReason openReason);
public SSTableReader finish(boolean openResult)
{
- setOpenResult(openResult);
+ this.setOpenResult(openResult);
txnProxy.finish();
observers.forEach(SSTableFlushObserver::complete);
return finished();
@@ -284,6 +209,7 @@
*/
public SSTableReader finished()
{
+ txnProxy.finalReaderAccessed = true;
return txnProxy.finalReader;
}
@@ -327,7 +253,9 @@
repairedAt,
pendingRepair,
isTransient,
- header);
+ header,
+ first.retainable().getKey(),
+ last.retainable().getKey());
}
protected StatsMetadata statsMetadata()
@@ -340,50 +268,8 @@
metadataCollector.release();
}
- public static void rename(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
- {
- for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY)))
- {
- FileUtils.renameWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component));
- }
-
- // do -Data last because -Data present should mean the sstable was completely renamed before crash
- FileUtils.renameWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA));
-
- // rename it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
- FileUtils.renameWithOutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY));
- }
-
- public static void copy(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
- {
- for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY)))
- {
- FileUtils.copyWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component));
- }
-
- // do -Data last because -Data present should mean the sstable was completely copied before crash
- FileUtils.copyWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA));
-
- // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
- FileUtils.copyWithOutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY));
- }
-
- public static void hardlink(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
- {
- for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY)))
- {
- FileUtils.createHardLinkWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component));
- }
-
- // do -Data last because -Data present should mean the sstable was completely copied before crash
- FileUtils.createHardLinkWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA));
-
- // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
- FileUtils.createHardLinkWithoutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY));
- }
-
/**
- * Parameters for calculating the expected size of an sstable. Exposed on memtable flush sets (i.e. collected
+ * Parameters for calculating the expected size of an SSTable. Exposed on memtable flush sets (i.e. collected
* subsets of a memtable that will be written to sstables).
*/
public interface SSTableSizeParameters
@@ -393,58 +279,225 @@
long dataSize();
}
- public static abstract class Factory
+ // due to lack of multiple inheritance, we use an inner class to proxy our Transactional implementation details
+ protected class TransactionalProxy extends AbstractTransactional
{
- public abstract long estimateSize(SSTableSizeParameters parameters);
+ // should be set during doPrepare()
+ private final Supplier<ImmutableList<Transactional>> transactionals;
- public abstract SSTableWriter open(Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- TableMetadataRef metadata,
- MetadataCollector metadataCollector,
- SerializationHeader header,
- Collection<SSTableFlushObserver> observers,
- LifecycleNewTracker lifecycleNewTracker);
+ private SSTableReader finalReader;
+ private boolean openResult;
+ private boolean finalReaderAccessed;
+
+ public TransactionalProxy(Supplier<ImmutableList<Transactional>> transactionals)
+ {
+ this.transactionals = transactionals;
+ }
+
+ // finalise our state on disk, including renaming
+ protected void doPrepare()
+ {
+ transactionals.get().forEach(Transactional::prepareToCommit);
+ new StatsComponent(finalizeMetadata()).save(descriptor);
+
+ // save the table of components
+ TOCComponent.appendTOC(descriptor, components);
+
+ if (openResult)
+ finalReader = openFinal(SSTableReader.OpenReason.NORMAL);
+ }
+
+ protected Throwable doCommit(Throwable accumulate)
+ {
+ for (Transactional t : transactionals.get().reverse())
+ accumulate = t.commit(accumulate);
+
+ return accumulate;
+ }
+
+ protected Throwable doAbort(Throwable accumulate)
+ {
+ for (Transactional t : transactionals.get())
+ accumulate = t.abort(accumulate);
+
+ if (!finalReaderAccessed && finalReader != null)
+ {
+ accumulate = Throwables.perform(accumulate, () -> finalReader.selfRef().release());
+ finalReader = null;
+ finalReaderAccessed = false;
+ }
+
+ return accumulate;
+ }
+
+ @Override
+ protected Throwable doPostCleanup(Throwable accumulate)
+ {
+ accumulate = super.doPostCleanup(accumulate);
+ accumulate = Throwables.close(accumulate, mmappedRegionsCache);
+ return accumulate;
+ }
}
- public static void guardCollectionSize(TableMetadata metadata, DecoratedKey partitionKey, Unfiltered unfiltered)
+ /**
+ * A builder of this sstable writer. It should be extended for each implementation with the specific fields.
+ *
+ * An implementation should open all the resources when {@link #build(LifecycleNewTracker, Owner)} and pass them
+ * in builder fields to the writer, so that the writer can access them via getters.
+ *
+ * @param <W> type of the sstable writer to be build with this builder
+ * @param <B> type of this builder
+ */
+ public abstract static class Builder<W extends SSTableWriter, B extends Builder<W, B>> extends SSTable.Builder<W, B>
{
- if (!Guardrails.collectionSize.enabled() && !Guardrails.itemsPerCollection.enabled())
- return;
+ private MetadataCollector metadataCollector;
+ private long keyCount;
+ private long repairedAt;
+ private TimeUUID pendingRepair;
+ private boolean transientSSTable;
+ private SerializationHeader serializationHeader;
+ private Collection<SSTableFlushObserver> flushObservers;
- if (!unfiltered.isRow() || SchemaConstants.isSystemKeyspace(metadata.keyspace))
- return;
-
- Row row = (Row) unfiltered;
- for (ColumnMetadata column : row.columns())
+ public B setMetadataCollector(MetadataCollector metadataCollector)
{
- if (!column.type.isCollection() || !column.type.isMultiCell())
- continue;
+ this.metadataCollector = metadataCollector;
+ return (B) this;
+ }
- ComplexColumnData cells = row.getComplexColumnData(column);
- if (cells == null)
- continue;
+ public B setKeyCount(long keyCount)
+ {
+ this.keyCount = keyCount;
+ return (B) this;
+ }
- ComplexColumnData liveCells = cells.purge(DeletionPurger.PURGE_ALL, FBUtilities.nowInSeconds());
- if (liveCells == null)
- continue;
+ public B setRepairedAt(long repairedAt)
+ {
+ this.repairedAt = repairedAt;
+ return (B) this;
+ }
- int cellsSize = liveCells.dataSize();
- int cellsCount = liveCells.cellsCount();
+ public B setPendingRepair(TimeUUID pendingRepair)
+ {
+ this.pendingRepair = pendingRepair;
+ return (B) this;
+ }
- if (!Guardrails.collectionSize.triggersOn(cellsSize, null) &&
- !Guardrails.itemsPerCollection.triggersOn(cellsCount, null))
- continue;
+ public B setTransientSSTable(boolean transientSSTable)
+ {
+ this.transientSSTable = transientSSTable;
+ return (B) this;
+ }
- String keyString = metadata.primaryKeyAsCQLLiteral(partitionKey.getKey(), row.clustering());
- String msg = String.format("%s in row %s in table %s",
- column.name.toString(),
- keyString,
- metadata);
- Guardrails.collectionSize.guard(cellsSize, msg, true, null);
- Guardrails.itemsPerCollection.guard(cellsCount, msg, true, null);
+ public B setSerializationHeader(SerializationHeader serializationHeader)
+ {
+ this.serializationHeader = serializationHeader;
+ return (B) this;
+ }
+
+ public B setFlushObservers(Collection<SSTableFlushObserver> flushObservers)
+ {
+ this.flushObservers = ImmutableList.copyOf(flushObservers);
+ return (B) this;
+ }
+
+ public B addDefaultComponents()
+ {
+ checkNotNull(getTableMetadataRef());
+
+ addComponents(ImmutableSet.of(Components.DATA, Components.STATS, Components.DIGEST, Components.TOC));
+
+ if (getTableMetadataRef().getLocal().params.compression.isEnabled())
+ {
+ addComponents(ImmutableSet.of(Components.COMPRESSION_INFO));
+ }
+ else
+ {
+ // it would feel safer to actually add this component later in maybeWriteDigest(),
+ // but the components are unmodifiable after construction
+ addComponents(ImmutableSet.of(Components.CRC));
+ }
+
+ return (B) this;
+ }
+
+ public B addFlushObserversForSecondaryIndexes(Collection<Index> indexes, OperationType operationType)
+ {
+ if (indexes == null)
+ return (B) this;
+
+ Collection<SSTableFlushObserver> current = this.flushObservers != null ? this.flushObservers : Collections.emptyList();
+ List<SSTableFlushObserver> observers = new ArrayList<>(indexes.size() + current.size());
+ observers.addAll(current);
+
+ for (Index index : indexes)
+ {
+ SSTableFlushObserver observer = index.getFlushObserver(descriptor, operationType);
+ if (observer != null)
+ {
+ observer.begin();
+ observers.add(observer);
+ }
+ }
+
+ return setFlushObservers(observers);
+ }
+
+ public MetadataCollector getMetadataCollector()
+ {
+ return metadataCollector;
+ }
+
+ public long getKeyCount()
+ {
+ return keyCount;
+ }
+
+ public long getRepairedAt()
+ {
+ return repairedAt;
+ }
+
+ public TimeUUID getPendingRepair()
+ {
+ return pendingRepair;
+ }
+
+ public boolean isTransientSSTable()
+ {
+ return transientSSTable;
+ }
+
+ public SerializationHeader getSerializationHeader()
+ {
+ return serializationHeader;
+ }
+
+ public Collection<SSTableFlushObserver> getFlushObservers()
+ {
+ return flushObservers;
+ }
+
+ public abstract MmappedRegionsCache getMmappedRegionsCache();
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ public W build(LifecycleNewTracker lifecycleNewTracker, Owner owner)
+ {
+ checkNotNull(getComponents());
+
+ validateRepairedMetadata(getRepairedAt(), getPendingRepair(), isTransientSSTable());
+
+ return buildInternal(lifecycleNewTracker, owner);
+ }
+
+ protected abstract W buildInternal(LifecycleNewTracker lifecycleNewTracker, Owner owner);
+
+ public SSTableZeroCopyWriter createZeroCopyWriter(LifecycleNewTracker lifecycleNewTracker, Owner owner)
+ {
+ return new SSTableZeroCopyWriter(this, lifecycleNewTracker, owner);
}
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
new file mode 100644
index 0000000..6634af3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static com.google.common.base.Preconditions.checkState;
+
+public abstract class SortedTablePartitionWriter implements AutoCloseable
+{
+ protected final UnfilteredSerializer unfilteredSerializer;
+
+ private final SerializationHeader header;
+ private final SequentialWriter writer;
+ private final SerializationHelper helper;
+ private final int version;
+
+ private long previousRowStart;
+ private long initialPosition;
+ private long headerLength;
+
+ protected long startPosition;
+ protected int written;
+
+ protected ClusteringPrefix<?> firstClustering;
+ protected ClusteringPrefix<?> lastClustering;
+
+ protected DeletionTime openMarker = DeletionTime.LIVE;
+ protected DeletionTime startOpenMarker = DeletionTime.LIVE;
+
+ // Sequence control, also used to add empty static row if `addStaticRow` is not called.
+ private enum State
+ {
+ AWAITING_PARTITION_HEADER,
+ AWAITING_STATIC_ROW,
+ AWAITING_ROWS,
+ COMPLETED
+ }
+
+ State state = State.AWAITING_PARTITION_HEADER;
+
+ protected SortedTablePartitionWriter(SerializationHeader header, SequentialWriter writer, Version version)
+ {
+ this.header = header;
+ this.writer = writer;
+ this.unfilteredSerializer = UnfilteredSerializer.serializer;
+ this.helper = new SerializationHelper(header);
+ this.version = version.correspondingMessagingVersion();
+ }
+
+ protected void reset()
+ {
+ this.initialPosition = writer.position();
+ this.startPosition = -1;
+ this.previousRowStart = 0;
+ this.written = 0;
+ this.firstClustering = null;
+ this.lastClustering = null;
+ this.openMarker = DeletionTime.LIVE;
+ this.headerLength = -1;
+ this.state = State.AWAITING_PARTITION_HEADER;
+ }
+
+ public long getHeaderLength()
+ {
+ return headerLength;
+ }
+
+ public void start(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException
+ {
+ if (state == State.COMPLETED)
+ reset();
+
+ checkState(state == State.AWAITING_PARTITION_HEADER);
+
+ ByteBufferUtil.writeWithShortLength(key.getKey(), writer);
+ DeletionTime.serializer.serialize(partitionLevelDeletion, writer);
+
+ if (!header.hasStatic())
+ {
+ this.headerLength = writer.position() - initialPosition;
+ state = State.AWAITING_ROWS;
+ return;
+ }
+
+ state = State.AWAITING_STATIC_ROW;
+ }
+
+ public void addStaticRow(Row staticRow) throws IOException
+ {
+ checkState(state == State.AWAITING_STATIC_ROW);
+ checkState(staticRow.isStatic());
+
+ UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version);
+
+ this.headerLength = writer.position() - initialPosition;
+ state = State.AWAITING_ROWS;
+ }
+
+ public void addUnfiltered(Unfiltered unfiltered) throws IOException
+ {
+ checkState(state == State.AWAITING_ROWS);
+
+ long pos = currentPosition();
+
+ if (firstClustering == null)
+ {
+ // Beginning of an index block. Remember the start and position
+ firstClustering = unfiltered.clustering();
+ startOpenMarker = openMarker;
+ startPosition = pos;
+ }
+
+ long unfilteredPosition = writer.position();
+ unfilteredSerializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version);
+
+ lastClustering = unfiltered.clustering();
+ previousRowStart = pos;
+ ++written;
+
+ if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
+ {
+ RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered;
+ openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : DeletionTime.LIVE;
+ }
+ }
+
+ protected long finish() throws IOException
+ {
+ checkState(state == State.AWAITING_ROWS);
+
+ state = State.COMPLETED;
+
+ long endPosition = currentPosition();
+ unfilteredSerializer.writeEndOfPartition(writer);
+
+ return endPosition;
+ }
+
+ protected long currentPosition()
+ {
+ return writer.position() - initialPosition;
+ }
+
+ public long getInitialPosition()
+ {
+ return initialPosition;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java
new file mode 100644
index 0000000..4b64754
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.IFilter;
+
+public abstract class SortedTableReaderLoadingBuilder<R extends SSTableReader, B extends SSTableReader.Builder<R, B>>
+extends SSTableReaderLoadingBuilder<R, B>
+{
+ private final static Logger logger = LoggerFactory.getLogger(SortedTableReaderLoadingBuilder.class);
+ private FileHandle.Builder dataFileBuilder;
+
+ public SortedTableReaderLoadingBuilder(SSTable.Builder<?, ?> builder)
+ {
+ super(builder);
+ }
+
+ protected IFilter loadFilter(ValidationMetadata validationMetadata)
+ {
+ return FilterComponent.maybeLoadBloomFilter(descriptor,
+ components,
+ tableMetadataRef.get(),
+ validationMetadata);
+ }
+
+ protected FileHandle.Builder dataFileBuilder(StatsMetadata statsMetadata)
+ {
+ assert this.dataFileBuilder == null || this.dataFileBuilder.file.equals(descriptor.fileFor(BtiFormat.Components.DATA));
+
+ logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(descriptor.fileFor(BtiFormat.Components.DATA).length()));
+
+ long recordSize = statsMetadata.estimatedPartitionSize.percentile(ioOptions.diskOptimizationEstimatePercentile);
+ int bufferSize = ioOptions.diskOptimizationStrategy.bufferSize(recordSize);
+
+ if (dataFileBuilder == null)
+ dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(BtiFormat.Components.DATA));
+
+ dataFileBuilder.bufferSize(bufferSize);
+ dataFileBuilder.withChunkCache(chunkCache);
+ dataFileBuilder.mmapped(ioOptions.defaultDiskAccessMode);
+
+ return dataFileBuilder;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
new file mode 100644
index 0000000..95c913a
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
@@ -0,0 +1,630 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.rows.AbstractCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Refs;
+import org.apache.cassandra.utils.memory.HeapCloner;
+
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+@NotThreadSafe
+public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> implements IScrubber
+{
+ private final static Logger logger = LoggerFactory.getLogger(SortedTableScrubber.class);
+
+ protected final ColumnFamilyStore cfs;
+ protected final LifecycleTransaction transaction;
+ protected final File destination;
+ protected final IScrubber.Options options;
+ protected final R sstable;
+ protected final OutputHandler outputHandler;
+ protected final boolean isCommutative;
+ protected final long expectedBloomFilterSize;
+ protected final ReadWriteLock fileAccessLock = new ReentrantReadWriteLock();
+ protected final RandomAccessReader dataFile;
+ protected final ScrubInfo scrubInfo;
+
+ protected final NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics = new NegativeLocalDeletionInfoMetrics();
+
+ private static final Comparator<Partition> partitionComparator = Comparator.comparing(Partition::partitionKey);
+ protected final SortedSet<Partition> outOfOrder = new TreeSet<>(partitionComparator);
+
+
+ protected int goodPartitions;
+ protected int badPartitions;
+ protected int emptyPartitions;
+
+
+ protected SortedTableScrubber(ColumnFamilyStore cfs,
+ LifecycleTransaction transaction,
+ OutputHandler outputHandler,
+ Options options)
+ {
+ this.sstable = (R) transaction.onlyOne();
+ Preconditions.checkNotNull(sstable.metadata());
+ assert sstable.metadata().keyspace.equals(cfs.keyspace.getName());
+ if (!sstable.descriptor.cfname.equals(cfs.metadata().name))
+ {
+ logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name);
+ }
+ try
+ {
+ sstable.metadata().validateCompatibility(cfs.metadata());
+ }
+ catch (ConfigurationException ex)
+ {
+ logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name);
+ }
+
+ this.cfs = cfs;
+ this.transaction = transaction;
+ this.outputHandler = outputHandler;
+ this.options = options;
+ this.destination = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForSSTable(sstable));
+ this.isCommutative = cfs.metadata().isCounter();
+
+ List<SSTableReader> toScrub = Collections.singletonList(sstable);
+
+ long approximateKeyCount;
+ try
+ {
+ approximateKeyCount = SSTableReader.getApproximateKeyCount(toScrub);
+ }
+ catch (RuntimeException ex)
+ {
+ approximateKeyCount = 0;
+ }
+ this.expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, approximateKeyCount);
+
+ // loop through each partition, deserializing to check for damage.
+ // We'll also loop through the index at the same time, using the position from the index to recover if the
+ // partition header (key or data size) is corrupt. (This means our position in the index file will be one
+ // partition "ahead" of the data file.)
+ this.dataFile = transaction.isOffline()
+ ? sstable.openDataReader()
+ : sstable.openDataReader(CompactionManager.instance.getRateLimiter());
+
+ this.scrubInfo = new ScrubInfo(dataFile, sstable, fileAccessLock.readLock());
+
+ if (options.reinsertOverflowedTTLRows)
+ outputHandler.output("Starting scrub with reinsert overflowed TTL option");
+ }
+
+ public static void deleteOrphanedComponents(Descriptor descriptor, Set<Component> components)
+ {
+ File dataFile = descriptor.fileFor(Components.DATA);
+ if (components.contains(Components.DATA) && dataFile.length() > 0)
+ // everything appears to be in order... moving on.
+ return;
+
+ // missing the DATA file! all components are orphaned
+ logger.warn("Removing orphans for {}: {}", descriptor, components);
+ for (Component component : components)
+ {
+ File file = descriptor.fileFor(component);
+ if (file.exists())
+ descriptor.fileFor(component).delete();
+ }
+ }
+
+ @Override
+ public void scrub()
+ {
+ List<SSTableReader> finished = new ArrayList<>();
+ outputHandler.output("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length()));
+ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge);
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
+ {
+ StatsMetadata metadata = sstable.getSSTableMetadata();
+ writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction));
+
+ scrubInternal(writer);
+
+ if (!outOfOrder.isEmpty())
+ finished.add(writeOutOfOrderPartitions(metadata));
+
+ // finish obsoletes the old sstable
+ transaction.obsoleteOriginals();
+ finished.addAll(writer.setRepairedAt(badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt).finish());
+ }
+ catch (IOException ex)
+ {
+ throw new RuntimeException(ex);
+ }
+ finally
+ {
+ if (transaction.isOffline())
+ finished.forEach(sstable -> sstable.selfRef().release());
+ }
+
+ outputSummary(finished);
+ }
+
+ protected abstract void scrubInternal(SSTableRewriter writer) throws IOException;
+
+ private void outputSummary(List<SSTableReader> finished)
+ {
+ if (!finished.isEmpty())
+ {
+ outputHandler.output("Scrub of %s complete: %d partitions in new sstable and %d empty (tombstoned) partitions dropped", sstable, goodPartitions, emptyPartitions);
+ if (negativeLocalDeletionInfoMetrics.fixedRows > 0)
+ outputHandler.output("Fixed %d rows with overflowed local deletion time.", negativeLocalDeletionInfoMetrics.fixedRows);
+ if (badPartitions > 0)
+ outputHandler.warn("Unable to recover %d partitions that were skipped. You can attempt manual recovery from the pre-scrub snapshot. You can also run nodetool repair to transfer the data from a healthy replica, if any", badPartitions);
+ }
+ else
+ {
+ if (badPartitions > 0)
+ outputHandler.warn("No valid partitions found while scrubbing %s; it is marked for deletion now. If you want to attempt manual recovery, you can find a copy in the pre-scrub snapshot", sstable);
+ else
+ outputHandler.output("Scrub of %s complete; looks like all %d partitions were tombstoned", sstable, emptyPartitions);
+ }
+ }
+
+ private SSTableReader writeOutOfOrderPartitions(StatsMetadata metadata)
+ {
+ // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time
+ long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
+ SSTableReader newInOrderSstable;
+ try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction))
+ {
+ for (Partition partition : outOfOrder)
+ inOrderWriter.append(partition.unfilteredIterator());
+ inOrderWriter.setRepairedAt(-1);
+ inOrderWriter.setMaxDataAge(sstable.maxDataAge);
+ newInOrderSstable = inOrderWriter.finish(true);
+ }
+ transaction.update(newInOrderSstable, false);
+ outputHandler.warn("%d out of order partition (or partitions without of order rows) found while scrubbing %s; " +
+ "Those have been written (in order) to a new sstable (%s)", outOfOrder.size(), sstable, newInOrderSstable);
+ return newInOrderSstable;
+ }
+
+ protected abstract UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename);
+
+ @Override
+ @VisibleForTesting
+ public ScrubResult scrubWithResult()
+ {
+ scrub();
+ return new ScrubResult(goodPartitions, badPartitions, emptyPartitions);
+ }
+
+ @Override
+ public CompactionInfo.Holder getScrubInfo()
+ {
+ return scrubInfo;
+ }
+
+ protected String keyString(DecoratedKey key)
+ {
+ if (key == null)
+ return "(unknown)";
+
+ try
+ {
+ return cfs.metadata().partitionKeyType.getString(key.getKey());
+ }
+ catch (Exception e)
+ {
+ return String.format("(corrupted; hex value: %s)", ByteBufferUtil.bytesToHex(key.getKey()));
+ }
+ }
+
+ protected boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewriter writer)
+ {
+ // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects
+ // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added
+ // to the outOfOrder set that will be later written to a new SSTable.
+ try (OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key), cfs.metadata().comparator);
+ UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath()))
+ {
+ if (prevKey != null && prevKey.compareTo(key) > 0)
+ {
+ saveOutOfOrderPartition(prevKey, key, iterator);
+ return false;
+ }
+
+ if (writer.tryAppend(iterator) == null)
+ emptyPartitions++;
+ else
+ goodPartitions++;
+
+ if (sstableIterator.hasRowsOutOfOrder())
+ {
+ outputHandler.warn("Out of order rows found in partition: %s", keyString(key));
+ outOfOrder.add(sstableIterator.getRowsOutOfOrder());
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Only wrap with {@link FixNegativeLocalDeletionTimeIterator} if {@link IScrubber.Options#reinsertOverflowedTTLRows} option
+ * is specified
+ */
+ private UnfilteredRowIterator getIterator(DecoratedKey key)
+ {
+ RowMergingSSTableIterator rowMergingIterator = new RowMergingSSTableIterator(SSTableIdentityIterator.create(sstable, dataFile, key), outputHandler);
+ if (options.reinsertOverflowedTTLRows)
+ return new FixNegativeLocalDeletionTimeIterator(rowMergingIterator, outputHandler, negativeLocalDeletionInfoMetrics);
+ else
+ return rowMergingIterator;
+ }
+
+ private void saveOutOfOrderPartition(DecoratedKey prevKey, DecoratedKey key, UnfilteredRowIterator iterator)
+ {
+ // TODO bitch if the row is too large? if it is there's not much we can do ...
+ outputHandler.warn("Out of order partition detected (%s found after %s)", keyString(key), keyString(prevKey));
+ outOfOrder.add(ImmutableBTreePartition.create(iterator));
+ }
+
+ protected static void throwIfFatal(Throwable th)
+ {
+ if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError))
+ throw (Error) th;
+ }
+
+ protected void throwIfCannotContinue(DecoratedKey key, Throwable th)
+ {
+ if (isCommutative && !options.skipCorrupted)
+ {
+ outputHandler.warn("An error occurred while scrubbing the partition with key '%s'. Skipping corrupt " +
+ "data in counter tables will result in undercounts for the affected " +
+ "counters (see CASSANDRA-2759 for more details), so by default the scrub will " +
+ "stop at this point. If you would like to skip the row anyway and continue " +
+ "scrubbing, re-run the scrub with the --skip-corrupted option.",
+ keyString(key));
+ throw new IOError(th);
+ }
+ }
+
+
+ public static class ScrubInfo extends CompactionInfo.Holder
+ {
+ private final RandomAccessReader dataFile;
+ private final SSTableReader sstable;
+ private final TimeUUID scrubCompactionId;
+ private final Lock fileReadLock;
+
+ public ScrubInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock)
+ {
+ this.dataFile = dataFile;
+ this.sstable = sstable;
+ this.fileReadLock = fileReadLock;
+ scrubCompactionId = nextTimeUUID();
+ }
+
+ public CompactionInfo getCompactionInfo()
+ {
+ fileReadLock.lock();
+ try
+ {
+ return new CompactionInfo(sstable.metadata(),
+ OperationType.SCRUB,
+ dataFile.getFilePointer(),
+ dataFile.length(),
+ scrubCompactionId,
+ ImmutableSet.of(sstable),
+ File.getPath(sstable.getFilename()).getParent().toString());
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ finally
+ {
+ fileReadLock.unlock();
+ }
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ }
+
+ /**
+ * In some case like CASSANDRA-12127 the cells might have been stored in the wrong order. This decorator check the
+ * cells order and collect the out-of-order cells to correct the problem.
+ */
+ private static final class OrderCheckerIterator extends AbstractIterator<Unfiltered> implements WrappingUnfilteredRowIterator
+ {
+ private final UnfilteredRowIterator iterator;
+ private final ClusteringComparator comparator;
+
+ private Unfiltered previous;
+
+ /**
+ * The partition containing the rows which are out of order.
+ */
+ private Partition rowsOutOfOrder;
+
+ public OrderCheckerIterator(UnfilteredRowIterator iterator, ClusteringComparator comparator)
+ {
+ this.iterator = iterator;
+ this.comparator = comparator;
+ }
+
+ @Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return iterator;
+ }
+
+ public boolean hasRowsOutOfOrder()
+ {
+ return rowsOutOfOrder != null;
+ }
+
+ public Partition getRowsOutOfOrder()
+ {
+ return rowsOutOfOrder;
+ }
+
+ @Override
+ protected Unfiltered computeNext()
+ {
+ if (!iterator.hasNext())
+ return endOfData();
+
+ Unfiltered next = iterator.next();
+
+ // If we detect that some rows are out of order we will store and sort the remaining ones to insert them
+ // in a separate SSTable.
+ if (previous != null && comparator.compare(next, previous) < 0)
+ {
+ rowsOutOfOrder = ImmutableBTreePartition.create(UnfilteredRowIterators.concat(next, iterator), false);
+ return endOfData();
+ }
+ previous = next;
+ return next;
+ }
+ }
+
+ /**
+ * During 2.x migration, under some circumstances rows might have gotten duplicated.
+ * Merging iterator merges rows with same clustering.
+ * <p>
+ * For more details, refer to CASSANDRA-12144.
+ */
+ private static class RowMergingSSTableIterator implements WrappingUnfilteredRowIterator
+ {
+ Unfiltered nextToOffer = null;
+ private final OutputHandler output;
+ private final UnfilteredRowIterator wrapped;
+
+ RowMergingSSTableIterator(UnfilteredRowIterator source, OutputHandler output)
+ {
+ this.wrapped = source;
+ this.output = output;
+ }
+
+ @Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return wrapped;
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ return nextToOffer != null || wrapped.hasNext();
+ }
+
+ @Override
+ public Unfiltered next()
+ {
+ Unfiltered next = nextToOffer != null ? nextToOffer : wrapped.next();
+
+ if (next.isRow())
+ {
+ boolean logged = false;
+ while (wrapped.hasNext())
+ {
+ Unfiltered peek = wrapped.next();
+ if (!peek.isRow() || !next.clustering().equals(peek.clustering()))
+ {
+ nextToOffer = peek; // Offer peek in next call
+ return next;
+ }
+
+ // Duplicate row, merge it.
+ next = Rows.merge((Row) next, (Row) peek);
+
+ if (!logged)
+ {
+ String partitionKey = metadata().partitionKeyType.getString(partitionKey().getKey());
+ output.warn("Duplicate row detected in %s.%s: %s %s", metadata().keyspace, metadata().name, partitionKey, next.clustering().toString(metadata()));
+ logged = true;
+ }
+ }
+ }
+
+ nextToOffer = null;
+ return next;
+ }
+ }
+
+ /**
+ * This iterator converts negative {@link AbstractCell#localDeletionTime()} into {@link AbstractCell#MAX_DELETION_TIME}
+ * <p>
+ * This is to recover entries with overflowed localExpirationTime due to CASSANDRA-14092
+ */
+ private static final class FixNegativeLocalDeletionTimeIterator extends AbstractIterator<Unfiltered> implements WrappingUnfilteredRowIterator
+ {
+ /**
+ * The decorated iterator.
+ */
+ private final UnfilteredRowIterator iterator;
+
+ private final OutputHandler outputHandler;
+ private final NegativeLocalDeletionInfoMetrics negativeLocalExpirationTimeMetrics;
+
+ public FixNegativeLocalDeletionTimeIterator(UnfilteredRowIterator iterator, OutputHandler outputHandler,
+ NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics)
+ {
+ this.iterator = iterator;
+ this.outputHandler = outputHandler;
+ this.negativeLocalExpirationTimeMetrics = negativeLocalDeletionInfoMetrics;
+ }
+
+ @Override
+ public UnfilteredRowIterator wrapped()
+ {
+ return iterator;
+ }
+
+ @Override
+ protected Unfiltered computeNext()
+ {
+ if (!iterator.hasNext())
+ return endOfData();
+
+ Unfiltered next = iterator.next();
+ if (!next.isRow())
+ return next;
+
+ if (hasNegativeLocalExpirationTime((Row) next))
+ {
+ outputHandler.debug("Found row with negative local expiration time: %s", next.toString(metadata(), false));
+ negativeLocalExpirationTimeMetrics.fixedRows++;
+ return fixNegativeLocalExpirationTime((Row) next);
+ }
+
+ return next;
+ }
+
+ private boolean hasNegativeLocalExpirationTime(Row next)
+ {
+ Row row = next;
+ if (row.primaryKeyLivenessInfo().isExpiring() && row.primaryKeyLivenessInfo().localExpirationTime() < 0)
+ {
+ return true;
+ }
+
+ for (ColumnData cd : row)
+ {
+ if (cd.column().isSimple())
+ {
+ Cell<?> cell = (Cell<?>) cd;
+ if (cell.isExpiring() && cell.localDeletionTime() < 0)
+ return true;
+ }
+ else
+ {
+ ComplexColumnData complexData = (ComplexColumnData) cd;
+ for (Cell<?> cell : complexData)
+ {
+ if (cell.isExpiring() && cell.localDeletionTime() < 0)
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ private Unfiltered fixNegativeLocalExpirationTime(Row row)
+ {
+ LivenessInfo livenessInfo = row.primaryKeyLivenessInfo();
+ if (livenessInfo.isExpiring() && livenessInfo.localExpirationTime() < 0)
+ livenessInfo = livenessInfo.withUpdatedTimestampAndLocalDeletionTime(livenessInfo.timestamp() + 1, AbstractCell.MAX_DELETION_TIME);
+
+ return row.transformAndFilter(livenessInfo, row.deletion(), cd -> {
+ if (cd.column().isSimple())
+ {
+ Cell cell = (Cell) cd;
+ return cell.isExpiring() && cell.localDeletionTime() < 0
+ ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME)
+ : cell;
+ }
+ else
+ {
+ ComplexColumnData complexData = (ComplexColumnData) cd;
+ return complexData.transformAndFilter(cell -> cell.isExpiring() && cell.localDeletionTime() < 0
+ ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME)
+ : cell);
+ }
+ }).clone(HeapCloner.instance);
+ }
+ }
+
+ private static class NegativeLocalDeletionInfoMetrics
+ {
+ public volatile int fixedRows = 0;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
new file mode 100644
index 0000000..fb3cb4e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
@@ -0,0 +1,528 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+import java.util.function.LongPredicate;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.TimeUUID;
+
+public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> implements IVerifier
+{
+ private final static Logger logger = LoggerFactory.getLogger(SortedTableVerifier.class);
+
+ protected final ColumnFamilyStore cfs;
+ protected final R sstable;
+
+ protected final ReadWriteLock fileAccessLock;
+ protected final RandomAccessReader dataFile;
+ protected final VerifyInfo verifyInfo;
+ protected final Options options;
+ protected final boolean isOffline;
+
+ /**
+ * Given a keyspace, return the set of local and pending token ranges. By default {@link StorageService#getLocalAndPendingRanges(String)}
+ * is expected, but for the standalone verifier case we can't use that, so this is here to allow the CLI to provide
+ * the token ranges.
+ */
+ protected final Function<String, ? extends Collection<Range<Token>>> tokenLookup;
+ protected int goodRows;
+
+ protected final OutputHandler outputHandler;
+
+ public SortedTableVerifier(ColumnFamilyStore cfs, R sstable, OutputHandler outputHandler, boolean isOffline, Options options)
+ {
+ this.cfs = cfs;
+ this.sstable = sstable;
+ this.outputHandler = outputHandler;
+
+ this.fileAccessLock = new ReentrantReadWriteLock();
+ this.dataFile = isOffline
+ ? sstable.openDataReader()
+ : sstable.openDataReader(CompactionManager.instance.getRateLimiter());
+ this.verifyInfo = new VerifyInfo(dataFile, sstable, fileAccessLock.readLock());
+ this.options = options;
+ this.isOffline = isOffline;
+ this.tokenLookup = options.tokenLookup;
+ }
+
+ protected void deserializeBloomFilter(SSTableReader sstable) throws IOException
+ {
+ try (IFilter filter = FilterComponent.load(sstable.descriptor)) {
+ if (filter != null)
+ logger.trace("Filter loaded for {}", sstable);
+ }
+ }
+
+ public CompactionInfo.Holder getVerifyInfo()
+ {
+ return verifyInfo;
+ }
+
+ protected void markAndThrow(Throwable cause)
+ {
+ markAndThrow(cause, true);
+ }
+
+ protected void markAndThrow(Throwable cause, boolean mutateRepaired)
+ {
+ if (mutateRepaired && options.mutateRepairStatus) // if we are able to mutate repaired flag, an incremental repair should be enough
+ {
+ try
+ {
+ sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient());
+ cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
+ }
+ catch (IOException ioe)
+ {
+ outputHandler.output("Error mutating repairedAt for SSTable %s, as part of markAndThrow", sstable.getFilename());
+ }
+ }
+ Exception e = new Exception(String.format("Invalid SSTable %s, please force %srepair", sstable.getFilename(), (mutateRepaired && options.mutateRepairStatus) ? "" : "a full "), cause);
+ if (options.invokeDiskFailurePolicy)
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ else
+ throw new RuntimeException(e);
+ }
+
+ public void verify()
+ {
+ verifySSTableVersion();
+
+ verifySSTableMetadata();
+
+ verifyIndex();
+
+ verifyBloomFilter();
+
+ if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner() instanceof LocalPartitioner))
+ {
+ if (verifyOwnedRanges() == 0)
+ return;
+ }
+
+ if (options.quick)
+ return;
+
+ if (verifyDigest() && !options.extendedVerification)
+ return;
+
+ verifySSTable();
+
+ outputHandler.output("Verify of %s succeeded. All %d rows read successfully", sstable, goodRows);
+ }
+
+ protected void verifyBloomFilter()
+ {
+ try
+ {
+ outputHandler.debug("Deserializing bloom filter for %s", sstable);
+ deserializeBloomFilter(sstable);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t);
+ markAndThrow(t);
+ }
+ }
+
+ protected void verifySSTableMetadata()
+ {
+ outputHandler.output("Deserializing sstable metadata for %s ", sstable);
+ try
+ {
+ StatsComponent statsComponent = StatsComponent.load(sstable.descriptor, MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
+ if (statsComponent.validationMetadata() != null &&
+ !statsComponent.validationMetadata().partitioner.equals(sstable.getPartitioner().getClass().getCanonicalName()))
+ throw new IOException("Partitioner does not match validation metadata");
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t);
+ markAndThrow(t, false);
+ }
+ }
+
+ protected void verifySSTableVersion()
+ {
+ outputHandler.output("Verifying %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length()));
+ if (options.checkVersion && !sstable.descriptor.version.isLatestVersion())
+ {
+ String msg = String.format("%s is not the latest version, run upgradesstables", sstable);
+ outputHandler.output(msg);
+ // don't use markAndThrow here because we don't want a CorruptSSTableException for this.
+ throw new RuntimeException(msg);
+ }
+ }
+
+ protected int verifyOwnedRanges()
+ {
+ List<Range<Token>> ownedRanges = Collections.emptyList();
+ outputHandler.debug("Checking that all tokens are owned by the current node");
+ try (KeyIterator iter = sstable.keyIterator())
+ {
+ ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace));
+ if (ownedRanges.isEmpty())
+ return 0;
+ RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
+ while (iter.hasNext())
+ {
+ DecoratedKey key = iter.next();
+ rangeOwnHelper.validate(key);
+ }
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t);
+ markAndThrow(t);
+ }
+
+ return ownedRanges.size();
+ }
+
+ protected boolean verifyDigest()
+ {
+ boolean passed = true;
+ // Verify will use the Digest files, which works for both compressed and uncompressed sstables
+ outputHandler.output("Checking computed hash of %s ", sstable);
+ try
+ {
+ DataIntegrityMetadata.FileDigestValidator validator = sstable.maybeGetDigestValidator();
+
+ if (validator != null)
+ {
+ validator.validate();
+ }
+ else
+ {
+ outputHandler.output("Data digest missing, assuming extended verification of disk values");
+ passed = false;
+ }
+ }
+ catch (IOException e)
+ {
+ outputHandler.warn(e);
+ markAndThrow(e);
+ }
+ return passed;
+ }
+
+ protected void verifySSTable()
+ {
+ outputHandler.output("Extended Verify requested, proceeding to inspect values");
+
+ try (VerifyController verifyController = new VerifyController(cfs);
+ KeyReader indexIterator = sstable.keyReader())
+ {
+ if (indexIterator.dataPosition() != 0)
+ markAndThrow(new RuntimeException("First row position from index != 0: " + indexIterator.dataPosition()));
+
+ List<Range<Token>> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace));
+ RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges);
+ DecoratedKey prevKey = null;
+
+ while (!dataFile.isEOF())
+ {
+
+ if (verifyInfo.isStopRequested())
+ throw new CompactionInterruptedException(verifyInfo.getCompactionInfo());
+
+ long rowStart = dataFile.getFilePointer();
+ outputHandler.debug("Reading row at %d", rowStart);
+
+ DecoratedKey key = null;
+ try
+ {
+ key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dataFile));
+ }
+ catch (Throwable th)
+ {
+ markAndThrow(th);
+ }
+
+ if (options.checkOwnsTokens && ownedRanges.size() > 0 && !(cfs.getPartitioner() instanceof LocalPartitioner))
+ {
+ try
+ {
+ rangeOwnHelper.validate(key);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t, "Key %s in sstable %s not owned by local ranges %s", key, sstable, ownedRanges);
+ markAndThrow(t);
+ }
+ }
+
+ ByteBuffer currentIndexKey = indexIterator.key();
+ long nextRowPositionFromIndex = 0;
+ try
+ {
+ nextRowPositionFromIndex = indexIterator.advance()
+ ? indexIterator.dataPosition()
+ : dataFile.length();
+ }
+ catch (Throwable th)
+ {
+ markAndThrow(th);
+ }
+
+ long dataStart = dataFile.getFilePointer();
+ long dataStartFromIndex = currentIndexKey == null
+ ? -1
+ : rowStart + 2 + currentIndexKey.remaining();
+
+ long dataSize = nextRowPositionFromIndex - dataStartFromIndex;
+ // avoid an NPE if key is null
+ String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey());
+ outputHandler.debug("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize));
+
+ try
+ {
+ if (key == null || dataSize > dataFile.length())
+ markAndThrow(new RuntimeException(String.format("key = %s, dataSize=%d, dataFile.length() = %d", key, dataSize, dataFile.length())));
+
+ try (UnfilteredRowIterator iterator = SSTableIdentityIterator.create(sstable, dataFile, key))
+ {
+ verifyPartition(key, iterator);
+ }
+
+ if ((prevKey != null && prevKey.compareTo(key) > 0) || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex)
+ markAndThrow(new RuntimeException("Key out of order: previous = " + prevKey + " : current = " + key));
+
+ goodRows++;
+ prevKey = key;
+
+
+ outputHandler.debug("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex);
+ dataFile.seek(nextRowPositionFromIndex);
+ }
+ catch (Throwable th)
+ {
+ markAndThrow(th);
+ }
+ }
+ }
+ catch (Throwable t)
+ {
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ protected abstract void verifyPartition(DecoratedKey key, UnfilteredRowIterator iterator);
+
+ protected void verifyIndex()
+ {
+ try
+ {
+ outputHandler.debug("Deserializing index for %s", sstable);
+ deserializeIndex(sstable);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t);
+ markAndThrow(t);
+ }
+ }
+
+ private void deserializeIndex(SSTableReader sstable) throws IOException
+ {
+ try (KeyReader it = sstable.keyReader())
+ {
+ ByteBuffer last = it.key();
+ while (it.advance()) last = it.key(); // no-op, just check if index is readable
+ if (!Objects.equals(last, sstable.last.getKey()))
+ throw new CorruptSSTableException(new IOException("Failed to read partition index"), it.toString());
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ fileAccessLock.writeLock().lock();
+ try
+ {
+ FileUtils.closeQuietly(dataFile);
+ }
+ finally
+ {
+ fileAccessLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Use the fact that check(...) is called with sorted tokens - we keep a pointer in to the normalized ranges
+ * and only bump the pointer if the key given is out of range. This is done to avoid calling .contains(..) many
+ * times for each key (with vnodes for example)
+ */
+ @VisibleForTesting
+ public static class RangeOwnHelper
+ {
+ private final List<Range<Token>> normalizedRanges;
+ private int rangeIndex = 0;
+ private DecoratedKey lastKey;
+
+ public RangeOwnHelper(List<Range<Token>> normalizedRanges)
+ {
+ this.normalizedRanges = normalizedRanges;
+ Range.assertNormalized(normalizedRanges);
+ }
+
+ /**
+ * check if the given key is contained in any of the given ranges
+ * <p>
+ * Must be called in sorted order - key should be increasing
+ *
+ * @param key the key
+ * @throws RuntimeException if the key is not contained
+ */
+ public void validate(DecoratedKey key)
+ {
+ if (!check(key))
+ throw new RuntimeException("Key " + key + " is not contained in the given ranges");
+ }
+
+ /**
+ * check if the given key is contained in any of the given ranges
+ * <p>
+ * Must be called in sorted order - key should be increasing
+ *
+ * @param key the key
+ * @return boolean
+ */
+ public boolean check(DecoratedKey key)
+ {
+ assert lastKey == null || key.compareTo(lastKey) > 0;
+ lastKey = key;
+
+ if (normalizedRanges.isEmpty()) // handle tests etc. where we don't have any ranges
+ return true;
+
+ if (rangeIndex > normalizedRanges.size() - 1)
+ throw new IllegalStateException("RangeOwnHelper can only be used to find the first out-of-range-token");
+
+ while (!normalizedRanges.get(rangeIndex).contains(key.getToken()))
+ {
+ rangeIndex++;
+ if (rangeIndex > normalizedRanges.size() - 1)
+ return false;
+ }
+
+ return true;
+ }
+ }
+
+ protected static class VerifyInfo extends CompactionInfo.Holder
+ {
+ private final RandomAccessReader dataFile;
+ private final SSTableReader sstable;
+ private final TimeUUID verificationCompactionId;
+ private final Lock fileReadLock;
+
+ public VerifyInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock)
+ {
+ this.dataFile = dataFile;
+ this.sstable = sstable;
+ this.fileReadLock = fileReadLock;
+ verificationCompactionId = TimeUUID.Generator.nextTimeUUID();
+ }
+
+ public CompactionInfo getCompactionInfo()
+ {
+ fileReadLock.lock();
+ try
+ {
+ return new CompactionInfo(sstable.metadata(),
+ OperationType.VERIFY,
+ dataFile.getFilePointer(),
+ dataFile.length(),
+ verificationCompactionId,
+ ImmutableSet.of(sstable));
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException();
+ }
+ finally
+ {
+ fileReadLock.unlock();
+ }
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ }
+
+ protected static class VerifyController extends CompactionController
+ {
+ public VerifyController(ColumnFamilyStore cfs)
+ {
+ super(cfs, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public LongPredicate getPurgeEvaluator(DecoratedKey key)
+ {
+ return time -> false;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
new file mode 100644
index 0000000..377169f
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.nio.BufferOverflowException;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionPurger;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.PartitionSerializationException;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A generic implementation of a writer which assumes the existence of some partition index and bloom filter.
+ */
+public abstract class SortedTableWriter<P extends SortedTablePartitionWriter> extends SSTableWriter
+{
+ private final static Logger logger = LoggerFactory.getLogger(SortedTableWriter.class);
+
+ // TODO dataWriter is not needed to be directly accessible - we can access everything we need for the dataWriter
+ // from a partition writer
+ protected final SequentialWriter dataWriter;
+ protected final P partitionWriter;
+ private final FileHandle.Builder dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.DATA));
+ private DecoratedKey lastWrittenKey;
+ private DataPosition dataMark;
+ private long lastEarlyOpenLength;
+
+ public SortedTableWriter(Builder<P, ?, ?> builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner)
+ {
+ super(builder, lifecycleNewTracker, owner);
+ checkNotNull(builder.getDataWriter());
+ checkNotNull(builder.getPartitionWriter());
+
+ this.dataWriter = builder.getDataWriter();
+ this.partitionWriter = builder.getPartitionWriter();
+ }
+
+ /**
+ * Appends partition data to this writer.
+ *
+ * @param partition the partition to write
+ * @return the created index entry if something was written, that is if {@code iterator} wasn't empty,
+ * {@code null} otherwise.
+ * @throws FSWriteError if write to the dataFile fails
+ */
+ @Override
+ public final AbstractRowIndexEntry append(UnfilteredRowIterator partition)
+ {
+ if (partition.isEmpty())
+ return null;
+
+ try
+ {
+ if (!verifyPartition(partition.partitionKey()))
+ return null;
+
+ startPartition(partition.partitionKey(), partition.partitionLevelDeletion());
+
+ AbstractRowIndexEntry indexEntry;
+ if (header.hasStatic())
+ addStaticRow(partition.partitionKey(), partition.staticRow());
+
+ while (partition.hasNext())
+ addUnfiltered(partition.partitionKey(), partition.next());
+
+ indexEntry = endPartition(partition.partitionKey(), partition.partitionLevelDeletion());
+
+ return indexEntry;
+ }
+ catch (BufferOverflowException boe)
+ {
+ throw new PartitionSerializationException(partition, boe);
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, getFilename());
+ }
+ }
+
+ private boolean verifyPartition(DecoratedKey key)
+ {
+ assert key != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values
+
+ if (key.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+ {
+ logger.error("Key size {} exceeds maximum of {}, skipping row", key.getKey().remaining(), FBUtilities.MAX_UNSIGNED_SHORT);
+ return false;
+ }
+
+ if (lastWrittenKey != null && lastWrittenKey.compareTo(key) >= 0)
+ throw new RuntimeException(String.format("Last written key %s >= current key %s, writing into %s", lastWrittenKey, key, getFilename()));
+
+ return true;
+ }
+
+ private void startPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException
+ {
+ partitionWriter.start(key, partitionLevelDeletion);
+ metadataCollector.updatePartitionDeletion(partitionLevelDeletion);
+
+ onStartPartition(key);
+ }
+
+ private void addStaticRow(DecoratedKey key, Row row) throws IOException
+ {
+ guardCollectionSize(key, row);
+
+ partitionWriter.addStaticRow(row);
+ if (!row.isEmpty())
+ Rows.collectStats(row, metadataCollector);
+
+ onStaticRow(row);
+ }
+
+ private void addUnfiltered(DecoratedKey key, Unfiltered unfiltered) throws IOException
+ {
+ if (unfiltered.isRow())
+ addRow(key, (Row) unfiltered);
+ else
+ addRangeTomstoneMarker((RangeTombstoneMarker) unfiltered);
+ }
+
+ private void addRow(DecoratedKey key, Row row) throws IOException
+ {
+ guardCollectionSize(key, row);
+
+ partitionWriter.addUnfiltered(row);
+ metadataCollector.updateClusteringValues(row.clustering());
+ Rows.collectStats(row, metadataCollector);
+
+ onRow(row);
+ }
+
+ private void addRangeTomstoneMarker(RangeTombstoneMarker marker) throws IOException
+ {
+ partitionWriter.addUnfiltered(marker);
+
+ metadataCollector.updateClusteringValuesByBoundOrBoundary(marker.clustering());
+ if (marker.isBoundary())
+ {
+ RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker) marker;
+ metadataCollector.update(bm.endDeletionTime());
+ metadataCollector.update(bm.startDeletionTime());
+ }
+ else
+ {
+ metadataCollector.update(((RangeTombstoneBoundMarker) marker).deletionTime());
+ }
+
+ onRangeTombstoneMarker(marker);
+ }
+
+ private AbstractRowIndexEntry endPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException
+ {
+ long finishResult = partitionWriter.finish();
+
+ long endPosition = dataWriter.position();
+ long rowSize = endPosition - partitionWriter.getInitialPosition();
+ guardPartitionSize(key, rowSize);
+ maybeLogLargePartitionWarning(key, rowSize);
+ maybeLogManyTombstonesWarning(key, metadataCollector.totalTombstones);
+ metadataCollector.addPartitionSizeInBytes(rowSize);
+ metadataCollector.addKey(key.getKey());
+ metadataCollector.addCellPerPartitionCount();
+
+ lastWrittenKey = key;
+ last = lastWrittenKey;
+ if (first == null)
+ first = lastWrittenKey;
+
+ if (logger.isTraceEnabled())
+ logger.trace("wrote {} at {}", key, endPosition);
+
+ return createRowIndexEntry(key, partitionLevelDeletion, finishResult);
+ }
+
+ protected void onStartPartition(DecoratedKey key)
+ {
+ notifyObservers(o -> o.startPartition(key, partitionWriter.getInitialPosition(), partitionWriter.getInitialPosition()));
+ }
+
+ protected void onStaticRow(Row row)
+ {
+ notifyObservers(o -> o.staticRow(row));
+ }
+
+ protected void onRow(Row row)
+ {
+ notifyObservers(o -> o.nextUnfilteredCluster(row));
+ }
+
+ protected void onRangeTombstoneMarker(RangeTombstoneMarker marker)
+ {
+ notifyObservers(o -> o.nextUnfilteredCluster(marker));
+ }
+
+ protected abstract AbstractRowIndexEntry createRowIndexEntry(DecoratedKey key, DeletionTime partitionLevelDeletion, long finishResult) throws IOException;
+
+ protected final void notifyObservers(Consumer<SSTableFlushObserver> action)
+ {
+ if (observers != null && !observers.isEmpty())
+ observers.forEach(action);
+ }
+
+ @Override
+ public void mark()
+ {
+ dataMark = dataWriter.mark();
+ }
+
+ @Override
+ public void resetAndTruncate()
+ {
+ dataWriter.resetAndTruncate(dataMark);
+ partitionWriter.reset();
+ }
+
+ @Override
+ public long getFilePointer()
+ {
+ return dataWriter.position();
+ }
+
+ @Override
+ public long getOnDiskFilePointer()
+ {
+ return dataWriter.getOnDiskFilePointer();
+ }
+
+ @Override
+ public long getEstimatedOnDiskBytesWritten()
+ {
+ return dataWriter.getEstimatedOnDiskBytesWritten();
+ }
+
+ protected FileHandle openDataFile(long lengthOverride, StatsMetadata statsMetadata)
+ {
+ int dataBufferSize = ioOptions.diskOptimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(ioOptions.diskOptimizationEstimatePercentile));
+
+
+ FileHandle dataFile;
+ try (CompressionMetadata compressionMetadata = compression ? ((CompressedSequentialWriter) dataWriter).open(lengthOverride) : null)
+ {
+ dataFile = dataFileBuilder.mmapped(ioOptions.defaultDiskAccessMode)
+ .withMmappedRegionsCache(mmappedRegionsCache)
+ .withChunkCache(chunkCache)
+ .withCompressionMetadata(compressionMetadata)
+ .bufferSize(dataBufferSize)
+ .withLengthOverride(lengthOverride)
+ .complete();
+ }
+
+ try
+ {
+ if (chunkCache != null)
+ {
+ if (lastEarlyOpenLength != 0 && dataFile.dataLength() > lastEarlyOpenLength)
+ chunkCache.invalidatePosition(dataFile, lastEarlyOpenLength);
+ }
+ lastEarlyOpenLength = dataFile.dataLength();
+ }
+ catch (RuntimeException | Error ex)
+ {
+ Throwables.closeNonNullAndAddSuppressed(ex, dataFile);
+ throw ex;
+ }
+
+ return dataFile;
+ }
+
+ private void guardPartitionSize(DecoratedKey key, long rowSize)
+ {
+ if (Guardrails.partitionSize.triggersOn(rowSize, null))
+ {
+ String what = String.format("%s.%s:%s on sstable %s",
+ metadata.keyspace,
+ metadata.name,
+ metadata().partitionKeyType.getString(key.getKey()),
+ getFilename());
+ Guardrails.partitionSize.guard(rowSize, what, true, null);
+ }
+ }
+
+ private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize)
+ {
+ if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
+ {
+ String keyString = metadata().partitionKeyType.getString(key.getKey());
+ logger.warn("Writing large partition {}/{}:{} ({}) to sstable {}", metadata.keyspace, metadata.name, keyString, FBUtilities.prettyPrintMemory(rowSize), getFilename());
+ }
+ }
+
+ private void maybeLogManyTombstonesWarning(DecoratedKey key, int tombstoneCount)
+ {
+ if (tombstoneCount > DatabaseDescriptor.getCompactionTombstoneWarningThreshold())
+ {
+ String keyString = metadata().partitionKeyType.getString(key.getKey());
+ logger.warn("Writing {} tombstones to {}/{}:{} in sstable {}", tombstoneCount, metadata.keyspace, metadata.name, keyString, getFilename());
+ }
+ }
+
+ private void guardCollectionSize(DecoratedKey partitionKey, Row row)
+ {
+ if (!Guardrails.collectionSize.enabled() && !Guardrails.itemsPerCollection.enabled())
+ return;
+
+ if (row.isEmpty() || SchemaConstants.isSystemKeyspace(metadata.keyspace))
+ return;
+
+ for (ColumnMetadata column : row.columns())
+ {
+ if (!column.type.isCollection() || !column.type.isMultiCell())
+ continue;
+
+ ComplexColumnData cells = row.getComplexColumnData(column);
+ if (cells == null)
+ continue;
+
+ ComplexColumnData liveCells = cells.purge(DeletionPurger.PURGE_ALL, FBUtilities.nowInSeconds());
+ if (liveCells == null)
+ continue;
+
+ int cellsSize = liveCells.dataSize();
+ int cellsCount = liveCells.cellsCount();
+
+ if (!Guardrails.collectionSize.triggersOn(cellsSize, null) &&
+ !Guardrails.itemsPerCollection.triggersOn(cellsCount, null))
+ continue;
+
+ String keyString = metadata.getLocal().primaryKeyAsCQLLiteral(partitionKey.getKey(), row.clustering());
+ String msg = String.format("%s in row %s in table %s",
+ column.name.toString(),
+ keyString,
+ metadata);
+ Guardrails.collectionSize.guard(cellsSize, msg, true, null);
+ Guardrails.itemsPerCollection.guard(cellsCount, msg, true, null);
+ }
+ }
+
+ protected static abstract class AbstractIndexWriter extends AbstractTransactional implements Transactional
+ {
+ protected final Descriptor descriptor;
+ protected final TableMetadataRef metadata;
+ protected final Set<Component> components;
+
+ protected final IFilter bf;
+
+ protected AbstractIndexWriter(Builder<?, ?, ?> b)
+ {
+ this.descriptor = b.descriptor;
+ this.metadata = b.getTableMetadataRef();
+ this.components = b.getComponents();
+
+ bf = FilterFactory.getFilter(b.getKeyCount(), b.getTableMetadataRef().getLocal().params.bloomFilterFpChance);
+ }
+
+ protected void flushBf()
+ {
+ if (components.contains(Components.FILTER))
+ {
+ try
+ {
+ FilterComponent.save(bf, descriptor, true);
+ }
+ catch (IOException ex)
+ {
+ throw new FSWriteError(ex, descriptor.fileFor(Components.FILTER));
+ }
+ }
+ }
+
+ protected void doPrepare()
+ {
+ flushBf();
+ }
+
+ @Override
+ protected Throwable doPostCleanup(Throwable accumulate)
+ {
+ accumulate = bf.close(accumulate);
+ return accumulate;
+ }
+
+ public IFilter getFilterCopy()
+ {
+ return bf.sharedCopy();
+ }
+ }
+
+ public abstract static class Builder<P extends SortedTablePartitionWriter,
+ W extends SortedTableWriter<P>,
+ B extends Builder<P, W, B>> extends SSTableWriter.Builder<W, B>
+ {
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ @Override
+ public B addDefaultComponents()
+ {
+ super.addDefaultComponents();
+
+ if (FilterComponent.shouldUseBloomFilter(getTableMetadataRef().getLocal().params.bloomFilterFpChance))
+ {
+ addComponents(ImmutableSet.of(SSTableFormat.Components.FILTER));
+ }
+
+ return (B) this;
+ }
+
+ public abstract SequentialWriter getDataWriter();
+
+ public abstract P getPartitionWriter();
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java b/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java
new file mode 100644
index 0000000..25042e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.exceptions.UnknownColumnException;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class StatsComponent
+{
+ public final Map<MetadataType, MetadataComponent> metadata;
+
+ public StatsComponent(Map<MetadataType, MetadataComponent> metadata)
+ {
+ this.metadata = ImmutableMap.copyOf(metadata);
+ }
+
+ public static StatsComponent load(Descriptor descriptor) throws IOException
+ {
+ return load(descriptor, MetadataType.values());
+ }
+
+ public static StatsComponent load(Descriptor descriptor, MetadataType... types) throws IOException
+ {
+ Map<MetadataType, MetadataComponent> metadata;
+ try
+ {
+ metadata = descriptor.getMetadataSerializer().deserialize(descriptor, EnumSet.copyOf(Arrays.asList(types)));
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, descriptor.fileFor(Components.STATS));
+ }
+
+ return new StatsComponent(metadata);
+ }
+
+ public SerializationHeader.Component serializationHeader()
+ {
+ return (SerializationHeader.Component) metadata.get(MetadataType.HEADER);
+ }
+
+ public SerializationHeader serializationHeader(TableMetadata metadata)
+ {
+ SerializationHeader.Component header = serializationHeader();
+ if (header != null)
+ {
+ try
+ {
+ return header.toHeader(metadata);
+ }
+ catch (UnknownColumnException ex)
+ {
+ throw new IllegalArgumentException(ex);
+ }
+ }
+
+ return null;
+ }
+
+ public CompactionMetadata compactionMetadata()
+ {
+ return (CompactionMetadata) metadata.get(MetadataType.COMPACTION);
+ }
+
+ public ValidationMetadata validationMetadata()
+ {
+ return (ValidationMetadata) metadata.get(MetadataType.VALIDATION);
+ }
+
+ public StatsMetadata statsMetadata()
+ {
+ return (StatsMetadata) metadata.get(MetadataType.STATS);
+ }
+
+ public void save(Descriptor desc)
+ {
+ File file = desc.fileFor(Components.STATS);
+ try (SequentialWriter out = new SequentialWriter(file, SequentialWriterOption.DEFAULT))
+ {
+ desc.getMetadataSerializer().serialize(metadata, out, desc.version);
+ out.finish();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, file.path());
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java
new file mode 100644
index 0000000..2915d36
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.io.FileNotFoundException;
+import java.io.IOError;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+
+import static org.apache.cassandra.io.util.File.WriteMode.APPEND;
+
+public class TOCComponent
+{
+ private static final Logger logger = LoggerFactory.getLogger(TOCComponent.class);
+
+ /**
+ * Reads the list of components from the TOC component.
+ *
+ * @return set of components found in the TOC
+ */
+ public static Set<Component> loadTOC(Descriptor descriptor) throws IOException
+ {
+ return loadTOC(descriptor, true);
+ }
+
+ /**
+ * Reads the list of components from the TOC component.
+ *
+ * @param skipMissing, skip adding the component to the returned set if the corresponding file is missing.
+ * @return set of components found in the TOC
+ */
+ public static Set<Component> loadTOC(Descriptor descriptor, boolean skipMissing) throws IOException
+ {
+ File tocFile = descriptor.fileFor(Components.TOC);
+ List<String> componentNames = Files.readAllLines(tocFile.toPath());
+ Set<Component> components = Sets.newHashSetWithExpectedSize(componentNames.size());
+ for (String componentName : componentNames)
+ {
+ Component component = Component.parse(componentName, descriptor.version.format);
+ if (skipMissing && !descriptor.fileFor(component).exists())
+ logger.error("Missing component: {}", descriptor.fileFor(component));
+ else
+ components.add(component);
+ }
+ return components;
+ }
+
+ /**
+ * Appends new component names to the TOC component.
+ */
+ public static void appendTOC(Descriptor descriptor, Collection<Component> components)
+ {
+ File tocFile = descriptor.fileFor(Components.TOC);
+ try (FileOutputStreamPlus out = tocFile.newOutputStream(APPEND);
+ PrintWriter w = new PrintWriter(out))
+ {
+ for (Component component : components)
+ w.println(component.name);
+ w.flush();
+ out.sync();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, tocFile);
+ }
+ }
+
+ public static Set<Component> loadOrCreate(Descriptor descriptor)
+ {
+ try
+ {
+ try
+ {
+ return TOCComponent.loadTOC(descriptor);
+ }
+ catch (FileNotFoundException | NoSuchFileException e)
+ {
+ Set<Component> components = descriptor.discoverComponents();
+ if (components.isEmpty())
+ return components; // sstable doesn't exist yet
+
+ components.add(Components.TOC);
+ TOCComponent.appendTOC(descriptor, components);
+ return components;
+ }
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index aa41b14..137979a 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -17,30 +17,31 @@
*/
package org.apache.cassandra.io.sstable.format;
+import java.util.Objects;
import java.util.regex.Pattern;
/**
* A set of feature flags associated with a SSTable format
- *
+ * <p>
* versions are denoted as [major][minor]. Minor versions must be forward-compatible:
* new fields are allowed in e.g. the metadata component, but fields can't be removed
* or have their size changed.
- *
+ * <p>
* Minor versions were introduced with version "hb" for Cassandra 1.0.3; prior to that,
* we always incremented the major version.
- *
*/
public abstract class Version
{
private static final Pattern VALIDATION = Pattern.compile("[a-z]+");
- protected final String version;
- protected final SSTableFormat format;
+ public final String version;
+ public final SSTableFormat<?, ?> format;
+
protected Version(SSTableFormat format, String version)
{
- this.format = format;
- this.version = version;
+ this.format = Objects.requireNonNull(format);
+ this.version = Objects.requireNonNull(version);
}
public abstract boolean isLatestVersion();
@@ -62,21 +63,30 @@
/**
* The old bloomfilter format serializes the data as BIG_ENDIAN long's, the new one uses the
* same format as in memory (serializes as bytes).
+ *
* @return True if the bloomfilter file is old serialization format
*/
public abstract boolean hasOldBfFormat();
+ /**
+ * @deprecated it is replaced by {@link #hasImprovedMinMax()} since 'nc' and to be completetly removed since 'oa'
+ */
+ @Deprecated
public abstract boolean hasAccurateMinMax();
- public String getVersion()
- {
- return version;
- }
+ /**
+ * @deprecated it is replaced by {@link #hasImprovedMinMax()} since 'nc' and to be completetly removed since 'oa'
+ */
+ @Deprecated
+ public abstract boolean hasLegacyMinMax();
- public SSTableFormat getSSTableFormat()
- {
- return format;
- }
+ public abstract boolean hasOriginatingHostId();
+
+ public abstract boolean hasImprovedMinMax();
+
+ public abstract boolean hasPartitionLevelDeletionsPresenceMarker();
+
+ public abstract boolean hasKeyRange();
/**
* @param ver SSTable version
@@ -89,6 +99,7 @@
}
abstract public boolean isCompatible();
+
abstract public boolean isCompatibleForStreaming();
@Override
@@ -97,24 +108,24 @@
return version;
}
- @Override
- public boolean equals(Object o)
+ public String toFormatAndVersionString()
{
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ return format.name() + '-' + version;
+ }
- Version version1 = (Version) o;
+ @Override
+ public boolean equals(Object other)
+ {
+ if (this == other) return true;
+ if (other == null || getClass() != other.getClass()) return false;
- if (version != null ? !version.equals(version1.version) : version1.version != null) return false;
-
- return true;
+ Version otherVersion = (Version) other;
+ return Objects.equals(version, otherVersion.version) && Objects.equals(format.name(), otherVersion.format.name());
}
@Override
public int hashCode()
{
- return version != null ? version.hashCode() : 0;
+ return Objects.hash(version, format.name());
}
-
- public abstract boolean hasOriginatingHostId();
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java b/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java
deleted file mode 100644
index 7d698a9..0000000
--- a/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.io.sstable.format;
-
-import java.util.List;
-import java.util.Objects;
-
-import com.google.common.base.Splitter;
-
-import org.apache.cassandra.io.sstable.Descriptor;
-
-/**
- * Groups a sstable {@link Version} with a {@link SSTableFormat.Type}.
- *
- * <p>Note that both information are currently necessary to identify the exact "format" of an sstable (without having
- * its {@link Descriptor}). In particular, while {@link Version} contains its {{@link SSTableFormat}}, you cannot get
- * the {{@link SSTableFormat.Type}} from that.
- */
-public final class VersionAndType
-{
- private static final Splitter splitOnDash = Splitter.on('-').omitEmptyStrings().trimResults();
-
- private final Version version;
- private final SSTableFormat.Type formatType;
-
- public VersionAndType(Version version, SSTableFormat.Type formatType)
- {
- this.version = version;
- this.formatType = formatType;
- }
-
- public Version version()
- {
- return version;
- }
-
- public SSTableFormat.Type formatType()
- {
- return formatType;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- VersionAndType that = (VersionAndType) o;
- return Objects.equals(version, that.version) &&
- formatType == that.formatType;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(version, formatType);
- }
-
- public static VersionAndType fromString(String versionAndType)
- {
- List<String> components = splitOnDash.splitToList(versionAndType);
- if (components.size() != 2)
- throw new IllegalArgumentException("Invalid VersionAndType string: " + versionAndType + " (should be of the form 'big-bc')");
-
- SSTableFormat.Type formatType = SSTableFormat.Type.validate(components.get(0));
- Version version = formatType.info.getVersion(components.get(1));
- return new VersionAndType(version, formatType);
- }
-
- @Override
- public String toString()
- {
- return formatType.name + '-' + version;
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index c84782f..88ca15c 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -17,33 +17,137 @@
*/
package org.apache.cassandra.io.sstable.format.big;
-import java.util.Collection;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.*;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.MetricsProviders;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics;
+import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.SortedTableScrubber;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryMetrics;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheMetrics;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.io.sstable.format.SSTableFormat.Components.DATA;
/**
* Legacy bigtable format
*/
-public class BigFormat implements SSTableFormat
+public class BigFormat extends AbstractSSTableFormat<BigTableReader, BigTableWriter>
{
- public static final BigFormat instance = new BigFormat();
- public static final Version latestVersion = new BigVersion(BigVersion.current_version);
- private static final SSTableReader.Factory readerFactory = new ReaderFactory();
- private static final SSTableWriter.Factory writerFactory = new WriterFactory();
+ private final static Logger logger = LoggerFactory.getLogger(BigFormat.class);
- private BigFormat()
+ public static final String NAME = "big";
+
+ private final Version latestVersion = new BigVersion(this, BigVersion.current_version);
+ private final BigTableReaderFactory readerFactory = new BigTableReaderFactory();
+ private final BigTableWriterFactory writerFactory = new BigTableWriterFactory();
+
+ public static class Components extends SSTableFormat.Components
{
+ public static class Types extends SSTableFormat.Components.Types
+ {
+ // index of the row keys with pointers to their positions in the data file
+ public static final Component.Type PRIMARY_INDEX = Component.Type.createSingleton("PRIMARY_INDEX", "Index.db", BigFormat.class);
+ // holds SSTable Index Summary (sampling of Index component)
+ public static final Component.Type SUMMARY = Component.Type.createSingleton("SUMMARY", "Summary.db", BigFormat.class);
+ }
+ public final static Component PRIMARY_INDEX = Types.PRIMARY_INDEX.getSingleton();
+ public final static Component SUMMARY = Types.SUMMARY.getSingleton();
+
+ private static final Set<Component> BATCH_COMPONENTS = ImmutableSet.of(DATA,
+ PRIMARY_INDEX,
+ COMPRESSION_INFO,
+ FILTER,
+ STATS);
+
+ private static final Set<Component> PRIMARY_COMPONENTS = ImmutableSet.of(DATA,
+ PRIMARY_INDEX);
+
+ private static final Set<Component> GENERATED_ON_LOAD_COMPONENTS = ImmutableSet.of(FILTER, SUMMARY);
+
+ private static final Set<Component> MUTABLE_COMPONENTS = ImmutableSet.of(STATS,
+ SUMMARY);
+
+ private static final Set<Component> UPLOAD_COMPONENTS = ImmutableSet.of(DATA,
+ PRIMARY_INDEX,
+ SUMMARY,
+ COMPRESSION_INFO,
+ STATS);
+
+ private static final Set<Component> STREAM_COMPONENTS = ImmutableSet.of(DATA,
+ PRIMARY_INDEX,
+ STATS,
+ COMPRESSION_INFO,
+ FILTER,
+ SUMMARY,
+ DIGEST,
+ CRC);
+
+ private static final Set<Component> ALL_COMPONENTS = ImmutableSet.of(DATA,
+ PRIMARY_INDEX,
+ STATS,
+ COMPRESSION_INFO,
+ FILTER,
+ SUMMARY,
+ DIGEST,
+ CRC,
+ TOC);
+ }
+
+ public BigFormat(Map<String, String> options)
+ {
+ super(NAME, options);
+ }
+
+ public static boolean is(SSTableFormat<?, ?> format)
+ {
+ return format.name().equals(NAME);
+ }
+
+ public static BigFormat getInstance()
+ {
+ return (BigFormat) Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(NAME), "Unknown SSTable format: " + NAME);
+ }
+
+ public static boolean isSelected()
+ {
+ return is(DatabaseDescriptor.getSelectedSSTableFormat());
}
@Override
@@ -55,28 +159,180 @@
@Override
public Version getVersion(String version)
{
- return new BigVersion(version);
+ return new BigVersion(this, version);
}
@Override
- public SSTableWriter.Factory getWriterFactory()
+ public BigTableWriterFactory getWriterFactory()
{
return writerFactory;
}
@Override
- public SSTableReader.Factory getReaderFactory()
+ public BigTableReaderFactory getReaderFactory()
{
return readerFactory;
}
@Override
- public RowIndexEntry.IndexSerializer getIndexSerializer(TableMetadata metadata, Version version, SerializationHeader header)
+ public Set<Component> allComponents()
{
- return new RowIndexEntry.Serializer(version, header);
+ return Components.ALL_COMPONENTS;
}
- static class WriterFactory extends SSTableWriter.Factory
+ @Override
+ public Set<Component> streamingComponents()
+ {
+ return Components.STREAM_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> primaryComponents()
+ {
+ return Components.PRIMARY_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> batchComponents()
+ {
+ return Components.BATCH_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> uploadComponents()
+ {
+ return Components.UPLOAD_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> mutableComponents()
+ {
+ return Components.MUTABLE_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> generatedOnLoadComponents()
+ {
+ return Components.GENERATED_ON_LOAD_COMPONENTS;
+ }
+
+ @Override
+ public SSTableFormat.KeyCacheValueSerializer<BigTableReader, RowIndexEntry> getKeyCacheValueSerializer()
+ {
+ return KeyCacheValueSerializer.instance;
+ }
+
+ @Override
+ public IScrubber getScrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, OutputHandler outputHandler, IScrubber.Options options)
+ {
+ Preconditions.checkArgument(cfs.metadata().equals(transaction.onlyOne().metadata()), "SSTable metadata does not match current definition");
+ return new BigTableScrubber(cfs, transaction, outputHandler, options);
+ }
+
+ @Override
+ public MetricsProviders getFormatSpecificMetricsProviders()
+ {
+ return BigTableSpecificMetricsProviders.instance;
+ }
+
+ @Override
+ public void deleteOrphanedComponents(Descriptor descriptor, Set<Component> components)
+ {
+ SortedTableScrubber.deleteOrphanedComponents(descriptor, components);
+ }
+
+ private void delete(Descriptor desc, List<Component> components)
+ {
+ logger.info("Deleting sstable: {}", desc);
+
+ if (components.remove(DATA))
+ components.add(0, DATA); // DATA component should be first
+ if (components.remove(Components.SUMMARY))
+ components.add(Components.SUMMARY); // SUMMARY component should be last (IDK why)
+
+ for (Component component : components)
+ {
+ logger.trace("Deleting component {} of {}", component, desc);
+ desc.fileFor(component).deleteIfExists();
+ }
+ }
+
+ @Override
+ public void delete(Descriptor desc)
+ {
+ try
+ {
+ // remove key cache entries for the sstable being deleted
+ Iterator<KeyCacheKey> it = CacheService.instance.keyCache.keyIterator();
+ while (it.hasNext())
+ {
+ KeyCacheKey key = it.next();
+ if (key.desc.equals(desc))
+ it.remove();
+ }
+
+ delete(desc, Lists.newArrayList(Sets.intersection(allComponents(), desc.discoverComponents())));
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ static class KeyCacheValueSerializer implements SSTableFormat.KeyCacheValueSerializer<BigTableReader, RowIndexEntry>
+ {
+ private final static KeyCacheValueSerializer instance = new KeyCacheValueSerializer();
+
+ @Override
+ public void skip(DataInputPlus input) throws IOException
+ {
+ RowIndexEntry.Serializer.skipForCache(input);
+ }
+
+ @Override
+ public RowIndexEntry deserialize(BigTableReader reader, DataInputPlus input) throws IOException
+ {
+ return reader.deserializeKeyCacheValue(input);
+ }
+
+ @Override
+ public void serialize(RowIndexEntry entry, DataOutputPlus output) throws IOException
+ {
+ entry.serializeForCache(output);
+ }
+ }
+
+ static class BigTableReaderFactory implements SSTableReaderFactory<BigTableReader, BigTableReader.Builder>
+ {
+ @Override
+ public BigTableReader.Builder builder(Descriptor descriptor)
+ {
+ return new BigTableReader.Builder(descriptor);
+ }
+
+ @Override
+ public SSTableReaderLoadingBuilder<BigTableReader, BigTableReader.Builder> loadingBuilder(Descriptor descriptor,
+ TableMetadataRef tableMetadataRef,
+ Set<Component> components)
+ {
+ return new BigSSTableReaderLoadingBuilder(new SSTable.Builder<>(descriptor).setTableMetadataRef(tableMetadataRef)
+ .setComponents(components));
+ }
+
+ @Override
+ public Pair<DecoratedKey, DecoratedKey> readKeyRange(Descriptor descriptor, IPartitioner partitioner) throws IOException
+ {
+ return IndexSummaryComponent.loadFirstAndLastKey(descriptor.fileFor(Components.SUMMARY), partitioner);
+ }
+
+ @Override
+ public Class<BigTableReader> getReaderClass()
+ {
+ return BigTableReader.class;
+ }
+ }
+
+ static class BigTableWriterFactory implements SSTableWriterFactory<BigTableWriter, BigTableWriter.Builder>
{
@Override
public long estimateSize(SSTableWriter.SSTableSizeParameters parameters)
@@ -88,40 +344,15 @@
}
@Override
- public SSTableWriter open(Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- TableMetadataRef metadata,
- MetadataCollector metadataCollector,
- SerializationHeader header,
- Collection<SSTableFlushObserver> observers,
- LifecycleNewTracker lifecycleNewTracker)
+ public BigTableWriter.Builder builder(Descriptor descriptor)
{
- SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient);
- return new BigTableWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers, lifecycleNewTracker);
+ return new BigTableWriter.Builder(descriptor);
}
}
- static class ReaderFactory extends SSTableReader.Factory
- {
- @Override
- public SSTableReader open(SSTableReaderBuilder builder)
- {
- return new BigTableReader(builder);
- }
- }
-
- // versions are denoted as [major][minor]. Minor versions must be forward-compatible:
- // new fields are allowed in e.g. the metadata component, but fields can't be removed
- // or have their size changed.
- //
- // Minor versions were introduced with version "hb" for Cassandra 1.0.3; prior to that,
- // we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "nb";
+ public static final String current_version = "nc";
public static final String earliest_supported_version = "ma";
// ma (3.0.0): swap bf hash order
@@ -133,19 +364,24 @@
// na (4.0-rc1): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format
// nb (4.0.0): originating host id
+ // nc (4.1): improved min/max, partition level deletion presence marker, key range (CASSANDRA-18134)
//
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
private final boolean isLatestVersion;
- public final int correspondingMessagingVersion;
+ private final int correspondingMessagingVersion;
private final boolean hasCommitLogLowerBound;
private final boolean hasCommitLogIntervals;
private final boolean hasAccurateMinMax;
+ private final boolean hasLegacyMinMax;
private final boolean hasOriginatingHostId;
- public final boolean hasMaxCompressedLength;
+ private final boolean hasMaxCompressedLength;
private final boolean hasPendingRepair;
private final boolean hasMetadataChecksum;
private final boolean hasIsTransient;
+ private final boolean hasImprovedMinMax;
+ private final boolean hasPartitionLevelDeletionPresenceMarker;
+ private final boolean hasKeyRange;
/**
* CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped)
@@ -153,22 +389,26 @@
*/
private final boolean hasOldBfFormat;
- BigVersion(String version)
+ BigVersion(BigFormat format, String version)
{
- super(instance, version);
+ super(format, version);
isLatestVersion = version.compareTo(current_version) == 0;
correspondingMessagingVersion = MessagingService.VERSION_30;
hasCommitLogLowerBound = version.compareTo("mb") >= 0;
hasCommitLogIntervals = version.compareTo("mc") >= 0;
- hasAccurateMinMax = version.compareTo("md") >= 0;
- hasOriginatingHostId = version.matches("(m[e-z])|(n[b-z])");
+ hasAccurateMinMax = version.matches("(m[d-z])|(n[a-z])"); // deprecated in 'nc' and to be removed in 'oa'
+ hasLegacyMinMax = version.matches("(m[a-z])|(n[a-z])"); // deprecated in 'nc' and to be removed in 'oa'
+ hasOriginatingHostId = version.matches("(m[e-z])") || version.compareTo("nb") >= 0;
hasMaxCompressedLength = version.compareTo("na") >= 0;
hasPendingRepair = version.compareTo("na") >= 0;
hasIsTransient = version.compareTo("na") >= 0;
hasMetadataChecksum = version.compareTo("na") >= 0;
hasOldBfFormat = version.compareTo("na") < 0;
+ hasImprovedMinMax = version.compareTo("nc") >= 0;
+ hasPartitionLevelDeletionPresenceMarker = version.compareTo("nc") >= 0;
+ hasKeyRange = version.compareTo("nc") >= 0;
}
@Override
@@ -178,6 +418,12 @@
}
@Override
+ public int correspondingMessagingVersion()
+ {
+ return correspondingMessagingVersion;
+ }
+
+ @Override
public boolean hasCommitLogLowerBound()
{
return hasCommitLogLowerBound;
@@ -189,6 +435,13 @@
return hasCommitLogIntervals;
}
+ @Override
+ public boolean hasMaxCompressedLength()
+ {
+ return hasMaxCompressedLength;
+ }
+
+ @Override
public boolean hasPendingRepair()
{
return hasPendingRepair;
@@ -201,23 +454,54 @@
}
@Override
- public int correspondingMessagingVersion()
- {
- return correspondingMessagingVersion;
- }
-
- @Override
public boolean hasMetadataChecksum()
{
return hasMetadataChecksum;
}
@Override
+ public boolean hasOldBfFormat()
+ {
+ return hasOldBfFormat;
+ }
+
+ @Override
public boolean hasAccurateMinMax()
{
return hasAccurateMinMax;
}
+ @Override
+ public boolean hasLegacyMinMax()
+ {
+ return hasLegacyMinMax;
+ }
+
+ @Override
+ public boolean hasOriginatingHostId()
+ {
+ return hasOriginatingHostId;
+ }
+
+ @Override
+ public boolean hasImprovedMinMax()
+ {
+ return hasImprovedMinMax;
+ }
+
+ @Override
+ public boolean hasPartitionLevelDeletionsPresenceMarker()
+ {
+ return hasPartitionLevelDeletionPresenceMarker;
+ }
+
+ @Override
+ public boolean hasKeyRange()
+ {
+ return hasKeyRange;
+ }
+
+ @Override
public boolean isCompatible()
{
return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
@@ -228,22 +512,36 @@
{
return isCompatible() && version.charAt(0) == current_version.charAt(0);
}
+ }
- public boolean hasOriginatingHostId()
+ private static class BigTableSpecificMetricsProviders implements MetricsProviders
+ {
+ private final static BigTableSpecificMetricsProviders instance = new BigTableSpecificMetricsProviders();
+
+ private final Iterable<GaugeProvider<?>> gaugeProviders = Iterables.concat(BloomFilterMetrics.instance.getGaugeProviders(),
+ IndexSummaryMetrics.instance.getGaugeProviders(),
+ KeyCacheMetrics.instance.getGaugeProviders());
+
+ @Override
+ public Iterable<GaugeProvider<?>> getGaugeProviders()
{
- return hasOriginatingHostId;
+ return gaugeProviders;
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public static class BigFormatFactory implements Factory
+ {
+ @Override
+ public String name()
+ {
+ return NAME;
}
@Override
- public boolean hasMaxCompressedLength()
+ public SSTableFormat<?, ?> getInstance(Map<String, String> options)
{
- return hasMaxCompressedLength;
- }
-
- @Override
- public boolean hasOldBfFormat()
- {
- return hasOldBfFormat;
+ return new BigFormat(options);
}
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
new file mode 100644
index 0000000..801982d
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.ISerializer;
+import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.io.sstable.format.SortedTablePartitionWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.SequentialWriter;
+
+/**
+ * Column index builder used by {@link org.apache.cassandra.io.sstable.format.big.BigTableWriter}.
+ * For index entries that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size},
+ * this uses the serialization logic as in {@link RowIndexEntry}.
+ */
+public class BigFormatPartitionWriter extends SortedTablePartitionWriter
+{
+ @VisibleForTesting
+ public static final int DEFAULT_GRANULARITY = 64 * 1024;
+
+ // used, if the row-index-entry reaches config column_index_cache_size
+ private DataOutputBuffer buffer;
+ // used to track the size of the serialized size of row-index-entry (unused for buffer)
+ private int indexSamplesSerializedSize;
+ // used, until the row-index-entry reaches config column_index_cache_size
+ private final List<IndexInfo> indexSamples = new ArrayList<>();
+
+ private DataOutputBuffer reusableBuffer;
+
+ private int columnIndexCount;
+ private int[] indexOffsets;
+
+ private final ISerializer<IndexInfo> idxSerializer;
+
+ private final int cacheSizeThreshold;
+ private final int indexSize;
+
+ BigFormatPartitionWriter(SerializationHeader header,
+ SequentialWriter writer,
+ Version version,
+ ISerializer<IndexInfo> indexInfoSerializer)
+ {
+ this(header, writer, version, indexInfoSerializer, DatabaseDescriptor.getColumnIndexCacheSize(), DatabaseDescriptor.getColumnIndexSize(DEFAULT_GRANULARITY));
+ }
+
+ BigFormatPartitionWriter(SerializationHeader header,
+ SequentialWriter writer,
+ Version version,
+ ISerializer<IndexInfo> indexInfoSerializer,
+ int cacheSizeThreshold,
+ int indexSize)
+ {
+ super(header, writer, version);
+ this.idxSerializer = indexInfoSerializer;
+ this.cacheSizeThreshold = cacheSizeThreshold;
+ this.indexSize = indexSize;
+ }
+
+ public void reset()
+ {
+ super.reset();
+ this.columnIndexCount = 0;
+ this.indexSamplesSerializedSize = 0;
+ this.indexSamples.clear();
+
+ if (this.buffer != null)
+ this.reusableBuffer = this.buffer;
+ this.buffer = null;
+ }
+
+ public int getColumnIndexCount()
+ {
+ return columnIndexCount;
+ }
+
+ public ByteBuffer buffer()
+ {
+ return buffer != null ? buffer.buffer() : null;
+ }
+
+ public List<IndexInfo> indexSamples()
+ {
+ if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= cacheSizeThreshold)
+ {
+ return indexSamples;
+ }
+
+ return null;
+ }
+
+ public int[] offsets()
+ {
+ return indexOffsets != null
+ ? Arrays.copyOf(indexOffsets, columnIndexCount)
+ : null;
+ }
+
+ private void addIndexBlock() throws IOException
+ {
+ IndexInfo cIndexInfo = new IndexInfo(firstClustering,
+ lastClustering,
+ startPosition,
+ currentPosition() - startPosition,
+ !openMarker.isLive() ? openMarker : null);
+
+ // indexOffsets is used for both shallow (ShallowIndexedEntry) and non-shallow IndexedEntry.
+ // For shallow ones, we need it to serialize the offsts in finish().
+ // For non-shallow ones, the offsts are passed into IndexedEntry, so we don't have to
+ // calculate the offsets again.
+
+ // indexOffsets contains the offsets of the serialized IndexInfo objects.
+ // I.e. indexOffsets[0] is always 0 so we don't have to deal with a special handling
+ // for index #0 and always subtracting 1 for the index (which could be error-prone).
+ if (indexOffsets == null)
+ indexOffsets = new int[10];
+ else
+ {
+ if (columnIndexCount >= indexOffsets.length)
+ indexOffsets = Arrays.copyOf(indexOffsets, indexOffsets.length + 10);
+
+ //the 0th element is always 0
+ if (columnIndexCount == 0)
+ {
+ indexOffsets[columnIndexCount] = 0;
+ }
+ else
+ {
+ indexOffsets[columnIndexCount] =
+ buffer != null
+ ? Ints.checkedCast(buffer.position())
+ : indexSamplesSerializedSize;
+ }
+ }
+ columnIndexCount++;
+
+ // First, we collect the IndexInfo objects until we reach Config.column_index_cache_size in an ArrayList.
+ // When column_index_cache_size is reached, we switch to byte-buffer mode.
+ if (buffer == null)
+ {
+ indexSamplesSerializedSize += idxSerializer.serializedSize(cIndexInfo);
+ if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) > cacheSizeThreshold)
+ {
+ buffer = reuseOrAllocateBuffer();
+ for (IndexInfo indexSample : indexSamples)
+ {
+ idxSerializer.serialize(indexSample, buffer);
+ }
+ }
+ else
+ {
+ indexSamples.add(cIndexInfo);
+ }
+ }
+ // don't put an else here since buffer may be allocated in preceding if block
+ if (buffer != null)
+ {
+ idxSerializer.serialize(cIndexInfo, buffer);
+ }
+
+ firstClustering = null;
+ }
+
+ private DataOutputBuffer reuseOrAllocateBuffer()
+ {
+ // Check whether a reusable DataOutputBuffer already exists for this
+ // ColumnIndex instance and return it.
+ if (reusableBuffer != null)
+ {
+ DataOutputBuffer buffer = reusableBuffer;
+ buffer.clear();
+ return buffer;
+ }
+ // don't use the standard RECYCLER as that only recycles up to 1MB and requires proper cleanup
+ return new DataOutputBuffer(cacheSizeThreshold * 2);
+ }
+
+ @Override
+ public void addUnfiltered(Unfiltered unfiltered) throws IOException
+ {
+ super.addUnfiltered(unfiltered);
+
+ // if we hit the column index size that we have to index after, go ahead and index it.
+ if (currentPosition() - startPosition >= indexSize)
+ addIndexBlock();
+ }
+
+ @Override
+ public long finish() throws IOException
+ {
+ long endPosition = super.finish();
+
+ // It's possible we add no rows, just a top level deletion
+ if (written == 0)
+ return endPosition;
+
+ // the last column may have fallen on an index boundary already. if not, index it explicitly.
+ if (firstClustering != null)
+ addIndexBlock();
+
+ // If we serialize the IndexInfo objects directly in the code above into 'buffer',
+ // we have to write the offsts to these here. The offsets have already been collected
+ // in indexOffsets[]. buffer is != null, if it exceeds Config.column_index_cache_size.
+ // In the other case, when buffer==null, the offsets are serialized in RowIndexEntry.IndexedEntry.serialize().
+ if (buffer != null)
+ {
+ for (int i = 0; i < columnIndexCount; i++)
+ buffer.writeInt(indexOffsets[i]);
+ }
+
+ // we should always have at least one computed index block, but we only write it out if there is more than that.
+ assert columnIndexCount > 0 && getHeaderLength() >= 0;
+
+ return endPosition;
+ }
+
+ public int indexInfoSerializedSize()
+ {
+ return buffer != null
+ ? buffer.buffer().limit()
+ : indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0);
+ }
+
+ @Override
+ public void close()
+ {
+ // no-op
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
new file mode 100644
index 0000000..1ef7206
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.OptionalInt;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
+import org.apache.cassandra.io.sstable.format.FilterComponent;
+import org.apache.cassandra.io.sstable.format.IndexComponent;
+import org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummary;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder;
+import org.apache.cassandra.io.sstable.keycache.KeyCache;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.util.DiskOptimizationStrategy;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Throwables;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class BigSSTableReaderLoadingBuilder extends SortedTableReaderLoadingBuilder<BigTableReader, BigTableReader.Builder>
+{
+ private final static Logger logger = LoggerFactory.getLogger(BigSSTableReaderLoadingBuilder.class);
+
+ private FileHandle.Builder indexFileBuilder;
+
+ public BigSSTableReaderLoadingBuilder(SSTable.Builder<?, ?> descriptor)
+ {
+ super(descriptor);
+ }
+
+ @Override
+ protected void openComponents(BigTableReader.Builder builder, SSTable.Owner owner, boolean validate, boolean online) throws IOException
+ {
+ try
+ {
+ if (online && builder.getTableMetadataRef().getLocal().params.caching.cacheKeys())
+ builder.setKeyCache(new KeyCache(CacheService.instance.keyCache));
+
+ StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION);
+ builder.setSerializationHeader(statsComponent.serializationHeader(builder.getTableMetadataRef().getLocal()));
+ checkArgument(!online || builder.getSerializationHeader() != null);
+
+ builder.setStatsMetadata(statsComponent.statsMetadata());
+ if (descriptor.version.hasKeyRange() && statsComponent.statsMetadata() != null)
+ {
+ builder.setFirst(tableMetadataRef.getLocal().partitioner.decorateKey(statsComponent.statsMetadata().firstKey));
+ builder.setLast(tableMetadataRef.getLocal().partitioner.decorateKey(statsComponent.statsMetadata().lastKey));
+ }
+
+ ValidationMetadata validationMetadata = statsComponent.validationMetadata();
+ validatePartitioner(builder.getTableMetadataRef().getLocal(), validationMetadata);
+
+ boolean filterNeeded = online;
+ if (filterNeeded)
+ builder.setFilter(loadFilter(validationMetadata));
+ boolean rebuildFilter = filterNeeded && builder.getFilter() == null;
+
+ boolean summaryNeeded = true;
+ if (summaryNeeded)
+ {
+ IndexSummaryComponent summaryComponent = loadSummary();
+ if (summaryComponent != null)
+ {
+ if (builder.getFirst() == null || builder.getLast() == null)
+ {
+ builder.setFirst(summaryComponent.first);
+ builder.setLast(summaryComponent.last);
+ }
+ builder.setIndexSummary(summaryComponent.indexSummary);
+ }
+ }
+ boolean rebuildSummary = summaryNeeded && builder.getIndexSummary() == null;
+
+ if (builder.getComponents().contains(Components.PRIMARY_INDEX) && (rebuildFilter || rebuildSummary))
+ {
+ try (FileHandle indexFile = indexFileBuilder(builder.getIndexSummary()).complete())
+ {
+ Pair<IFilter, IndexSummaryComponent> filterAndSummary = buildSummaryAndBloomFilter(indexFile, builder.getSerializationHeader(), rebuildFilter, rebuildSummary, owner != null ? owner.getMetrics() : null);
+ IFilter filter = filterAndSummary.left;
+ IndexSummaryComponent summaryComponent = filterAndSummary.right;
+
+ if (summaryComponent != null)
+ {
+ builder.setFirst(summaryComponent.first);
+ builder.setLast(summaryComponent.last);
+ builder.setIndexSummary(summaryComponent.indexSummary);
+
+ if (online)
+ summaryComponent.save(descriptor.fileFor(Components.SUMMARY), false);
+ }
+
+ if (filter != null)
+ {
+ builder.setFilter(filter);
+
+ if (online)
+ FilterComponent.save(filter, descriptor, false);
+ }
+ }
+ }
+
+ try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components))
+ {
+ builder.setDataFile(dataFileBuilder(builder.getStatsMetadata()).withCompressionMetadata(compressionMetadata).complete());
+ }
+
+ if (builder.getFilter() == null)
+ builder.setFilter(FilterFactory.AlwaysPresent);
+
+ if (builder.getComponents().contains(Components.PRIMARY_INDEX))
+ builder.setIndexFile(indexFileBuilder(builder.getIndexSummary()).complete());
+ }
+ catch (IOException | RuntimeException | Error ex)
+ {
+ Throwables.closeNonNullAndAddSuppressed(ex, builder.getDataFile(), builder.getIndexFile(), builder.getFilter(), builder.getIndexSummary());
+ throw ex;
+ }
+ }
+
+ @Override
+ public KeyReader buildKeyReader(TableMetrics tableMetrics) throws IOException
+ {
+ StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION);
+ SerializationHeader header = statsComponent.serializationHeader(tableMetadataRef.getLocal());
+ try (FileHandle indexFile = indexFileBuilder(null).complete())
+ {
+ return createKeyReader(indexFile, header, tableMetrics);
+ }
+ }
+
+ private KeyReader createKeyReader(FileHandle indexFile, SerializationHeader serializationHeader, TableMetrics tableMetrics) throws IOException
+ {
+ checkNotNull(indexFile);
+ checkNotNull(serializationHeader);
+
+ RowIndexEntry.IndexSerializer serializer = new RowIndexEntry.Serializer(descriptor.version, serializationHeader, tableMetrics);
+ return BigTableKeyReader.create(indexFile, serializer);
+ }
+
+ /**
+ * Go through the index and optionally rebuild the index summary and Bloom filter.
+ *
+ * @param rebuildFilter true if Bloom filter should be rebuilt
+ * @param rebuildSummary true if index summary, first and last keys should be rebuilt
+ * @return a pair of created filter and index summary component (or nulls if some of them were not created)
+ */
+ @SuppressWarnings("resource")
+ private Pair<IFilter, IndexSummaryComponent> buildSummaryAndBloomFilter(FileHandle indexFile,
+ SerializationHeader serializationHeader,
+ boolean rebuildFilter,
+ boolean rebuildSummary,
+ TableMetrics tableMetrics) throws IOException
+ {
+ checkNotNull(indexFile);
+ checkNotNull(serializationHeader);
+
+ DecoratedKey first = null, key = null;
+ IFilter bf = null;
+ IndexSummary indexSummary = null;
+
+ // we read the positions in a BRAF, so we don't have to worry about an entry spanning a mmap boundary.
+ try (KeyReader keyReader = createKeyReader(indexFile, serializationHeader, tableMetrics))
+ {
+ long estimatedRowsNumber = rebuildFilter || rebuildSummary ? estimateRowsFromIndex(indexFile) : 0;
+
+ if (rebuildFilter)
+ bf = FilterFactory.getFilter(estimatedRowsNumber, tableMetadataRef.getLocal().params.bloomFilterFpChance);
+
+ try (IndexSummaryBuilder summaryBuilder = !rebuildSummary ? null : new IndexSummaryBuilder(estimatedRowsNumber,
+ tableMetadataRef.getLocal().params.minIndexInterval,
+ Downsampling.BASE_SAMPLING_LEVEL))
+ {
+ while (!keyReader.isExhausted())
+ {
+ key = tableMetadataRef.getLocal().partitioner.decorateKey(keyReader.key());
+ if (rebuildSummary)
+ {
+ if (first == null)
+ first = key;
+ summaryBuilder.maybeAddEntry(key, keyReader.keyPositionForSecondaryIndex());
+ }
+
+ if (rebuildFilter)
+ bf.add(key);
+
+ keyReader.advance();
+ }
+
+ if (rebuildSummary)
+ indexSummary = summaryBuilder.build(tableMetadataRef.getLocal().partitioner);
+ }
+ }
+ catch (IOException | RuntimeException | Error ex)
+ {
+ Throwables.closeNonNullAndAddSuppressed(ex, indexSummary, bf);
+ throw ex;
+ }
+
+ assert rebuildSummary || indexSummary == null;
+ return Pair.create(bf, rebuildSummary ? new IndexSummaryComponent(indexSummary, first, key) : null);
+ }
+
+ /**
+ * Load index summary, first key and last key from Summary.db file if it exists.
+ * <p>
+ * if loaded index summary has different index interval from current value stored in schema,
+ * then Summary.db file will be deleted and need to be rebuilt.
+ */
+ private IndexSummaryComponent loadSummary()
+ {
+ IndexSummaryComponent summaryComponent = null;
+ try
+ {
+ if (components.contains(Components.SUMMARY))
+ summaryComponent = IndexSummaryComponent.loadOrDeleteCorrupted(descriptor.fileFor(Components.SUMMARY), tableMetadataRef.get());
+
+ if (summaryComponent == null)
+ logger.debug("Index summary file is missing: {}", descriptor.fileFor(Components.SUMMARY));
+ }
+ catch (IOException ex)
+ {
+ logger.debug("Index summary file is corrupted: " + descriptor.fileFor(Components.SUMMARY), ex);
+ }
+
+ return summaryComponent;
+ }
+
+ /**
+ * @return An estimate of the number of keys contained in the given index file.
+ */
+ public long estimateRowsFromIndex(FileHandle indexFile) throws IOException
+ {
+ checkNotNull(indexFile);
+
+ try (RandomAccessReader indexReader = indexFile.createReader())
+ {
+ // collect sizes for the first 10000 keys, or first 10 mebibytes of data
+ final int samplesCap = 10000;
+ final int bytesCap = (int) Math.min(10000000, indexReader.length());
+ int keys = 0;
+ while (indexReader.getFilePointer() < bytesCap && keys < samplesCap)
+ {
+ ByteBufferUtil.skipShortLength(indexReader);
+ RowIndexEntry.Serializer.skip(indexReader, descriptor.version);
+ keys++;
+ }
+ assert keys > 0 && indexReader.getFilePointer() > 0 && indexReader.length() > 0 : "Unexpected empty index file: " + indexReader;
+ long estimatedRows = indexReader.length() / (indexReader.getFilePointer() / keys);
+ indexReader.seek(0);
+ return estimatedRows;
+ }
+ }
+
+ private FileHandle.Builder indexFileBuilder(IndexSummary indexSummary)
+ {
+ assert this.indexFileBuilder == null || this.indexFileBuilder.file.equals(descriptor.fileFor(Components.PRIMARY_INDEX));
+
+ long indexFileLength = descriptor.fileFor(Components.PRIMARY_INDEX).length();
+ OptionalInt indexBufferSize = indexSummary != null ? OptionalInt.of(ioOptions.diskOptimizationStrategy.bufferSize(indexFileLength / indexSummary.size()))
+ : OptionalInt.empty();
+
+ if (indexFileBuilder == null)
+ indexFileBuilder = IndexComponent.fileBuilder(descriptor.fileFor(Components.PRIMARY_INDEX), ioOptions, chunkCache)
+ .bufferSize(indexBufferSize.orElse(DiskOptimizationStrategy.MAX_BUFFER_SIZE));
+
+ indexBufferSize.ifPresent(indexFileBuilder::bufferSize);
+
+ return indexFileBuilder;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java
new file mode 100644
index 0000000..06c4e63
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry.IndexSerializer;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Throwables;
+
+@NotThreadSafe
+public class BigTableKeyReader implements KeyReader
+{
+ private final FileHandle indexFile;
+ private final RandomAccessReader indexFileReader;
+ private final IndexSerializer rowIndexEntrySerializer;
+ private final long initialPosition;
+
+ private ByteBuffer key;
+ private long dataPosition;
+ private long keyPosition;
+
+ private BigTableKeyReader(FileHandle indexFile,
+ RandomAccessReader indexFileReader,
+ IndexSerializer rowIndexEntrySerializer)
+ {
+ this.indexFile = indexFile;
+ this.indexFileReader = indexFileReader;
+ this.rowIndexEntrySerializer = rowIndexEntrySerializer;
+ this.initialPosition = indexFileReader.getFilePointer();
+ }
+
+ public static BigTableKeyReader create(RandomAccessReader indexFileReader, IndexSerializer serializer) throws IOException
+ {
+ BigTableKeyReader iterator = new BigTableKeyReader(null, indexFileReader, serializer);
+ try
+ {
+ iterator.advance();
+ return iterator;
+ }
+ catch (IOException | RuntimeException ex)
+ {
+ iterator.close();
+ throw ex;
+ }
+ }
+
+ @SuppressWarnings({ "resource" })
+ public static BigTableKeyReader create(FileHandle indexFile, IndexSerializer serializer) throws IOException
+ {
+ FileHandle iFile = null;
+ RandomAccessReader reader = null;
+ BigTableKeyReader iterator = null;
+ try
+ {
+ iFile = indexFile.sharedCopy();
+ reader = iFile.createReader();
+ iterator = new BigTableKeyReader(iFile, reader, serializer);
+ iterator.advance();
+ return iterator;
+ }
+ catch (IOException | RuntimeException ex)
+ {
+ if (iterator != null)
+ {
+ iterator.close();
+ }
+ else
+ {
+ Throwables.closeNonNullAndAddSuppressed(ex, reader, iFile);
+ }
+ throw ex;
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ key = null;
+ dataPosition = -1;
+ keyPosition = -1;
+ FileUtils.closeQuietly(indexFileReader);
+ FileUtils.closeQuietly(indexFile);
+ }
+
+ @Override
+ public boolean advance() throws IOException
+ {
+ if (!indexFileReader.isEOF())
+ {
+ keyPosition = indexFileReader.getFilePointer();
+ key = ByteBufferUtil.readWithShortLength(indexFileReader);
+ dataPosition = rowIndexEntrySerializer.deserializePositionAndSkip(indexFileReader);
+ return true;
+ }
+ else
+ {
+ keyPosition = -1;
+ dataPosition = -1;
+ key = null;
+ return false;
+ }
+ }
+
+ @Override
+ public boolean isExhausted()
+ {
+ return key == null && dataPosition < 0;
+ }
+
+ @Override
+ public ByteBuffer key()
+ {
+ return key;
+ }
+
+ @Override
+ public long keyPositionForSecondaryIndex()
+ {
+ return keyPosition;
+ }
+
+ @Override
+ public long dataPosition()
+ {
+ return dataPosition;
+ }
+
+ public long indexPosition()
+ {
+ return indexFileReader.getFilePointer();
+ }
+
+ public void indexPosition(long position) throws IOException
+ {
+ if (position > indexLength())
+ throw new IndexOutOfBoundsException("The requested position exceeds the index length");
+ indexFileReader.seek(position);
+ key = null;
+ keyPosition = 0;
+ dataPosition = 0;
+ advance();
+ }
+
+ public long indexLength()
+ {
+ return indexFileReader.length();
+ }
+
+ @Override
+ public void reset() throws IOException
+ {
+ indexFileReader.seek(initialPosition);
+ key = null;
+ keyPosition = 0;
+ dataPosition = 0;
+ advance();
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("BigTable-PartitionIndexIterator(%s)", indexFile.path());
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
index e0edd7a..0984cc9 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
@@ -19,42 +19,110 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
-import org.apache.cassandra.io.sstable.format.SSTableReaderBuilder;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.SSTableIterator;
-import org.apache.cassandra.db.columniterator.SSTableReversedIterator;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener.SelectionReason;
+import org.apache.cassandra.io.sstable.SSTableReadsListener.SkippingReason;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SelectionReason;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SkippingReason;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummary;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport;
+import org.apache.cassandra.io.sstable.keycache.KeyCache;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.OutputHandler;
+
+import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull;
/**
* SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
* Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
*/
-public class BigTableReader extends SSTableReader
+public class BigTableReader extends SSTableReaderWithFilter implements IndexSummarySupport<BigTableReader>,
+ KeyCacheSupport<BigTableReader>
{
private static final Logger logger = LoggerFactory.getLogger(BigTableReader.class);
- BigTableReader(SSTableReaderBuilder builder)
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+ private final IndexSummary indexSummary;
+ private final FileHandle ifile;
+
+ private final KeyCache keyCache;
+
+ public BigTableReader(Builder builder, SSTable.Owner owner)
{
- super(builder);
+ super(builder, owner);
+ this.ifile = builder.getIndexFile();
+ this.indexSummary = builder.getIndexSummary();
+ this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(descriptor.version, header, owner != null ? owner.getMetrics() : null);
+ this.keyCache = Objects.requireNonNull(builder.getKeyCache());
+ }
+
+ @Override
+ protected List<AutoCloseable> setupInstance(boolean trackHotness)
+ {
+ ArrayList<AutoCloseable> closeables = Lists.newArrayList(indexSummary, ifile);
+ closeables.addAll(super.setupInstance(trackHotness));
+ return closeables;
+ }
+
+ @Override
+ public void releaseInMemoryComponents()
+ {
+ closeInternalComponent(indexSummary);
+ assert indexSummary.isCleanedUp();
+ }
+
+ @Override
+ public IndexSummary getIndexSummary()
+ {
+ return indexSummary;
}
public UnfilteredRowIterator rowIterator(DecoratedKey key,
@@ -63,18 +131,18 @@
boolean reversed,
SSTableReadsListener listener)
{
- RowIndexEntry rie = getPosition(key, SSTableReader.Operator.EQ, listener);
+ RowIndexEntry rie = getRowIndexEntry(key, SSTableReader.Operator.EQ, true, listener);
return rowIterator(null, key, rie, slices, selectedColumns, reversed);
}
- @SuppressWarnings("resource")
public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
{
if (indexEntry == null)
return UnfilteredRowIterators.noRowsIterator(metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed);
- return reversed
- ? new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, ifile)
- : new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, ifile);
+ else if (reversed)
+ return new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, ifile);
+ else
+ return new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, ifile);
}
@Override
@@ -83,6 +151,12 @@
return BigTableScanner.getScanner(this, columns, dataRange, listener);
}
+ @Override
+ public KeyReader keyReader() throws IOException
+ {
+ return BigTableKeyReader.create(ifile, rowIndexEntrySerializer);
+ }
+
/**
* Direct I/O SSTableScanner over an iterator of bounds.
*
@@ -118,97 +192,133 @@
return getScanner();
}
-
- @SuppressWarnings("resource") // caller to close
+ /**
+ * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
+ */
@Override
- public UnfilteredRowIterator simpleIterator(FileDataInput dfile, DecoratedKey key, RowIndexEntry position, boolean tombstoneOnly)
+ public DecoratedKey firstKeyBeyond(PartitionPosition token)
{
- return SSTableIdentityIterator.create(this, dfile, position, key, tombstoneOnly);
+ if (token.compareTo(first) < 0)
+ return first;
+
+ long sampledPosition = getIndexScanPosition(token);
+
+ if (ifile == null)
+ return null;
+
+ String path = null;
+ try (FileDataInput in = ifile.createReader(sampledPosition))
+ {
+ path = in.getPath();
+ while (!in.isEOF())
+ {
+ ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+ DecoratedKey indexDecoratedKey = decorateKey(indexKey);
+ if (indexDecoratedKey.compareTo(token) > 0)
+ return indexDecoratedKey;
+
+ RowIndexEntry.Serializer.skip(in, descriptor.version);
+ }
+ }
+ catch (IOException e)
+ {
+ markSuspect();
+ throw new CorruptSSTableException(e, path);
+ }
+
+ return null;
}
/**
+ * Retrieves the position while updating the key cache and the stats.
+ *
* @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
- * allow key selection by token bounds but only if op != * EQ
- * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
- * @param updateCacheAndStats true if updating stats and cache
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ */
+ public final RowIndexEntry getRowIndexEntry(PartitionPosition key, Operator op)
+ {
+ return getRowIndexEntry(key, op, true, SSTableReadsListener.NOOP_LISTENER);
+ }
+
+ /**
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param operator The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateStats true if updating stats and cache
* @return The index entry corresponding to the key, or null if the key is not present
*/
- protected RowIndexEntry getPosition(PartitionPosition key,
- Operator op,
- boolean updateCacheAndStats,
- boolean permitMatchPastLast,
- SSTableReadsListener listener)
+ @Override
+ public RowIndexEntry getRowIndexEntry(PartitionPosition key,
+ Operator operator,
+ boolean updateStats,
+ SSTableReadsListener listener)
{
// Having no index file is impossible in a normal operation. The only way it might happen is running
- // Scrubber that does not really rely onto this method.
+ // Scrubber that does not really rely on this method.
if (ifile == null)
- {
return null;
- }
- if (op == Operator.EQ)
- {
- assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key
- if (!bf.isPresent((DecoratedKey)key))
- {
- listener.onSSTableSkipped(this, SkippingReason.BLOOM_FILTER);
- Tracing.trace("Bloom filter allows skipping sstable {}", descriptor.id);
- bloomFilterTracker.addTrueNegative();
- return null;
- }
- }
-
- // next, the key cache (only make sense for valid row key)
- if ((op == Operator.EQ || op == Operator.GE) && (key instanceof DecoratedKey))
- {
- DecoratedKey decoratedKey = (DecoratedKey) key;
- RowIndexEntry cachedPosition = getCachedPosition(decoratedKey, updateCacheAndStats);
- if (cachedPosition != null)
- {
- // we do not need to track "true positive" for Bloom Filter here because it has been already tracked
- // inside getCachedPosition method
- listener.onSSTableSelected(this, cachedPosition, SelectionReason.KEY_CACHE_HIT);
- Tracing.trace("Key cache hit for sstable {}", descriptor.id);
- return cachedPosition;
- }
- }
+ Operator searchOp = operator;
// check the smallest and greatest keys in the sstable to see if it can't be present
boolean skip = false;
if (key.compareTo(first) < 0)
{
- if (op == Operator.EQ)
+ if (searchOp == Operator.EQ)
+ {
skip = true;
+ }
else
+ {
key = first;
-
- op = Operator.EQ;
+ searchOp = Operator.GE; // since op != EQ, bloom filter will be skipped; first key is included so no reason to check bloom filter
+ }
}
else
{
int l = last.compareTo(key);
- // l <= 0 => we may be looking past the end of the file; we then narrow our behaviour to:
- // 1) skipping if strictly greater for GE and EQ;
- // 2) skipping if equal and searching GT, and we aren't permitting matching past last
- skip = l <= 0 && (l < 0 || (!permitMatchPastLast && op == Operator.GT));
+ skip = l < 0 // out of range, skip
+ || l == 0 && searchOp == Operator.GT; // search entry > key, but key is the last in range, so skip
+ if (l == 0)
+ searchOp = Operator.GE; // since op != EQ, bloom filter will be skipped, last key is included so no reason to check bloom filter
}
if (skip)
{
- if (op == Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addFalsePositive();
- listener.onSSTableSkipped(this, SkippingReason.MIN_MAX_KEYS);
- Tracing.trace("Check against min and max keys allows skipping sstable {}", descriptor.id);
+ notifySkipped(SkippingReason.MIN_MAX_KEYS, listener, operator, updateStats);
return null;
}
+ if (searchOp == Operator.EQ)
+ {
+ assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key
+ if (!isPresentInFilter((IFilter.FilterKey) key))
+ {
+ notifySkipped(SkippingReason.BLOOM_FILTER, listener, operator, updateStats);
+ return null;
+ }
+ }
+
+ // next, the key cache (only make sense for valid row key)
+ if ((searchOp == Operator.EQ || searchOp == Operator.GE) && (key instanceof DecoratedKey))
+ {
+ DecoratedKey decoratedKey = (DecoratedKey) key;
+ AbstractRowIndexEntry cachedPosition = getCachedPosition(decoratedKey, updateStats);
+ if (cachedPosition != null && cachedPosition.getSSTableFormat() == descriptor.getFormat())
+ {
+ notifySelected(SelectionReason.KEY_CACHE_HIT, listener, operator, updateStats, cachedPosition);
+ return (RowIndexEntry) cachedPosition;
+ }
+ }
+
int binarySearchResult = indexSummary.binarySearch(key);
- long sampledPosition = getIndexScanPositionFromBinarySearchResult(binarySearchResult, indexSummary);
- int sampledIndex = getIndexSummaryIndexFromBinarySearchResult(binarySearchResult);
+ long sampledPosition = indexSummary.getScanPositionFromBinarySearchResult(binarySearchResult);
+ int sampledIndex = IndexSummary.getIndexFromBinarySearchResult(binarySearchResult);
int effectiveInterval = indexSummary.getEffectiveIndexIntervalAfterIndex(sampledIndex);
// scan the on-disk index, starting at the nearest sampled position.
- // The check against IndexInterval is to be exit the loop in the EQ case when the key looked for is not present
+ // The check against IndexInterval is to be exited the loop in the EQ case when the key looked for is not present
// (bloom filter false positive). But note that for non-EQ cases, we might need to check the first key of the
// next index position because the searched key can be greater the last key of the index interval checked if it
// is lesser than the first key of next interval (and in that case we must return the position of the first key
@@ -228,7 +338,7 @@
boolean exactMatch; // is the current position an exact match for the key, suitable for caching
// Compare raw keys if possible for performance, otherwise compare decorated keys.
- if (op == Operator.EQ && i <= effectiveInterval)
+ if (searchOp == Operator.EQ && i <= effectiveInterval)
{
opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey());
}
@@ -236,15 +346,12 @@
{
DecoratedKey indexDecoratedKey = decorateKey(indexKey);
int comparison = indexDecoratedKey.compareTo(key);
- int v = op.apply(comparison);
+ int v = searchOp.apply(comparison);
opSatisfied = (v == 0);
exactMatch = (comparison == 0);
if (v < 0)
{
- if (op == SSTableReader.Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addFalsePositive();
- listener.onSSTableSkipped(this, SkippingReason.PARTITION_INDEX_LOOKUP);
- Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.id);
+ notifySkipped(SkippingReason.PARTITION_INDEX_LOOKUP, listener, operator, updateStats);
return null;
}
}
@@ -253,10 +360,10 @@
{
// read data position from index entry
RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in);
- if (exactMatch && updateCacheAndStats)
+ if (exactMatch && updateStats)
{
assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key
- DecoratedKey decoratedKey = (DecoratedKey)key;
+ DecoratedKey decoratedKey = (DecoratedKey) key;
if (logger.isTraceEnabled())
{
@@ -272,10 +379,7 @@
// store exact match for the key
cacheKey(decoratedKey, indexEntry);
}
- if (op == Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addTruePositive();
- listener.onSSTableSelected(this, indexEntry, SelectionReason.INDEX_ENTRY_FOUND);
- Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndexCount(), descriptor.id);
+ notifySelected(SelectionReason.INDEX_ENTRY_FOUND, listener, operator, updateStats, indexEntry);
return indexEntry;
}
@@ -288,12 +392,354 @@
throw new CorruptSSTableException(e, path);
}
- if (op == SSTableReader.Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addFalsePositive();
- listener.onSSTableSkipped(this, SkippingReason.INDEX_ENTRY_NOT_FOUND);
- Tracing.trace("Partition index lookup complete (bloom filter false positive) for sstable {}", descriptor.id);
+ notifySkipped(SkippingReason.INDEX_ENTRY_NOT_FOUND, listener, operator, updateStats);
return null;
}
+ /**
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateCacheAndStats true if updating stats and cache
+ * @return The index entry corresponding to the key, or null if the key is not present
+ */
+ @Override
+ protected long getPosition(PartitionPosition key,
+ Operator op,
+ boolean updateCacheAndStats,
+ SSTableReadsListener listener)
+ {
+ RowIndexEntry rowIndexEntry = getRowIndexEntry(key, op, updateCacheAndStats, listener);
+ return rowIndexEntry != null ? rowIndexEntry.position : -1;
+ }
+ @Override
+ public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException
+ {
+ DecoratedKey key;
+ try (FileDataInput in = ifile.createReader(keyPositionFromSecondaryIndex))
+ {
+ if (in.isEOF())
+ return null;
+
+ key = decorateKey(ByteBufferUtil.readWithShortLength(in));
+
+ // hint read path about key location if caching is enabled
+ // this saves index summary lookup and index file iteration which whould be pretty costly
+ // especially in presence of promoted column indexes
+ cacheKey(key, rowIndexEntrySerializer.deserialize(in));
+ }
+
+ return key;
+ }
+
+ @Override
+ public RowIndexEntry deserializeKeyCacheValue(DataInputPlus input) throws IOException
+ {
+ return rowIndexEntrySerializer.deserializeForCache(input);
+ }
+
+ @Override
+ public ClusteringBound<?> getLowerBoundPrefixFromCache(DecoratedKey partitionKey, boolean isReversed)
+ {
+ AbstractRowIndexEntry rie = getCachedPosition(partitionKey, false);
+ if (!(rie instanceof RowIndexEntry))
+ return null;
+
+ RowIndexEntry rowIndexEntry = (RowIndexEntry) rie;
+ if (!rowIndexEntry.indexOnHeap())
+ return null;
+
+ try (RowIndexEntry.IndexInfoRetriever onHeapRetriever = rowIndexEntry.openWithIndex(null))
+ {
+ IndexInfo columns = onHeapRetriever.columnsIndex(isReversed ? rowIndexEntry.blockCount() - 1 : 0);
+ ClusteringBound<?> bound = isReversed ? columns.lastName.asEndBound() : columns.firstName.asStartBound();
+ UnfilteredRowIteratorWithLowerBound.assertBoundSize(bound, this);
+ return bound.artificialLowerBound(isReversed);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("should never occur", e);
+ }
+ }
+
+ /**
+ * @return An estimate of the number of keys in this SSTable based on the index summary.
+ */
+ @Override
+ public long estimatedKeys()
+ {
+ return indexSummary.getEstimatedKeyCount();
+ }
+
+ /**
+ * @return An estimate of the number of keys for given ranges in this SSTable.
+ */
+ @Override
+ public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
+ {
+ long sampleKeyCount = 0;
+ List<IndexesBounds> sampleIndexes = indexSummary.getSampleIndexesForRanges(ranges);
+ for (IndexesBounds sampleIndexRange : sampleIndexes)
+ sampleKeyCount += (sampleIndexRange.upperPosition - sampleIndexRange.lowerPosition + 1);
+
+ // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling
+ long estimatedKeys = sampleKeyCount * ((long) Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel();
+ return Math.max(1, estimatedKeys);
+ }
+
+ /**
+ * Returns whether the number of entries in the IndexSummary > 2. At full sampling, this is approximately
+ * 1/INDEX_INTERVALth of the keys in this SSTable.
+ */
+ @Override
+ public boolean isEstimationInformative()
+ {
+ return indexSummary.size() > 2;
+ }
+
+ @Override
+ public Iterable<DecoratedKey> getKeySamples(final Range<Token> range)
+ {
+ return Iterables.transform(indexSummary.getKeySamples(range), bytes -> decorateKey(ByteBuffer.wrap(bytes)));
+ }
+
+ public RandomAccessReader openIndexReader()
+ {
+ if (ifile != null)
+ return ifile.createReader();
+ return null;
+ }
+
+ public FileHandle getIndexFile()
+ {
+ return ifile;
+ }
+
+ @Override
+ public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options)
+ {
+ Preconditions.checkArgument(cfs.metadata().equals(metadata()));
+ return new BigTableVerifier(cfs, this, outputHandler, isOffline, options);
+ }
+
+ /**
+ * Gets the position with the index file to start scanning to find the given key (at most indexInterval keys away,
+ * modulo downsampling of the index summary). Always returns a {@code value >= 0}
+ */
+ long getIndexScanPosition(PartitionPosition key)
+ {
+ if (openReason == OpenReason.MOVED_START && key.compareTo(first) < 0)
+ key = first;
+
+ return indexSummary.getScanPosition(key);
+ }
+
+ protected final Builder unbuildTo(Builder builder, boolean sharedCopy)
+ {
+ Builder b = super.unbuildTo(builder, sharedCopy);
+ if (builder.getIndexFile() == null)
+ b.setIndexFile(sharedCopy ? sharedCopyOrNull(ifile) : ifile);
+ if (builder.getIndexSummary() == null)
+ b.setIndexSummary(sharedCopy ? sharedCopyOrNull(indexSummary) : indexSummary);
+
+ b.setKeyCache(keyCache);
+
+ return b;
+ }
+
+ @VisibleForTesting
+ @Override
+ public SSTableReaderWithFilter cloneAndReplace(IFilter filter)
+ {
+ return unbuildTo(new Builder(descriptor).setFilter(filter), true).build(owner().orElse(null), true, true);
+ }
+
+ /**
+ * Clone this reader with the provided start and open reason, and set the clone as replacement.
+ *
+ * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive
+ * opening of compaction results).
+ * @param reason the {@code OpenReason} for the replacement.
+ * @return the cloned reader. That reader is set as a replacement by the method.
+ */
+ private SSTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason)
+ {
+ return unbuildTo(new Builder(descriptor), true)
+ .setFirst(newFirst)
+ .setOpenReason(reason)
+ .build(owner().orElse(null), true, true);
+ }
+
+ /**
+ * Clone this reader with the new values and set the clone as replacement.
+ *
+ * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive
+ * opening of compaction results).
+ * @param reason the {@code OpenReason} for the replacement.
+ * @param newSummary the index summary for the replacement.
+ * @return the cloned reader. That reader is set as a replacement by the method.
+ */
+ private BigTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason, IndexSummary newSummary)
+ {
+ return unbuildTo(new Builder(descriptor).setIndexSummary(newSummary), true)
+ .setIndexSummary(newSummary)
+ .setFirst(newFirst)
+ .setOpenReason(reason)
+ .build(owner().orElse(null), true, true);
+ }
+
+ public SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart)
+ {
+ return runWithLock(ignored -> cloneAndReplace(restoredStart, OpenReason.NORMAL));
+ }
+
+ public SSTableReader cloneWithNewStart(DecoratedKey newStart)
+ {
+ return runWithLock(ignored -> {
+ assert openReason != OpenReason.EARLY;
+ // TODO: merge with caller's firstKeyBeyond() work,to save time
+ if (newStart.compareTo(first) > 0)
+ {
+ Map<FileHandle, Long> handleAndPositions = new LinkedHashMap<>(2);
+ if (dfile != null)
+ handleAndPositions.put(dfile, getPosition(newStart, Operator.EQ));
+ if (ifile != null)
+ handleAndPositions.put(ifile, getIndexScanPosition(newStart));
+ runOnClose(() -> handleAndPositions.forEach(FileHandle::dropPageCache));
+ }
+
+ return cloneAndReplace(newStart, OpenReason.MOVED_START);
+ });
+ }
+
+ /**
+ * Returns a new SSTableReader with the same properties as this SSTableReader except that a new IndexSummary will
+ * be built at the target samplingLevel. This (original) SSTableReader instance will be marked as replaced, have
+ * its DeletingTask removed, and have its periodic read-meter sync task cancelled.
+ *
+ * @param samplingLevel the desired sampling level for the index summary on the new SSTableReader
+ * @return a new SSTableReader
+ */
+ @SuppressWarnings("resource")
+ public BigTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
+ {
+ assert openReason != OpenReason.EARLY;
+
+ int minIndexInterval = metadata().params.minIndexInterval;
+ int maxIndexInterval = metadata().params.maxIndexInterval;
+ double effectiveInterval = indexSummary.getEffectiveIndexInterval();
+
+ IndexSummary newSummary;
+
+ // We have to rebuild the summary from the on-disk primary index in three cases:
+ // 1. The sampling level went up, so we need to read more entries off disk
+ // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary
+ // at full sampling (and consequently at any other sampling level)
+ // 3. The max_index_interval was lowered, forcing us to raise the sampling level
+ if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval)
+ {
+ newSummary = buildSummaryAtLevel(samplingLevel);
+ }
+ else if (samplingLevel < indexSummary.getSamplingLevel())
+ {
+ // we can use the existing index summary to make a smaller one
+ newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, getPartitioner());
+ }
+ else
+ {
+ throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " +
+ "no adjustments to min/max_index_interval");
+ }
+
+ // Always save the resampled index with lock to avoid racing with entire-sstable streaming
+ return runWithLock(ignored -> {
+ new IndexSummaryComponent(newSummary, first, last).save(descriptor.fileFor(Components.SUMMARY), true);
+ return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary);
+ });
+ }
+
+ private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
+ {
+ // we read the positions in a BRAF, so we don't have to worry about an entry spanning a mmap boundary.
+ RandomAccessReader primaryIndex = RandomAccessReader.open(descriptor.fileFor(Components.PRIMARY_INDEX));
+ try
+ {
+ long indexSize = primaryIndex.length();
+ try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata().params.minIndexInterval, newSamplingLevel))
+ {
+ long indexPosition;
+ while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
+ {
+ summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
+ RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version);
+ }
+
+ return summaryBuilder.build(getPartitioner());
+ }
+ }
+ finally
+ {
+ FileUtils.closeQuietly(primaryIndex);
+ }
+ }
+
+ @Override
+ public KeyCache getKeyCache()
+ {
+ return this.keyCache;
+ }
+
+ public static class Builder extends SSTableReaderWithFilter.Builder<BigTableReader, Builder>
+ {
+ private static final Logger logger = LoggerFactory.getLogger(Builder.class);
+
+ private IndexSummary indexSummary;
+ private FileHandle indexFile;
+ private KeyCache keyCache = KeyCache.NO_CACHE;
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ public Builder setIndexFile(FileHandle indexFile)
+ {
+ this.indexFile = indexFile;
+ return this;
+ }
+
+ public Builder setIndexSummary(IndexSummary indexSummary)
+ {
+ this.indexSummary = indexSummary;
+ return this;
+ }
+
+ public Builder setKeyCache(KeyCache keyCache)
+ {
+ this.keyCache = keyCache;
+ return this;
+ }
+
+ public IndexSummary getIndexSummary()
+ {
+ return indexSummary;
+ }
+
+ public FileHandle getIndexFile()
+ {
+ return indexFile;
+ }
+
+ public KeyCache getKeyCache()
+ {
+ return keyCache;
+ }
+
+ @Override
+ protected BigTableReader buildInternal(Owner owner)
+ {
+ return new BigTableReader(this, owner);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
index 235b9b1..887d997 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
@@ -18,63 +18,45 @@
package org.apache.cassandra.io.sstable.format.big;
import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Collection;
+import java.util.Iterator;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.utils.AbstractIterator;
-
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterators;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.AbstractBounds.Boundary;
-import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableScanner;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.utils.ByteBufferUtil;
-import static org.apache.cassandra.dht.AbstractBounds.isEmpty;
-import static org.apache.cassandra.dht.AbstractBounds.maxLeft;
-import static org.apache.cassandra.dht.AbstractBounds.minRight;
-
-public class BigTableScanner implements ISSTableScanner
+public class BigTableScanner extends SSTableScanner<BigTableReader, RowIndexEntry, BigTableScanner.BigScanningIterator>
{
- private final AtomicBoolean isClosed = new AtomicBoolean(false);
- protected final RandomAccessReader dfile;
protected final RandomAccessReader ifile;
- public final SSTableReader sstable;
- private final Iterator<AbstractBounds<PartitionPosition>> rangeIterator;
private AbstractBounds<PartitionPosition> currentRange;
- private final ColumnFilter columns;
- private final DataRange dataRange;
private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final SSTableReadsListener listener;
- private long startScan = -1;
- private long bytesScanned = 0;
-
- protected Iterator<UnfilteredRowIterator> iterator;
// Full scan of the sstables
- public static ISSTableScanner getScanner(SSTableReader sstable)
+ public static ISSTableScanner getScanner(BigTableReader sstable)
{
return getScanner(sstable, Iterators.singletonIterator(fullRange(sstable)));
}
- public static ISSTableScanner getScanner(SSTableReader sstable,
+ public static ISSTableScanner getScanner(BigTableReader sstable,
ColumnFilter columns,
DataRange dataRange,
SSTableReadsListener listener)
@@ -82,99 +64,25 @@
return new BigTableScanner(sstable, columns, dataRange, makeBounds(sstable, dataRange).iterator(), listener);
}
- public static ISSTableScanner getScanner(SSTableReader sstable, Collection<Range<Token>> tokenRanges)
+ public static ISSTableScanner getScanner(BigTableReader sstable, Collection<Range<Token>> tokenRanges)
{
- // We want to avoid allocating a SSTableScanner if the range don't overlap the sstable (#5249)
- List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(tokenRanges);
- if (positions.isEmpty())
- return new EmptySSTableScanner(sstable);
-
return getScanner(sstable, makeBounds(sstable, tokenRanges).iterator());
}
- public static ISSTableScanner getScanner(SSTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
+ public static ISSTableScanner getScanner(BigTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
{
return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata()), null, rangeIterator, SSTableReadsListener.NOOP_LISTENER);
}
- private BigTableScanner(SSTableReader sstable,
+ private BigTableScanner(BigTableReader sstable,
ColumnFilter columns,
DataRange dataRange,
Iterator<AbstractBounds<PartitionPosition>> rangeIterator,
SSTableReadsListener listener)
{
- assert sstable != null;
-
- this.dfile = sstable.openDataReader();
+ super(sstable, columns, dataRange, rangeIterator, listener);
this.ifile = sstable.openIndexReader();
- this.sstable = sstable;
- this.columns = columns;
- this.dataRange = dataRange;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata(),
- sstable.descriptor.version,
- sstable.header);
- this.rangeIterator = rangeIterator;
- this.listener = listener;
- }
-
- private static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReader sstable, Collection<Range<Token>> tokenRanges)
- {
- List<AbstractBounds<PartitionPosition>> boundsList = new ArrayList<>(tokenRanges.size());
- for (Range<Token> range : Range.normalize(tokenRanges))
- addRange(sstable, Range.makeRowRange(range), boundsList);
- return boundsList;
- }
-
- private static List<AbstractBounds<PartitionPosition>> makeBounds(SSTableReader sstable, DataRange dataRange)
- {
- List<AbstractBounds<PartitionPosition>> boundsList = new ArrayList<>(2);
- addRange(sstable, dataRange.keyRange(), boundsList);
- return boundsList;
- }
-
- private static AbstractBounds<PartitionPosition> fullRange(SSTableReader sstable)
- {
- return new Bounds<PartitionPosition>(sstable.first, sstable.last);
- }
-
- private static void addRange(SSTableReader sstable, AbstractBounds<PartitionPosition> requested, List<AbstractBounds<PartitionPosition>> boundsList)
- {
- if (requested instanceof Range && ((Range)requested).isWrapAround())
- {
- if (requested.right.compareTo(sstable.first) >= 0)
- {
- // since we wrap, we must contain the whole sstable prior to stopKey()
- Boundary<PartitionPosition> left = new Boundary<PartitionPosition>(sstable.first, true);
- Boundary<PartitionPosition> right;
- right = requested.rightBoundary();
- right = minRight(right, sstable.last, true);
- if (!isEmpty(left, right))
- boundsList.add(AbstractBounds.bounds(left, right));
- }
- if (requested.left.compareTo(sstable.last) <= 0)
- {
- // since we wrap, we must contain the whole sstable after dataRange.startKey()
- Boundary<PartitionPosition> right = new Boundary<PartitionPosition>(sstable.last, true);
- Boundary<PartitionPosition> left;
- left = requested.leftBoundary();
- left = maxLeft(left, sstable.first, true);
- if (!isEmpty(left, right))
- boundsList.add(AbstractBounds.bounds(left, right));
- }
- }
- else
- {
- assert requested.left.compareTo(requested.right) <= 0 || requested.right.isMinimum();
- Boundary<PartitionPosition> left, right;
- left = requested.leftBoundary();
- right = requested.rightBoundary();
- left = maxLeft(left, sstable.first, true);
- // apparently isWrapAround() doesn't count Bounds that extend to the limit (min) as wrapping
- right = requested.right.isMinimum() ? new Boundary<PartitionPosition>(sstable.last, true)
- : minRight(right, sstable.last, true);
- if (!isEmpty(left, right))
- boundsList.add(AbstractBounds.bounds(left, right));
- }
+ this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, sstable.owner().map(SSTable.Owner::getMetrics).orElse(null));
}
private void seekToCurrentRangeStart()
@@ -209,177 +117,88 @@
}
}
- public void close()
+ protected void doClose() throws IOException
{
- try
- {
- if (isClosed.compareAndSet(false, true))
- FileUtils.close(dfile, ifile);
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
+ FileUtils.close(dfile, ifile);
}
- public long getLengthInBytes()
+ protected BigScanningIterator doCreateIterator()
{
- return dfile.length();
+ return new BigScanningIterator();
}
- public long getCurrentPosition()
- {
- return dfile.getFilePointer();
- }
-
- public long getBytesScanned()
- {
- return bytesScanned;
- }
-
- public long getCompressedLengthInBytes()
- {
- return sstable.onDiskLength();
- }
-
- public Set<SSTableReader> getBackingSSTables()
- {
- return ImmutableSet.of(sstable);
- }
-
-
- public TableMetadata metadata()
- {
- return sstable.metadata();
- }
-
- public boolean hasNext()
- {
- if (iterator == null)
- iterator = createIterator();
- return iterator.hasNext();
- }
-
- public UnfilteredRowIterator next()
- {
- if (iterator == null)
- iterator = createIterator();
- return iterator.next();
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
-
- private Iterator<UnfilteredRowIterator> createIterator()
- {
- this.listener.onScanningStarted(sstable);
- return new KeyScanningIterator();
- }
-
- protected class KeyScanningIterator extends AbstractIterator<UnfilteredRowIterator>
+ protected class BigScanningIterator extends SSTableScanner<BigTableReader, RowIndexEntry, BigTableScanner.BigScanningIterator>.BaseKeyScanningIterator
{
private DecoratedKey nextKey;
private RowIndexEntry nextEntry;
- private DecoratedKey currentKey;
- private RowIndexEntry currentEntry;
- protected UnfilteredRowIterator computeNext()
+ protected boolean prepareToIterateRow() throws IOException
{
- try
+ if (nextEntry == null)
{
- if (nextEntry == null)
+ do
{
- do
- {
- if (startScan != -1)
- bytesScanned += dfile.getFilePointer() - startScan;
+ if (startScan != -1)
+ bytesScanned += dfile.getFilePointer() - startScan;
- // we're starting the first range or we just passed the end of the previous range
- if (!rangeIterator.hasNext())
- return endOfData();
+ // we're starting the first range or we just passed the end of the previous range
+ if (!rangeIterator.hasNext())
+ return false;
- currentRange = rangeIterator.next();
- seekToCurrentRangeStart();
- startScan = dfile.getFilePointer();
+ currentRange = rangeIterator.next();
+ seekToCurrentRangeStart();
+ startScan = dfile.getFilePointer();
- if (ifile.isEOF())
- return endOfData();
+ if (ifile.isEOF())
+ return false;
- currentKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- currentEntry = rowIndexEntrySerializer.deserialize(ifile);
- } while (!currentRange.contains(currentKey));
- }
- else
+ currentKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
+ currentEntry = rowIndexEntrySerializer.deserialize(ifile);
+ } while (!currentRange.contains(currentKey));
+ }
+ else
+ {
+ // we're in the middle of a range
+ currentKey = nextKey;
+ currentEntry = nextEntry;
+ }
+
+ if (ifile.isEOF())
+ {
+ nextEntry = null;
+ nextKey = null;
+ }
+ else
+ {
+ // we need the position of the start of the next key, regardless of whether it falls in the current range
+ nextKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
+ nextEntry = rowIndexEntrySerializer.deserialize(ifile);
+
+ if (!currentRange.contains(nextKey))
{
- // we're in the middle of a range
- currentKey = nextKey;
- currentEntry = nextEntry;
- }
-
- if (ifile.isEOF())
- {
- nextEntry = null;
nextKey = null;
+ nextEntry = null;
}
- else
- {
- // we need the position of the start of the next key, regardless of whether it falls in the current range
- nextKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- nextEntry = rowIndexEntrySerializer.deserialize(ifile);
-
- if (!currentRange.contains(nextKey))
- {
- nextKey = null;
- nextEntry = null;
- }
- }
-
- /*
- * For a given partition key, we want to avoid hitting the data
- * file unless we're explicitely asked to. This is important
- * for PartitionRangeReadCommand#checkCacheFilter.
- */
- return new LazilyInitializedUnfilteredRowIterator(currentKey)
- {
- protected UnfilteredRowIterator initializeIterator()
- {
-
- if (startScan != -1)
- bytesScanned += dfile.getFilePointer() - startScan;
-
- try
- {
- if (dataRange == null)
- {
- dfile.seek(currentEntry.position);
- startScan = dfile.getFilePointer();
- ByteBufferUtil.skipShortLength(dfile); // key
- return SSTableIdentityIterator.create(sstable, dfile, partitionKey());
- }
- else
- {
- startScan = dfile.getFilePointer();
- }
-
- ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(partitionKey());
- return sstable.rowIterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed());
- }
- catch (CorruptSSTableException | IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
- }
- };
}
- catch (CorruptSSTableException | IOException e)
+ return true;
+ }
+
+ protected UnfilteredRowIterator getRowIterator(RowIndexEntry rowIndexEntry, DecoratedKey key) throws IOException
+ {
+ if (dataRange == null)
{
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
+ dfile.seek(rowIndexEntry.position);
+ startScan = dfile.getFilePointer();
+ ByteBufferUtil.skipShortLength(dfile); // key
+ return SSTableIdentityIterator.create(sstable, dfile, key);
}
+ else
+ {
+ startScan = dfile.getFilePointer();
+ }
+
+ ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
+ return sstable.rowIterator(dfile, key, rowIndexEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed());
}
}
@@ -392,59 +211,4 @@
" sstable=" + sstable +
")";
}
-
- public static class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator implements ISSTableScanner
- {
- private final SSTableReader sstable;
-
- public EmptySSTableScanner(SSTableReader sstable)
- {
- this.sstable = sstable;
- }
-
- public long getLengthInBytes()
- {
- return 0;
- }
-
- public long getCurrentPosition()
- {
- return 0;
- }
-
- public long getBytesScanned()
- {
- return 0;
- }
-
- public long getCompressedLengthInBytes()
- {
- return 0;
- }
-
- public Set<SSTableReader> getBackingSSTables()
- {
- return ImmutableSet.of(sstable);
- }
-
- public TableMetadata metadata()
- {
- return sstable.metadata();
- }
-
- public boolean hasNext()
- {
- return false;
- }
-
- public UnfilteredRowIterator next()
- {
- return null;
- }
-
- public int getMinLocalDeletionTime()
- {
- return DeletionTime.LIVE.localDeletionTime();
- }
- }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java
new file mode 100644
index 0000000..dc991f4
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.format.SortedTableScrubber;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.OutputHandler;
+
+public class BigTableScrubber extends SortedTableScrubber<BigTableReader> implements IScrubber
+{
+ private final boolean isIndex;
+
+ private final RandomAccessReader indexFile;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+
+ private ByteBuffer currentIndexKey;
+ private ByteBuffer nextIndexKey;
+ private long currentPartitionPositionFromIndex;
+ private long nextPartitionPositionFromIndex;
+
+ public BigTableScrubber(ColumnFamilyStore cfs,
+ LifecycleTransaction transaction,
+ OutputHandler outputHandler,
+ Options options)
+ {
+ super(cfs, transaction, outputHandler, options);
+
+ this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, cfs.getMetrics());
+
+ boolean hasIndexFile = sstable.descriptor.fileFor(Components.PRIMARY_INDEX).exists();
+ this.isIndex = cfs.isIndex();
+ if (!hasIndexFile)
+ {
+ // if there's any corruption in the -Data.db then partitions can't be skipped over. but it's worth a shot.
+ outputHandler.warn("Missing component: %s", sstable.descriptor.fileFor(Components.PRIMARY_INDEX));
+ }
+
+ this.indexFile = hasIndexFile
+ ? RandomAccessReader.open(sstable.descriptor.fileFor(Components.PRIMARY_INDEX))
+ : null;
+
+ this.currentPartitionPositionFromIndex = 0;
+ this.nextPartitionPositionFromIndex = 0;
+ }
+
+ @Override
+ protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
+ @Override
+ protected void scrubInternal(SSTableRewriter writer) throws IOException
+ {
+ try
+ {
+ nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
+ if (indexAvailable())
+ {
+ // throw away variable, so we don't have a side effect in the assertion
+ long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
+ assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
+ }
+ }
+ catch (Throwable ex)
+ {
+ throwIfFatal(ex);
+ nextIndexKey = null;
+ nextPartitionPositionFromIndex = dataFile.length();
+ if (indexFile != null)
+ indexFile.seek(indexFile.length());
+ }
+
+ DecoratedKey prevKey = null;
+
+ while (!dataFile.isEOF())
+ {
+ if (scrubInfo.isStopRequested())
+ throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
+
+ long partitionStart = dataFile.getFilePointer();
+ outputHandler.debug("Reading row at %d", partitionStart);
+
+ DecoratedKey key = null;
+ try
+ {
+ ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile);
+ if (!cfs.metadata.getLocal().isIndex())
+ cfs.metadata.getLocal().partitionKeyType.validate(raw);
+ key = sstable.decorateKey(raw);
+ }
+ catch (Throwable th)
+ {
+ throwIfFatal(th);
+ // check for null key below
+ }
+
+ long dataStartFromIndex = -1;
+ long dataSizeFromIndex = -1;
+ updateIndexKey();
+
+ if (indexAvailable())
+ {
+ if (currentIndexKey != null)
+ {
+ dataStartFromIndex = currentPartitionPositionFromIndex + 2 + currentIndexKey.remaining();
+ dataSizeFromIndex = nextPartitionPositionFromIndex - dataStartFromIndex;
+ }
+ }
+
+ long dataStart = dataFile.getFilePointer();
+
+ String keyName = key == null ? "(unreadable key)" : keyString(key);
+ outputHandler.debug("partition %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSizeFromIndex));
+ assert currentIndexKey != null || !indexAvailable();
+
+ try
+ {
+ if (key == null)
+ throw new IOError(new IOException("Unable to read partition key from data file"));
+
+ if (currentIndexKey != null && !key.getKey().equals(currentIndexKey))
+ {
+ throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)",
+ //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
+ "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey))));
+ }
+
+ if (indexFile != null && dataSizeFromIndex > dataFile.length())
+ throw new IOError(new IOException("Impossible partition size (greater than file length): " + dataSizeFromIndex));
+
+ if (indexFile != null && dataStart != dataStartFromIndex)
+ outputHandler.warn("Data file partition position %d differs from index file row position %d", dataStart, dataStartFromIndex);
+
+ if (tryAppend(prevKey, key, writer))
+ prevKey = key;
+ }
+ catch (Throwable th)
+ {
+ throwIfFatal(th);
+ outputHandler.warn(th, "Error reading partition %s (stacktrace follows):", keyName);
+
+ if (currentIndexKey != null
+ && (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex))
+ {
+
+ outputHandler.output("Retrying from partition index; data is %s bytes starting at %s",
+ dataSizeFromIndex, dataStartFromIndex);
+ key = sstable.decorateKey(currentIndexKey);
+ try
+ {
+ if (!cfs.metadata.getLocal().isIndex())
+ cfs.metadata.getLocal().partitionKeyType.validate(key.getKey());
+ dataFile.seek(dataStartFromIndex);
+
+ if (tryAppend(prevKey, key, writer))
+ prevKey = key;
+ }
+ catch (Throwable th2)
+ {
+ throwIfFatal(th2);
+ throwIfCannotContinue(key, th2);
+
+ outputHandler.warn(th2, "Retry failed too. Skipping to next partition (retry's stacktrace follows)");
+ badPartitions++;
+ if (!seekToNextPartition())
+ break;
+ }
+ }
+ else
+ {
+ throwIfCannotContinue(key, th);
+
+ outputHandler.warn("Partition starting at position %d is unreadable; skipping to next", dataStart);
+ badPartitions++;
+ if (currentIndexKey != null)
+ if (!seekToNextPartition())
+ break;
+ }
+ }
+ }
+ }
+
+ private void updateIndexKey()
+ {
+ currentIndexKey = nextIndexKey;
+ currentPartitionPositionFromIndex = nextPartitionPositionFromIndex;
+ try
+ {
+ nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile);
+
+ nextPartitionPositionFromIndex = !indexAvailable()
+ ? dataFile.length()
+ : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
+ }
+ catch (Throwable th)
+ {
+ JVMStabilityInspector.inspectThrowable(th);
+ outputHandler.warn(th, "Error reading index file");
+ nextIndexKey = null;
+ nextPartitionPositionFromIndex = dataFile.length();
+ }
+ }
+
+ private boolean indexAvailable()
+ {
+ return indexFile != null && !indexFile.isEOF();
+ }
+
+ private boolean seekToNextPartition()
+ {
+ while (nextPartitionPositionFromIndex < dataFile.length())
+ {
+ try
+ {
+ dataFile.seek(nextPartitionPositionFromIndex);
+ return true;
+ }
+ catch (Throwable th)
+ {
+ throwIfFatal(th);
+ outputHandler.warn(th, "Failed to seek to next partition position %d", nextPartitionPositionFromIndex);
+ badPartitions++;
+ }
+
+ updateIndexKey();
+ }
+
+ return false;
+ }
+
+ @Override
+ protected void throwIfCannotContinue(DecoratedKey key, Throwable th)
+ {
+ if (isIndex)
+ {
+ outputHandler.warn("An error occurred while scrubbing the partition with key '%s' for an index table. " +
+ "Scrubbing will abort for this table and the index will be rebuilt.", keyString(key));
+ throw new IOError(th);
+ }
+
+ super.throwIfCannotContinue(key, th);
+ }
+
+ @Override
+ public void close()
+ {
+ fileAccessLock.writeLock().lock();
+ try
+ {
+ FileUtils.closeQuietly(dataFile);
+ FileUtils.closeQuietly(indexFile);
+ }
+ finally
+ {
+ fileAccessLock.writeLock().unlock();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java
new file mode 100644
index 0000000..70df3e1
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.nio.file.NoSuchFileException;
+import java.time.Instant;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SortedTableVerifier;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.OutputHandler;
+
+public class BigTableVerifier extends SortedTableVerifier<BigTableReader> implements IVerifier
+{
+ public BigTableVerifier(ColumnFamilyStore cfs, BigTableReader sstable, OutputHandler outputHandler, boolean isOffline, Options options)
+ {
+ super(cfs, sstable, outputHandler, isOffline, options);
+ }
+
+ protected void verifyPartition(DecoratedKey key, UnfilteredRowIterator iterator)
+ {
+ Row first = null;
+ int duplicateRows = 0;
+ long minTimestamp = Long.MAX_VALUE;
+ long maxTimestamp = Long.MIN_VALUE;
+ while (iterator.hasNext())
+ {
+ Unfiltered uf = iterator.next();
+ if (uf.isRow())
+ {
+ Row row = (Row) uf;
+ if (first != null && first.clustering().equals(row.clustering()))
+ {
+ duplicateRows++;
+ for (Cell cell : row.cells())
+ {
+ maxTimestamp = Math.max(cell.timestamp(), maxTimestamp);
+ minTimestamp = Math.min(cell.timestamp(), minTimestamp);
+ }
+ }
+ else
+ {
+ if (duplicateRows > 0)
+ logDuplicates(key, first, duplicateRows, minTimestamp, maxTimestamp);
+ duplicateRows = 0;
+ first = row;
+ maxTimestamp = Long.MIN_VALUE;
+ minTimestamp = Long.MAX_VALUE;
+ }
+ }
+ }
+ if (duplicateRows > 0)
+ logDuplicates(key, first, duplicateRows, minTimestamp, maxTimestamp);
+ }
+
+ private void verifyIndexSummary()
+ {
+ try
+ {
+ outputHandler.debug("Deserializing index summary for %s", sstable);
+ deserializeIndexSummary(sstable);
+ }
+ catch (Throwable t)
+ {
+ outputHandler.output("Index summary is corrupt - if it is removed it will get rebuilt on startup %s", sstable.descriptor.fileFor(Components.SUMMARY));
+ outputHandler.warn(t);
+ markAndThrow(t, false);
+ }
+ }
+
+ protected void verifyIndex()
+ {
+ verifyIndexSummary();
+ super.verifyIndex();
+ }
+
+ private void logDuplicates(DecoratedKey key, Row first, int duplicateRows, long minTimestamp, long maxTimestamp)
+ {
+ String keyString = sstable.metadata().partitionKeyType.getString(key.getKey());
+ long firstMaxTs = Long.MIN_VALUE;
+ long firstMinTs = Long.MAX_VALUE;
+ for (Cell cell : first.cells())
+ {
+ firstMaxTs = Math.max(firstMaxTs, cell.timestamp());
+ firstMinTs = Math.min(firstMinTs, cell.timestamp());
+ }
+ outputHandler.output("%d duplicate rows found for [%s %s] in %s.%s (%s), timestamps: [first row (%s, %s)], [duplicates (%s, %s, eq:%b)]",
+ duplicateRows,
+ keyString, first.clustering().toString(sstable.metadata()),
+ sstable.metadata().keyspace,
+ sstable.metadata().name,
+ sstable,
+ dateString(firstMinTs), dateString(firstMaxTs),
+ dateString(minTimestamp), dateString(maxTimestamp), minTimestamp == maxTimestamp);
+ }
+
+ private String dateString(long time)
+ {
+ return Instant.ofEpochMilli(TimeUnit.MICROSECONDS.toMillis(time)).toString();
+ }
+
+ private void deserializeIndexSummary(SSTableReader sstable) throws IOException
+ {
+ IndexSummaryComponent summaryComponent = IndexSummaryComponent.load(sstable.descriptor.fileFor(Components.SUMMARY), cfs.metadata());
+ if (summaryComponent == null)
+ throw new NoSuchFileException("Index summary component of sstable " + sstable.descriptor + " is missing");
+ FileUtils.closeQuietly(summaryComponent.indexSummary);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index e8dff32..5ae52cf 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -17,539 +17,268 @@
*/
package org.apache.cassandra.io.sstable.format.big;
-
import java.io.IOException;
-import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.stream.Stream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Consumer;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.compress.CompressedSequentialWriter;
-import org.apache.cassandra.io.compress.ICompressor;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.DataComponent;
+import org.apache.cassandra.io.sstable.format.IndexComponent;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
-import org.apache.cassandra.utils.concurrent.Transactional;
+import org.apache.cassandra.io.sstable.format.SortedTableWriter;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummary;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder;
+import org.apache.cassandra.io.sstable.keycache.KeyCache;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.MmappedRegionsCache;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.Throwables;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.cassandra.io.util.FileHandle.Builder.NO_LENGTH_OVERRIDE;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-public class BigTableWriter extends SSTableWriter
+public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter>
{
private static final Logger logger = LoggerFactory.getLogger(BigTableWriter.class);
- private final ColumnIndex columnIndexWriter;
- private final IndexWriter iwriter;
- private final FileHandle.Builder dbuilder;
- protected final SequentialWriter dataFile;
- private DecoratedKey lastWrittenKey;
- private DataPosition dataMark;
- private long lastEarlyOpenLength = 0;
- private final Optional<ChunkCache> chunkCache = Optional.ofNullable(ChunkCache.instance);
+ private final IndexWriter indexWriter;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+ private final Map<DecoratedKey, AbstractRowIndexEntry> cachedKeys = new HashMap<>();
+ private final boolean shouldMigrateKeyCache;
- private final SequentialWriterOption writerOption = SequentialWriterOption.newBuilder()
- .trickleFsync(DatabaseDescriptor.getTrickleFsync())
- .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024)
- .build();
-
- public BigTableWriter(Descriptor descriptor,
- long keyCount,
- long repairedAt,
- TimeUUID pendingRepair,
- boolean isTransient,
- TableMetadataRef metadata,
- MetadataCollector metadataCollector,
- SerializationHeader header,
- Collection<SSTableFlushObserver> observers,
- LifecycleNewTracker lifecycleNewTracker)
+ public BigTableWriter(Builder builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner)
{
- super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers);
- lifecycleNewTracker.trackNew(this); // must track before any files are created
+ super(builder, lifecycleNewTracker, owner);
+ checkNotNull(builder.getRowIndexEntrySerializer());
+ checkNotNull(builder.getIndexWriter());
- if (compression)
- {
- final CompressionParams compressionParams = compressionFor(lifecycleNewTracker.opType());
-
- dataFile = new CompressedSequentialWriter(new File(getFilename()),
- descriptor.filenameFor(Component.COMPRESSION_INFO),
- new File(descriptor.filenameFor(Component.DIGEST)),
- writerOption,
- compressionParams,
- metadataCollector);
- }
- else
- {
- dataFile = new ChecksummedSequentialWriter(new File(getFilename()),
- new File(descriptor.filenameFor(Component.CRC)),
- new File(descriptor.filenameFor(Component.DIGEST)),
- writerOption);
- }
- dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(compression)
- .mmapped(DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap);
- chunkCache.ifPresent(dbuilder::withChunkCache);
- iwriter = new IndexWriter(keyCount);
-
- columnIndexWriter = new ColumnIndex(this.header, dataFile, descriptor.version, this.observers, getRowIndexEntrySerializer().indexInfoSerializer());
+ this.rowIndexEntrySerializer = builder.getRowIndexEntrySerializer();
+ this.indexWriter = builder.getIndexWriter();
+ this.shouldMigrateKeyCache = DatabaseDescriptor.shouldMigrateKeycacheOnCompaction()
+ && lifecycleNewTracker instanceof ILifecycleTransaction
+ && !((ILifecycleTransaction) lifecycleNewTracker).isOffline();
}
- /**
- * Given an OpType, determine the correct Compression Parameters
- * @param opType
- * @return {@link org.apache.cassandra.schema.CompressionParams}
- */
- private CompressionParams compressionFor(final OperationType opType)
- {
- CompressionParams compressionParams = metadata.getLocal().params.compression;
- final ICompressor compressor = compressionParams.getSstableCompressor();
-
- if (null != compressor && opType == OperationType.FLUSH)
- {
- // When we are flushing out of the memtable throughput of the compressor is critical as flushes,
- // especially of large tables, can queue up and potentially block writes.
- // This optimization allows us to fall back to a faster compressor if a particular
- // compression algorithm indicates we should. See CASSANDRA-15379 for more details.
- switch (DatabaseDescriptor.getFlushCompression())
- {
- // It is relatively easier to insert a Noop compressor than to disable compressed writing
- // entirely as the "compression" member field is provided outside the scope of this class.
- // It may make sense in the future to refactor the ownership of the compression flag so that
- // We can bypass the CompressedSequentialWriter in this case entirely.
- case none:
- compressionParams = CompressionParams.NOOP;
- break;
- case fast:
- if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION))
- {
- // The default compressor is generally fast (LZ4 with 16KiB block size)
- compressionParams = CompressionParams.DEFAULT;
- break;
- }
- case table:
- default:
- }
- }
- return compressionParams;
- }
-
+ @Override
public void mark()
{
- dataMark = dataFile.mark();
- iwriter.mark();
+ super.mark();
+ indexWriter.mark();
}
+ @Override
public void resetAndTruncate()
{
- dataFile.resetAndTruncate(dataMark);
- iwriter.resetAndTruncate();
+ super.resetAndTruncate();
+ indexWriter.resetAndTruncate();
}
- /**
- * Perform sanity checks on @param decoratedKey and @return the position in the data file before any data is written
- */
- protected long beforeAppend(DecoratedKey decoratedKey)
+ @Override
+ protected void onStartPartition(DecoratedKey key)
{
- assert decoratedKey != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values
- if (lastWrittenKey != null && lastWrittenKey.compareTo(decoratedKey) >= 0)
- throw new RuntimeException("Last written key " + lastWrittenKey + " >= current key " + decoratedKey + " writing into " + getFilename());
- return (lastWrittenKey == null) ? 0 : dataFile.position();
+ notifyObservers(o -> o.startPartition(key, partitionWriter.getInitialPosition(), indexWriter.writer.position()));
}
- private void afterAppend(DecoratedKey decoratedKey, long dataEnd, RowIndexEntry index, ByteBuffer indexInfo) throws IOException
+ @Override
+ protected RowIndexEntry createRowIndexEntry(DecoratedKey key, DeletionTime partitionLevelDeletion, long finishResult) throws IOException
{
- metadataCollector.addKey(decoratedKey.getKey());
- lastWrittenKey = decoratedKey;
- last = lastWrittenKey;
- if (first == null)
- first = lastWrittenKey;
+ // afterAppend() writes the partition key before the first RowIndexEntry - so we have to add it's
+ // serialized size to the index-writer position
+ long indexFilePosition = ByteBufferUtil.serializedSizeWithShortLength(key.getKey()) + indexWriter.writer.position();
- if (logger.isTraceEnabled())
- logger.trace("wrote {} at {}", decoratedKey, dataEnd);
- iwriter.append(decoratedKey, index, dataEnd, indexInfo);
- }
+ RowIndexEntry entry = RowIndexEntry.create(partitionWriter.getInitialPosition(),
+ indexFilePosition,
+ partitionLevelDeletion,
+ partitionWriter.getHeaderLength(),
+ partitionWriter.getColumnIndexCount(),
+ partitionWriter.indexInfoSerializedSize(),
+ partitionWriter.indexSamples(),
+ partitionWriter.offsets(),
+ rowIndexEntrySerializer.indexInfoSerializer());
- /**
- * Appends partition data to this writer.
- *
- * @param iterator the partition to write
- * @return the created index entry if something was written, that is if {@code iterator}
- * wasn't empty, {@code null} otherwise.
- *
- * @throws FSWriteError if a write to the dataFile fails
- */
- public RowIndexEntry append(UnfilteredRowIterator iterator)
- {
- DecoratedKey key = iterator.partitionKey();
+ indexWriter.append(key, entry, dataWriter.position(), partitionWriter.buffer());
- if (key.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+ if (shouldMigrateKeyCache)
{
- logger.error("Key size {} exceeds maximum of {}, skipping row", key.getKey().remaining(), FBUtilities.MAX_UNSIGNED_SHORT);
- return null;
- }
-
- if (iterator.isEmpty())
- return null;
-
- long startPosition = beforeAppend(key);
- observers.forEach((o) -> o.startPartition(key, iwriter.indexFile.position()));
-
- //Reuse the writer for each row
- columnIndexWriter.reset();
-
- try (UnfilteredRowIterator collecting = Transformation.apply(iterator, new StatsCollector(metadataCollector)))
- {
- columnIndexWriter.buildRowIndex(collecting);
-
- // afterAppend() writes the partition key before the first RowIndexEntry - so we have to add it's
- // serialized size to the index-writer position
- long indexFilePosition = ByteBufferUtil.serializedSizeWithShortLength(key.getKey()) + iwriter.indexFile.position();
-
- RowIndexEntry entry = RowIndexEntry.create(startPosition, indexFilePosition,
- collecting.partitionLevelDeletion(),
- columnIndexWriter.headerLength,
- columnIndexWriter.columnIndexCount,
- columnIndexWriter.indexInfoSerializedSize(),
- columnIndexWriter.indexSamples(),
- columnIndexWriter.offsets(),
- getRowIndexEntrySerializer().indexInfoSerializer());
-
- long endPosition = dataFile.position();
- long rowSize = endPosition - startPosition;
- maybeLogLargePartitionWarning(key, rowSize);
- maybeLogManyTombstonesWarning(key, metadataCollector.totalTombstones);
- metadataCollector.addPartitionSizeInBytes(rowSize);
- afterAppend(key, endPosition, entry, columnIndexWriter.buffer());
- return entry;
- }
- catch (BufferOverflowException boe)
- {
- throw new PartitionSerializationException(iterator, boe);
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, dataFile.getPath());
- }
- }
-
- private RowIndexEntry.IndexSerializer<IndexInfo> getRowIndexEntrySerializer()
- {
- return (RowIndexEntry.IndexSerializer<IndexInfo>) rowIndexEntrySerializer;
- }
-
- private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize)
- {
- if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
- {
- String keyString = metadata().partitionKeyType.getString(key.getKey());
- logger.warn("Writing large partition {}/{}:{} ({}) to sstable {}", metadata.keyspace, metadata.name, keyString, FBUtilities.prettyPrintMemory(rowSize), getFilename());
- }
- }
-
- private void maybeLogManyTombstonesWarning(DecoratedKey key, int tombstoneCount)
- {
- if (tombstoneCount > DatabaseDescriptor.getCompactionTombstoneWarningThreshold())
- {
- String keyString = metadata().partitionKeyType.getString(key.getKey());
- logger.warn("Writing {} tombstones to {}/{}:{} in sstable {}", tombstoneCount, metadata.keyspace, metadata.name, keyString, getFilename());
- }
- }
-
- private static class StatsCollector extends Transformation
- {
- private final MetadataCollector collector;
- private int cellCount;
-
- StatsCollector(MetadataCollector collector)
- {
- this.collector = collector;
- }
-
- @Override
- public Row applyToStatic(Row row)
- {
- if (!row.isEmpty())
- cellCount += Rows.collectStats(row, collector);
- return row;
- }
-
- @Override
- public Row applyToRow(Row row)
- {
- collector.updateClusteringValues(row.clustering());
- cellCount += Rows.collectStats(row, collector);
- return row;
- }
-
- @Override
- public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
- {
- collector.updateClusteringValues(marker.clustering());
- if (marker.isBoundary())
+ for (SSTableReader reader : ((ILifecycleTransaction) lifecycleNewTracker).originals())
{
- RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)marker;
- collector.update(bm.endDeletionTime());
- collector.update(bm.startDeletionTime());
+ if (reader instanceof KeyCacheSupport<?> && ((KeyCacheSupport<?>) reader).getCachedPosition(key, false) != null)
+ {
+ cachedKeys.put(key, entry);
+ break;
+ }
}
- else
- {
- collector.update(((RangeTombstoneBoundMarker)marker).deletionTime());
- }
- return marker;
}
- @Override
- public void onPartitionClose()
- {
- collector.addCellPerPartitionCount(cellCount);
- }
-
- @Override
- public DeletionTime applyToDeletion(DeletionTime deletionTime)
- {
- collector.update(deletionTime);
- return deletionTime;
- }
+ return entry;
}
@SuppressWarnings("resource")
- public SSTableReader openEarly()
+ private BigTableReader openInternal(IndexSummaryBuilder.ReadableBoundary boundary, SSTableReader.OpenReason openReason)
{
- // find the max (exclusive) readable key
- IndexSummaryBuilder.ReadableBoundary boundary = iwriter.getMaxReadable();
- if (boundary == null)
- return null;
+ assert boundary == null || (boundary.indexLength > 0 && boundary.dataLength > 0);
+ IFilter filter = null;
IndexSummary indexSummary = null;
- FileHandle ifile = null;
- FileHandle dfile = null;
- SSTableReader sstable = null;
+ FileHandle dataFile = null;
+ FileHandle indexFile = null;
+ BigTableReader.Builder builder = unbuildTo(new BigTableReader.Builder(descriptor), true).setMaxDataAge(maxDataAge)
+ .setSerializationHeader(header)
+ .setOpenReason(openReason)
+ .setFirst(first)
+ .setLast(boundary != null ? boundary.lastKey : last);
+
+ BigTableReader reader;
try
{
- StatsMetadata stats = statsMetadata();
- assert boundary.indexLength > 0 && boundary.dataLength > 0;
- // open the reader early
- indexSummary = iwriter.summary.build(metadata().partitioner, boundary);
- long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
- int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size());
- ifile = iwriter.builder.bufferSize(indexBufferSize).complete(boundary.indexLength);
- if (compression)
- dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(boundary.dataLength));
- int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
- dfile = dbuilder.bufferSize(dataBufferSize).complete(boundary.dataLength);
- invalidateCacheAtBoundary(dfile);
- sstable = SSTableReader.internalOpen(descriptor,
- components, metadata,
- ifile, dfile,
- indexSummary,
- iwriter.bf.sharedCopy(),
- maxDataAge,
- stats,
- SSTableReader.OpenReason.EARLY,
- header);
- // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed)
- sstable.first = getMinimalKey(first);
- sstable.last = getMinimalKey(boundary.lastKey);
- return sstable;
+ builder.setStatsMetadata(statsMetadata());
+
+ EstimatedHistogram partitionSizeHistogram = builder.getStatsMetadata().estimatedPartitionSize;
+ if (boundary != null)
+ {
+ if (partitionSizeHistogram.isOverflowed())
+ {
+ logger.warn("Estimated partition size histogram for '{}' is overflowed ({} values greater than {}). " +
+ "Clearing the overflow bucket to allow for degraded mean and percentile calculations...",
+ descriptor, partitionSizeHistogram.overflowCount(), partitionSizeHistogram.getLargestBucketOffset());
+ partitionSizeHistogram.clearOverflow();
+ }
+ }
+
+ filter = indexWriter.getFilterCopy();
+ builder.setFilter(filter);
+ indexSummary = indexWriter.summary.build(metadata().partitioner, boundary);
+ builder.setIndexSummary(indexSummary);
+
+ long indexFileLength = descriptor.fileFor(Components.PRIMARY_INDEX).length();
+ int indexBufferSize = ioOptions.diskOptimizationStrategy.bufferSize(indexFileLength / builder.getIndexSummary().size());
+ FileHandle.Builder indexFileBuilder = indexWriter.builder;
+ indexFile = indexFileBuilder.bufferSize(indexBufferSize)
+ .withLengthOverride(boundary != null ? boundary.indexLength : NO_LENGTH_OVERRIDE)
+ .complete();
+ builder.setIndexFile(indexFile);
+ dataFile = openDataFile(boundary != null ? boundary.dataLength : NO_LENGTH_OVERRIDE, builder.getStatsMetadata());
+ builder.setDataFile(dataFile);
+ builder.setKeyCache(metadata().params.caching.cacheKeys() ? new KeyCache(CacheService.instance.keyCache) : KeyCache.NO_CACHE);
+
+ reader = builder.build(owner().orElse(null), true, true);
}
catch (Throwable t)
{
JVMStabilityInspector.inspectThrowable(t);
- // If we successfully created our sstable, we can rely on its InstanceTidier to clean things up for us
- if (sstable != null)
- sstable.selfRef().release();
- else
- Stream.of(indexSummary, ifile, dfile).filter(Objects::nonNull).forEach(SharedCloseableImpl::close);
+ Throwables.closeNonNullAndAddSuppressed(t, dataFile, indexFile, indexSummary, filter);
throw t;
}
+
+ try
+ {
+ for (Map.Entry<DecoratedKey, AbstractRowIndexEntry> cachedKey : cachedKeys.entrySet())
+ reader.cacheKey(cachedKey.getKey(), cachedKey.getValue());
+
+ // clearing the collected cache keys so that we will not have to cache them again when opening partial or
+ // final later - cache key refer only to the descriptor, not to the particular SSTableReader instance.
+ cachedKeys.clear();
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+
+ return reader;
}
- void invalidateCacheAtBoundary(FileHandle dfile)
+ @Override
+ public void openEarly(Consumer<SSTableReader> doWhenReady)
{
- chunkCache.ifPresent(cache -> {
- if (lastEarlyOpenLength != 0 && dfile.dataLength() > lastEarlyOpenLength)
- cache.invalidatePosition(dfile, lastEarlyOpenLength);
- });
- lastEarlyOpenLength = dfile.dataLength();
+ // find the max (exclusive) readable key
+ IndexSummaryBuilder.ReadableBoundary boundary = indexWriter.getMaxReadable();
+ if (boundary == null)
+ return;
+
+ doWhenReady.accept(openInternal(boundary, SSTableReader.OpenReason.EARLY));
}
+ @Override
public SSTableReader openFinalEarly()
{
// we must ensure the data is completely flushed to disk
- dataFile.sync();
- iwriter.indexFile.sync();
+ dataWriter.sync();
+ indexWriter.writer.sync();
return openFinal(SSTableReader.OpenReason.EARLY);
}
- @SuppressWarnings("resource")
- private SSTableReader openFinal(SSTableReader.OpenReason openReason)
+ @Override
+ public SSTableReader openFinal(SSTableReader.OpenReason openReason)
{
if (maxDataAge < 0)
maxDataAge = currentTimeMillis();
- IndexSummary indexSummary = null;
- FileHandle ifile = null;
- FileHandle dfile = null;
- SSTableReader sstable = null;
-
- try
- {
- StatsMetadata stats = statsMetadata();
- // finalize in-memory state for the reader
- indexSummary = iwriter.summary.build(metadata().partitioner);
- long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
- int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
- int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size());
- ifile = iwriter.builder.bufferSize(indexBufferSize).complete();
- if (compression)
- dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(0));
- dfile = dbuilder.bufferSize(dataBufferSize).complete();
- invalidateCacheAtBoundary(dfile);
- sstable = SSTableReader.internalOpen(descriptor,
- components,
- metadata,
- ifile,
- dfile,
- indexSummary,
- iwriter.bf.sharedCopy(),
- maxDataAge,
- stats,
- openReason,
- header);
- sstable.first = getMinimalKey(first);
- sstable.last = getMinimalKey(last);
- return sstable;
- }
- catch (Throwable t)
- {
- JVMStabilityInspector.inspectThrowable(t);
- // If we successfully created our sstable, we can rely on its InstanceTidier to clean things up for us
- if (sstable != null)
- sstable.selfRef().release();
- else
- Stream.of(indexSummary, ifile, dfile).filter(Objects::nonNull).forEach(SharedCloseableImpl::close);
- throw t;
- }
+ return openInternal(null, openReason);
}
+ @Override
protected SSTableWriter.TransactionalProxy txnProxy()
{
- return new TransactionalProxy();
- }
-
- class TransactionalProxy extends SSTableWriter.TransactionalProxy
- {
- // finalise our state on disk, including renaming
- protected void doPrepare()
- {
- iwriter.prepareToCommit();
-
- // write sstable statistics
- dataFile.prepareToCommit();
- writeMetadata(descriptor, finalizeMetadata());
-
- // save the table of components
- SSTable.appendTOC(descriptor, components);
-
- if (openResult)
- finalReader = openFinal(SSTableReader.OpenReason.NORMAL);
- }
-
- protected Throwable doCommit(Throwable accumulate)
- {
- accumulate = dataFile.commit(accumulate);
- accumulate = iwriter.commit(accumulate);
- return accumulate;
- }
-
- @Override
- protected Throwable doPostCleanup(Throwable accumulate)
- {
- accumulate = dbuilder.close(accumulate);
- return accumulate;
- }
-
- protected Throwable doAbort(Throwable accumulate)
- {
- accumulate = iwriter.abort(accumulate);
- accumulate = dataFile.abort(accumulate);
- return accumulate;
- }
- }
-
- private void writeMetadata(Descriptor desc, Map<MetadataType, MetadataComponent> components)
- {
- File file = new File(desc.filenameFor(Component.STATS));
- try (SequentialWriter out = new SequentialWriter(file, writerOption))
- {
- desc.getMetadataSerializer().serialize(components, out, desc.version);
- out.finish();
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, file.path());
- }
- }
-
- public long getFilePointer()
- {
- return dataFile.position();
- }
-
- public long getOnDiskFilePointer()
- {
- return dataFile.getOnDiskFilePointer();
- }
-
- public long getEstimatedOnDiskBytesWritten()
- {
- return dataFile.getEstimatedOnDiskBytesWritten();
+ return new SSTableWriter.TransactionalProxy(() -> FBUtilities.immutableListWithFilteredNulls(indexWriter, dataWriter));
}
/**
* Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
*/
- class IndexWriter extends AbstractTransactional implements Transactional
+ static class IndexWriter extends SortedTableWriter.AbstractIndexWriter
{
- private final SequentialWriter indexFile;
- public final FileHandle.Builder builder;
- public final IndexSummaryBuilder summary;
- public final IFilter bf;
- private DataPosition mark;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- IndexWriter(long keyCount)
+ final SequentialWriter writer;
+ final FileHandle.Builder builder;
+ final IndexSummaryBuilder summary;
+ private DataPosition mark;
+ private DecoratedKey first;
+ private DecoratedKey last;
+
+ protected IndexWriter(Builder b)
{
- indexFile = new SequentialWriter(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), writerOption);
- builder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap);
- chunkCache.ifPresent(builder::withChunkCache);
- summary = new IndexSummaryBuilder(keyCount, metadata().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
- bf = FilterFactory.getFilter(keyCount, metadata().params.bloomFilterFpChance);
+ super(b);
+ this.rowIndexEntrySerializer = b.getRowIndexEntrySerializer();
+ writer = new SequentialWriter(b.descriptor.fileFor(Components.PRIMARY_INDEX), b.getIOOptions().writerOptions);
+ builder = IndexComponent.fileBuilder(Components.PRIMARY_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache());
+ summary = new IndexSummaryBuilder(b.getKeyCount(), b.getTableMetadataRef().getLocal().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
// register listeners to be alerted when the data files are flushed
- indexFile.setPostFlushListener(() -> summary.markIndexSynced(indexFile.getLastFlushOffset()));
- dataFile.setPostFlushListener(() -> summary.markDataSynced(dataFile.getLastFlushOffset()));
+ writer.setPostFlushListener(() -> summary.markIndexSynced(writer.getLastFlushOffset()));
+ @SuppressWarnings("resource")
+ SequentialWriter dataWriter = b.getDataWriter();
+ dataWriter.setPostFlushListener(() -> summary.markDataSynced(dataWriter.getLastFlushOffset()));
}
// finds the last (-offset) decorated key that can be guaranteed to occur fully in the flushed portion of the index file
@@ -561,17 +290,21 @@
public void append(DecoratedKey key, RowIndexEntry indexEntry, long dataEnd, ByteBuffer indexInfo) throws IOException
{
bf.add(key);
- long indexStart = indexFile.position();
+ if (first == null)
+ first = key;
+ last = key;
+
+ long indexStart = writer.position();
try
{
- ByteBufferUtil.writeWithShortLength(key.getKey(), indexFile);
- rowIndexEntrySerializer.serialize(indexEntry, indexFile, indexInfo);
+ ByteBufferUtil.writeWithShortLength(key.getKey(), writer);
+ rowIndexEntrySerializer.serialize(indexEntry, writer, indexInfo);
}
catch (IOException e)
{
- throw new FSWriteError(e, indexFile.getPath());
+ throw new FSWriteError(e, writer.getPath());
}
- long indexEnd = indexFile.position();
+ long indexEnd = writer.position();
if (logger.isTraceEnabled())
logger.trace("wrote index entry: {} at {}", indexEntry, indexStart);
@@ -579,75 +312,152 @@
summary.maybeAddEntry(key, indexStart, indexEnd, dataEnd);
}
- /**
- * Closes the index and bloomfilter, making the public state of this writer valid for consumption.
- */
- void flushBf()
- {
- if (components.contains(Component.FILTER))
- {
- String path = descriptor.filenameFor(Component.FILTER);
- try (FileOutputStreamPlus stream = new FileOutputStreamPlus(path))
- {
- // bloom filter
- BloomFilterSerializer.serialize((BloomFilter) bf, stream);
- stream.flush();
- stream.sync();
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, path);
- }
- }
- }
-
public void mark()
{
- mark = indexFile.mark();
+ mark = writer.mark();
}
public void resetAndTruncate()
{
// we can't un-set the bloom filter addition, but extra keys in there are harmless.
- // we can't reset dbuilder either, but that is the last thing called in afterappend so
+ // we can't reset dbuilder either, but that is the last thing called in afterappend, so
// we assume that if that worked then we won't be trying to reset.
- indexFile.resetAndTruncate(mark);
+ writer.resetAndTruncate(mark);
}
protected void doPrepare()
{
- flushBf();
+ checkNotNull(first);
+ checkNotNull(last);
+
+ super.doPrepare();
// truncate index file
- long position = indexFile.position();
- indexFile.prepareToCommit();
- FileUtils.truncate(indexFile.getPath(), position);
+ long position = writer.position();
+ writer.prepareToCommit();
+ FileUtils.truncate(writer.getPath(), position);
// save summary
summary.prepareToCommit();
- try (IndexSummary indexSummary = summary.build(getPartitioner()))
+ try (IndexSummary indexSummary = summary.build(metadata.getLocal().partitioner))
{
- SSTableReader.saveSummary(descriptor, first, last, indexSummary);
+ new IndexSummaryComponent(indexSummary, first, last).save(descriptor.fileFor(Components.SUMMARY), true);
+ }
+ catch (IOException ex)
+ {
+ logger.warn("Failed to save index summary", ex);
}
}
protected Throwable doCommit(Throwable accumulate)
{
- return indexFile.commit(accumulate);
+ return writer.commit(accumulate);
}
protected Throwable doAbort(Throwable accumulate)
{
- return summary.close(indexFile.abort(accumulate));
+ return summary.close(writer.abort(accumulate));
}
@Override
protected Throwable doPostCleanup(Throwable accumulate)
{
+ accumulate = super.doPostCleanup(accumulate);
accumulate = summary.close(accumulate);
- accumulate = bf.close(accumulate);
- accumulate = builder.close(accumulate);
return accumulate;
}
}
+
+ public static class Builder extends SortedTableWriter.Builder<BigFormatPartitionWriter, BigTableWriter, Builder>
+ {
+ private RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+ private IndexWriter indexWriter;
+ private SequentialWriter dataWriter;
+ private BigFormatPartitionWriter partitionWriter;
+ private MmappedRegionsCache mmappedRegionsCache;
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ @Override
+ public Builder addDefaultComponents()
+ {
+ super.addDefaultComponents();
+
+ addComponents(ImmutableSet.of(Components.PRIMARY_INDEX, Components.SUMMARY));
+
+ return this;
+ }
+
+ // The following getters for the resources opened by buildInternal method can be only used during the lifetime of
+ // that method - that is, during the construction of the sstable.
+
+ @Override
+ public MmappedRegionsCache getMmappedRegionsCache()
+ {
+ return ensuringInBuildInternalContext(mmappedRegionsCache);
+ }
+
+ @Override
+ public SequentialWriter getDataWriter()
+ {
+ return ensuringInBuildInternalContext(dataWriter);
+ }
+
+ @Override
+ public BigFormatPartitionWriter getPartitionWriter()
+ {
+ return ensuringInBuildInternalContext(partitionWriter);
+ }
+
+ public RowIndexEntry.IndexSerializer getRowIndexEntrySerializer()
+ {
+ return ensuringInBuildInternalContext(rowIndexEntrySerializer);
+ }
+
+ public IndexWriter getIndexWriter()
+ {
+ return ensuringInBuildInternalContext(indexWriter);
+ }
+
+ private <T> T ensuringInBuildInternalContext(T value)
+ {
+ Preconditions.checkState(value != null, "This getter can be used only during the lifetime of the sstable constructor. Do not use it directly.");
+ return value;
+ }
+
+ @Override
+ protected BigTableWriter buildInternal(LifecycleNewTracker lifecycleNewTracker, Owner owner)
+ {
+ try
+ {
+ mmappedRegionsCache = new MmappedRegionsCache();
+ rowIndexEntrySerializer = new RowIndexEntry.Serializer(descriptor.version, getSerializationHeader(), owner != null ? owner.getMetrics() : null);
+ dataWriter = DataComponent.buildWriter(descriptor,
+ getTableMetadataRef().getLocal(),
+ getIOOptions().writerOptions,
+ getMetadataCollector(),
+ lifecycleNewTracker.opType(),
+ getIOOptions().flushCompression);
+ indexWriter = new IndexWriter(this);
+ partitionWriter = new BigFormatPartitionWriter(getSerializationHeader(), dataWriter, descriptor.version, rowIndexEntrySerializer.indexInfoSerializer());
+ return new BigTableWriter(this, lifecycleNewTracker, owner);
+ }
+ catch (RuntimeException | Error ex)
+ {
+ Throwables.closeAndAddSuppressed(ex, partitionWriter, indexWriter, dataWriter, mmappedRegionsCache);
+ throw ex;
+ }
+ finally
+ {
+ rowIndexEntrySerializer = null;
+ indexWriter = null;
+ dataWriter = null;
+ partitionWriter = null;
+ mmappedRegionsCache = null;
+ }
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java
deleted file mode 100644
index f640349..0000000
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format.big;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.channels.ClosedChannelException;
-import java.util.Collection;
-import java.util.EnumMap;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.io.util.File;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.compress.BufferType;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.io.util.SequentialWriterOption;
-import org.apache.cassandra.net.AsyncStreamingInputPlus;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.schema.TableMetadataRef;
-
-import static java.lang.String.format;
-import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
-
-public class BigTableZeroCopyWriter extends SSTable implements SSTableMultiWriter
-{
- private static final Logger logger = LoggerFactory.getLogger(BigTableZeroCopyWriter.class);
-
- private final TableMetadataRef metadata;
- private volatile SSTableReader finalReader;
- private final Map<Component.Type, SequentialWriter> componentWriters;
-
- private static final SequentialWriterOption WRITER_OPTION =
- SequentialWriterOption.newBuilder()
- .trickleFsync(false)
- .bufferSize(2 << 20)
- .bufferType(BufferType.OFF_HEAP)
- .build();
-
- private static final ImmutableSet<Component> SUPPORTED_COMPONENTS =
- ImmutableSet.of(Component.DATA,
- Component.PRIMARY_INDEX,
- Component.SUMMARY,
- Component.STATS,
- Component.COMPRESSION_INFO,
- Component.FILTER,
- Component.DIGEST,
- Component.CRC);
-
- public BigTableZeroCopyWriter(Descriptor descriptor,
- TableMetadataRef metadata,
- LifecycleNewTracker lifecycleNewTracker,
- final Collection<Component> components)
- {
- super(descriptor, ImmutableSet.copyOf(components), metadata, DatabaseDescriptor.getDiskOptimizationStrategy());
-
- lifecycleNewTracker.trackNew(this);
- this.metadata = metadata;
- this.componentWriters = new EnumMap<>(Component.Type.class);
-
- if (!SUPPORTED_COMPONENTS.containsAll(components))
- throw new AssertionError(format("Unsupported streaming component detected %s",
- Sets.difference(ImmutableSet.copyOf(components), SUPPORTED_COMPONENTS)));
-
- for (Component c : components)
- componentWriters.put(c.type, makeWriter(descriptor, c));
- }
-
- private static SequentialWriter makeWriter(Descriptor descriptor, Component component)
- {
- return new SequentialWriter(new File(descriptor.filenameFor(component)), WRITER_OPTION, false);
- }
-
- private void write(DataInputPlus in, long size, SequentialWriter out) throws FSWriteError
- {
- final int BUFFER_SIZE = 1 << 20;
- long bytesRead = 0;
- byte[] buff = new byte[BUFFER_SIZE];
- try
- {
- while (bytesRead < size)
- {
- int toRead = (int) Math.min(size - bytesRead, BUFFER_SIZE);
- in.readFully(buff, 0, toRead);
- int count = Math.min(toRead, BUFFER_SIZE);
- out.write(buff, 0, count);
- bytesRead += count;
- }
- out.sync(); // finish will also call sync(). Leaving here to get stuff flushed as early as possible
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, out.getPath());
- }
- }
-
- @Override
- public boolean append(UnfilteredRowIterator partition)
- {
- throw new UnsupportedOperationException("Operation not supported by BigTableBlockWriter");
- }
-
- @Override
- public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
- {
- return finish(openResult);
- }
-
- @Override
- public Collection<SSTableReader> finish(boolean openResult)
- {
- setOpenResult(openResult);
-
- for (SequentialWriter writer : componentWriters.values())
- writer.finish();
-
- return finished();
- }
-
- @Override
- public Collection<SSTableReader> finished()
- {
- if (finalReader == null)
- finalReader = SSTableReader.open(descriptor, components, metadata);
-
- return ImmutableList.of(finalReader);
- }
-
- @Override
- public SSTableMultiWriter setOpenResult(boolean openResult)
- {
- return null;
- }
-
- @Override
- public long getFilePointer()
- {
- return 0;
- }
-
- @Override
- public TableId getTableId()
- {
- return metadata.id;
- }
-
- @Override
- public Throwable commit(Throwable accumulate)
- {
- for (SequentialWriter writer : componentWriters.values())
- accumulate = writer.commit(accumulate);
- return accumulate;
- }
-
- @Override
- public Throwable abort(Throwable accumulate)
- {
- for (SequentialWriter writer : componentWriters.values())
- accumulate = writer.abort(accumulate);
- return accumulate;
- }
-
- @Override
- public void prepareToCommit()
- {
- for (SequentialWriter writer : componentWriters.values())
- writer.prepareToCommit();
- }
-
- @Override
- public void close()
- {
- for (SequentialWriter writer : componentWriters.values())
- writer.close();
- }
-
- public void writeComponent(Component.Type type, DataInputPlus in, long size) throws ClosedChannelException
- {
- logger.info("Writing component {} to {} length {}", type, componentWriters.get(type).getPath(), prettyPrintMemory(size));
-
- if (in instanceof AsyncStreamingInputPlus)
- write((AsyncStreamingInputPlus) in, size, componentWriters.get(type));
- else
- write(in, size, componentWriters.get(type));
- }
-
- private void write(AsyncStreamingInputPlus in, long size, SequentialWriter writer) throws ClosedChannelException
- {
- logger.info("Block Writing component to {} length {}", writer.getPath(), prettyPrintMemory(size));
-
- try
- {
- in.consume(writer::writeDirectlyToChannel, size);
- writer.sync();
- }
- catch (EOFException e)
- {
- in.close();
- }
- catch (ClosedChannelException e)
- {
- // FSWriteError triggers disk failure policy, but if we get a connection issue we do not want to do that
- // so rethrow so the error handling logic higher up is able to deal with this
- // see CASSANDRA-17116
- throw e;
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, writer.getPath());
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java
new file mode 100644
index 0000000..c7697cf
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.io.sstable.AbstractSSTableIterator;
+import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.FileHandle;
+
+// Used by indexed readers to store where they are of the index.
+public class IndexState implements AutoCloseable
+{
+ private final AbstractSSTableIterator<RowIndexEntry>.AbstractReader reader;
+ private final ClusteringComparator comparator;
+
+ private final RowIndexEntry indexEntry;
+ private final RowIndexEntry.IndexInfoRetriever indexInfoRetriever;
+ private final boolean reversed;
+
+ private int currentIndexIdx;
+
+ // Marks the beginning of the block corresponding to currentIndexIdx.
+ private DataPosition mark;
+
+ public IndexState(AbstractSSTableIterator<RowIndexEntry>.AbstractReader reader, ClusteringComparator comparator, RowIndexEntry indexEntry, boolean reversed, FileHandle indexFile)
+ {
+ this.reader = reader;
+ this.comparator = comparator;
+ this.indexEntry = indexEntry;
+ this.indexInfoRetriever = indexEntry.openWithIndex(indexFile);
+ this.reversed = reversed;
+ this.currentIndexIdx = reversed ? indexEntry.blockCount() : -1;
+ }
+
+ public boolean isDone()
+ {
+ return reversed ? currentIndexIdx < 0 : currentIndexIdx >= indexEntry.blockCount();
+ }
+
+ // Sets the reader to the beginning of blockIdx.
+ public void setToBlock(int blockIdx) throws IOException
+ {
+ if (blockIdx >= 0 && blockIdx < indexEntry.blockCount())
+ {
+ reader.seekToPosition(columnOffset(blockIdx));
+ mark = reader.file.mark();
+ reader.deserializer.clearState();
+ }
+
+ currentIndexIdx = blockIdx;
+ reader.openMarker = blockIdx > 0 ? index(blockIdx - 1).endOpenMarker : null;
+ }
+
+ private long columnOffset(int i) throws IOException
+ {
+ return indexEntry.position + index(i).offset;
+ }
+
+ public int blocksCount()
+ {
+ return indexEntry.blockCount();
+ }
+
+ // Update the block idx based on the current reader position if we're past the current block.
+ // This only makes sense for forward iteration (for reverse ones, when we reach the end of a block we
+ // should seek to the previous one, not update the index state and continue).
+ public void updateBlock() throws IOException
+ {
+ assert !reversed;
+
+ // If we get here with currentBlockIdx < 0, it means setToBlock() has never been called, so it means
+ // we're about to read from the beginning of the partition, but haven't "prepared" the IndexState yet.
+ // Do so by setting us on the first block.
+ if (currentIndexIdx < 0)
+ {
+ setToBlock(0);
+ return;
+ }
+
+ while (currentIndexIdx + 1 < indexEntry.blockCount() && isPastCurrentBlock())
+ {
+ reader.openMarker = currentIndex().endOpenMarker;
+ ++currentIndexIdx;
+
+ // We have to set the mark, and we have to set it at the beginning of the block. So if we're not at the beginning of the block, this forces us to a weird seek dance.
+ // This can only happen when reading old file however.
+ long startOfBlock = columnOffset(currentIndexIdx);
+ long currentFilePointer = reader.file.getFilePointer();
+ if (startOfBlock == currentFilePointer)
+ {
+ mark = reader.file.mark();
+ }
+ else
+ {
+ reader.seekToPosition(startOfBlock);
+ mark = reader.file.mark();
+ reader.seekToPosition(currentFilePointer);
+ }
+ }
+ }
+
+ // Check if we've crossed an index boundary (based on the mark on the beginning of the index block).
+ public boolean isPastCurrentBlock() throws IOException
+ {
+ assert reader.deserializer != null;
+ return reader.file.bytesPastMark(mark) >= currentIndex().width;
+ }
+
+ public int currentBlockIdx()
+ {
+ return currentIndexIdx;
+ }
+
+ public IndexInfo currentIndex() throws IOException
+ {
+ return index(currentIndexIdx);
+ }
+
+ public IndexInfo index(int i) throws IOException
+ {
+ return indexInfoRetriever.columnsIndex(i);
+ }
+
+ // Finds the index of the first block containing the provided bound, starting at the provided index.
+ // Will be -1 if the bound is before any block, and blocksCount() if it is after every block.
+ public int findBlockIndex(ClusteringBound<?> bound, int fromIdx) throws IOException
+ {
+ if (bound.isBottom())
+ return -1;
+ if (bound.isTop())
+ return blocksCount();
+
+ return indexFor(bound, fromIdx);
+ }
+
+ public int indexFor(ClusteringPrefix<?> name, int lastIndex) throws IOException
+ {
+ IndexInfo target = new IndexInfo(name, name, 0, 0, null);
+ /*
+ Take the example from the unit test, and say your index looks like this:
+ [0..5][10..15][20..25]
+ and you look for the slice [13..17].
+
+ When doing forward slice, we are doing a binary search comparing 13 (the start of the query)
+ to the lastName part of the index slot. You'll end up with the "first" slot, going from left to right,
+ that may contain the start.
+
+ When doing a reverse slice, we do the same thing, only using as a start column the end of the query,
+ i.e. 17 in this example, compared to the firstName part of the index slots. bsearch will give us the
+ first slot where firstName > start ([20..25] here), so we subtract an extra one to get the slot just before.
+ */
+ int startIdx = 0;
+ int endIdx = indexEntry.blockCount() - 1;
+
+ if (reversed)
+ {
+ if (lastIndex < endIdx)
+ {
+ endIdx = lastIndex;
+ }
+ }
+ else
+ {
+ if (lastIndex > 0)
+ {
+ startIdx = lastIndex;
+ }
+ }
+
+ int index = binarySearch(target, comparator.indexComparator(reversed), startIdx, endIdx);
+ return (index < 0 ? -index - (reversed ? 2 : 1) : index);
+ }
+
+ private int binarySearch(IndexInfo key, Comparator<IndexInfo> c, int low, int high) throws IOException
+ {
+ while (low <= high)
+ {
+ int mid = (low + high) >>> 1;
+ IndexInfo midVal = index(mid);
+ int cmp = c.compare(midVal, key);
+
+ if (cmp < 0)
+ low = mid + 1;
+ else if (cmp > 0)
+ high = mid - 1;
+ else
+ return mid;
+ }
+ return -(low + 1);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("IndexState(indexSize=%d, currentBlock=%d, reversed=%b)", indexEntry.blockCount(), currentIndexIdx, reversed);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ indexInfoRetriever.close();
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexSummaryComponent.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexSummaryComponent.java
new file mode 100644
index 0000000..b3a40cb
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexSummaryComponent.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummary;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+
+public class IndexSummaryComponent
+{
+ private static final Logger logger = LoggerFactory.getLogger(IndexSummaryComponent.class);
+
+ public final IndexSummary indexSummary;
+ public final DecoratedKey first;
+ public final DecoratedKey last;
+
+ public IndexSummaryComponent(IndexSummary indexSummary, DecoratedKey first, DecoratedKey last)
+ {
+ this.indexSummary = indexSummary;
+ this.first = first;
+ this.last = last;
+ }
+
+ public static Pair<DecoratedKey, DecoratedKey> loadFirstAndLastKey(File summaryFile, IPartitioner partitioner) throws IOException
+ {
+ if (!summaryFile.exists())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("Index summary {} does not exist", summaryFile.absolutePath());
+ return null;
+ }
+
+ try (FileInputStreamPlus iStream = summaryFile.newInputStream())
+ {
+ return new IndexSummary.IndexSummarySerializer().deserializeFirstLastKey(iStream, partitioner);
+ }
+ }
+
+ /**
+ * Load index summary, first key and last key from Summary.db file if it exists.
+ * <p>
+ * if loaded index summary has different index interval from current value stored in schema,
+ * then Summary.db file will be deleted and need to be rebuilt.
+ */
+ @SuppressWarnings("resource")
+ public static IndexSummaryComponent load(File summaryFile, TableMetadata metadata) throws IOException
+ {
+ if (!summaryFile.exists())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("Index summary {} does not exist", summaryFile.absolutePath());
+ return null;
+ }
+
+ IndexSummary summary = null;
+ try (FileInputStreamPlus iStream = summaryFile.newInputStream())
+ {
+ summary = IndexSummary.serializer.deserialize(iStream,
+ metadata.partitioner,
+ metadata.params.minIndexInterval,
+ metadata.params.maxIndexInterval);
+ DecoratedKey first = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
+ DecoratedKey last = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
+
+ return new IndexSummaryComponent(summary, first, last);
+ }
+ catch (IOException ex)
+ {
+ if (summary != null)
+ summary.close();
+
+ throw new IOException(String.format("Cannot deserialize index summary from %s", summaryFile), ex);
+ }
+ }
+
+ public static IndexSummaryComponent loadOrDeleteCorrupted(File summaryFile, TableMetadata metadata) throws IOException
+ {
+ try
+ {
+ return load(summaryFile, metadata);
+ }
+ catch (IOException ex)
+ {
+ summaryFile.deleteIfExists();
+ throw ex;
+ }
+ }
+
+ /**
+ * Save index summary to Summary.db file.
+ */
+ public void save(File summaryFile, boolean deleteOnFailure) throws IOException
+ {
+ if (summaryFile.exists())
+ summaryFile.delete();
+
+ try (DataOutputStreamPlus oStream = summaryFile.newOutputStream(File.WriteMode.OVERWRITE))
+ {
+ IndexSummary.serializer.serialize(indexSummary, oStream);
+ ByteBufferUtil.writeWithLength(first.getKey(), oStream);
+ ByteBufferUtil.writeWithLength(last.getKey(), oStream);
+ }
+ catch (IOException ex)
+ {
+ if (deleteOnFailure)
+ summaryFile.deleteIfExists();
+ throw new IOException("Failed to save index summary to " + summaryFile, ex);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java
new file mode 100644
index 0000000..4ebfee3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java
@@ -0,0 +1,887 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.codahale.metrics.Histogram;
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ArrayClustering;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.MessageParams;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RejectException;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.ISerializer;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.TrackedDataInputPlus;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+import org.apache.cassandra.metrics.MetricNameFactory;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.ParamType;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.vint.VIntCoding;
+import org.github.jamm.Unmetered;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Binary format of {@code RowIndexEntry} is defined as follows:
+ * {@code
+ * (long) position (64 bit long, vint encoded)
+ * (int) serialized size of data that follows (32 bit int, vint encoded)
+ * -- following for indexed entries only (so serialized size > 0)
+ * (int) DeletionTime.localDeletionTime
+ * (long) DeletionTime.markedForDeletionAt
+ * (int) number of IndexInfo objects (32 bit int, vint encoded)
+ * (*) serialized IndexInfo objects, see below
+ * (*) offsets of serialized IndexInfo objects, since version "ma" (3.0)
+ * Each IndexInfo object's offset is relative to the first IndexInfo object.
+ * }
+ * <p>
+ * See {@link IndexInfo} for a description of the serialized format.
+ * </p>
+ *
+ * <p>
+ * For each partition, the layout of the index file looks like this:
+ * </p>
+ * <ol>
+ * <li>partition key - prefixed with {@code short} length</li>
+ * <li>serialized {@code RowIndexEntry} objects</li>
+ * </ol>
+ *
+ * <p>
+ * Generally, we distinguish between index entries that have <i>index
+ * samples</i> (list of {@link IndexInfo} objects) and those who don't.
+ * For each <i>portion</i> of data for a single partition in the data file,
+ * an index sample is created. The size of that <i>portion</i> is defined
+ * by {@link org.apache.cassandra.config.Config#column_index_size}.
+ * </p>
+ * <p>
+ * Index entries with less than 2 index samples, will just store the
+ * position in the data file.
+ * </p>
+ * <p>
+ * Note: legacy sstables for index entries are those sstable formats that
+ * do <i>not</i> have an offsets table to index samples ({@link IndexInfo}
+ * objects). These are those sstables created on Cassandra versions
+ * earlier than 3.0.
+ * </p>
+ * <p>
+ * For index entries with index samples we store the index samples
+ * ({@link IndexInfo} objects). The bigger the partition, the more
+ * index samples are created. Since a huge amount of index samples
+ * will "pollute" the heap and cause huge GC pressure, Cassandra 3.6
+ * (CASSANDRA-11206) distinguishes between index entries with an
+ * "acceptable" amount of index samples per partition and those
+ * with an "enormous" amount of index samples. The barrier
+ * is controlled by the configuration parameter
+ * {@link org.apache.cassandra.config.Config#column_index_cache_size}.
+ * Index entries with a total serialized size of index samples up to
+ * {@code column_index_cache_size} will be held in an array.
+ * Index entries exceeding that value will always be accessed from
+ * disk.
+ * </p>
+ * <p>
+ * This results in these classes:
+ * </p>
+ * <ul>
+ * <li>{@link RowIndexEntry} just stores the offset in the data file.</li>
+ * <li>{@link IndexedEntry} is for index entries with index samples
+ * and used for both current and legacy sstables, which do not exceed
+ * {@link org.apache.cassandra.config.Config#column_index_cache_size}.</li>
+ * <li>{@link ShallowIndexedEntry} is for index entries with index samples
+ * that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size}
+ * for sstables with an offset table to the index samples.</li>
+ * </ul>
+ * <p>
+ * Since access to index samples on disk (obviously) requires some file
+ * reader, that functionality is encapsulated in implementations of
+ * {@link IndexInfoRetriever}. There is an implementation to access
+ * index samples of legacy sstables (without the offsets table),
+ * an implementation of access sstables with an offsets table.
+ * </p>
+ * <p>
+ * Until now (Cassandra 3.x), we still support reading from <i>legacy</i> sstables -
+ * i.e. sstables created by Cassandra < 3.0 (see {@link org.apache.cassandra.io.sstable.format.big.BigFormat}.
+ * </p>
+ *
+ */
+public class RowIndexEntry extends AbstractRowIndexEntry
+{
+ private static final BigFormat FORMAT = BigFormat.getInstance();
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new RowIndexEntry(0));
+
+ // constants for type of row-index-entry as serialized for saved-cache
+ static final int CACHE_NOT_INDEXED = 0;
+ static final int CACHE_INDEXED = 1;
+ static final int CACHE_INDEXED_SHALLOW = 2;
+
+ static final Histogram indexEntrySizeHistogram;
+ static final Histogram indexInfoCountHistogram;
+ static final Histogram indexInfoGetsHistogram;
+ static final Histogram indexInfoReadsHistogram;
+ static
+ {
+ MetricNameFactory factory = new DefaultNameFactory("Index", "RowIndexEntry");
+ indexEntrySizeHistogram = Metrics.histogram(factory.createMetricName("IndexedEntrySize"), false);
+ indexInfoCountHistogram = Metrics.histogram(factory.createMetricName("IndexInfoCount"), false);
+ indexInfoGetsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoGets"), false);
+ indexInfoReadsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoReads"), false);
+ }
+
+ public RowIndexEntry(long position)
+ {
+ super(position);
+ }
+
+ /**
+ * @return true if this index entry contains the row-level tombstone and column summary. Otherwise,
+ * caller should fetch these from the row header.
+ */
+ @Override
+ public boolean isIndexed()
+ {
+ return blockCount() > 1;
+ }
+
+ public boolean indexOnHeap()
+ {
+ return false;
+ }
+
+ @Override
+ public DeletionTime deletionTime()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int blockCount()
+ {
+ return 0;
+ }
+
+ @Override
+ public BigFormat getSSTableFormat()
+ {
+ return FORMAT;
+ }
+
+ @Override
+ public long unsharedHeapSize()
+ {
+ return EMPTY_SIZE;
+ }
+
+ /**
+ * @param dataFilePosition position of the partition in the {@link Components.Types#DATA} file
+ * @param indexFilePosition position in the {@link Components.Types#PRIMARY_INDEX} of the {@link RowIndexEntry}
+ * @param deletionTime deletion time of {@link RowIndexEntry}
+ * @param headerLength deletion time of {@link RowIndexEntry}
+ * @param columnIndexCount number of {@link IndexInfo} entries in the {@link RowIndexEntry}
+ * @param indexedPartSize serialized size of all serialized {@link IndexInfo} objects and their offsets
+ * @param indexSamples list with IndexInfo offsets (if total serialized size is less than {@link org.apache.cassandra.config.Config#column_index_cache_size}
+ * @param offsets offsets of IndexInfo offsets
+ * @param idxInfoSerializer the {@link IndexInfo} serializer
+ */
+ public static RowIndexEntry create(long dataFilePosition, long indexFilePosition,
+ DeletionTime deletionTime, long headerLength, int columnIndexCount,
+ int indexedPartSize,
+ List<IndexInfo> indexSamples, int[] offsets,
+ ISerializer<IndexInfo> idxInfoSerializer)
+ {
+ // If the "partition building code" in BigTableWriter.append() via ColumnIndex returns a list
+ // of IndexInfo objects, which is the case if the serialized size is less than
+ // Config.column_index_cache_size, AND we have more than one IndexInfo object, we
+ // construct an IndexedEntry object. (note: indexSamples.size() and columnIndexCount have the same meaning)
+ if (indexSamples != null && indexSamples.size() > 1)
+ return new IndexedEntry(dataFilePosition, deletionTime, headerLength,
+ indexSamples.toArray(new IndexInfo[indexSamples.size()]), offsets,
+ indexedPartSize, idxInfoSerializer);
+ // Here we have to decide whether we have serialized IndexInfo objects that exceeds
+ // Config.column_index_cache_size (not exceeding case covered above).
+ // Such a "big" indexed-entry is represented as a shallow one.
+ if (columnIndexCount > 1)
+ return new ShallowIndexedEntry(dataFilePosition, indexFilePosition,
+ deletionTime, headerLength, columnIndexCount,
+ indexedPartSize, idxInfoSerializer);
+ // Last case is that there are no index samples.
+ return new RowIndexEntry(dataFilePosition);
+ }
+
+ public IndexInfoRetriever openWithIndex(FileHandle indexFile)
+ {
+ return null;
+ }
+
+ public interface IndexSerializer
+ {
+ void serialize(RowIndexEntry rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException;
+
+ RowIndexEntry deserialize(DataInputPlus in, long indexFilePosition) throws IOException;
+ default RowIndexEntry deserialize(RandomAccessReader reader) throws IOException
+ {
+ return deserialize(reader, reader.getFilePointer());
+
+ }
+
+ default RowIndexEntry deserialize(FileDataInput input) throws IOException
+ {
+ return deserialize(input, input.getFilePointer());
+
+ }
+
+ void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException;
+
+ RowIndexEntry deserializeForCache(DataInputPlus in) throws IOException;
+
+ long deserializePositionAndSkip(DataInputPlus in) throws IOException;
+
+ ISerializer indexInfoSerializer();
+ }
+
+ public static final class Serializer implements IndexSerializer
+ {
+ private final IndexInfo.Serializer idxInfoSerializer;
+ private final Version version;
+ private final TableMetrics tableMetrics;
+
+ public Serializer(Version version, SerializationHeader header, TableMetrics tableMetrics)
+ {
+ this.idxInfoSerializer = IndexInfo.serializer(version, header);
+ this.version = version;
+ this.tableMetrics = tableMetrics;
+ }
+
+ @Override
+ public IndexInfo.Serializer indexInfoSerializer()
+ {
+ return idxInfoSerializer;
+ }
+
+ @Override
+ public void serialize(RowIndexEntry rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException
+ {
+ rie.serialize(out, indexInfo);
+ }
+
+ @Override
+ public void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException
+ {
+ rie.serializeForCache(out);
+ }
+
+ @Override
+ public RowIndexEntry deserializeForCache(DataInputPlus in) throws IOException
+ {
+ long position = in.readUnsignedVInt();
+
+ switch (in.readByte())
+ {
+ case CACHE_NOT_INDEXED:
+ return new RowIndexEntry(position);
+ case CACHE_INDEXED:
+ return new IndexedEntry(position, in, idxInfoSerializer);
+ case CACHE_INDEXED_SHALLOW:
+ return new ShallowIndexedEntry(position, in, idxInfoSerializer);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ public static void skipForCache(DataInputPlus in) throws IOException
+ {
+ in.readUnsignedVInt();
+ switch (in.readByte())
+ {
+ case CACHE_NOT_INDEXED:
+ break;
+ case CACHE_INDEXED:
+ IndexedEntry.skipForCache(in);
+ break;
+ case CACHE_INDEXED_SHALLOW:
+ ShallowIndexedEntry.skipForCache(in);
+ break;
+ default:
+ assert false;
+ }
+ }
+
+ @Override
+ public RowIndexEntry deserialize(DataInputPlus in, long indexFilePosition) throws IOException
+ {
+ long position = in.readUnsignedVInt();
+
+ int size = in.readUnsignedVInt32();
+ if (size == 0)
+ {
+ return new RowIndexEntry(position);
+ }
+ else
+ {
+ long headerLength = in.readUnsignedVInt();
+ DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
+ int columnsIndexCount = in.readUnsignedVInt32();
+
+ checkSize(columnsIndexCount, size);
+
+ int indexedPartSize = size - serializedSize(deletionTime, headerLength, columnsIndexCount);
+
+ if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
+ {
+ return new IndexedEntry(position, in, deletionTime, headerLength, columnsIndexCount,
+ idxInfoSerializer, indexedPartSize);
+ }
+ else
+ {
+ in.skipBytes(indexedPartSize);
+
+ return new ShallowIndexedEntry(position,
+ indexFilePosition,
+ deletionTime, headerLength, columnsIndexCount,
+ indexedPartSize, idxInfoSerializer);
+ }
+ }
+ }
+
+ private void checkSize(int entries, int bytes)
+ {
+ ReadCommand command = ReadCommand.getCommand();
+ if (command == null || SchemaConstants.isSystemKeyspace(command.metadata().keyspace) || !DatabaseDescriptor.getReadThresholdsEnabled())
+ return;
+
+ DataStorageSpec.LongBytesBound warnThreshold = DatabaseDescriptor.getRowIndexReadSizeWarnThreshold();
+ DataStorageSpec.LongBytesBound failThreshold = DatabaseDescriptor.getRowIndexReadSizeFailThreshold();
+ if (warnThreshold == null && failThreshold == null)
+ return;
+
+ long estimatedMemory = estimateMaterializedIndexSize(entries, bytes);
+ if (tableMetrics != null)
+ tableMetrics.rowIndexSize.update(estimatedMemory);
+
+ if (failThreshold != null && estimatedMemory > failThreshold.toBytes())
+ {
+ String msg = String.format("Query %s attempted to access a large RowIndexEntry estimated to be %d bytes " +
+ "in-memory (total entries: %d, total bytes: %d) but the max allowed is %s;" +
+ " query aborted (see row_index_read_size_fail_threshold)",
+ command.toCQLString(), estimatedMemory, entries, bytes, failThreshold);
+ MessageParams.remove(ParamType.ROW_INDEX_READ_SIZE_WARN);
+ MessageParams.add(ParamType.ROW_INDEX_READ_SIZE_FAIL, estimatedMemory);
+
+ throw new RowIndexEntryReadSizeTooLargeException(msg);
+ }
+ else if (warnThreshold != null && estimatedMemory > warnThreshold.toBytes())
+ {
+ // use addIfLarger rather than add as a previous partition may be larger than this one
+ Long current = MessageParams.get(ParamType.ROW_INDEX_READ_SIZE_WARN);
+ if (current == null || current.compareTo(estimatedMemory) < 0)
+ MessageParams.add(ParamType.ROW_INDEX_READ_SIZE_WARN, estimatedMemory);
+ }
+ }
+
+ private static long estimateMaterializedIndexSize(int entries, int bytes)
+ {
+ long overhead = IndexInfo.EMPTY_SIZE
+ + ArrayClustering.EMPTY_SIZE
+ + DeletionTime.EMPTY_SIZE;
+
+ return (overhead * entries) + bytes;
+ }
+
+ @Override
+ public long deserializePositionAndSkip(DataInputPlus in) throws IOException
+ {
+ long position = in.readUnsignedVInt();
+
+ int size = in.readUnsignedVInt32();
+ if (size > 0)
+ in.skipBytesFully(size);
+
+ return position;
+ }
+
+ /**
+ * Reads only the data 'position' of the index entry and returns it. Note that this left 'in' in the middle
+ * of reading an entry, so this is only useful if you know what you are doing and in most case 'deserialize'
+ * should be used instead.
+ */
+ public static long readPosition(DataInputPlus in) throws IOException
+ {
+ return in.readUnsignedVInt();
+ }
+
+ public static void skip(DataInputPlus in, Version version) throws IOException
+ {
+ readPosition(in);
+ skipPromotedIndex(in);
+ }
+
+ private static void skipPromotedIndex(DataInputPlus in) throws IOException
+ {
+ int size = in.readUnsignedVInt32();
+ if (size <= 0)
+ return;
+
+ in.skipBytesFully(size);
+ }
+ }
+
+ private static int serializedSize(DeletionTime deletionTime, long headerLength, int columnIndexCount)
+ {
+ return TypeSizes.sizeofUnsignedVInt(headerLength)
+ + (int) DeletionTime.serializer.serializedSize(deletionTime)
+ + TypeSizes.sizeofUnsignedVInt(columnIndexCount);
+ }
+
+ public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
+ {
+ out.writeUnsignedVInt(position);
+
+ out.writeUnsignedVInt32(0);
+ }
+
+ public void serializeForCache(DataOutputPlus out) throws IOException
+ {
+ out.writeUnsignedVInt(position);
+
+ out.writeByte(CACHE_NOT_INDEXED);
+ }
+
+ /**
+ * An entry in the row index for a row whose columns are indexed - used for both legacy and current formats.
+ */
+ private static final class IndexedEntry extends RowIndexEntry
+ {
+ private static final long BASE_SIZE;
+
+ static
+ {
+ BASE_SIZE = ObjectSizes.measure(new IndexedEntry(0, DeletionTime.LIVE, 0, null, null, 0, null));
+ }
+
+ private final DeletionTime deletionTime;
+ private final long headerLength;
+
+ private final IndexInfo[] columnsIndex;
+ private final int[] offsets;
+ private final int indexedPartSize;
+ @Unmetered
+ private final ISerializer<IndexInfo> idxInfoSerializer;
+
+ private IndexedEntry(long dataFilePosition, DeletionTime deletionTime, long headerLength,
+ IndexInfo[] columnsIndex, int[] offsets,
+ int indexedPartSize, ISerializer<IndexInfo> idxInfoSerializer)
+ {
+ super(dataFilePosition);
+
+ this.headerLength = headerLength;
+ this.deletionTime = deletionTime;
+
+ this.columnsIndex = columnsIndex;
+ this.offsets = offsets;
+ this.indexedPartSize = indexedPartSize;
+ this.idxInfoSerializer = idxInfoSerializer;
+ }
+
+ private IndexedEntry(long dataFilePosition, DataInputPlus in,
+ DeletionTime deletionTime, long headerLength, int columnIndexCount,
+ IndexInfo.Serializer idxInfoSerializer, int indexedPartSize) throws IOException
+ {
+ super(dataFilePosition);
+
+ this.headerLength = headerLength;
+ this.deletionTime = deletionTime;
+ int columnsIndexCount = columnIndexCount;
+
+ this.columnsIndex = new IndexInfo[columnsIndexCount];
+ for (int i = 0; i < columnsIndexCount; i++)
+ this.columnsIndex[i] = idxInfoSerializer.deserialize(in);
+
+ this.offsets = new int[this.columnsIndex.length];
+ for (int i = 0; i < offsets.length; i++)
+ offsets[i] = in.readInt();
+
+ this.indexedPartSize = indexedPartSize;
+
+ this.idxInfoSerializer = idxInfoSerializer;
+ }
+
+ /**
+ * Constructor called from {@link Serializer#deserializeForCache(org.apache.cassandra.io.util.DataInputPlus)}.
+ */
+ private IndexedEntry(long dataFilePosition, DataInputPlus in, ISerializer<IndexInfo> idxInfoSerializer) throws IOException
+ {
+ super(dataFilePosition);
+
+ this.headerLength = in.readUnsignedVInt();
+ this.deletionTime = DeletionTime.serializer.deserialize(in);
+ int columnsIndexCount = in.readUnsignedVInt32();
+
+ TrackedDataInputPlus trackedIn = new TrackedDataInputPlus(in);
+
+ this.columnsIndex = new IndexInfo[columnsIndexCount];
+ for (int i = 0; i < columnsIndexCount; i++)
+ this.columnsIndex[i] = idxInfoSerializer.deserialize(trackedIn);
+
+ this.offsets = null;
+
+ this.indexedPartSize = (int) trackedIn.getBytesRead();
+
+ this.idxInfoSerializer = idxInfoSerializer;
+ }
+
+ @Override
+ public boolean indexOnHeap()
+ {
+ return true;
+ }
+
+ @Override
+ public int blockCount()
+ {
+ return columnsIndex.length;
+ }
+
+ @Override
+ public DeletionTime deletionTime()
+ {
+ return deletionTime;
+ }
+
+ @Override
+ public IndexInfoRetriever openWithIndex(FileHandle indexFile)
+ {
+ indexEntrySizeHistogram.update(serializedSize(deletionTime, headerLength, columnsIndex.length) + indexedPartSize);
+ indexInfoCountHistogram.update(columnsIndex.length);
+ return new IndexInfoRetriever()
+ {
+ private int retrievals;
+
+ @Override
+ public IndexInfo columnsIndex(int index)
+ {
+ retrievals++;
+ return columnsIndex[index];
+ }
+
+ @Override
+ public void close()
+ {
+ indexInfoGetsHistogram.update(retrievals);
+ }
+ };
+ }
+
+ @Override
+ public long unsharedHeapSize()
+ {
+ long entrySize = 0;
+ for (IndexInfo idx : columnsIndex)
+ entrySize += idx.unsharedHeapSize();
+ return BASE_SIZE
+ + entrySize
+ + ObjectSizes.sizeOfReferenceArray(columnsIndex.length);
+ }
+
+ @Override
+ public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
+ {
+ assert indexedPartSize != Integer.MIN_VALUE;
+
+ out.writeUnsignedVInt(position);
+
+ out.writeUnsignedVInt32(serializedSize(deletionTime, headerLength, columnsIndex.length) + indexedPartSize);
+
+ out.writeUnsignedVInt(headerLength);
+ DeletionTime.serializer.serialize(deletionTime, out);
+ out.writeUnsignedVInt32(columnsIndex.length);
+ for (IndexInfo info : columnsIndex)
+ idxInfoSerializer.serialize(info, out);
+ for (int offset : offsets)
+ out.writeInt(offset);
+ }
+
+ @Override
+ public void serializeForCache(DataOutputPlus out) throws IOException
+ {
+ out.writeUnsignedVInt(position);
+ out.writeByte(CACHE_INDEXED);
+
+ out.writeUnsignedVInt(headerLength);
+ DeletionTime.serializer.serialize(deletionTime, out);
+ out.writeUnsignedVInt32(blockCount());
+
+ for (IndexInfo indexInfo : columnsIndex)
+ idxInfoSerializer.serialize(indexInfo, out);
+ }
+
+ static void skipForCache(DataInputPlus in) throws IOException
+ {
+ in.readUnsignedVInt();
+ DeletionTime.serializer.skip(in);
+ in.readUnsignedVInt();
+
+ in.readUnsignedVInt();
+ }
+ }
+
+ /**
+ * An entry in the row index for a row whose columns are indexed and the {@link IndexInfo} objects
+ * are not read into the key cache.
+ */
+ private static final class ShallowIndexedEntry extends RowIndexEntry
+ {
+ private static final long BASE_SIZE;
+
+ static
+ {
+ BASE_SIZE = ObjectSizes.measure(new ShallowIndexedEntry(0, 0, DeletionTime.LIVE, 0, 10, 0, null));
+ }
+
+ private final long indexFilePosition;
+
+ private final DeletionTime deletionTime;
+ private final long headerLength;
+ private final int columnsIndexCount;
+
+ private final int indexedPartSize;
+ private final int offsetsOffset;
+ @Unmetered
+ private final ISerializer<IndexInfo> idxInfoSerializer;
+ private final int fieldsSerializedSize;
+
+ /**
+ * See {@link #create(long, long, DeletionTime, long, int, int, List, int[], ISerializer)} for a description
+ * of the parameters.
+ */
+ private ShallowIndexedEntry(long dataFilePosition, long indexFilePosition,
+ DeletionTime deletionTime, long headerLength, int columnIndexCount,
+ int indexedPartSize, ISerializer<IndexInfo> idxInfoSerializer)
+ {
+ super(dataFilePosition);
+
+ assert columnIndexCount > 1;
+
+ this.indexFilePosition = indexFilePosition;
+ this.headerLength = headerLength;
+ this.deletionTime = deletionTime;
+ this.columnsIndexCount = columnIndexCount;
+
+ this.indexedPartSize = indexedPartSize;
+ this.idxInfoSerializer = idxInfoSerializer;
+
+ this.fieldsSerializedSize = serializedSize(deletionTime, headerLength, columnIndexCount);
+ this.offsetsOffset = indexedPartSize + fieldsSerializedSize - columnsIndexCount * TypeSizes.INT_SIZE;
+ }
+
+ /**
+ * Constructor for key-cache deserialization
+ */
+ private ShallowIndexedEntry(long dataFilePosition, DataInputPlus in, IndexInfo.Serializer idxInfoSerializer) throws IOException
+ {
+ super(dataFilePosition);
+
+ this.indexFilePosition = in.readUnsignedVInt();
+
+ this.headerLength = in.readUnsignedVInt();
+ this.deletionTime = DeletionTime.serializer.deserialize(in);
+ this.columnsIndexCount = in.readUnsignedVInt32();
+
+ this.indexedPartSize = in.readUnsignedVInt32();
+
+ this.idxInfoSerializer = idxInfoSerializer;
+
+ this.fieldsSerializedSize = serializedSize(deletionTime, headerLength, columnsIndexCount);
+ this.offsetsOffset = indexedPartSize + fieldsSerializedSize - columnsIndexCount * TypeSizes.INT_SIZE;
+ }
+
+ @Override
+ public int blockCount()
+ {
+ return columnsIndexCount;
+ }
+
+ @Override
+ public DeletionTime deletionTime()
+ {
+ return deletionTime;
+ }
+
+ @Override
+ public IndexInfoRetriever openWithIndex(FileHandle indexFile)
+ {
+ indexEntrySizeHistogram.update(indexedPartSize + fieldsSerializedSize);
+ indexInfoCountHistogram.update(columnsIndexCount);
+ return new ShallowInfoRetriever(indexFilePosition +
+ VIntCoding.computeUnsignedVIntSize(position) +
+ VIntCoding.computeUnsignedVIntSize(indexedPartSize + fieldsSerializedSize) +
+ fieldsSerializedSize,
+ offsetsOffset - fieldsSerializedSize,
+ indexFile.createReader(), idxInfoSerializer);
+ }
+
+ @Override
+ public long unsharedHeapSize()
+ {
+ return BASE_SIZE;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException
+ {
+ out.writeUnsignedVInt(position);
+
+ out.writeUnsignedVInt32(fieldsSerializedSize + indexInfo.limit());
+
+ out.writeUnsignedVInt(headerLength);
+ DeletionTime.serializer.serialize(deletionTime, out);
+ out.writeUnsignedVInt32(columnsIndexCount);
+
+ out.write(indexInfo);
+ }
+
+ @Override
+ public void serializeForCache(DataOutputPlus out) throws IOException
+ {
+ out.writeUnsignedVInt(position);
+ out.writeByte(CACHE_INDEXED_SHALLOW);
+
+ out.writeUnsignedVInt(indexFilePosition);
+
+ out.writeUnsignedVInt(headerLength);
+ DeletionTime.serializer.serialize(deletionTime, out);
+ out.writeUnsignedVInt32(columnsIndexCount);
+
+ out.writeUnsignedVInt32(indexedPartSize);
+ }
+
+ static void skipForCache(DataInputPlus in) throws IOException
+ {
+ in.readUnsignedVInt();
+
+ in.readUnsignedVInt();
+ DeletionTime.serializer.skip(in);
+ in.readUnsignedVInt();
+
+ in.readUnsignedVInt();
+ }
+ }
+
+ private static final class ShallowInfoRetriever extends FileIndexInfoRetriever
+ {
+ private final int offsetsOffset;
+
+ private ShallowInfoRetriever(long indexInfoFilePosition, int offsetsOffset,
+ FileDataInput indexReader, ISerializer<IndexInfo> idxInfoSerializer)
+ {
+ super(indexInfoFilePosition, indexReader, idxInfoSerializer);
+ this.offsetsOffset = offsetsOffset;
+ }
+
+ @Override
+ IndexInfo fetchIndex(int index) throws IOException
+ {
+ // seek to position in "offsets to IndexInfo" table
+ indexReader.seek(indexInfoFilePosition + offsetsOffset + index * TypeSizes.INT_SIZE);
+
+ // read offset of IndexInfo
+ int indexInfoPos = indexReader.readInt();
+
+ // seek to posision of IndexInfo
+ indexReader.seek(indexInfoFilePosition + indexInfoPos);
+
+ // finally, deserialize IndexInfo
+ return idxInfoSerializer.deserialize(indexReader);
+ }
+ }
+
+ /**
+ * Base class to access {@link IndexInfo} objects.
+ */
+ public interface IndexInfoRetriever extends AutoCloseable
+ {
+ IndexInfo columnsIndex(int index) throws IOException;
+
+ void close() throws IOException;
+ }
+
+ /**
+ * Base class to access {@link IndexInfo} objects on disk that keeps already
+ * read {@link IndexInfo} on heap.
+ */
+ private abstract static class FileIndexInfoRetriever implements IndexInfoRetriever
+ {
+ final long indexInfoFilePosition;
+ final ISerializer<IndexInfo> idxInfoSerializer;
+ final FileDataInput indexReader;
+ int retrievals;
+
+ /**
+ *
+ * @param indexInfoFilePosition offset of first serialized {@link IndexInfo} object
+ * @param indexReader file data input to access the index file, closed by this instance
+ * @param idxInfoSerializer the index serializer to deserialize {@link IndexInfo} objects
+ */
+ FileIndexInfoRetriever(long indexInfoFilePosition, FileDataInput indexReader, ISerializer<IndexInfo> idxInfoSerializer)
+ {
+ this.indexInfoFilePosition = indexInfoFilePosition;
+ this.idxInfoSerializer = idxInfoSerializer;
+ this.indexReader = indexReader;
+ }
+
+ @Override
+ public final IndexInfo columnsIndex(int index) throws IOException
+ {
+ retrievals++;
+ return fetchIndex(index);
+ }
+
+ abstract IndexInfo fetchIndex(int index) throws IOException;
+
+ @Override
+ public void close() throws IOException
+ {
+ indexReader.close();
+
+ indexInfoGetsHistogram.update(retrievals);
+ indexInfoReadsHistogram.update(retrievals);
+ }
+ }
+
+ public static class RowIndexEntryReadSizeTooLargeException extends RejectException
+ {
+ public RowIndexEntryReadSizeTooLargeException(String message)
+ {
+ super(message);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableIterator.java
new file mode 100644
index 0000000..0415d0a
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableIterator.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.sstable.AbstractSSTableIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+
+/**
+ * A Cell Iterator over SSTable
+ */
+public class SSTableIterator extends AbstractSSTableIterator<RowIndexEntry>
+{
+ /**
+ * The index of the slice being processed.
+ */
+ private int slice;
+
+ public SSTableIterator(SSTableReader sstable,
+ FileDataInput file,
+ DecoratedKey key,
+ RowIndexEntry indexEntry,
+ Slices slices,
+ ColumnFilter columns,
+ FileHandle ifile)
+ {
+ super(sstable, file, key, indexEntry, slices, columns, ifile);
+ }
+
+ @SuppressWarnings("resource") // caller to close
+ protected Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ return indexEntry.isIndexed()
+ ? new ForwardIndexedReader(indexEntry, file, shouldCloseFile)
+ : new ForwardReader(file, shouldCloseFile);
+ }
+
+ protected int nextSliceIndex()
+ {
+ int next = slice;
+ slice++;
+ return next;
+ }
+
+ protected boolean hasMoreSlices()
+ {
+ return slice < slices.size();
+ }
+
+ public boolean isReverseOrder()
+ {
+ return false;
+ }
+
+ private class ForwardIndexedReader extends ForwardReader
+ {
+ private final IndexState indexState;
+
+ private int lastBlockIdx; // the last index block that has data for the current query
+
+ private ForwardIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ this.indexState = new IndexState(this, metadata.comparator, indexEntry, false, ifile);
+ this.lastBlockIdx = indexState.blocksCount(); // if we never call setForSlice, that's where we want to stop
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ super.close();
+ this.indexState.close();
+ }
+
+ @Override
+ public void setForSlice(Slice slice) throws IOException
+ {
+ super.setForSlice(slice);
+
+ // if our previous slicing already got us the biggest row in the sstable, we're done
+ if (indexState.isDone())
+ {
+ sliceDone = true;
+ return;
+ }
+
+ // Find the first index block we'll need to read for the slice.
+ int startIdx = indexState.findBlockIndex(slice.start(), indexState.currentBlockIdx());
+ if (startIdx >= indexState.blocksCount())
+ {
+ sliceDone = true;
+ return;
+ }
+
+ // Find the last index block we'll need to read for the slice.
+ lastBlockIdx = indexState.findBlockIndex(slice.end(), startIdx);
+
+ // If the slice end is before the very first block, we have nothing for that slice
+ if (lastBlockIdx < 0)
+ {
+ assert startIdx < 0;
+ sliceDone = true;
+ return;
+ }
+
+ // If we start before the very first block, just read from the first one.
+ if (startIdx < 0)
+ startIdx = 0;
+
+ // If that's the last block we were reading, we're already where we want to be. Otherwise,
+ // seek to that first block
+ if (startIdx != indexState.currentBlockIdx())
+ indexState.setToBlock(startIdx);
+
+ // The index search is based on the last name of the index blocks, so at that point we have that:
+ // 1) indexes[currentIdx - 1].lastName < slice.start <= indexes[currentIdx].lastName
+ // 2) indexes[lastBlockIdx - 1].lastName < slice.end <= indexes[lastBlockIdx].lastName
+ // so if currentIdx == lastBlockIdx and slice.end < indexes[currentIdx].firstName, we're guaranteed that the
+ // whole slice is between the previous block end and this block start, and thus has no corresponding
+ // data. One exception is if the previous block ends with an openMarker as it will cover our slice
+ // and we need to return it.
+ if (indexState.currentBlockIdx() == lastBlockIdx
+ && metadata().comparator.compare(slice.end(), indexState.currentIndex().firstName) < 0
+ && openMarker == null)
+ {
+ sliceDone = true;
+ }
+ }
+
+ @Override
+ protected Unfiltered computeNext() throws IOException
+ {
+ while (true)
+ {
+ // Our previous read might have made us cross an index block boundary. If so, update our informations.
+ // If we read from the beginning of the partition, this is also what will initialize the index state.
+ indexState.updateBlock();
+
+ // Return the next unfiltered unless we've reached the end, or we're beyond our slice
+ // end (note that unless we're on the last block for the slice, there is no point
+ // in checking the slice end).
+ if (indexState.isDone()
+ || indexState.currentBlockIdx() > lastBlockIdx
+ || !deserializer.hasNext()
+ || (indexState.currentBlockIdx() == lastBlockIdx && deserializer.compareNextTo(end) >= 0))
+ return null;
+
+
+ Unfiltered next = deserializer.readNext();
+ UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable);
+ // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
+ if (next.isEmpty())
+ continue;
+
+ if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
+ updateOpenMarker((RangeTombstoneMarker) next);
+ return next;
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableReversedIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableReversedIterator.java
new file mode 100644
index 0000000..14e2538
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableReversedIterator.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.MutableDeletionInfo;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.sstable.AbstractSSTableIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.btree.BTree;
+
+/**
+ * A Cell Iterator in reversed clustering order over SSTable
+ */
+public class SSTableReversedIterator extends AbstractSSTableIterator<RowIndexEntry>
+{
+ /**
+ * The index of the slice being processed.
+ */
+ private int slice;
+
+ public SSTableReversedIterator(SSTableReader sstable,
+ FileDataInput file,
+ DecoratedKey key,
+ RowIndexEntry indexEntry,
+ Slices slices,
+ ColumnFilter columns,
+ FileHandle ifile)
+ {
+ super(sstable, file, key, indexEntry, slices, columns, ifile);
+ }
+
+ @SuppressWarnings("resource") // caller to close
+ protected Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ return indexEntry.isIndexed()
+ ? new ReverseIndexedReader(indexEntry, file, shouldCloseFile)
+ : new ReverseReader(file, shouldCloseFile);
+ }
+
+ public boolean isReverseOrder()
+ {
+ return true;
+ }
+
+ protected int nextSliceIndex()
+ {
+ int next = slice;
+ slice++;
+ return slices.size() - (next + 1);
+ }
+
+ protected boolean hasMoreSlices()
+ {
+ return slice < slices.size();
+ }
+
+ private class ReverseReader extends AbstractReader
+ {
+ protected ReusablePartitionData buffer;
+ protected Iterator<Unfiltered> iterator;
+
+ // Set in loadFromDisk () and used in setIterator to handle range tombstone extending on multiple index block. See
+ // loadFromDisk for details. Note that those are always false for non-indexed readers.
+ protected boolean skipFirstIteratedItem;
+ protected boolean skipLastIteratedItem;
+
+ private ReverseReader(FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ }
+
+ protected ReusablePartitionData createBuffer(int blocksCount)
+ {
+ int estimatedRowCount = 16;
+ int columnCount = metadata().regularColumns().size();
+ if (columnCount == 0 || metadata().clusteringColumns().isEmpty())
+ {
+ estimatedRowCount = 1;
+ }
+ else
+ {
+ try
+ {
+ // To avoid wasted resizing we guess-estimate the number of rows we're likely to read. For that
+ // we use the stats on the number of rows per partition for that sstable.
+ // FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows,
+ // we divide by the number of regular columns the table has. We should fix once we collect the
+ // stats on rows
+ int estimatedRowsPerPartition = (int)(sstable.getEstimatedCellPerPartitionCount().percentile(0.75) / columnCount);
+ estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1);
+ }
+ catch (IllegalStateException e)
+ {
+ // The EstimatedHistogram mean() method can throw this (if it overflows). While such overflow
+ // shouldn't happen, it's not worth taking the risk of letting the exception bubble up.
+ }
+ }
+ return new ReusablePartitionData(metadata(), partitionKey(), columns(), estimatedRowCount);
+ }
+
+ public void setForSlice(Slice slice) throws IOException
+ {
+ // If we have read the data, just create the iterator for the slice. Otherwise, read the data.
+ if (buffer == null)
+ {
+ buffer = createBuffer(1);
+ // Note that we can reuse that buffer between slices (we could alternatively re-read from disk
+ // every time, but that feels more wasteful) so we want to include everything from the beginning.
+ // We can stop at the slice end however since any following slice will be before that.
+ loadFromDisk(null, slice.end(), false, false);
+ }
+ setIterator(slice);
+ }
+
+ protected void setIterator(Slice slice)
+ {
+ assert buffer != null;
+ iterator = buffer.built.unfilteredIterator(columns, Slices.with(metadata().comparator, slice), true);
+
+ if (!iterator.hasNext())
+ return;
+
+ if (skipFirstIteratedItem)
+ iterator.next();
+
+ if (skipLastIteratedItem)
+ iterator = new SkipLastIterator(iterator);
+ }
+
+ protected boolean hasNextInternal() throws IOException
+ {
+ // If we've never called setForSlice, we're reading everything
+ if (iterator == null)
+ setForSlice(Slice.ALL);
+
+ return iterator.hasNext();
+ }
+
+ protected Unfiltered nextInternal() throws IOException
+ {
+ if (!hasNext())
+ throw new NoSuchElementException();
+ return iterator.next();
+ }
+
+ protected boolean stopReadingDisk() throws IOException
+ {
+ return false;
+ }
+
+ // Reads the unfiltered from disk and load them into the reader buffer. It stops reading when either the partition
+ // is fully read, or when stopReadingDisk() returns true.
+ protected void loadFromDisk(ClusteringBound<?> start,
+ ClusteringBound<?> end,
+ boolean hasPreviousBlock,
+ boolean hasNextBlock) throws IOException
+ {
+ // start != null means it's the block covering the beginning of the slice, so it has to be the last block for this slice.
+ assert start == null || !hasNextBlock;
+
+ buffer.reset();
+ skipFirstIteratedItem = false;
+ skipLastIteratedItem = false;
+
+ // If the start might be in this block, skip everything that comes before it.
+ if (start != null)
+ {
+ while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0 && !stopReadingDisk())
+ {
+ if (deserializer.nextIsRow())
+ deserializer.skipNext();
+ else
+ updateOpenMarker((RangeTombstoneMarker)deserializer.readNext());
+ }
+ }
+
+ // If we have an open marker, it's either one from what we just skipped or it's one that open in the next (or
+ // one of the next) index block (if openMarker == openMarkerAtStartOfBlock).
+ if (openMarker != null)
+ {
+ // We have to feed a marker to the buffer, because that marker is likely to be close later and ImmtableBTreePartition
+ // doesn't take kindly to marker that comes without their counterpart. If that's the last block we're gonna read (for
+ // the current slice at least) it's easy because we'll want to return that open marker at the end of the data in this
+ // block anyway, so we have nothing more to do than adding it to the buffer.
+ // If it's not the last block however, in which case we know we'll have start == null, it means this marker is really
+ // open in a next block and so while we do need to add it the buffer for the reason mentioned above, we don't
+ // want to "return" it just yet, we'll wait until we reach it in the next blocks. That's why we trigger
+ // skipLastIteratedItem in that case (this is first item of the block, but we're iterating in reverse order
+ // so it will be last returned by the iterator).
+ ClusteringBound<?> markerStart = start == null ? BufferClusteringBound.BOTTOM : start;
+ buffer.add(new RangeTombstoneBoundMarker(markerStart, openMarker));
+ if (hasNextBlock)
+ skipLastIteratedItem = true;
+ }
+
+ // Now deserialize everything until we reach our requested end (if we have one)
+ // See SSTableIterator.ForwardRead.computeNext() for why this is a strict inequality below: this is the same
+ // reasoning here.
+ while (deserializer.hasNext()
+ && (end == null || deserializer.compareNextTo(end) < 0)
+ && !stopReadingDisk())
+ {
+ Unfiltered unfiltered = deserializer.readNext();
+ UnfilteredValidation.maybeValidateUnfiltered(unfiltered, metadata(), key, sstable);
+ // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
+ if (!unfiltered.isEmpty())
+ buffer.add(unfiltered);
+
+ if (unfiltered.isRangeTombstoneMarker())
+ updateOpenMarker((RangeTombstoneMarker)unfiltered);
+ }
+
+ // If we have an open marker, we should close it before finishing
+ if (openMarker != null)
+ {
+ // This is the reverse problem than the one at the start of the block. Namely, if it's the first block
+ // we deserialize for the slice (the one covering the slice end basically), then it's easy, we just want
+ // to add the close marker to the buffer and return it normally.
+ // If it's note our first block (for the slice) however, it means that marker closed in a previously read
+ // block and we have already returned it. So while we should still add it to the buffer for the sake of
+ // not breaking ImmutableBTreePartition, we should skip it when returning from the iterator, hence the
+ // skipFirstIteratedItem (this is the last item of the block, but we're iterating in reverse order so it will
+ // be the first returned by the iterator).
+ ClusteringBound<?> markerEnd = end == null ? BufferClusteringBound.TOP : end;
+ buffer.add(new RangeTombstoneBoundMarker(markerEnd, openMarker));
+ if (hasPreviousBlock)
+ skipFirstIteratedItem = true;
+ }
+
+ buffer.build();
+ }
+ }
+
+ private class ReverseIndexedReader extends ReverseReader
+ {
+ private final IndexState indexState;
+
+ // The slice we're currently iterating over
+ private Slice slice;
+ // The last index block to consider for the slice
+ private int lastBlockIdx;
+
+ private ReverseIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ this.indexState = new IndexState(this, metadata.comparator, indexEntry, true, ifile);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ super.close();
+ this.indexState.close();
+ }
+
+ @Override
+ public void setForSlice(Slice slice) throws IOException
+ {
+ this.slice = slice;
+
+ // if our previous slicing already got us past the beginning of the sstable, we're done
+ if (indexState.isDone())
+ {
+ iterator = Collections.emptyIterator();
+ return;
+ }
+
+ // Find the first index block we'll need to read for the slice.
+ int startIdx = indexState.findBlockIndex(slice.end(), indexState.currentBlockIdx());
+ if (startIdx < 0)
+ {
+ iterator = Collections.emptyIterator();
+ indexState.setToBlock(startIdx);
+ return;
+ }
+
+ lastBlockIdx = indexState.findBlockIndex(slice.start(), startIdx);
+
+ // If the last block to look (in reverse order) is after the very last block, we have nothing for that slice
+ if (lastBlockIdx >= indexState.blocksCount())
+ {
+ assert startIdx >= indexState.blocksCount();
+ iterator = Collections.emptyIterator();
+ return;
+ }
+
+ // If we start (in reverse order) after the very last block, just read from the last one.
+ if (startIdx >= indexState.blocksCount())
+ startIdx = indexState.blocksCount() - 1;
+
+ // Note that even if we were already set on the proper block (which would happen if the previous slice
+ // requested ended on the same block this one start), we can't reuse it because when reading the previous
+ // slice we've only read that block from the previous slice start. Re-reading also handles
+ // skipFirstIteratedItem/skipLastIteratedItem that we would need to handle otherwise.
+ indexState.setToBlock(startIdx);
+
+ readCurrentBlock(false, startIdx != lastBlockIdx);
+ }
+
+ @Override
+ protected boolean hasNextInternal() throws IOException
+ {
+ if (super.hasNextInternal())
+ return true;
+
+ while (true)
+ {
+ // We have nothing more for our current block, move the next one (so the one before on disk).
+ int nextBlockIdx = indexState.currentBlockIdx() - 1;
+ if (nextBlockIdx < 0 || nextBlockIdx < lastBlockIdx)
+ return false;
+
+ // The slice start can be in
+ indexState.setToBlock(nextBlockIdx);
+ readCurrentBlock(true, nextBlockIdx != lastBlockIdx);
+
+ // If an indexed block only contains data for a dropped column, the iterator will be empty, even
+ // though we may still have data to read in subsequent blocks
+
+ // also, for pre-3.0 storage formats, index blocks that only contain a single row and that row crosses
+ // index boundaries, the iterator will be empty even though we haven't read everything we're intending
+ // to read. In that case, we want to read the next index block. This shouldn't be possible in 3.0+
+ // formats (see next comment)
+ if (!iterator.hasNext() && nextBlockIdx > lastBlockIdx)
+ {
+ continue;
+ }
+
+ return iterator.hasNext();
+ }
+ }
+
+ /**
+ * Reads the current block, the last one we've set.
+ *
+ * @param hasPreviousBlock is whether we have already read a previous block for the current slice.
+ * @param hasNextBlock is whether we have more blocks to read for the current slice.
+ */
+ private void readCurrentBlock(boolean hasPreviousBlock, boolean hasNextBlock) throws IOException
+ {
+ if (buffer == null)
+ buffer = createBuffer(indexState.blocksCount());
+
+ // The slice start (resp. slice end) is only meaningful on the last (resp. first) block read (since again,
+ // we read blocks in reverse order).
+ boolean canIncludeSliceStart = !hasNextBlock;
+ boolean canIncludeSliceEnd = !hasPreviousBlock;
+
+ loadFromDisk(canIncludeSliceStart ? slice.start() : null,
+ canIncludeSliceEnd ? slice.end() : null,
+ hasPreviousBlock,
+ hasNextBlock);
+ setIterator(slice);
+ }
+
+ @Override
+ protected boolean stopReadingDisk() throws IOException
+ {
+ return indexState.isPastCurrentBlock();
+ }
+ }
+
+ private class ReusablePartitionData
+ {
+ private final TableMetadata metadata;
+ private final DecoratedKey partitionKey;
+ private final RegularAndStaticColumns columns;
+
+ private MutableDeletionInfo.Builder deletionBuilder;
+ private MutableDeletionInfo deletionInfo;
+ private BTree.Builder<Row> rowBuilder;
+ private ImmutableBTreePartition built;
+
+ private ReusablePartitionData(TableMetadata metadata,
+ DecoratedKey partitionKey,
+ RegularAndStaticColumns columns,
+ int initialRowCapacity)
+ {
+ this.metadata = metadata;
+ this.partitionKey = partitionKey;
+ this.columns = columns;
+ this.rowBuilder = BTree.builder(metadata.comparator, initialRowCapacity);
+ }
+
+
+ public void add(Unfiltered unfiltered)
+ {
+ if (unfiltered.isRow())
+ rowBuilder.add((Row)unfiltered);
+ else
+ deletionBuilder.add((RangeTombstoneMarker)unfiltered);
+ }
+
+ public void reset()
+ {
+ built = null;
+ rowBuilder.reuse();
+ deletionBuilder = MutableDeletionInfo.builder(partitionLevelDeletion, metadata().comparator, false);
+ }
+
+ public void build()
+ {
+ deletionInfo = deletionBuilder.build();
+ built = new ImmutableBTreePartition(metadata, partitionKey, columns, Rows.EMPTY_STATIC_ROW, rowBuilder.build(),
+ deletionInfo, EncodingStats.NO_STATS);
+ deletionBuilder = null;
+ }
+ }
+
+ private static class SkipLastIterator extends AbstractIterator<Unfiltered>
+ {
+ private final Iterator<Unfiltered> iterator;
+
+ private SkipLastIterator(Iterator<Unfiltered> iterator)
+ {
+ this.iterator = iterator;
+ }
+
+ protected Unfiltered computeNext()
+ {
+ if (!iterator.hasNext())
+ return endOfData();
+
+ Unfiltered next = iterator.next();
+ return iterator.hasNext() ? next : endOfData();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java
new file mode 100644
index 0000000..45cba4f
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.MetricsProviders;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics;
+import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.SortedTableScrubber;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Bigtable format with trie indices. See BTIFormat.md for the format documentation.
+ */
+public class BtiFormat extends AbstractSSTableFormat<BtiTableReader, BtiTableWriter>
+{
+ private final static Logger logger = LoggerFactory.getLogger(BtiFormat.class);
+
+ public static final String NAME = "bti";
+
+ private final Version latestVersion = new BtiVersion(this, BtiVersion.current_version);
+ static final BtiTableReaderFactory readerFactory = new BtiTableReaderFactory();
+ static final BtiTableWriterFactory writerFactory = new BtiTableWriterFactory();
+
+ public static class Components extends SSTableFormat.Components
+ {
+ public static class Types extends AbstractSSTableFormat.Components.Types
+ {
+ public static final Component.Type PARTITION_INDEX = Component.Type.createSingleton("PARTITION_INDEX", "Partitions.db", BtiFormat.class);
+ public static final Component.Type ROW_INDEX = Component.Type.createSingleton("ROW_INDEX", "Rows.db", BtiFormat.class);
+ }
+
+ public final static Component PARTITION_INDEX = Types.PARTITION_INDEX.getSingleton();
+
+ public final static Component ROW_INDEX = Types.ROW_INDEX.getSingleton();
+
+ private final static Set<Component> STREAMING_COMPONENTS = ImmutableSet.of(DATA,
+ PARTITION_INDEX,
+ ROW_INDEX,
+ STATS,
+ COMPRESSION_INFO,
+ FILTER,
+ DIGEST,
+ CRC);
+
+ private final static Set<Component> PRIMARY_COMPONENTS = ImmutableSet.of(DATA,
+ PARTITION_INDEX);
+
+ private final static Set<Component> MUTABLE_COMPONENTS = ImmutableSet.of(STATS);
+
+ private static final Set<Component> UPLOAD_COMPONENTS = ImmutableSet.of(DATA,
+ PARTITION_INDEX,
+ ROW_INDEX,
+ COMPRESSION_INFO,
+ STATS);
+
+ private static final Set<Component> BATCH_COMPONENTS = ImmutableSet.of(DATA,
+ PARTITION_INDEX,
+ ROW_INDEX,
+ COMPRESSION_INFO,
+ FILTER,
+ STATS);
+
+ private final static Set<Component> ALL_COMPONENTS = ImmutableSet.of(DATA,
+ PARTITION_INDEX,
+ ROW_INDEX,
+ STATS,
+ COMPRESSION_INFO,
+ FILTER,
+ DIGEST,
+ CRC,
+ TOC);
+
+ private final static Set<Component> GENERATED_ON_LOAD_COMPONENTS = ImmutableSet.of(FILTER);
+ }
+
+
+ public BtiFormat(Map<String, String> options)
+ {
+ super(NAME, options);
+ }
+
+ public static boolean is(SSTableFormat<?, ?> format)
+ {
+ return format.name().equals(NAME);
+ }
+
+ public static boolean isSelected()
+ {
+ return is(DatabaseDescriptor.getSelectedSSTableFormat());
+ }
+
+ @Override
+ public Version getLatestVersion()
+ {
+ return latestVersion;
+ }
+
+ @Override
+ public Version getVersion(String version)
+ {
+ return new BtiVersion(this, version);
+ }
+
+ @Override
+ public BtiTableWriterFactory getWriterFactory()
+ {
+ return writerFactory;
+ }
+
+ @Override
+ public BtiTableReaderFactory getReaderFactory()
+ {
+ return readerFactory;
+ }
+
+ @Override
+ public Set<Component> streamingComponents()
+ {
+ return Components.STREAMING_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> primaryComponents()
+ {
+ return Components.PRIMARY_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> batchComponents()
+ {
+ return Components.BATCH_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> uploadComponents()
+ {
+ return Components.UPLOAD_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> mutableComponents()
+ {
+ return Components.MUTABLE_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> allComponents()
+ {
+ return Components.ALL_COMPONENTS;
+ }
+
+ @Override
+ public Set<Component> generatedOnLoadComponents()
+ {
+ return Components.GENERATED_ON_LOAD_COMPONENTS;
+ }
+
+ @Override
+ public SSTableFormat.KeyCacheValueSerializer<BtiTableReader, TrieIndexEntry> getKeyCacheValueSerializer()
+ {
+ throw new AssertionError("BTI sstables do not use key cache");
+ }
+
+ @Override
+ public IScrubber getScrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, OutputHandler outputHandler, IScrubber.Options options)
+ {
+ Preconditions.checkArgument(cfs.metadata().equals(transaction.onlyOne().metadata()), "SSTable metadata does not match current definition");
+ return new BtiTableScrubber(cfs, transaction, outputHandler, options);
+ }
+
+ @Override
+ public MetricsProviders getFormatSpecificMetricsProviders()
+ {
+ return BtiTableSpecificMetricsProviders.instance;
+ }
+
+ @Override
+ public void deleteOrphanedComponents(Descriptor descriptor, Set<Component> components)
+ {
+ SortedTableScrubber.deleteOrphanedComponents(descriptor, components);
+ }
+
+ private void delete(Descriptor desc, List<Component> components)
+ {
+ logger.info("Deleting sstable: {}", desc);
+
+ if (components.remove(SSTableFormat.Components.DATA))
+ components.add(0, SSTableFormat.Components.DATA); // DATA component should be first
+
+ for (Component component : components)
+ {
+ logger.trace("Deleting component {} of {}", component, desc);
+ desc.fileFor(component).deleteIfExists();
+ }
+ }
+
+ @Override
+ public void delete(Descriptor desc)
+ {
+ try
+ {
+ delete(desc, Lists.newArrayList(Sets.intersection(allComponents(), desc.discoverComponents())));
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ }
+ }
+
+ static class BtiTableReaderFactory implements SSTableReaderFactory<BtiTableReader, BtiTableReader.Builder>
+ {
+ @Override
+ public SSTableReader.Builder<BtiTableReader, BtiTableReader.Builder> builder(Descriptor descriptor)
+ {
+ return new BtiTableReader.Builder(descriptor);
+ }
+
+ @Override
+ public SSTableReaderLoadingBuilder<BtiTableReader, BtiTableReader.Builder> loadingBuilder(Descriptor descriptor, TableMetadataRef tableMetadataRef, Set<Component> components)
+ {
+ return new BtiTableReaderLoadingBuilder(new SSTable.Builder<>(descriptor).setTableMetadataRef(tableMetadataRef)
+ .setComponents(components));
+ }
+
+ @Override
+ public Pair<DecoratedKey, DecoratedKey> readKeyRange(Descriptor descriptor, IPartitioner partitioner) throws IOException
+ {
+ return PartitionIndex.readFirstAndLastKey(descriptor.fileFor(Components.PARTITION_INDEX), partitioner);
+ }
+
+ @Override
+ public Class<BtiTableReader> getReaderClass()
+ {
+ return BtiTableReader.class;
+ }
+ }
+
+ static class BtiTableWriterFactory implements SSTableWriterFactory<BtiTableWriter, BtiTableWriter.Builder>
+ {
+ @Override
+ public BtiTableWriter.Builder builder(Descriptor descriptor)
+ {
+ return new BtiTableWriter.Builder(descriptor);
+ }
+
+ @Override
+ public long estimateSize(SSTableWriter.SSTableSizeParameters parameters)
+ {
+ return (long) ((parameters.partitionCount() * 8 // index entries
+ + parameters.partitionKeysSize() // keys in data file
+ + parameters.dataSize()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
+ }
+
+ static class BtiVersion extends Version
+ {
+ public static final String current_version = "da";
+ public static final String earliest_supported_version = "da";
+
+ // versions aa-cz are not supported in OSS
+ // da (5.0): initial version of the BIT format
+ // NOTE: when adding a new version, please add that to LegacySSTableTest, too.
+
+ private final boolean isLatestVersion;
+
+ private final int correspondingMessagingVersion;
+
+ BtiVersion(BtiFormat format, String version)
+ {
+ super(format, version);
+
+ isLatestVersion = version.compareTo(current_version) == 0;
+ correspondingMessagingVersion = MessagingService.VERSION_40;
+ }
+
+ @Override
+ public boolean isLatestVersion()
+ {
+ return isLatestVersion;
+ }
+
+ @Override
+ public int correspondingMessagingVersion()
+ {
+ return correspondingMessagingVersion;
+ }
+
+ @Override
+ public boolean hasCommitLogLowerBound()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasCommitLogIntervals()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasMaxCompressedLength()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasPendingRepair()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasIsTransient()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasMetadataChecksum()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasOldBfFormat()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean hasAccurateMinMax()
+ {
+ return true;
+ }
+
+ public boolean hasLegacyMinMax()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean hasOriginatingHostId()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasImprovedMinMax() {
+ return true;
+ }
+
+ @Override
+ public boolean hasPartitionLevelDeletionsPresenceMarker()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean hasKeyRange()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean isCompatible()
+ {
+ return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
+ }
+
+ @Override
+ public boolean isCompatibleForStreaming()
+ {
+ return isCompatible() && version.charAt(0) == current_version.charAt(0);
+ }
+
+ }
+
+ private static class BtiTableSpecificMetricsProviders implements MetricsProviders
+ {
+ private final static BtiTableSpecificMetricsProviders instance = new BtiTableSpecificMetricsProviders();
+
+ private final Iterable<GaugeProvider<?>> gaugeProviders = BloomFilterMetrics.instance.getGaugeProviders();
+
+ @Override
+ public Iterable<GaugeProvider<?>> getGaugeProviders()
+ {
+ return gaugeProviders;
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public static class BtiFormatFactory implements Factory
+ {
+ @Override
+ public String name()
+ {
+ return NAME;
+ }
+
+ @Override
+ public SSTableFormat<?, ?> getInstance(Map<String, String> options)
+ {
+ return new BtiFormat(options);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.md b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.md
new file mode 100644
index 0000000..74c883a
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.md
@@ -0,0 +1,1010 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Big Trie-Indexed (BTI) SSTable format
+
+This document describes the BTI SSTable format, which is introduced to
+Cassandra with [CEP-25](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-25%3A+Trie-indexed+SSTable+format).
+
+The format is called BTI, which stands for "Big Trie-Indexed", because it shares
+the data format of the existing BIG format and only changes the primary indexes
+inside SSTables. The format relies on byte order and tries, and uses a combination
+of features to make the indexing structures compact and efficient. The paragraphs
+below describe the format's features and mechanisms together with the motivation
+behind them, and conclude with detailed description of the on-disk format.
+
+# Prerequisites
+
+## Byte-comparable types
+
+The property of being byte-comparable (also called byte-ordered) for a
+key denotes that there is a serialisation of that key to a sequence of
+bytes where the lexicographic comparison of the unsigned bytes produces
+the same result as performing a typed comparison of the key.
+
+For Cassandra, such a representation is given by
+[CASSANDRA-6936](https://issues.apache.org/jira/browse/CASSANDRA-6936).
+Detailed description of the mechanics of the translation are provided in
+the [included documentation](../../../../utils/bytecomparable/ByteComparable.md).
+
+## Tries
+
+A trie is a data structure that describes a mapping between sequences
+and associated values. It is very similar to a deterministic finite
+state automaton, with the main difference that an automaton is allowed
+to have cycles, while a trie is not.
+
+Because the theory and main usage of the structure is for encoding words
+of a language, the trie terminology talks about "characters", "words"
+and "alphabet", which in our case map to bytes of the byte-ordered
+representation, the sequence that encodes it, and the possible values of
+a byte[^1].
+
+[^1]: For simplicity this description assumes we directly map a byte to
+a character. Other options are also possible (e.g. using hex digits
+as the alphabet and two transitions per byte).
+
+A trie can be defined as a tree graph in which vertices are states, some
+of which can be final and contain associated information, and where
+edges are labelled with characters. A valid word in the trie is encoded
+by a path starting from the root of the trie where each edge is labelled
+with the next character of the word, and ending in a final state which
+contains the 'payload' associated with the word.
+
+```mermaid
+graph TD
+ Node_(( ))
+ style Node_ fill:darkgrey
+ Node_ --"a"--> Node_a(((a)))
+ Node_a --"l"--> Node_al(( ))
+ Node_al --"l"--> Node_all(( ))
+ Node_all --"o"--> Node_allo(( ))
+ Node_allo --"w"--> Node_allow(((allow)))
+ Node_a --"n"--> Node_an(((an)))
+ Node_an --"d"--> Node_and(((and)))
+ Node_an --"y"--> Node_any(((any)))
+ Node_a --"r"--> Node_ar(( ))
+ Node_ar --"e"--> Node_are(((are)))
+ Node_a --"s"--> Node_as(((as)))
+ Node_ --"n"--> Node_n(( ))
+ Node_n --"o"--> Node_no(( ))
+ Node_no --"d"--> Node_nod(( ))
+ Node_nod --"e"--> Node_node(((node)))
+ Node_ --"o"--> Node_o(( ))
+ Node_o --"f"--> Node_of(((of)))
+ Node_o --"n"--> Node_on(((on)))
+ Node_ --"t"--> Node_t(( ))
+ Node_t --"h"--> Node_th(( ))
+ Node_th --"e"--> Node_the(((the)))
+ Node_th --"i"--> Node_thi(( ))
+ Node_thi --"s"--> Node_this(((this)))
+ Node_t --"o"--> Node_to(((to)))
+ Node_t --"r"--> Node_tr(( ))
+ Node_tr --"i"--> Node_tri(( ))
+ Node_tri --"e"--> Node_trie(((trie)))
+ Node_t --"y"--> Node_ty(( ))
+ Node_ty --"p"--> Node_typ(( ))
+ Node_typ --"e"--> Node_type(( ))
+ Node_type --"s"--> Node_types(((types)))
+ Node_ --"w"--> Node_w(( ))
+ Node_w --"i"--> Node_wi(( ))
+ Node_wi --"t"--> Node_wit(( ))
+ Node_wit --"h"--> Node_with(((with)))
+ Node_with --"o"--> Node_witho(( ))
+ Node_witho --"u"--> Node_withou(( ))
+ Node_withou --"t"--> Node_without(((without)))
+```
+
+This means that in a constructed trie finding the payload associated
+with a word is a matter of following the edges (also called
+"transitions") from the initial state labelled with the consecutive
+characters of the word, and retrieving the payload associated with the
+state at which we end up. If that's not a final state, or if at any
+point in this we did not find a transition in the trie matching the
+character, the trie does not have an association for the word. The
+complexity of lookup is thus _O_(len(word)) transitions, where the cost of
+taking a transition is usually constant, thus this complexity is
+theoretically optimal.
+
+From a storage space perspective, one of the main benefits of a trie as
+a data structure for storing a map is the fact that it completely avoids
+storing redundant prefixes. All words that start with the same sequence
+store a representation of that sequence only once. If prefixes are
+commonly shared, this can save a great deal of space.
+
+When the items stored in a trie are lexicographically (=byte) ordered, a
+trie is also an ordered structure. A trie can be walked in order and it
+is also possible to efficiently list the items between two given keys.
+
+In fact, one can efficiently (and lazily) apply set algebra over tries,
+and slicing can be seen as a simple application of intersection, where
+the intersecting trie is generated on the fly. The set operations
+benefit from the same prefix-sharing effect — we apply union /
+intersection / difference to a state, which has the effect of applying
+the operation to all words that share the prefix denoted by that state.
+
+```mermaid
+graph TD
+ Node_(( ))
+ style Node_ fill:darkgrey
+ Node_ --"a"--> Node_a(((a)))
+ style Node_a stroke:lightgrey,color:lightgrey
+ linkStyle 0 stroke:lightgrey,color:lightgrey
+ Node_a --"l"--> Node_al(( ))
+ style Node_al stroke:lightgrey,color:lightgrey
+ linkStyle 1 stroke:lightgrey,color:lightgrey
+ Node_al --"l"--> Node_all(( ))
+ style Node_all stroke:lightgrey,color:lightgrey
+ linkStyle 2 stroke:lightgrey,color:lightgrey
+ Node_all --"o"--> Node_allo(( ))
+ style Node_allo stroke:lightgrey,color:lightgrey
+ linkStyle 3 stroke:lightgrey,color:lightgrey
+ Node_allo --"w"--> Node_allow(((allow)))
+ style Node_allow stroke:lightgrey,color:lightgrey
+ linkStyle 4 stroke:lightgrey,color:lightgrey
+ Node_a --"n"--> Node_an(((an)))
+ style Node_an stroke:lightgrey,color:lightgrey
+ linkStyle 5 stroke:lightgrey,color:lightgrey
+ Node_an --"d"--> Node_and(((and)))
+ style Node_and stroke:lightgrey,color:lightgrey
+ linkStyle 6 stroke:lightgrey,color:lightgrey
+ Node_an --"y"--> Node_any(((any)))
+ style Node_any stroke:lightgrey,color:lightgrey
+ linkStyle 7 stroke:lightgrey,color:lightgrey
+ Node_a --"r"--> Node_ar(( ))
+ style Node_ar stroke:lightgrey,color:lightgrey
+ linkStyle 8 stroke:lightgrey,color:lightgrey
+ Node_ar --"e"--> Node_are(((are)))
+ style Node_are stroke:lightgrey,color:lightgrey
+ linkStyle 9 stroke:lightgrey,color:lightgrey
+ Node_a --"s"--> Node_as(((as)))
+ style Node_as stroke:lightgrey,color:lightgrey
+ linkStyle 10 stroke:lightgrey,color:lightgrey
+ Node_ --"n"--> Node_n(( ))
+ Node_n --"o"--> Node_no(( ))
+ Node_no --"d"--> Node_nod(( ))
+ Node_nod --"e"--> Node_node(((node)))
+ Node_ --"o"--> Node_o(( ))
+ Node_o --"f"--> Node_of(((of)))
+ Node_o --"n"--> Node_on(((on)))
+ Node_ --"t"--> Node_t(( ))
+ Node_t --"h"--> Node_th(( ))
+ Node_th --"e"--> Node_the(((the)))
+ Node_th --"i"--> Node_thi(( ))
+ Node_thi --"s"--> Node_this(((this)))
+ style Node_this stroke:lightgrey,color:lightgrey
+ linkStyle 22 stroke:lightgrey,color:lightgrey
+ Node_t --"o"--> Node_to(((to)))
+ style Node_to stroke:lightgrey,color:lightgrey
+ linkStyle 23 stroke:lightgrey,color:lightgrey
+ Node_t --"r"--> Node_tr(( ))
+ style Node_tr stroke:lightgrey,color:lightgrey
+ linkStyle 24 stroke:lightgrey,color:lightgrey
+ Node_tr --"i"--> Node_tri(( ))
+ style Node_tri stroke:lightgrey,color:lightgrey
+ linkStyle 25 stroke:lightgrey,color:lightgrey
+ Node_tri --"e"--> Node_trie(((trie)))
+ style Node_trie stroke:lightgrey,color:lightgrey
+ linkStyle 26 stroke:lightgrey,color:lightgrey
+ Node_t --"y"--> Node_ty(( ))
+ style Node_ty stroke:lightgrey,color:lightgrey
+ linkStyle 27 stroke:lightgrey,color:lightgrey
+ Node_ty --"p"--> Node_typ(( ))
+ style Node_typ stroke:lightgrey,color:lightgrey
+ linkStyle 28 stroke:lightgrey,color:lightgrey
+ Node_typ --"e"--> Node_type(( ))
+ style Node_type stroke:lightgrey,color:lightgrey
+ linkStyle 29 stroke:lightgrey,color:lightgrey
+ Node_type --"s"--> Node_types(((types)))
+ style Node_types stroke:lightgrey,color:lightgrey
+ linkStyle 30 stroke:lightgrey,color:lightgrey
+ Node_ --"w"--> Node_w(( ))
+ style Node_w stroke:lightgrey,color:lightgrey
+ linkStyle 31 stroke:lightgrey,color:lightgrey
+ Node_w --"i"--> Node_wi(( ))
+ style Node_wi stroke:lightgrey,color:lightgrey
+ linkStyle 32 stroke:lightgrey,color:lightgrey
+ Node_wi --"t"--> Node_wit(( ))
+ style Node_wit stroke:lightgrey,color:lightgrey
+ linkStyle 33 stroke:lightgrey,color:lightgrey
+ Node_wit --"h"--> Node_with(((with)))
+ style Node_with stroke:lightgrey,color:lightgrey
+ linkStyle 34 stroke:lightgrey,color:lightgrey
+ Node_with --"o"--> Node_witho(( ))
+ style Node_witho stroke:lightgrey,color:lightgrey
+ linkStyle 35 stroke:lightgrey,color:lightgrey
+ Node_witho --"u"--> Node_withou(( ))
+ style Node_withou stroke:lightgrey,color:lightgrey
+ linkStyle 36 stroke:lightgrey,color:lightgrey
+ Node_withou --"t"--> Node_without(((without)))
+ style Node_without stroke:lightgrey,color:lightgrey
+ linkStyle 37 stroke:lightgrey,color:lightgrey
+
+```
+
+(An example of slicing the trie above with the range "bit"-"thing".
+Processing only applies on boundary nodes (root, "t", "th", "thi"),
+where we throw away the transitions outside the range. Subtries like the
+ones for "n" and "o" fall completely between "b" and "t" thus are fully
+inside the range and can be processed without any further restrictions.)
+
+A trie can be used as a modifiable in-memory data structure where one
+can add and remove individual elements. It can also be constructed from
+sorted input, incrementally storing the data directly to disk and
+building an efficient read-only on-disk data structure.
+
+For more formal information about the concept and applications of tries
+and finite state automata, try [Introduction to Automata Theory,
+Languages, and Computation](https://books.google.com/books?id=dVipBwAAQBAJ).
+There are many variations of the concept, and of the implementation of
+states and transitions that can be put to use to achieve even further
+efficiency gains; some of these will be detailed below.
+
+# Indexing with tries
+
+Since a trie is generally an ordered byte source to payload map, we can
+apply the concept directly to the components of Cassandra that are most
+affected by the inefficiency of using comparison-based structures: the
+indices.
+
+This can be done in the following way:
+
+- When we write the index, we map each key into its byte-ordered
+ representation and create an on-disk trie of byte-ordered
+ representations of keys mapping into positions in the data file.
+
+- When we need an exact match for a key, we create a (lazily
+ generated) byte-ordered representation of the key and look for it
+ in the trie.
+
+ - If we find a match, we know the data file position.
+
+ - If there is no match, there is no data associated with the key.
+
+- When we need a greater-than/greater-or-equal match, we use the
+ byte-ordered representation to create a path that leads to the
+ first matching data position in the sstable.
+
+ - We can then use this path to iterate the greater keys in the
+ sstable.
+
+This works, but isn't very efficient. Lookup in it is _O_(len(key)),
+which can even mean that many seeks on disk, and we have to store
+a transition (which defines the size of the structure) for every
+non-prefix character in the dataset.
+
+We can do much better.
+
+## Trimming the fat
+
+The primary purpose of the index is to find a position in the data file
+for the given key. It needs to be able to find the correct position for
+any existing key, but there is no need for it to be exact on keys that
+are not present in the file — since our data files contain a copy of
+the key at the start of each partition, we can simply check if the key
+we are searching for matches the key at the position returned by the
+index.
+
+This allows us to use a simple optimization: instead of storing the full
+key in the index trie, we can store only a prefix of the key that is
+unique among all partitions in the table. This means that we have
+intermediate nodes in the trie only if a prefix is shared by multiple
+keys, which normally reduces the number of nodes and transitions in the
+trie to about 2*n*.
+
+```mermaid
+graph TD
+ Node_(( ))
+ style Node_ fill:darkgrey
+ Node_ --"a"--> Node_a((( )))
+ Node_a --"l"--> Node_al((( )))
+ Node_a --"n"--> Node_an((( )))
+ Node_an --"d"--> Node_and((( )))
+ Node_an --"y"--> Node_any((( )))
+ Node_a --"r"--> Node_ar((( )))
+ Node_a --"s"--> Node_as((( )))
+ Node_ --"n"--> Node_n((( )))
+ Node_ --"o"--> Node_o(( ))
+ Node_o --"f"--> Node_of((( )))
+ Node_o --"n"--> Node_on((( )))
+ Node_ --"t"--> Node_t(( ))
+ Node_t --"h"--> Node_th(( ))
+ Node_th --"e"--> Node_the((( )))
+ Node_th --"i"--> Node_thi((( )))
+ Node_t --"o"--> Node_to((( )))
+ Node_t --"r"--> Node_tr((( )))
+ Node_t --"y"--> Node_ty((( )))
+ Node_ --"w"--> Node_w(( ))
+ Node_w --"ith"--> Node_with((( )))
+ Node_with --"o"--> Node_without((( )))
+```
+
+This also reduces the number of steps we need to take in the trie. In a
+well-balanced key set (such as the one where the byte-ordered key starts
+with a hash as in Murmur or Random-partitioned primary keys) the lookup
+complexity becomes _O_(log _n_) transitions[^2].
+
+[^2]: For comparison, the complexity of binary search in a sorted
+primary index is also _O_(log _n_), but in key comparisons whose
+complexity on average in a well-balanced key set is another _O_(log _n_)
+for a total _O_(log<sup>2</sup> _n_).
+
+## Taking hardware into account
+
+The point above improves the number of transitions significantly, but
+the out-of-cache efficiency is still pretty bad if we have to read a new
+disk page every time we examine a node. Fortunately we can take some
+extra care during construction to make sure we make the most of every
+disk page brought up during lookup.
+
+The idea of this is to pack wide sections of the trie in pages, so that
+every time we open a page we can be certain to be able to follow several
+transitions before leaving that page.
+
+```mermaid
+graph TD
+ subgraph p1 [ ]
+ Node_(( ))
+ style Node_ fill:darkgrey
+ Node_ --"a"--> Node_a((( )))
+ Node_ --"t"--> Node_t(( ))
+ end
+
+ subgraph p2 [ ]
+ Node_a --"l"--> Node_al((( )))
+ Node_a --"n"--> Node_an((( )))
+ Node_an --"d"--> Node_and((( )))
+ Node_an --"y"--> Node_any((( )))
+ end
+
+ subgraph p3 [ ]
+ Node_a --"r"--> Node_ar((( )))
+ Node_a --"s"--> Node_as((( )))
+ Node_ --"n"--> Node_n((( )))
+ end
+
+ subgraph p4 [ ]
+ Node_ --"o"--> Node_o(( ))
+ Node_o --"f"--> Node_of((( )))
+ Node_o --"n"--> Node_on((( )))
+ Node_t --"o"--> Node_to((( )))
+ end
+
+ subgraph p5 [ ]
+ Node_t --"h"--> Node_th(( ))
+ Node_th --"e"--> Node_the((( )))
+ Node_th --"i"--> Node_thi((( )))
+ Node_t --"r"--> Node_tr((( )))
+ end
+
+ subgraph p6 [ ]
+ Node_t --"y"--> Node_ty((( )))
+ Node_ --"w"--> Node_w(( ))
+ Node_w --"ith"--> Node_with((( )))
+ Node_with --"o"--> Node_without((( )))
+ end
+
+ p2 ~~~ p3 ~~~ p4 ~~~ p5 ~~~ p6
+```
+
+One way to generate something like this is to start from the root and do
+a breadth-first walk, placing the encountered nodes on disk until a page
+is filled and their target transitions in a queue for which the process
+is repeated to fill other pages.
+
+Another approach, more suitable to our application because it can be
+done as part of the incremental construction process, is to do the
+packing from the bottom up — when the incremental construction
+algorithm completes a node we do not immediately write it, but wait
+until we have formed a branch that is bigger than a page. When this
+happens we lay out the node's children (each smaller than a page but
+root of a biggest branch that would fit) and let the parent node be
+treated like a leaf from there on. In turn it will become part of a
+branch that is bigger than a page and will be laid packaged together
+with its related nodes, resulting in a picture similar to the above.
+
+In fact the bottom-up process has a little performance benefit over the
+top-down: with the top-down construction the root page is full and leaf
+pages take combinations of unrelated smaller branches; with the
+bottom-up the leaf pages take as much information as possible about a
+branch, while the root often remains unfilled. For the best possible
+out-of-cache efficiency we would prefer the set of non-leaf pages to be
+as small as possible. Having larger leaf page branches means more of the
+trie data is in the leaf branches and thus the size of that intermediate
+node set is smaller.
+
+See [`IncrementalTrieWriterPageAware`](../../../tries/IncrementalDeepTrieWriterPageAware.java)
+for details on how the page-aware
+trie construction is implemented.
+
+## Storing the trie
+
+Another interesting question about the format of the trie is how one
+stores the information about the transitions in a node. If we want to
+maintain that the size of the structure is proportional to the number of
+overall transitions, we need to be able to store node transitions
+sparsely. Typically this is done using a list of transition characters
+and binary searching among them to make a transition.
+
+This binary search can theoretically be taken to use constant time
+(since the alphabet size is small and predefined), but isn't the most
+efficient operation in practice due to the unpredictable branch
+instructions necessary for its implementation. It is preferable to avoid
+it as much as possible.
+
+To do this, and to shave a few additional bytes in common cases, our
+implementation of on-disk tries uses typed nodes. A node can be:
+
+- Final with no transitions (`PAYLOAD_ONLY`).
+
+- Having one transition (`SINGLE`), which has to store only the
+ character and target for that transition.
+
+- Having a binary-searched list of transitions (`SPARSE`), where the
+ number of characters, each character and the targets are stored.
+
+- Having a consecutive range of transitions (`DENSE`), where the first
+ and last character and targets are stored, possibly including some
+ null transitions.
+
+We use one byte per node to store four bits of node type as well as four
+bits of payload information.
+
+In a well-balanced and populated trie the nodes where lookup spends most
+time (the nodes closest to the root) are `DENSE` nodes, where finding the
+target for the transition is a direct calculation from the code of the
+character. On the other hand, most of the nodes (the ones closest to the
+leaves) are `PAYLOAD_ONLY`, `SINGLE` or `SPARSE` to avoid taking any more
+space than necessary.
+
+The main objective for the trie storage format is to achieve the
+smallest possible packing (and thus smallest cache usage and fewest disk
+reads), thus we choose the type that results in the smallest
+representation of the node. `DENSE` type gets chosen naturally when its
+encoding (which avoids storing the character list but may include null
+targets) is smaller than `SPARSE`.
+
+## Pointer Sizes
+
+The next optimization we make in the storage format is based on the fact
+that most nodes in the trie are in the lower levels of the tree and thus
+close to leaves. As such, the distance between the node and its target
+transitions when laid out during the construction process is small and
+thus it is a huge win to store pointers as distances with variable size.
+
+This is even more true for the page-aware layout we use — all internal
+transitions within the page (i.e. >99% of all transitions in the trie!)
+can be stored using just an offset within the page, using just 12 bits.
+
+This is heavily used via further specialization of the node types: e.g.
+we have `DENSE_12`, `DENSE_16` to `DENSE_40` as well as `DENSE_LONG`
+subtypes which differ in the size of pointer they use.
+
+# Primary indexing in the BTI format
+
+The purpose of the primary index of an sstable is to be able to map a
+key containing partition and clustering components to a position in the
+sstable data file which holds the relevant row or the closest row with a
+greater key and enables iteration of rows from that point on.
+
+Partition keys are normally fully specified, while clustering keys are
+often given partially or via a comparison relation. They are also
+treated differently by all the infrastructure and have historically had
+different index structures; we chose to retain this distinction for the
+time being and implement similar replacement structures using tries.
+
+## Partition index implementation details
+
+The primary purpose of the partition index is to map a specified
+partition key to a row index for the partition. It also needs to support
+iteration from a (possibly partially specified) partition position. The
+description below details mapping only; iteration is a trivial
+application of the trie machinery to the described structure.
+
+In addition to wide partitions where a row index is mandatory, Cassandra
+is often used for tables where the partitions have only a
+couple of rows, including also ones where the partition key is the only
+component of the primary key, i.e. where row and partition are the same
+thing. For these situations it makes no sense to actually have a row
+index and the partition index should point directly to the data.
+
+The application of tries to Cassandra's partition index uses the trie
+infrastructure described above to create a trie mapping unique
+byte-ordered partition key prefixes to either:
+
+- A position in the row index file which contains the index of the
+ rows within that partition, or
+
+- A position in the data file containing the relevant partition (if a
+ row index for it is not necessary).
+
+A single table can have both indexed and non-indexed rows. For
+efficiency the partition index stores the position as a single long,
+using its sign bit to differentiate between the two options[^3]. This
+value is stored with variable length — more precisely, we use the four
+bits provided in the node type byte to store the length of the pointer.
+
+[^3]: It needs to differentiate between 0 with index and 0 without
+index, however, so we use ~pos instead of -pos to encode
+direct-to-data mappings. This still allows sign expansion
+instructions to be used to convert e.g. `int` to `long`.
+
+Lookup in this index is accomplished by converting the decorated
+partition key to its byte-ordered representation and following the
+transitions for its bytes while the trie has any. If at any point the
+trie does not offer a transition for the next byte but is not a leaf
+node, the sstable does not contain a mapping for the given key.
+
+If a leaf of the trie is reached, then the prefix of the partition key
+matches some content in the file, but we are not yet sure if it is a
+full match for the partition key. The leaf node points to a place in the
+row index or data file. In either case the first bytes at the specified
+position contain a serialization of the partition key, which we can
+compare to the key being mapped. If it matches, we have found the
+partition. If not, since the stored prefixes are unique, no data for
+this partition exists in this sstable.
+
+### Efficiency
+
+If everything is in cache this lookup is extremely efficient: it follows
+a few transitions in `DENSE` nodes plus one or two binary searches in
+`SPARSE` or `SINGLE`, and finishes with a direct comparison of a byte buffer
+with contents of a file. No object allocation or deserialization is
+necessary.
+
+If not all data is in cache, the performance of this lookup most heavily
+depends on the number of pages that must be fetched from persistent
+storage. The expectation on which this implementation is based, is that
+if an sstable is in use all non-leaf pages of the index will tend to
+remain cached. If that expectation is met, lookup will only require
+fetching one leaf index page and one data/row index page for the full
+key comparison. On a match the latter fetch will be required anyway,
+since we would want to read the data at that position.
+
+An important consideration in the design of this feature was to make
+sure there is no situation in which the trie indices perform worse than
+the earlier code, thus we should aim to do at most as many reads. The
+number of random accesses for the earlier index implementation where an
+index summary is forced in memory is one _seek_ required to start
+reading from the partition index (though usually multiple consecutive
+pages need to be read), and one seek needed to start reading the actual
+data. Since the index summary ends up being of similar size to the
+non-leaf pages of the trie index, the memory usage and number of seeks
+for the trie index on match ends up being the same but we read less data
+and do much less processing.
+
+On mismatch, though, we may be making one additional seek. However, we
+can drastically reduce the chance of mismatch, which we currently do in
+two ways:
+
+- By using a bloom filter before lookup. The chance of getting a bloom
+ filter hit as well as a prefix match for the wrong key is pretty
+ low and gets lower with increasing sstable size.
+
+- By storing some of the key hash bits that are not part of the token
+ at the payload node and comparing them with the mapped key's hash
+ bits.
+
+Currently we use a combination of both by default as the best performing
+option. The user can disable or choose to have a smaller bloom filter,
+and the code also supports indices that do not contain hash bits (though
+to reduce configuration complexity we do not have plans to expose that
+option).
+
+For fully cold sstables we have to perform more random fetches from disk
+than the earlier implementation, but we read less. Testing showed that
+having a bloom filter is enough to make the trie index faster; if a
+bloom filter is not present, we try going through the byte contents of
+the index file on boot to prefetch it which ends up taking not too long
+(since it is read sequentially rather than randomly) and boosting cold
+performance dramatically.
+
+### Building and early open
+
+The partition index is built using the page-aware incremental
+construction described earlier, where we also delay writing each key
+until we have seen the next so that we can find the shortest prefix that
+is enough to differentiate it from the previous and next keys (this also
+differentiates it from all others in the sstable because the contents
+are sorted). Only that prefix is written to the trie.
+
+One last complication is the support for early opening of sstables which
+allows newly-compacted tables to gradually occupy the page cache. Though
+the index building is incremental, the partially-written trie is not
+usable directly because the root of the trie as well as the path from it
+to the last written nodes is not yet present in the file.
+
+This problem can be easily overcome, though, by dumping these
+intermediate nodes to an in-memory buffer (without the need for
+page-aware packing) and forming an index by attaching this buffer at the
+end of the partially written file using
+[`TailOverridingRebufferer`](../../../util/TailOverridingRebufferer.java).
+
+## Row index implementation details
+
+Unlike the partition index, the main use of the row index is to iterate
+from a given clustering key in forward or reverse direction (where exact
+key lookup is just a special case).
+
+Rows are often very small (they could contain a single int or no columns
+at all) and thus there is a real possibility for the row indices to
+become bigger than the data they represent. This is not a desirable
+outcome, which is part of the reason why Cassandra's row index has
+historically operated on blocks of rows rather than indexing every row
+in the partition. This is a concern we also have with the trie-based
+index, thus we also index blocks of rows (by default, a block of rows
+that is at least 16kb in size — this will be called the index
+_granularity_ below, specified by the `column_index_size`
+`cassandra.yaml` parameter).
+
+Our row index implementation thus creates a map from clustering keys or
+prefixes to the data position at the start of the index block which is
+the earliest that could contain a key equal or greater than the given
+one. Additionally, if there is an active deletion at the beginning of
+the block, the index must specify it so that it can be taken into
+account when merging data from multiple sstables.
+
+Each index block will contain at least one key, but generally it will
+have different first and last keys. We don't store these keys, but
+instead we index the positions between blocks by storing a "separator",
+some key that is greater than the last key of the previous block and
+smaller than or equal to the first key of the next[^4]. Then, when we
+look up a given clustering, we follow its bytes as long as we can in the
+trie and we can be certain that all blocks before the closest
+less-than-or-equal entry in the trie cannot contain any data that is
+greater than or equal to the given key.
+
+[^4]: Another way to interpret this is that we index the start of each
+block only, but for efficiency we don't use the first key of the
+block as its beginning, but instead something closer to the last key
+of the previous block (but still greater than it).
+
+It may happen that the identified block actually doesn't contain any
+matching data (e.g. because the looked-up key ends up between the last
+key in the block and the separator), but this only affects efficiency as
+the iteration mechanism does not expect the data position returned by
+the index to be guaranteed to start with elements that fit the criteria;
+it would only have to walk a whole block forward to find the matching
+key.
+
+It is important to keep the number of these false positives low, and at
+the same time we aim for the smallest possible size of the index for a
+given granularity. The choice of separator affects this balance[^5]; the
+option we use, as a good tradeoff in the vein of the unique prefix
+approach used in the partition index, is to use the shortest prefix of
+the next block's beginning key that separates it from the previous
+block's end key, adjusted so that the last byte of it is 1 greater than
+that end key.
+
+[^5]: For example, the best separator for false positives is the next
+possible byte sequence after the previous block's final key, which
+is obtained by adding a 00 byte to its end. This, however, means all
+the bytes of the byte-ordered representation of this key must be
+present in the index, which inflates the index's size and lookup
+complexity.
+
+For example, if block 2 covers "something" to "somewhere" and block 3
+— "sorry" to "tease", then the sequence "son" is used as the separator
+between blocks 2 and 3. This leaves things like "sommelier" in the area
+that triggers false positives, but stores and has to walk just three
+bytes to find the starting point for iteration.
+
+### Efficiency
+
+Finding the candidate block in the trie involves walking the byte
+ordered representation of the clustering key in the trie and finding the
+closest less-than-or-equal value. The number of steps is proportional to
+the length of the separators — the lower their number the shorter that
+sequence is, though we can't expect _O_(log _n_) complexity since there may
+be many items sharing the same long prefixes (e.g. if there are long
+strings in the components of the clustering keys before the last). Even
+so, such repeating prefixes are addressed very well by the page-packing
+and `SINGLE_NOPAYLOAD_4` node type, resulting in very efficient walks.
+
+After this step we also perform a linear walk within the data file to
+find the actual start of the matching data. This is usually costlier and
+may involve object allocation and deserialization.
+
+The tradeoff between the size of the index and the time it takes to find
+the relevant rows is controlled by the index granularity. The lower it
+is, the more efficient lookup (especially exact match lookup) becomes at
+the expense of bigger index size. The 16kb default is chosen pretty
+conservatively[^6]; if users don't mind bigger indices something like 4,
+2 or 1kb granularity should be quite a bit more efficient. It is also
+possible to index every row by choosing a granularity of 0kb; at these
+settings in-cache trie-indexed sstables tend to outperform
+`ConcurrentSkipListMap` memtables for reads.
+
+[^6]: This was chosen with the aim to match the size of the trie index
+compared to the earlier version of the row index at its default
+granularity of 64kb.
+
+### Reverse lookup
+
+To perform a reverse lookup, we can use the same mechanism as above
+(with greater-than-or-equal) to find the initial block for the
+iteration. However, in the forward direction we could simply walk the
+data file to find the next rows, but this isn't possible going
+backwards.
+
+To solve this problem the index helps the iteration machinery by
+providing an iterator of index blocks in reverse order. For each index
+block the iteration walks it forward and creates a stack of all its row
+positions, then starts issuing rows by popping and examining rows from
+that stack. When the stack is exhausted it requests the previous block
+from the index and applies the same procedure there.
+
+# Code structure
+
+The implementation is mostly in two packages, `o.a.c.io.tries` contains
+the generic code to construct and read on-disk tries, and
+`o.a.c.io.sstable.format.bti`, which implements the specifics of the
+format and the two indexes.
+
+## Building tries
+
+Tries are built from sorted keys using an [`IncrementalTrieWriter`](../../../tries/IncrementalTrieWriter.java).
+The code contains three implementations with increasing complexity:
+- [`IncrementalTrieWriterSimple`](../../../tries/IncrementalTrieWriterSimple.java)
+ implements simple incremental construction of tries from sorted input,
+- [`IncrementalTrieWriterPageAware`](../../../tries/IncrementalTrieWriterPageAware.java)
+ adds packing of nodes to disk pages,
+- [`IncrementalDeepTrieWriterPageAware`](../../../tries/IncrementalDeepTrieWriterPageAware.java)
+ adds the ability to transition to on-heap recursion for all stages of the construction
+ process to be able to handle very large keys.
+
+Only the latter is used, but we provide (and test) the other two as a form of
+documentation.
+
+The builders take a `TrieSerializer` as parameter, which determines how the nodes
+are written. The indexes implement this using `TrieNode`, writing any payload they
+need immediately after the node serialization.
+
+## Reading tries
+
+The BTI format tries are used directly in their on-disk format. To achieve this,
+all node types are implemented as static objects in `TrieNode`. Reading nodes in
+a file is encapsulated in [`Walker`](../../../tries/Walker.java),
+which provides a method to `go` to a specific node and use it, i.e.
+get any associated data, search in the children list and
+follow transitions to children. It also provides functionality to find the
+mapping for a given key, floors and ceilings as well as some combinations.
+Iterating the payloads between two key bounds is implemented by
+[`ValueIterator`](../../../tries/ValueIterator.java),
+and [`ReverseValueIterator`](../../../tries/ReverseValueIterator.java).
+
+Special care is given to prefixes to make sure the semantics of searches matches
+what the format needs.
+
+## SSTable format implementation
+
+The two indexes are implemented, respectively, by [`PartitionIndex`](PartitionIndex.java)
+/[`PartitionIndexBuilder`](PartitionIndexBuilder.java)
+and [`RowIndexReader`](RowIndexReader.java)/[`RowIndexWriter`](RowIndexWriter.java).
+The format implementation extends the filtered
+base class and follows the structure of the BIG implementation, where
+all references to the primary index are replaced with calls to these two
+classes.
+
+# Index file format in BTI
+
+## Trie nodes
+Implemented in [`TrieNode.java`](../../../tries/TrieNode.java)
+
+Nodes start with four bits of node type, followed by 4 payload bits
+(_pb_), which are 0 if the node has no associated payload; otherwise the
+node type gives an option to compute the starting position for the
+payload (_ppos_) from the starting position of the node (_npos_).
+The layout of the node depends on its type.
+
+`PAYLOAD_ONLY` nodes:
+
+- 4 type bits, 0
+
+- 4 payload bits
+
+- payload if _pb_ ≠ 0, _ppos_ is _npos_ + 1
+
+`SINGLE_NOPAYLOAD_4` and `SINGLE_NOPAYLOAD_12` nodes:
+
+- 4 type bits
+
+- 4 pointer bits
+
+- 8 pointer bits (for `SINGLE_NOPAYLOAD_12`)
+
+- 8 bits transition byte
+
+- _pb_ is assumed 0
+
+`SINGLE_8/16`:
+
+- 4 type bits
+
+- 4 payload bits
+
+- 8 bits transition byte
+
+- 8/16 pointer bits
+
+- payload if _pb_ ≠ 0, _ppos_ is _npos_ + 3/4
+
+`SPARSE_8/12/16/24/40`:
+
+- 4 type bits
+
+- 4 payload bits
+
+- 8 bit child count
+
+- 8 bits per child, the transition bytes
+
+- 8/12/16/24/40 bits per child, the pointers
+
+- payload if _pb_ ≠ 0, _ppos_ is _npos_ + 2 + (2/2.5/3/4/6)*(_child
+ count_) (rounded up)
+
+`DENSE_12/16/24/32/40/LONG`:
+
+- 4 type bits
+
+- 4 payload bits
+
+- 8 bit start byte value
+
+- 8 bit _length_-1
+
+- _length_ * 12/16/24/32/40/64 bits per child, the pointers
+
+- payload if _pb_ ≠ 0, _ppos_ is _npos_ + 3 + (1.5/2/3/4/5/8)*(_length_)
+ (rounded up)
+
+This is the space taken by each node type (_CS_ stands for child span,
+i.e. largest - smallest + 1, _CC_ is child count):
+
+|Type | Size in bytes excl. payload |Size for 1 child|Size for 9 dense children (01-08, 10)|Size for 10 sparse children (01 + i*10)|Why the type is needed |
+|:--------------------|:----------------------------|---------------:|-------------------:|-------------------:|:-------------------------------|
+|`PAYLOAD_ONLY` | 1 | - | - | - | Leaves dominate the trie |
+|`SINGLE_NOPAYLOAD_4` | 2 |2 | - | - | Single-transition chains |
+|`SINGLE_8` | 3 |3 | - | - | Payload within chain |
+|`SPARSE_8` | 2 + _CC_ * 2 |4 | 20 | 22 | Most common type after leaves |
+|`SINGLE_NOPAYLOAD_12`| 3 |3 | - | - | 12 bits cover all in-page transitions |
+|`SPARSE_12` | 2 + _CC_ * 2.5 |5 | 25 | 27 | Size of sparse is proportional to number of children |
+|`DENSE_12` | 3 + _CS_ * 1.5 |5 | 18 | 140 | Lookup in dense is faster, size smaller if few holes |
+|`SINGLE_16` | 4 |4 | - | - | |
+|`SPARSE_16` | 2 + _CC_ * 3 |5 | 29 | 32 | |
+|`DENSE_16` | 3 + _CS_ * 2 |5 | 23 | 185 | |
+|`SPARSE_24` | 2 + _CC_ * 4 |6 | 38 | 42 | |
+|`DENSE_24` | 3 + _CS_ * 3 |6 | 33 | 276 | |
+|`DENSE_32` | 3 + _CS_ * 4 |7 | 43 | 367 | Nodes with big subtrees are usually dense |
+|`SPARSE_40` | 2 + _CC_ * 6 |8 | 56 | 62 | |
+|`DENSE_40` | 3 + _CS_ * 5 |8 | 53 | 458 | |
+|`DENSE_LONG` | 3 + _CS_ * 8 |11 | 83 | 731 | Catch-all |
+
+All pointers are stored as distances, and since all tries are written
+from the bottom up (and hence a child is always before the parent in the
+file), the distance is subtracted from the position of the current node
+to obtain the position of the child node.
+
+Note: All nodes are placed in such a way that they do not cross a page
+boundary. I.e. if a reader (e.g. [`Walker`](../../../tries/Walker.java))
+is positioned at a node, it is
+guaranteed that all reads of the node's data can complete without
+requiring a different page to be fetched from disk.
+
+## Partition index
+Implemented in [`PartitionIndex.java`](PartitionIndex.java)
+
+Layout:
+```
+[nodes page, 4096 bytes]
+...
+[nodes page, 4096 bytes]
+[nodes page including root node, ≤4096 bytes]
+[smallest key, with short length]
+[largest key, with short length]
+[smallest key pos, long]
+[key count, long]
+[root pos, long]
+```
+
+The SSTable's partition index is stored in the -Partitions.db file. The
+file itself is written from the bottom up, and its "header" is at the
+end of the file.
+
+More precisely, the last 3 longs in the file contain:
+
+- A file position where the smallest and greatest key are written.
+
+- The exact number of keys in the file.
+
+- A file position for the root node of the index.
+
+These three longs are preceded by the serialization of the first and
+last key, and before that are the trie contents.
+
+To find a match for the key, start at the root position, decode the node
+(see the "Trie nodes" section above) and follow the transitions
+according to the bytes of the byte-ordered representation of the key
+while the node has children and there are bytes left in the key.
+
+If a leaf node is reached, that node contains the following payload:
+
+- If _pb_ < 8, let
+
+ - _idxpos_ be the sign-extended integer value of length _pb_ at
+ _ppos_
+
+- If _pb_ ≥ 8 (always the case in Cassandra 5 files), let
+
+ - _hash_ be the byte at _ppos_
+
+ - _idxpos_ be the sign-extended integer value of length _pb-7_ at
+ _ppos+1_
+
+If at any step there is no transition corresponding to the byte of the
+key, or if _hash_ is present and the lowest-order byte of the key's hash
+value does not match it, the index and sstable have no mapping for the
+given key.
+
+Otherwise _idxpos_ specifies:
+
+- if _idxpos_ ≥ 0, the row index file contains an index for the
+ given key (see row index section below) at position _idxpos_
+
+- otherwise, the data associated with the key starts at position
+ ~_idxpos_ in the data file
+
+In either case the content in the respective file starts with the
+serialization of the partition key, which must be compared with the key
+requested to ensure they match.
+
+## Row index
+Implemented in [`RowIndexReader.java`](RowIndexReader.java)
+
+Layout:
+```
+[row index, padded to page boundary]
+...
+[row index, padded to page boundary]
+```
+Where each row index contains:
+```
+[nodes page, 4096 bytes]
+...
+[nodes page, 4096 bytes]
+[nodes page including root node, ≤4096 bytes]
+[partition key, with short length]
+[position of the partition in the data file, unsigned vint]
+[position of the root node, vint encoding the difference between the
+start of the data file position and the position of the root node]
+[number of rows in the partition, unsigned vint]
+[partition deletion time, 12 bytes]
+```
+
+The entries in the partition index point to the position at the start of
+the partition key.
+
+The payload reachable at the end of a traversal contains:
+
+- Integer of _pb_&7 bytes specifying the offset within the partition
+ where the relevant row is contained
+
+- If _pb_ ≥ 8, 12 bytes of deletion time active at the start of the
+ row index block
+
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java
new file mode 100644
index 0000000..c568320
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.sstable.format.SortedTablePartitionWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
+import org.apache.cassandra.io.util.SequentialWriter;
+
+/**
+ * Partition writer used by {@link BtiTableWriter}.
+ * <p>
+ * Writes all passed data to the given SequentialWriter and if necessary builds a RowIndex by constructing an entry
+ * for each row within a partition that follows {@link org.apache.cassandra.config.Config#column_index_size} of written
+ * data.
+ */
+class BtiFormatPartitionWriter extends SortedTablePartitionWriter
+{
+ private static final int DEFAULT_GRANULARITY = 16 * 1024;
+ private final RowIndexWriter rowTrie;
+ private final int rowIndexBlockSize;
+ private int rowIndexBlockCount;
+
+ BtiFormatPartitionWriter(SerializationHeader header,
+ ClusteringComparator comparator,
+ SequentialWriter dataWriter,
+ SequentialWriter rowIndexWriter,
+ Version version)
+ {
+ this(header, comparator, dataWriter, rowIndexWriter, version, DatabaseDescriptor.getColumnIndexSize(DEFAULT_GRANULARITY));
+ }
+
+
+ BtiFormatPartitionWriter(SerializationHeader header,
+ ClusteringComparator comparator,
+ SequentialWriter dataWriter,
+ SequentialWriter rowIndexWriter,
+ Version version,
+ int rowIndexBlockSize)
+ {
+ super(header, dataWriter, version);
+ this.rowIndexBlockSize = rowIndexBlockSize;
+ this.rowTrie = new RowIndexWriter(comparator, rowIndexWriter);
+ }
+
+ @Override
+ public void reset()
+ {
+ super.reset();
+ rowTrie.reset();
+ rowIndexBlockCount = 0;
+ }
+
+ @Override
+ public void addUnfiltered(Unfiltered unfiltered) throws IOException
+ {
+ super.addUnfiltered(unfiltered);
+
+ // if we hit the column index size that we have to index after, go ahead and index it.
+ if (currentPosition() - startPosition >= rowIndexBlockSize)
+ addIndexBlock();
+ }
+
+ @Override
+ public void close()
+ {
+ rowTrie.close();
+ }
+
+ public long finish() throws IOException
+ {
+ long endPosition = super.finish();
+
+ // the last row may have fallen on an index boundary already. if not, index the last block explicitly.
+ if (rowIndexBlockCount > 0 && firstClustering != null)
+ addIndexBlock();
+
+ if (rowIndexBlockCount > 1)
+ {
+ return rowTrie.complete(endPosition);
+ }
+ else
+ {
+ // Otherwise we don't complete the trie as an index of one block adds no information and we are better off
+ // without a row index for such partitions. Even if we did write something to the file (which shouldn't be
+ // the case as the first entry has an empty key and root isn't filled), that's not a problem.
+ return -1;
+ }
+ }
+
+ protected void addIndexBlock() throws IOException
+ {
+ IndexInfo cIndexInfo = new IndexInfo(startPosition, startOpenMarker);
+ rowTrie.add(firstClustering, lastClustering, cIndexInfo);
+ firstClustering = null;
+ ++rowIndexBlockCount;
+ }
+
+ public int getRowIndexBlockCount()
+ {
+ return rowIndexBlockCount;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
new file mode 100644
index 0000000..172dcde
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener.SelectionReason;
+import org.apache.cassandra.io.sstable.SSTableReadsListener.SkippingReason;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.OutputHandler;
+
+import static org.apache.cassandra.io.sstable.format.SSTableReader.Operator.EQ;
+import static org.apache.cassandra.io.sstable.format.SSTableReader.Operator.GE;
+import static org.apache.cassandra.io.sstable.format.SSTableReader.Operator.GT;
+import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull;
+
+/**
+ * Reader of SSTable files in BTI format (see {@link BtiFormat}), written by {@link BtiTableWriter}.
+ * <p>
+ * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
+ * Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
+ */
+public class BtiTableReader extends SSTableReaderWithFilter
+{
+ private final FileHandle rowIndexFile;
+ private final PartitionIndex partitionIndex;
+
+ public BtiTableReader(Builder builder, SSTable.Owner owner)
+ {
+ super(builder, owner);
+ this.rowIndexFile = builder.getRowIndexFile();
+ this.partitionIndex = builder.getPartitionIndex();
+ }
+
+ protected final Builder unbuildTo(Builder builder, boolean sharedCopy)
+ {
+ Builder b = super.unbuildTo(builder, sharedCopy);
+ if (builder.getPartitionIndex() == null)
+ b.setPartitionIndex(sharedCopy ? sharedCopyOrNull(partitionIndex) : partitionIndex);
+ if (builder.getRowIndexFile() == null)
+ b.setRowIndexFile(sharedCopy ? sharedCopyOrNull(rowIndexFile) : rowIndexFile);
+
+ return b;
+ }
+
+ @Override
+ protected List<AutoCloseable> setupInstance(boolean trackHotness)
+ {
+ ArrayList<AutoCloseable> closeables = Lists.newArrayList(rowIndexFile, partitionIndex);
+ closeables.addAll(super.setupInstance(trackHotness));
+ return closeables;
+ }
+
+ /**
+ * Whether to filter out data before {@link #first}. Needed for sources of data in a compaction, where the relevant
+ * output is opened early -- in this case the sstable's start is changed, but the data can still be found in the
+ * file. Range and point queries must filter it out.
+ */
+ protected boolean filterFirst()
+ {
+ return openReason == OpenReason.MOVED_START;
+ }
+
+ /**
+ * Whether to filter out data after {@link #last}. Early-open sstables may contain data beyond the switch point
+ * (because an early-opened sstable is not ready until buffers have been flushed), and leaving that data visible
+ * will give a redundant copy with all associated overheads.
+ */
+ protected boolean filterLast()
+ {
+ return openReason == OpenReason.EARLY && partitionIndex instanceof PartitionIndexEarly;
+ }
+
+ public long estimatedKeys()
+ {
+ return partitionIndex == null ? 0 : partitionIndex.size();
+ }
+
+ @Override
+ protected TrieIndexEntry getRowIndexEntry(PartitionPosition key,
+ Operator operator,
+ boolean updateStats,
+ SSTableReadsListener listener)
+ {
+ PartitionPosition searchKey;
+ Operator searchOp;
+
+ if (operator == EQ)
+ return getExactPosition((DecoratedKey) key, listener, updateStats);
+
+ if (operator == GT || operator == GE)
+ {
+ if (filterLast() && last.compareTo(key) < 0)
+ {
+ notifySkipped(SkippingReason.MIN_MAX_KEYS, listener, operator, updateStats);
+ return null;
+ }
+ boolean filteredLeft = (filterFirst() && first.compareTo(key) > 0);
+ searchKey = filteredLeft ? first : key;
+ searchOp = filteredLeft ? GE : operator;
+
+ try (PartitionIndex.Reader reader = partitionIndex.openReader())
+ {
+ TrieIndexEntry rie = reader.ceiling(searchKey, (pos, assumeNoMatch, compareKey) -> retrieveEntryIfAcceptable(searchOp, compareKey, pos, assumeNoMatch));
+ if (rie != null)
+ notifySelected(SelectionReason.INDEX_ENTRY_FOUND, listener, operator, updateStats, rie);
+ else
+ notifySkipped(SkippingReason.INDEX_ENTRY_NOT_FOUND, listener, operator, updateStats);
+ return rie;
+ }
+ catch (IOException e)
+ {
+ markSuspect();
+ throw new CorruptSSTableException(e, rowIndexFile.path());
+ }
+ }
+
+ throw new IllegalArgumentException("Invalid op: " + operator);
+ }
+
+ /**
+ * Called by {@link #getRowIndexEntry} above (via Reader.ceiling/floor) to check if the position satisfies the full
+ * key constraint. This is called once if there is a prefix match (which can be in any relationship with the sought
+ * key, thus assumeNoMatch: false), and if it returns null it is called again for the closest greater position
+ * (with assumeNoMatch: true).
+ * Returns the index entry at this position, or null if the search op rejects it.
+ */
+ private TrieIndexEntry retrieveEntryIfAcceptable(Operator searchOp, PartitionPosition searchKey, long pos, boolean assumeNoMatch) throws IOException
+ {
+ if (pos >= 0)
+ {
+ try (FileDataInput in = rowIndexFile.createReader(pos))
+ {
+ if (assumeNoMatch)
+ ByteBufferUtil.skipShortLength(in);
+ else
+ {
+ ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+ DecoratedKey decorated = decorateKey(indexKey);
+ if (searchOp.apply(decorated.compareTo(searchKey)) != 0)
+ return null;
+ }
+ return TrieIndexEntry.deserialize(in, in.getFilePointer());
+ }
+ }
+ else
+ {
+ pos = ~pos;
+ if (!assumeNoMatch)
+ {
+ try (FileDataInput in = dfile.createReader(pos))
+ {
+ ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+ DecoratedKey decorated = decorateKey(indexKey);
+ if (searchOp.apply(decorated.compareTo(searchKey)) != 0)
+ return null;
+ }
+ }
+ return new TrieIndexEntry(pos);
+ }
+ }
+
+ @Override
+ public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException
+ {
+ try (RandomAccessReader reader = openDataReader())
+ {
+ reader.seek(keyPositionFromSecondaryIndex);
+ if (reader.isEOF())
+ return null;
+ return decorateKey(ByteBufferUtil.readWithShortLength(reader));
+ }
+ }
+
+ TrieIndexEntry getExactPosition(DecoratedKey dk,
+ SSTableReadsListener listener,
+ boolean updateStats)
+ {
+ if ((filterFirst() && first.compareTo(dk) > 0) || (filterLast() && last.compareTo(dk) < 0))
+ {
+ notifySkipped(SkippingReason.MIN_MAX_KEYS, listener, EQ, updateStats);
+ return null;
+ }
+
+ if (!isPresentInFilter(dk))
+ {
+ notifySkipped(SkippingReason.BLOOM_FILTER, listener, EQ, updateStats);
+ return null;
+ }
+
+ try (PartitionIndex.Reader reader = partitionIndex.openReader())
+ {
+ long indexPos = reader.exactCandidate(dk);
+ if (indexPos == PartitionIndex.NOT_FOUND)
+ {
+ notifySkipped(SkippingReason.PARTITION_INDEX_LOOKUP, listener, EQ, updateStats);
+ return null;
+ }
+
+ FileHandle fh;
+ long seekPosition;
+ if (indexPos >= 0)
+ {
+ fh = rowIndexFile;
+ seekPosition = indexPos;
+ }
+ else
+ {
+ fh = dfile;
+ seekPosition = ~indexPos;
+ }
+
+ try (FileDataInput in = fh.createReader(seekPosition))
+ {
+ if (ByteBufferUtil.equalsWithShortLength(in, dk.getKey()))
+ {
+ TrieIndexEntry rie = indexPos >= 0 ? TrieIndexEntry.deserialize(in, in.getFilePointer())
+ : new TrieIndexEntry(~indexPos);
+ notifySelected(SelectionReason.INDEX_ENTRY_FOUND, listener, EQ, updateStats, rie);
+ return rie;
+ }
+ else
+ {
+ notifySkipped(SkippingReason.INDEX_ENTRY_NOT_FOUND, listener, EQ, updateStats);
+ return null;
+ }
+ }
+ }
+ catch (IOException | IllegalArgumentException | ArrayIndexOutOfBoundsException | AssertionError e)
+ {
+ markSuspect();
+ throw new CorruptSSTableException(e, rowIndexFile.path());
+ }
+ }
+
+ /**
+ * Create a PartitionIterator listing all partitions within the given bounds.
+ * This method relies on its caller to prepare the bounds correctly.
+ *
+ * @param bounds A range of keys. Must not be a wraparound range, and will not be checked against
+ * the sstable's bounds (i.e. this will return data before a moved start or after an early-open limit)
+ */
+ PartitionIterator coveredKeysIterator(AbstractBounds<PartitionPosition> bounds) throws IOException
+ {
+ return PartitionIterator.create(partitionIndex,
+ metadata().partitioner,
+ rowIndexFile,
+ dfile,
+ bounds.left, bounds.inclusiveLeft() ? -1 : 0,
+ bounds.right, bounds.inclusiveRight() ? 0 : -1);
+ }
+
+ public ScrubPartitionIterator scrubPartitionsIterator() throws IOException
+ {
+ return new ScrubIterator(partitionIndex, rowIndexFile);
+ }
+
+ @Override
+ public PartitionIterator keyReader() throws IOException
+ {
+ return PartitionIterator.create(partitionIndex, metadata().partitioner, rowIndexFile, dfile);
+ }
+
+ @Override
+ public Iterable<DecoratedKey> getKeySamples(final Range<Token> range)
+ {
+ // BTI does not support key sampling as it would involve walking the index or data file.
+ // Validator has an alternate solution for empty key sample lists.
+ return Collections.emptyList();
+ }
+
+ @Override
+ public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
+ {
+ // Estimate the number of partitions by calculating the bytes of the sstable that are covered by the specified
+ // ranges and using the mean partition size to obtain a number of partitions from that.
+ long selectedDataSize = 0;
+ for (Range<Token> range : Range.normalize(ranges))
+ {
+ PartitionPosition left = range.left.minKeyBound();
+ if (left.compareTo(first) <= 0)
+ left = null;
+ else if (left.compareTo(last) > 0)
+ continue; // no intersection
+
+ PartitionPosition right = range.right.minKeyBound();
+ if (range.right.isMinimum() || right.compareTo(last) >= 0)
+ right = null;
+ else if (right.compareTo(first) < 0)
+ continue; // no intersection
+
+ if (left == null && right == null)
+ return partitionIndex.size(); // sstable is fully covered, return full partition count to avoid rounding errors
+
+ if (left == null && filterFirst())
+ left = first;
+ if (right == null && filterLast())
+ right = last;
+
+ long startPos = left != null ? getPosition(left, GE) : 0;
+ long endPos = right != null ? getPosition(right, GE) : uncompressedLength();
+ selectedDataSize += endPos - startPos;
+ }
+ return Math.round(selectedDataSize / sstableMetadata.estimatedPartitionSize.rawMean());
+ }
+
+
+ @Override
+ public UnfilteredRowIterator rowIterator(DecoratedKey key,
+ Slices slices,
+ ColumnFilter selectedColumns,
+ boolean reversed,
+ SSTableReadsListener listener)
+ {
+ return rowIterator(null, key, getExactPosition(key, listener, true), slices, selectedColumns, reversed);
+ }
+
+ public UnfilteredRowIterator rowIterator(FileDataInput dataFileInput,
+ DecoratedKey key,
+ TrieIndexEntry indexEntry,
+ Slices slices,
+ ColumnFilter selectedColumns,
+ boolean reversed)
+ {
+ if (indexEntry == null)
+ return UnfilteredRowIterators.noRowsIterator(metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed);
+
+ if (reversed)
+ return new SSTableReversedIterator(this, dataFileInput, key, indexEntry, slices, selectedColumns, rowIndexFile);
+ else
+ return new SSTableIterator(this, dataFileInput, key, indexEntry, slices, selectedColumns, rowIndexFile);
+ }
+
+ @Override
+ public ISSTableScanner getScanner()
+ {
+ return BtiTableScanner.getScanner(this);
+ }
+
+ @Override
+ public ISSTableScanner getScanner(Collection<Range<Token>> ranges)
+ {
+ if (ranges != null)
+ return BtiTableScanner.getScanner(this, ranges);
+ else
+ return getScanner();
+ }
+
+ @Override
+ public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
+ {
+ return BtiTableScanner.getScanner(this, rangeIterator);
+ }
+
+ @VisibleForTesting
+ @Override
+ public BtiTableReader cloneAndReplace(IFilter filter)
+ {
+ return unbuildTo(new Builder(descriptor).setFilter(filter), true).build(owner().orElse(null), true, true);
+ }
+
+ @Override
+ public BtiTableReader cloneWithRestoredStart(DecoratedKey restoredStart)
+ {
+ return runWithLock(ignored -> cloneAndReplace(restoredStart, OpenReason.NORMAL));
+ }
+
+ @Override
+ public BtiTableReader cloneWithNewStart(DecoratedKey newStart)
+ {
+ return runWithLock(d -> {
+ assert openReason != OpenReason.EARLY : "Cannot open early an early-open SSTable";
+ if (newStart.compareTo(first) > 0)
+ {
+ final long dataStart = getPosition(newStart, Operator.EQ);
+ runOnClose(() -> dfile.dropPageCache(dataStart));
+ }
+
+ return cloneAndReplace(newStart, OpenReason.MOVED_START);
+ });
+ }
+
+ /**
+ * Clone this reader with the provided start and open reason, and set the clone as replacement.
+ *
+ * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive
+ * opening of compaction results).
+ * @param reason the {@code OpenReason} for the replacement.
+ * @return the cloned reader. That reader is set as a replacement by the method.
+ */
+ private BtiTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason)
+ {
+ return unbuildTo(new Builder(descriptor), true)
+ .setFirst(newFirst)
+ .setOpenReason(reason)
+ .build(owner().orElse(null), true, true);
+ }
+
+ @Override
+ public DecoratedKey firstKeyBeyond(PartitionPosition token)
+ {
+ try
+ {
+ TrieIndexEntry pos = getRowIndexEntry(token, Operator.GT, true, SSTableReadsListener.NOOP_LISTENER);
+ if (pos == null)
+ return null;
+
+ try (FileDataInput in = dfile.createReader(pos.position))
+ {
+ ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+ return decorateKey(indexKey);
+ }
+ }
+ catch (IOException e)
+ {
+ markSuspect();
+ throw new CorruptSSTableException(e, dfile.path());
+ }
+ }
+
+ @Override
+ public void releaseInMemoryComponents()
+ {
+ closeInternalComponent(partitionIndex);
+ }
+
+ @Override
+ public boolean isEstimationInformative()
+ {
+ return true;
+ }
+
+ @Override
+ public UnfilteredPartitionIterator partitionIterator(ColumnFilter columnFilter, DataRange dataRange, SSTableReadsListener listener)
+ {
+ return BtiTableScanner.getScanner(this, columnFilter, dataRange, listener);
+ }
+
+ @Override
+ public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options)
+ {
+ Preconditions.checkArgument(cfs.metadata().equals(metadata()));
+ return new BtiTableVerifier(cfs, this, outputHandler, isOffline, options);
+ }
+
+ public static class Builder extends SSTableReaderWithFilter.Builder<BtiTableReader, Builder>
+ {
+ private PartitionIndex partitionIndex;
+ private FileHandle rowIndexFile;
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ public Builder setRowIndexFile(FileHandle rowIndexFile)
+ {
+ this.rowIndexFile = rowIndexFile;
+ return this;
+ }
+
+ public Builder setPartitionIndex(PartitionIndex partitionIndex)
+ {
+ this.partitionIndex = partitionIndex;
+ return this;
+ }
+
+ public PartitionIndex getPartitionIndex()
+ {
+ return partitionIndex;
+ }
+
+ public FileHandle getRowIndexFile()
+ {
+ return rowIndexFile;
+ }
+
+ @Override
+ protected BtiTableReader buildInternal(Owner owner)
+ {
+ return new BtiTableReader(this, owner);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java
new file mode 100644
index 0000000..1be6925
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
+import org.apache.cassandra.io.sstable.format.FilterComponent;
+import org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat.Components;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.Throwables;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class BtiTableReaderLoadingBuilder extends SortedTableReaderLoadingBuilder<BtiTableReader, BtiTableReader.Builder>
+{
+ private final static Logger logger = LoggerFactory.getLogger(BtiTableReaderLoadingBuilder.class);
+
+ private FileHandle.Builder partitionIndexFileBuilder;
+ private FileHandle.Builder rowIndexFileBuilder;
+
+ public BtiTableReaderLoadingBuilder(SSTable.Builder<?, ?> builder)
+ {
+ super(builder);
+ }
+
+ @Override
+ public KeyReader buildKeyReader(TableMetrics tableMetrics) throws IOException
+ {
+ StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION);
+ return createKeyReader(statsComponent.statsMetadata());
+ }
+
+ private KeyReader createKeyReader(StatsMetadata statsMetadata) throws IOException
+ {
+ checkNotNull(statsMetadata);
+
+ try (PartitionIndex index = PartitionIndex.load(partitionIndexFileBuilder(), tableMetadataRef.getLocal().partitioner, false);
+ CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components);
+ FileHandle dFile = dataFileBuilder(statsMetadata).withCompressionMetadata(compressionMetadata).complete();
+ FileHandle riFile = rowIndexFileBuilder().complete())
+ {
+ return PartitionIterator.create(index,
+ tableMetadataRef.getLocal().partitioner,
+ riFile,
+ dFile);
+ }
+ }
+
+ @Override
+ protected void openComponents(BtiTableReader.Builder builder, SSTable.Owner owner, boolean validate, boolean online) throws IOException
+ {
+ try
+ {
+ StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.VALIDATION, MetadataType.HEADER);
+ builder.setSerializationHeader(statsComponent.serializationHeader(builder.getTableMetadataRef().getLocal()));
+ checkArgument(!online || builder.getSerializationHeader() != null);
+
+ builder.setStatsMetadata(statsComponent.statsMetadata());
+ ValidationMetadata validationMetadata = statsComponent.validationMetadata();
+ validatePartitioner(builder.getTableMetadataRef().getLocal(), validationMetadata);
+
+ boolean filterNeeded = online;
+ if (filterNeeded)
+ builder.setFilter(loadFilter(validationMetadata));
+ boolean rebuildFilter = filterNeeded && builder.getFilter() == null;
+
+ if (builder.getComponents().contains(Components.PARTITION_INDEX) && builder.getComponents().contains(Components.ROW_INDEX) && rebuildFilter)
+ {
+ @SuppressWarnings({ "resource", "RedundantSuppression" })
+ IFilter filter = buildBloomFilter(statsComponent.statsMetadata());
+ builder.setFilter(filter);
+ FilterComponent.save(filter, descriptor, false);
+ }
+
+ if (builder.getFilter() == null)
+ builder.setFilter(FilterFactory.AlwaysPresent);
+
+ if (builder.getComponents().contains(Components.ROW_INDEX))
+ builder.setRowIndexFile(rowIndexFileBuilder().complete());
+
+ if (descriptor.version.hasKeyRange() && builder.getStatsMetadata() != null)
+ {
+ IPartitioner partitioner = tableMetadataRef.getLocal().partitioner;
+ builder.setFirst(partitioner.decorateKey(builder.getStatsMetadata().firstKey));
+ builder.setLast(partitioner.decorateKey(builder.getStatsMetadata().lastKey));
+ }
+
+ if (builder.getComponents().contains(Components.PARTITION_INDEX))
+ {
+ builder.setPartitionIndex(openPartitionIndex(builder.getFilter().isInformative()));
+ if (builder.getFirst() == null || builder.getLast() == null)
+ {
+ builder.setFirst(builder.getPartitionIndex().firstKey());
+ builder.setLast(builder.getPartitionIndex().lastKey());
+ }
+ }
+
+ try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components))
+ {
+ builder.setDataFile(dataFileBuilder(builder.getStatsMetadata()).withCompressionMetadata(compressionMetadata).complete());
+ }
+ }
+ catch (IOException | RuntimeException | Error ex)
+ {
+ // in case of failure, close only those components which have been opened in this try-catch block
+ Throwables.closeAndAddSuppressed(ex, builder.getPartitionIndex(), builder.getRowIndexFile(), builder.getDataFile(), builder.getFilter());
+ throw ex;
+ }
+ }
+
+ private IFilter buildBloomFilter(StatsMetadata statsMetadata) throws IOException
+ {
+ IFilter bf = null;
+
+ try (KeyReader keyReader = createKeyReader(statsMetadata))
+ {
+ bf = FilterFactory.getFilter(statsMetadata.totalRows, tableMetadataRef.getLocal().params.bloomFilterFpChance);
+
+ while (!keyReader.isExhausted())
+ {
+ DecoratedKey key = tableMetadataRef.getLocal().partitioner.decorateKey(keyReader.key());
+ bf.add(key);
+
+ keyReader.advance();
+ }
+ }
+ catch (IOException | RuntimeException | Error ex)
+ {
+ Throwables.closeAndAddSuppressed(ex, bf);
+ throw ex;
+ }
+
+ return bf;
+ }
+
+ private PartitionIndex openPartitionIndex(boolean preload) throws IOException
+ {
+ try (FileHandle indexFile = partitionIndexFileBuilder().complete())
+ {
+ return PartitionIndex.load(indexFile, tableMetadataRef.getLocal().partitioner, preload);
+ }
+ catch (IOException ex)
+ {
+ logger.debug("Partition index file is corrupted: " + descriptor.fileFor(Components.PARTITION_INDEX), ex);
+ throw ex;
+ }
+ }
+
+ private FileHandle.Builder rowIndexFileBuilder()
+ {
+ assert rowIndexFileBuilder == null || rowIndexFileBuilder.file.equals(descriptor.fileFor(Components.ROW_INDEX));
+
+ if (rowIndexFileBuilder == null)
+ rowIndexFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.ROW_INDEX));
+
+ rowIndexFileBuilder.withChunkCache(chunkCache);
+ rowIndexFileBuilder.mmapped(ioOptions.indexDiskAccessMode);
+
+ return rowIndexFileBuilder;
+ }
+
+ private FileHandle.Builder partitionIndexFileBuilder()
+ {
+ assert partitionIndexFileBuilder == null || partitionIndexFileBuilder.file.equals(descriptor.fileFor(Components.PARTITION_INDEX));
+
+ if (partitionIndexFileBuilder == null)
+ partitionIndexFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.PARTITION_INDEX));
+
+ partitionIndexFileBuilder.withChunkCache(chunkCache);
+ partitionIndexFileBuilder.mmapped(ioOptions.indexDiskAccessMode);
+
+ return partitionIndexFileBuilder;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java
new file mode 100644
index 0000000..a9f862c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableScanner;
+import org.apache.cassandra.io.util.FileUtils;
+
+public class BtiTableScanner extends SSTableScanner<BtiTableReader, TrieIndexEntry, BtiTableScanner.BtiScanningIterator>
+{
+ // Full scan of the sstables
+ public static BtiTableScanner getScanner(BtiTableReader sstable)
+ {
+ return getScanner(sstable, Iterators.singletonIterator(fullRange(sstable)));
+ }
+
+ public static BtiTableScanner getScanner(BtiTableReader sstable,
+ ColumnFilter columns,
+ DataRange dataRange,
+ SSTableReadsListener listener)
+ {
+ return new BtiTableScanner(sstable, columns, dataRange, makeBounds(sstable, dataRange).iterator(), listener);
+ }
+
+ public static BtiTableScanner getScanner(BtiTableReader sstable, Collection<Range<Token>> tokenRanges)
+ {
+ return getScanner(sstable, makeBounds(sstable, tokenRanges).iterator());
+ }
+
+ public static BtiTableScanner getScanner(BtiTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
+ {
+ return new BtiTableScanner(sstable, ColumnFilter.all(sstable.metadata()), null, rangeIterator, SSTableReadsListener.NOOP_LISTENER);
+ }
+
+ private BtiTableScanner(BtiTableReader sstable,
+ ColumnFilter columns,
+ DataRange dataRange,
+ Iterator<AbstractBounds<PartitionPosition>> rangeIterator,
+ SSTableReadsListener listener)
+ {
+ super(sstable, columns, dataRange, rangeIterator, listener);
+ }
+
+ protected void doClose() throws IOException
+ {
+ FileUtils.close(dfile, iterator);
+ }
+
+ @Override
+ protected BtiScanningIterator doCreateIterator()
+ {
+ return new BtiScanningIterator();
+ }
+
+ protected class BtiScanningIterator extends SSTableScanner<BtiTableReader, TrieIndexEntry, BtiTableScanner.BtiScanningIterator>.BaseKeyScanningIterator implements Closeable
+ {
+ private PartitionIterator iterator;
+
+ @Override
+ protected boolean prepareToIterateRow() throws IOException
+ {
+ while (true)
+ {
+ if (startScan != -1)
+ bytesScanned += getCurrentPosition() - startScan;
+
+ if (iterator != null)
+ {
+ currentEntry = iterator.entry();
+ currentKey = iterator.decoratedKey();
+ if (currentEntry != null)
+ {
+ iterator.advance();
+ return true;
+ }
+ iterator.close();
+ iterator = null;
+ }
+
+ // try next range
+ if (!rangeIterator.hasNext())
+ return false;
+ iterator = sstable.coveredKeysIterator(rangeIterator.next());
+ }
+ }
+
+ @Override
+ protected UnfilteredRowIterator getRowIterator(TrieIndexEntry indexEntry, DecoratedKey key)
+ {
+ if (dataRange == null)
+ {
+ return sstable.simpleIterator(dfile, key, indexEntry.position, false);
+ }
+ else
+ {
+ ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
+ return sstable.rowIterator(dfile, key, indexEntry, filter.getSlices(BtiTableScanner.this.metadata()), columns, filter.isReversed());
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ super.close(); // can't throw
+ if (iterator != null)
+ iterator.close();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java
new file mode 100644
index 0000000..238ed7e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.SSTableRewriter;
+import org.apache.cassandra.io.sstable.format.SortedTableScrubber;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat.Components;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Throwables;
+
+public class BtiTableScrubber extends SortedTableScrubber<BtiTableReader> implements IScrubber
+{
+ private final boolean isIndex;
+ private final AbstractType<?> partitionKeyType;
+ private ScrubPartitionIterator indexIterator;
+
+ public BtiTableScrubber(ColumnFamilyStore cfs,
+ LifecycleTransaction transaction,
+ OutputHandler outputHandler,
+ IScrubber.Options options)
+ {
+ super(cfs, transaction, outputHandler, options);
+
+ boolean hasIndexFile = sstable.getComponents().contains(Components.PARTITION_INDEX);
+ this.isIndex = cfs.isIndex();
+ this.partitionKeyType = cfs.metadata.get().partitionKeyType;
+ if (!hasIndexFile)
+ {
+ // if there's any corruption in the -Data.db then partitions can't be skipped over. but it's worth a shot.
+ outputHandler.warn("Missing index component");
+ }
+
+ try
+ {
+ this.indexIterator = hasIndexFile
+ ? openIndexIterator()
+ : null;
+ }
+ catch (RuntimeException ex)
+ {
+ outputHandler.warn("Detected corruption in the index file - cannot open index iterator", ex);
+ }
+ }
+
+ private ScrubPartitionIterator openIndexIterator()
+ {
+ try
+ {
+ return sstable.scrubPartitionsIterator();
+ }
+ catch (Throwable t)
+ {
+ outputHandler.warn(t, "Index is unreadable, scrubbing will continue without index.");
+ }
+ return null;
+ }
+
+ @Override
+ protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
+ @Override
+ public void scrubInternal(SSTableRewriter writer)
+ {
+ if (indexAvailable() && indexIterator.dataPosition() != 0)
+ {
+ outputHandler.warn("First position reported by index should be 0, was " +
+ indexIterator.dataPosition() +
+ ", continuing without index.");
+ indexIterator.close();
+ indexIterator = null;
+ }
+
+ DecoratedKey prevKey = null;
+
+ while (!dataFile.isEOF())
+ {
+ if (scrubInfo.isStopRequested())
+ throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
+
+ // position in a data file where the partition starts
+ long dataStart = dataFile.getFilePointer();
+ outputHandler.debug("Reading row at %d", dataStart);
+
+ DecoratedKey key = null;
+ Throwable keyReadError = null;
+ try
+ {
+ ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile);
+ if (!isIndex)
+ partitionKeyType.validate(raw);
+ key = sstable.decorateKey(raw);
+ }
+ catch (Throwable th)
+ {
+ keyReadError = th;
+ throwIfFatal(th);
+ // check for null key below
+ }
+
+ // position of the partition in a data file, it points to the beginning of the partition key
+ long dataStartFromIndex = -1;
+ // size of the partition (including partition key)
+ long dataSizeFromIndex = -1;
+ ByteBuffer currentIndexKey = null;
+ if (indexAvailable())
+ {
+ currentIndexKey = indexIterator.key();
+ dataStartFromIndex = indexIterator.dataPosition();
+ if (!indexIterator.isExhausted())
+ {
+ try
+ {
+ indexIterator.advance();
+ if (!indexIterator.isExhausted())
+ dataSizeFromIndex = indexIterator.dataPosition() - dataStartFromIndex;
+ }
+ catch (Throwable th)
+ {
+ throwIfFatal(th);
+ outputHandler.warn(th,
+ "Failed to advance to the next index position. Index is corrupted. " +
+ "Continuing without the index. Last position read is %d.",
+ indexIterator.dataPosition());
+ indexIterator.close();
+ indexIterator = null;
+ currentIndexKey = null;
+ dataStartFromIndex = -1;
+ dataSizeFromIndex = -1;
+ }
+ }
+ }
+
+ String keyName = key == null ? "(unreadable key)" : keyString(key);
+ outputHandler.debug("partition %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSizeFromIndex));
+
+ try
+ {
+ if (key == null)
+ throw new IOError(new IOException("Unable to read partition key from data file", keyReadError));
+
+ if (currentIndexKey != null && !key.getKey().equals(currentIndexKey))
+ {
+ throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)",
+ ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
+ }
+
+ if (indexIterator != null && dataSizeFromIndex > dataFile.length())
+ throw new IOError(new IOException("Impossible partition size (greater than file length): " + dataSizeFromIndex));
+
+ if (indexIterator != null && dataStart != dataStartFromIndex)
+ outputHandler.warn("Data file partition position %d differs from index file row position %d", dataStart, dataStartFromIndex);
+
+ if (tryAppend(prevKey, key, writer))
+ prevKey = key;
+ }
+ catch (Throwable th)
+ {
+ throwIfFatal(th);
+ outputHandler.warn(th, "Error reading partition %s (stacktrace follows):", keyName);
+
+ if (currentIndexKey != null
+ && (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex))
+ {
+
+ // position where the row should start in a data file (right after the partition key)
+ long rowStartFromIndex = dataStartFromIndex + TypeSizes.SHORT_SIZE + currentIndexKey.remaining();
+ outputHandler.output("Retrying from partition index; data is %s bytes starting at %s",
+ dataSizeFromIndex, rowStartFromIndex);
+ key = sstable.decorateKey(currentIndexKey);
+ try
+ {
+ if (!isIndex)
+ partitionKeyType.validate(key.getKey());
+ dataFile.seek(rowStartFromIndex);
+
+ if (tryAppend(prevKey, key, writer))
+ prevKey = key;
+ }
+ catch (Throwable th2)
+ {
+ throwIfFatal(th2);
+ throwIfCannotContinue(key, th2);
+
+ outputHandler.warn(th2, "Retry failed too. Skipping to next partition (retry's stacktrace follows)");
+ badPartitions++;
+ if (!seekToNextPartition())
+ break;
+ }
+ }
+ else
+ {
+ throwIfCannotContinue(key, th);
+
+ badPartitions++;
+ if (indexIterator != null)
+ {
+ outputHandler.warn("Partition starting at position %d is unreadable; skipping to next", dataStart);
+ if (!seekToNextPartition())
+ break;
+ }
+ else
+ {
+ outputHandler.warn("Unrecoverable error while scrubbing %s." +
+ "Scrubbing cannot continue. The sstable will be marked for deletion. " +
+ "You can attempt manual recovery from the pre-scrub snapshot. " +
+ "You can also run nodetool repair to transfer the data from a healthy replica, if any.",
+ sstable);
+ // There's no way to resync and continue. Give up.
+ break;
+ }
+ }
+ }
+ }
+ }
+
+
+ private boolean indexAvailable()
+ {
+ return indexIterator != null && !indexIterator.isExhausted();
+ }
+
+ private boolean seekToNextPartition()
+ {
+ while (indexAvailable())
+ {
+ long nextRowPositionFromIndex = indexIterator.dataPosition();
+
+ try
+ {
+ dataFile.seek(nextRowPositionFromIndex);
+ return true;
+ }
+ catch (Throwable th)
+ {
+ throwIfFatal(th);
+ outputHandler.warn(th, "Failed to seek to next row position %d", nextRowPositionFromIndex);
+ badPartitions++;
+ }
+
+ try
+ {
+ indexIterator.advance();
+ }
+ catch (Throwable th)
+ {
+ outputHandler.warn(th, "Failed to go to the next entry in index");
+ throw Throwables.cleaned(th);
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ protected void throwIfCannotContinue(DecoratedKey key, Throwable th)
+ {
+ if (isIndex)
+ {
+ outputHandler.warn("An error occurred while scrubbing the partition with key '%s' for an index table. " +
+ "Scrubbing will abort for this table and the index will be rebuilt.", keyString(key));
+ throw new IOError(th);
+ }
+
+ super.throwIfCannotContinue(key, th);
+ }
+
+ @Override
+ public void close()
+ {
+ fileAccessLock.writeLock().lock();
+ try
+ {
+ FileUtils.closeQuietly(dataFile);
+ FileUtils.closeQuietly(indexIterator);
+ }
+ finally
+ {
+ fileAccessLock.writeLock().unlock();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java
new file mode 100644
index 0000000..6125af8
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.format.SortedTableVerifier;
+import org.apache.cassandra.utils.OutputHandler;
+
+public class BtiTableVerifier extends SortedTableVerifier<BtiTableReader> implements IVerifier
+{
+ public BtiTableVerifier(ColumnFamilyStore cfs, BtiTableReader sstable, OutputHandler outputHandler, boolean isOffline, Options options)
+ {
+ super(cfs, sstable, outputHandler, isOffline, options);
+ }
+
+ protected void verifyPartition(DecoratedKey key, UnfilteredRowIterator iterator)
+ {
+ // The trie writers abort if supplied with badly ordered or duplicate row keys. Verification is not necessary.
+ // no-op, just open and close partition.
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
new file mode 100644
index 0000000..ea343fd
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.DataComponent;
+import org.apache.cassandra.io.sstable.format.IndexComponent;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader.OpenReason;
+import org.apache.cassandra.io.sstable.format.SortedTableWriter;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat.Components;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.MmappedRegionsCache;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+import static org.apache.cassandra.io.util.FileHandle.Builder.NO_LENGTH_OVERRIDE;
+
+/**
+ * Writes SSTables in BTI format (see {@link BtiFormat}), which can be read by {@link BtiTableReader}.
+ */
+@VisibleForTesting
+public class BtiTableWriter extends SortedTableWriter<BtiFormatPartitionWriter>
+{
+ private static final Logger logger = LoggerFactory.getLogger(BtiTableWriter.class);
+
+ private final BtiFormatPartitionWriter partitionWriter;
+ private final IndexWriter iwriter;
+
+ public BtiTableWriter(Builder builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner)
+ {
+ super(builder, lifecycleNewTracker, owner);
+ this.iwriter = builder.getIndexWriter();
+ this.partitionWriter = builder.getPartitionWriter();
+ }
+
+ @Override
+ public void mark()
+ {
+ super.mark();
+ iwriter.mark();
+ }
+
+ @Override
+ public void resetAndTruncate()
+ {
+ super.resetAndTruncate();
+ iwriter.resetAndTruncate();
+ }
+
+ @Override
+ protected TrieIndexEntry createRowIndexEntry(DecoratedKey key, DeletionTime partitionLevelDeletion, long finishResult) throws IOException
+ {
+ TrieIndexEntry entry = TrieIndexEntry.create(partitionWriter.getInitialPosition(),
+ finishResult,
+ partitionLevelDeletion,
+ partitionWriter.getRowIndexBlockCount());
+ iwriter.append(key, entry);
+ return entry;
+ }
+
+ @SuppressWarnings({"resource", "RedundantSuppression"})
+ private BtiTableReader openInternal(OpenReason openReason, boolean isFinal, Supplier<PartitionIndex> partitionIndexSupplier)
+ {
+ IFilter filter = null;
+ FileHandle dataFile = null;
+ PartitionIndex partitionIndex = null;
+ FileHandle rowIndexFile = null;
+
+ BtiTableReader.Builder builder = unbuildTo(new BtiTableReader.Builder(descriptor), true).setMaxDataAge(maxDataAge)
+ .setSerializationHeader(header)
+ .setOpenReason(openReason);
+
+ try
+ {
+ builder.setStatsMetadata(statsMetadata());
+
+ partitionIndex = partitionIndexSupplier.get();
+ rowIndexFile = iwriter.rowIndexFHBuilder.complete();
+ dataFile = openDataFile(isFinal ? NO_LENGTH_OVERRIDE : dataWriter.getLastFlushOffset(), builder.getStatsMetadata());
+ filter = iwriter.getFilterCopy();
+
+ return builder.setPartitionIndex(partitionIndex)
+ .setFirst(partitionIndex.firstKey())
+ .setLast(partitionIndex.lastKey())
+ .setRowIndexFile(rowIndexFile)
+ .setDataFile(dataFile)
+ .setFilter(filter)
+ .build(owner().orElse(null), true, true);
+ }
+ catch (RuntimeException | Error ex)
+ {
+ JVMStabilityInspector.inspectThrowable(ex);
+ Throwables.closeNonNullAndAddSuppressed(ex, filter, dataFile, rowIndexFile, partitionIndex);
+ throw ex;
+ }
+ }
+
+ @Override
+ public void openEarly(Consumer<SSTableReader> callWhenReady)
+ {
+ long dataLength = dataWriter.position();
+ iwriter.buildPartial(dataLength, partitionIndex ->
+ {
+ iwriter.rowIndexFHBuilder.withLengthOverride(iwriter.rowIndexWriter.getLastFlushOffset());
+ BtiTableReader reader = openInternal(OpenReason.EARLY, false, () -> partitionIndex);
+ callWhenReady.accept(reader);
+ });
+ }
+
+ @Override
+ public SSTableReader openFinalEarly()
+ {
+ // we must ensure the data is completely flushed to disk
+ iwriter.complete(); // This will be called by completedPartitionIndex() below too, but we want it done now to
+ // ensure outstanding openEarly actions are not triggered.
+ dataWriter.sync();
+ iwriter.rowIndexWriter.sync();
+ // Note: Nothing must be written to any of the files after this point, as the chunk cache could pick up and
+ // retain a partially-written page.
+
+ return openFinal(OpenReason.EARLY);
+ }
+
+ @Override
+ @SuppressWarnings({"resource", "RedundantSuppression"})
+ protected SSTableReader openFinal(OpenReason openReason)
+ {
+
+ if (maxDataAge < 0)
+ maxDataAge = Clock.Global.currentTimeMillis();
+
+ return openInternal(openReason, true, iwriter::completedPartitionIndex);
+ }
+
+ @Override
+ protected TransactionalProxy txnProxy()
+ {
+ return new TransactionalProxy(() -> FBUtilities.immutableListWithFilteredNulls(iwriter, dataWriter));
+ }
+
+ private class TransactionalProxy extends SortedTableWriter<BtiFormatPartitionWriter>.TransactionalProxy
+ {
+ public TransactionalProxy(Supplier<ImmutableList<Transactional>> transactionals)
+ {
+ super(transactionals);
+ }
+
+ @Override
+ protected Throwable doPostCleanup(Throwable accumulate)
+ {
+ accumulate = Throwables.close(accumulate, partitionWriter);
+ accumulate = super.doPostCleanup(accumulate);
+ return accumulate;
+ }
+ }
+
+ /**
+ * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
+ */
+ static class IndexWriter extends SortedTableWriter.AbstractIndexWriter
+ {
+ final SequentialWriter rowIndexWriter;
+ private final FileHandle.Builder rowIndexFHBuilder;
+ private final SequentialWriter partitionIndexWriter;
+ private final FileHandle.Builder partitionIndexFHBuilder;
+ private final PartitionIndexBuilder partitionIndex;
+ boolean partitionIndexCompleted = false;
+ private DataPosition riMark;
+ private DataPosition piMark;
+
+ IndexWriter(Builder b)
+ {
+ super(b);
+ rowIndexWriter = new SequentialWriter(descriptor.fileFor(Components.ROW_INDEX), b.getIOOptions().writerOptions);
+ rowIndexFHBuilder = IndexComponent.fileBuilder(Components.ROW_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache());
+ partitionIndexWriter = new SequentialWriter(descriptor.fileFor(Components.PARTITION_INDEX), b.getIOOptions().writerOptions);
+ partitionIndexFHBuilder = IndexComponent.fileBuilder(Components.PARTITION_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache());
+ partitionIndex = new PartitionIndexBuilder(partitionIndexWriter, partitionIndexFHBuilder);
+ // register listeners to be alerted when the data files are flushed
+ partitionIndexWriter.setPostFlushListener(() -> partitionIndex.markPartitionIndexSynced(partitionIndexWriter.getLastFlushOffset()));
+ rowIndexWriter.setPostFlushListener(() -> partitionIndex.markRowIndexSynced(rowIndexWriter.getLastFlushOffset()));
+ @SuppressWarnings({"resource", "RedundantSuppression"})
+ SequentialWriter dataWriter = b.getDataWriter();
+ dataWriter.setPostFlushListener(() -> partitionIndex.markDataSynced(dataWriter.getLastFlushOffset()));
+ }
+
+ public long append(DecoratedKey key, AbstractRowIndexEntry indexEntry) throws IOException
+ {
+ bf.add(key);
+ long position;
+ if (indexEntry.isIndexed())
+ {
+ long indexStart = rowIndexWriter.position();
+ try
+ {
+ ByteBufferUtil.writeWithShortLength(key.getKey(), rowIndexWriter);
+ ((TrieIndexEntry) indexEntry).serialize(rowIndexWriter, rowIndexWriter.position());
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, rowIndexWriter.getFile());
+ }
+
+ if (logger.isTraceEnabled())
+ logger.trace("wrote index entry: {} at {}", indexEntry, indexStart);
+ position = indexStart;
+ }
+ else
+ {
+ // Write data position directly in trie.
+ position = ~indexEntry.position;
+ }
+ partitionIndex.addEntry(key, position);
+ return position;
+ }
+
+ public boolean buildPartial(long dataPosition, Consumer<PartitionIndex> callWhenReady)
+ {
+ return partitionIndex.buildPartial(callWhenReady, rowIndexWriter.position(), dataPosition);
+ }
+
+ public void mark()
+ {
+ riMark = rowIndexWriter.mark();
+ piMark = partitionIndexWriter.mark();
+ }
+
+ public void resetAndTruncate()
+ {
+ // we can't un-set the bloom filter addition, but extra keys in there are harmless.
+ // we can't reset dbuilder either, but that is the last thing called in after append, so
+ // we assume that if that worked then we won't be trying to reset.
+ rowIndexWriter.resetAndTruncate(riMark);
+ partitionIndexWriter.resetAndTruncate(piMark);
+ }
+
+ protected void doPrepare()
+ {
+ flushBf();
+
+ // truncate index file
+ rowIndexWriter.prepareToCommit();
+ rowIndexFHBuilder.withLengthOverride(rowIndexWriter.getLastFlushOffset());
+
+ complete();
+ }
+
+ void complete() throws FSWriteError
+ {
+ if (partitionIndexCompleted)
+ return;
+
+ try
+ {
+ partitionIndex.complete();
+ partitionIndexCompleted = true;
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, partitionIndexWriter.getFile());
+ }
+ }
+
+ PartitionIndex completedPartitionIndex()
+ {
+ complete();
+ rowIndexFHBuilder.withLengthOverride(0);
+ partitionIndexFHBuilder.withLengthOverride(0);
+ try
+ {
+ return PartitionIndex.load(partitionIndexFHBuilder, metadata.getLocal().partitioner, false);
+ }
+ catch (IOException e)
+ {
+ throw new FSReadError(e, partitionIndexWriter.getFile());
+ }
+ }
+
+ protected Throwable doCommit(Throwable accumulate)
+ {
+ return rowIndexWriter.commit(accumulate);
+ }
+
+ protected Throwable doAbort(Throwable accumulate)
+ {
+ return rowIndexWriter.abort(accumulate);
+ }
+
+ @Override
+ protected Throwable doPostCleanup(Throwable accumulate)
+ {
+ return Throwables.close(accumulate, bf, partitionIndex, rowIndexWriter, partitionIndexWriter);
+ }
+ }
+
+ public static class Builder extends SortedTableWriter.Builder<BtiFormatPartitionWriter, BtiTableWriter, Builder>
+ {
+ private SequentialWriter dataWriter;
+ private BtiFormatPartitionWriter partitionWriter;
+ private IndexWriter indexWriter;
+ private MmappedRegionsCache mmappedRegionsCache;
+
+ public Builder(Descriptor descriptor)
+ {
+ super(descriptor);
+ }
+
+ // The following getters for the resources opened by buildInternal method can be only used during the lifetime of
+ // that method - that is, during the construction of the sstable.
+
+ @Override
+ public MmappedRegionsCache getMmappedRegionsCache()
+ {
+ return ensuringInBuildInternalContext(mmappedRegionsCache);
+ }
+
+ @Override
+ public SequentialWriter getDataWriter()
+ {
+ return ensuringInBuildInternalContext(dataWriter);
+ }
+
+ @Override
+ public BtiFormatPartitionWriter getPartitionWriter()
+ {
+ return ensuringInBuildInternalContext(partitionWriter);
+ }
+
+ public IndexWriter getIndexWriter()
+ {
+ return ensuringInBuildInternalContext(indexWriter);
+ }
+
+ private <T> T ensuringInBuildInternalContext(T value)
+ {
+ Preconditions.checkState(value != null, "This getter can be used only during the lifetime of the sstable constructor. Do not use it directly.");
+ return value;
+ }
+
+ @Override
+ public Builder addDefaultComponents()
+ {
+ super.addDefaultComponents();
+
+ addComponents(ImmutableSet.of(Components.PARTITION_INDEX, Components.ROW_INDEX));
+
+ return this;
+ }
+
+ @Override
+ protected BtiTableWriter buildInternal(LifecycleNewTracker lifecycleNewTracker, Owner owner)
+ {
+ try
+ {
+ mmappedRegionsCache = new MmappedRegionsCache();
+ dataWriter = DataComponent.buildWriter(descriptor,
+ getTableMetadataRef().getLocal(),
+ getIOOptions().writerOptions,
+ getMetadataCollector(),
+ lifecycleNewTracker.opType(),
+ getIOOptions().flushCompression);
+
+ indexWriter = new IndexWriter(this);
+ partitionWriter = new BtiFormatPartitionWriter(getSerializationHeader(),
+ getTableMetadataRef().getLocal().comparator,
+ dataWriter,
+ indexWriter.rowIndexWriter,
+ descriptor.version);
+
+
+ return new BtiTableWriter(this, lifecycleNewTracker, owner);
+ }
+ catch (RuntimeException | Error ex)
+ {
+ Throwables.closeAndAddSuppressed(ex, partitionWriter, indexWriter, dataWriter, mmappedRegionsCache);
+ throw ex;
+ }
+ finally
+ {
+ partitionWriter = null;
+ indexWriter = null;
+ dataWriter = null;
+ mmappedRegionsCache = null;
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
new file mode 100644
index 0000000..71a3434
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
@@ -0,0 +1,454 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.ByteBuffer;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.tries.SerializationNode;
+import org.apache.cassandra.io.tries.TrieNode;
+import org.apache.cassandra.io.tries.TrieSerializer;
+import org.apache.cassandra.io.tries.ValueIterator;
+import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.SizedInts;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.SharedCloseable;
+
+/**
+ * This class holds the partition index as an on-disk trie mapping unique prefixes of decorated keys to:
+ * <ul>
+ * <li>data file position if the partition is small enough to not need an index
+ * <li>row index file position if the partition has a row index
+ * </ul>plus<ul>
+ * <li>the last 8 bits of the key's filter hash which is used to filter out mismatched keys without reading the key
+ * </ul>
+ * To avoid having to create an object to carry the result, the two are distinguished by sign. Direct-to-dfile entries
+ * are recorded as ~position (~ instead of - to differentiate 0 in ifile from 0 in dfile).
+ * <p>
+ * In either case the contents of the file at this position start with a serialization of the key which can be used
+ * to verify the correct key is found.
+ * <p>
+ * The indexes are created by {@link PartitionIndexBuilder}. To read the index one must obtain a thread-unsafe
+ * {@link Reader} or {@link IndexPosIterator}.
+ */
+@VisibleForTesting
+public class PartitionIndex implements SharedCloseable
+{
+ private static final Logger logger = LoggerFactory.getLogger(PartitionIndex.class);
+
+ private final FileHandle fh;
+ private final long keyCount;
+ private final DecoratedKey first;
+ private final DecoratedKey last;
+ private final long root;
+
+ public static final long NOT_FOUND = Long.MIN_VALUE;
+ public static final int FOOTER_LENGTH = 3 * 8;
+ private static final int FLAG_HAS_HASH_BYTE = 8;
+
+ @VisibleForTesting
+ public PartitionIndex(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last)
+ {
+ this.keyCount = keyCount;
+ this.fh = fh.sharedCopy();
+ this.first = first;
+ this.last = last;
+ this.root = trieRoot;
+ }
+
+ private PartitionIndex(PartitionIndex src)
+ {
+ this(src.fh, src.root, src.keyCount, src.first, src.last);
+ }
+
+ static class Payload
+ {
+ final long position;
+ final short hashBits;
+
+ public Payload(long position, short hashBits)
+ {
+ this.position = position;
+ assert this.position != NOT_FOUND : "Partition position " + NOT_FOUND + " is not valid.";
+ this.hashBits = hashBits;
+ }
+ }
+
+ static final PartitionIndexSerializer TRIE_SERIALIZER = new PartitionIndexSerializer();
+
+ private static class PartitionIndexSerializer implements TrieSerializer<Payload, DataOutputPlus>
+ {
+ public int sizeofNode(SerializationNode<Payload> node, long nodePosition)
+ {
+ return TrieNode.typeFor(node, nodePosition).sizeofNode(node) +
+ (node.payload() != null ? 1 + SizedInts.nonZeroSize(node.payload().position) : 0);
+ }
+
+ @Override
+ public void write(DataOutputPlus dest, SerializationNode<Payload> node, long nodePosition) throws IOException
+ {
+ write(dest, TrieNode.typeFor(node, nodePosition), node, nodePosition);
+ }
+
+ private void write(DataOutputPlus dest, TrieNode type, SerializationNode<Payload> node, long nodePosition) throws IOException
+ {
+ Payload payload = node.payload();
+ if (payload != null)
+ {
+ int size = SizedInts.nonZeroSize(payload.position);
+ // The reader supports payloads both with (payloadBits between 8 and 15) and without (payloadBits
+ // between 1 and 7) hash. To not introduce undue configuration complexity, we always write a hash.
+ int payloadBits = FLAG_HAS_HASH_BYTE + (size - 1);
+ type.serialize(dest, node, payloadBits, nodePosition);
+ dest.writeByte(payload.hashBits);
+ SizedInts.write(dest, payload.position, size);
+ }
+ else
+ type.serialize(dest, node, 0, nodePosition);
+ }
+ }
+
+ public long size()
+ {
+ return keyCount;
+ }
+
+ public DecoratedKey firstKey()
+ {
+ return first;
+ }
+
+ public DecoratedKey lastKey()
+ {
+ return last;
+ }
+
+ @Override
+ public PartitionIndex sharedCopy()
+ {
+ return new PartitionIndex(this);
+ }
+
+ @Override
+ public void addTo(Ref.IdentityCollection identities)
+ {
+ fh.addTo(identities);
+ }
+
+ public static PartitionIndex load(FileHandle.Builder fhBuilder,
+ IPartitioner partitioner,
+ boolean preload) throws IOException
+ {
+ try (FileHandle fh = fhBuilder.complete())
+ {
+ return load(fh, partitioner, preload);
+ }
+ }
+
+ public static Pair<DecoratedKey, DecoratedKey> readFirstAndLastKey(File file, IPartitioner partitioner) throws IOException
+ {
+ try (PartitionIndex index = load(new FileHandle.Builder(file), partitioner, false))
+ {
+ return Pair.create(index.firstKey(), index.lastKey());
+ }
+ }
+
+ public static PartitionIndex load(FileHandle fh, IPartitioner partitioner, boolean preload) throws IOException
+ {
+ try (FileDataInput rdr = fh.createReader(fh.dataLength() - FOOTER_LENGTH))
+ {
+ long firstPos = rdr.readLong();
+ long keyCount = rdr.readLong();
+ long root = rdr.readLong();
+ rdr.seek(firstPos);
+ DecoratedKey first = partitioner != null ? partitioner.decorateKey(ByteBufferUtil.readWithShortLength(rdr)) : null;
+ DecoratedKey last = partitioner != null ? partitioner.decorateKey(ByteBufferUtil.readWithShortLength(rdr)) : null;
+ if (preload)
+ {
+ int csum = 0;
+ // force a read of all the pages of the index
+ for (long pos = 0; pos < fh.dataLength(); pos += PageAware.PAGE_SIZE)
+ {
+ rdr.seek(pos);
+ csum += rdr.readByte();
+ }
+ logger.trace("Checksum {}", csum); // Note: trace is required so that reads aren't optimized away.
+ }
+
+ return new PartitionIndex(fh, root, keyCount, first, last);
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ fh.close();
+ }
+
+ @Override
+ public Throwable close(Throwable accumulate)
+ {
+ return fh.close(accumulate);
+ }
+
+ public Reader openReader()
+ {
+ return new Reader(this);
+ }
+
+ protected IndexPosIterator allKeysIterator()
+ {
+ return new IndexPosIterator(this);
+ }
+
+ protected Rebufferer instantiateRebufferer()
+ {
+ return fh.instantiateRebufferer(null);
+ }
+
+
+ /**
+ * @return the file handle to the file on disk. This is needed for locking the index in RAM.
+ */
+ FileHandle getFileHandle()
+ {
+ return fh;
+ }
+
+ private static long getIndexPos(ByteBuffer contents, int payloadPos, int bytes)
+ {
+ if (bytes >= FLAG_HAS_HASH_BYTE)
+ {
+ ++payloadPos;
+ bytes -= FLAG_HAS_HASH_BYTE - 1;
+ }
+ if (bytes == 0)
+ return NOT_FOUND;
+ return SizedInts.read(contents, payloadPos, bytes);
+ }
+
+ public interface Acceptor<ArgType, ResultType>
+ {
+ ResultType accept(long position, boolean assumeNoMatch, ArgType v) throws IOException;
+ }
+
+ /**
+ * Provides methods to read the partition index trie.
+ * Thread-unsafe, uses class members to store lookup state.
+ */
+ public static class Reader extends Walker<Reader>
+ {
+ protected Reader(PartitionIndex index)
+ {
+ super(index.instantiateRebufferer(), index.root);
+ }
+
+ /**
+ * Finds a candidate for an exact key search. Returns an ifile (if positive) or dfile (if negative, using ~)
+ * position. The position returned has a low chance of being a different entry, but only if the sought key
+ * is not present in the file.
+ */
+ public long exactCandidate(DecoratedKey key)
+ {
+ // A hit must be a prefix of the byte-comparable representation of the key.
+ int b = follow(key);
+ // If the prefix ended in a node with children it is only acceptable if it is a full match.
+ if (b != ByteSource.END_OF_STREAM && hasChildren())
+ return NOT_FOUND;
+ if (!checkHashBits(key.filterHashLowerBits()))
+ return NOT_FOUND;
+ return getCurrentIndexPos();
+ }
+
+ final boolean checkHashBits(short hashBits)
+ {
+ int bytes = payloadFlags();
+ if (bytes < FLAG_HAS_HASH_BYTE)
+ return bytes > 0;
+ return (buf.get(payloadPosition()) == (byte) hashBits);
+ }
+
+ public <ResultType> ResultType ceiling(PartitionPosition key, Acceptor<PartitionPosition, ResultType> acceptor) throws IOException
+ {
+ // Look for a prefix of the key. If there is one, the key it stands for could be less, equal, or greater
+ // than the required value so try that first.
+ int b = followWithGreater(key);
+ // If the prefix ended in a node with children it is only acceptable if it is a full match.
+ if (!hasChildren() || b == ByteSource.END_OF_STREAM)
+ {
+ long indexPos = getCurrentIndexPos();
+ if (indexPos != NOT_FOUND)
+ {
+ ResultType res = acceptor.accept(indexPos, false, key);
+ if (res != null)
+ return res;
+ }
+ }
+ // If that was not found, the closest greater value can be used instead, and we know that
+ // it stands for a key greater than the argument.
+ if (greaterBranch == NONE)
+ return null;
+ goMin(greaterBranch);
+ long indexPos = getCurrentIndexPos();
+ if (indexPos == NOT_FOUND)
+ return null;
+
+ return acceptor.accept(indexPos, true, key);
+ }
+
+
+ public <ResultType> ResultType floor(PartitionPosition key, Acceptor<PartitionPosition, ResultType> acceptor) throws IOException
+ {
+ // Check for a prefix and find closest smaller branch.
+ Long indexPos = prefixAndNeighbours(key, Reader::getSpecificIndexPos);
+
+ if (indexPos != null && indexPos != NOT_FOUND)
+ {
+ ResultType res = acceptor.accept(indexPos, false, key);
+ if (res != null)
+ return res;
+ }
+
+ // Otherwise return the IndexInfo for the closest entry of the smaller branch (which is the max of lesserBranch).
+ // Note (see prefixAndNeighbours): since we accept prefix matches above, at this point there cannot be another
+ // prefix match that is closer than max(lesserBranch).
+ if (lesserBranch == NONE)
+ return null;
+ goMax(lesserBranch);
+ indexPos = getCurrentIndexPos();
+ if (indexPos == NOT_FOUND)
+ return null;
+
+ return acceptor.accept(indexPos, true, key);
+ }
+
+
+ public Long getSpecificIndexPos(int pos, int bits)
+ {
+ return getIndexPos(buf, pos, bits);
+ }
+
+ public long getCurrentIndexPos()
+ {
+ return getIndexPos(buf, payloadPosition(), payloadFlags());
+ }
+
+ public long getLastIndexPosition()
+ {
+ goMax(root);
+ return getCurrentIndexPos();
+ }
+
+ /**
+ * To be used only in analysis.
+ */
+ @SuppressWarnings("unused")
+ protected int payloadSize()
+ {
+ int bytes = payloadFlags();
+ return bytes > 7 ? bytes - 6 : bytes;
+ }
+ }
+
+ /**
+ * Iterator of index positions covered between two keys. Since we store prefixes only, the first and last returned
+ * values can be outside the span (and inclusiveness is not given as we cannot verify it).
+ */
+ public static class IndexPosIterator extends ValueIterator<IndexPosIterator>
+ {
+ static final long INVALID = -1;
+ long pos = INVALID;
+
+ /**
+ * @param index PartitionIndex to use for the iteration.
+ * <p>
+ * Note: For performance reasons this class does not keep a reference of the index. Caller must ensure a
+ * reference is held for the lifetime of this object.
+ */
+ public IndexPosIterator(PartitionIndex index)
+ {
+ super(index.instantiateRebufferer(), index.root);
+ }
+
+ IndexPosIterator(PartitionIndex index, PartitionPosition start, PartitionPosition end)
+ {
+ super(index.instantiateRebufferer(), index.root, start, end, true);
+ }
+
+ /**
+ * Returns the position in the row index or data file.
+ */
+ protected long nextIndexPos()
+ {
+ // without missing positions, we save and reuse the unreturned position.
+ if (pos == INVALID)
+ {
+ pos = nextPayloadedNode();
+ if (pos == INVALID)
+ return NOT_FOUND;
+ }
+
+ go(pos);
+
+ pos = INVALID; // make sure next time we call nextPayloadedNode() again
+ return getIndexPos(buf, payloadPosition(), payloadFlags()); // this should not throw
+ }
+ }
+
+ /**
+ * debug/test code
+ */
+ @VisibleForTesting
+ public void dumpTrie(String fileName)
+ {
+ try(PrintStream ps = new PrintStream(fileName))
+ {
+ dumpTrie(ps);
+ }
+ catch (Throwable t)
+ {
+ logger.warn("Failed to dump trie to {} due to exception {}", fileName, t);
+ }
+ }
+
+ private void dumpTrie(PrintStream out)
+ {
+ try (Reader rdr = openReader())
+ {
+ rdr.dumpTrie(out, (buf, ppos, pbits) -> Long.toString(getIndexPos(buf, ppos, pbits)));
+ }
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java
new file mode 100644
index 0000000..9af43cb
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.io.tries.IncrementalTrieWriter;
+import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Partition index builder: stores index or data positions in an incrementally built, page aware on-disk trie.
+ * <p>
+ * The files created by this builder are read by {@link PartitionIndex}.
+ */
+class PartitionIndexBuilder implements AutoCloseable
+{
+ private final SequentialWriter writer;
+ private final IncrementalTrieWriter<PartitionIndex.Payload> trieWriter;
+ private final FileHandle.Builder fhBuilder;
+
+ // the last synced data file position
+ private long dataSyncPosition;
+ // the last synced row index file position
+ private long rowIndexSyncPosition;
+ // the last synced partition index file position
+ private long partitionIndexSyncPosition;
+
+ // Partial index can only be used after all three files have been synced to the required positions.
+ private long partialIndexDataEnd;
+ private long partialIndexRowEnd;
+ private long partialIndexPartitionEnd;
+ private IncrementalTrieWriter.PartialTail partialIndexTail;
+ private Consumer<PartitionIndex> partialIndexConsumer;
+ private DecoratedKey partialIndexLastKey;
+
+ private int lastDiffPoint;
+ private DecoratedKey firstKey;
+ private DecoratedKey lastKey;
+ private DecoratedKey lastWrittenKey;
+ private PartitionIndex.Payload lastPayload;
+
+ public PartitionIndexBuilder(SequentialWriter writer, FileHandle.Builder fhBuilder)
+ {
+ this.writer = writer;
+ this.trieWriter = IncrementalTrieWriter.open(PartitionIndex.TRIE_SERIALIZER, writer);
+ this.fhBuilder = fhBuilder;
+ }
+
+ /*
+ * Called when partition index has been flushed to the given position.
+ * If this makes all required positions for a partial view flushed, this will call the partialIndexConsumer.
+ */
+ public void markPartitionIndexSynced(long upToPosition)
+ {
+ partitionIndexSyncPosition = upToPosition;
+ refreshReadableBoundary();
+ }
+
+ /*
+ * Called when row index has been flushed to the given position.
+ * If this makes all required positions for a partial view flushed, this will call the partialIndexConsumer.
+ */
+ public void markRowIndexSynced(long upToPosition)
+ {
+ rowIndexSyncPosition = upToPosition;
+ refreshReadableBoundary();
+ }
+
+ /*
+ * Called when data file has been flushed to the given position.
+ * If this makes all required positions for a partial view flushed, this will call the partialIndexConsumer.
+ */
+ public void markDataSynced(long upToPosition)
+ {
+ dataSyncPosition = upToPosition;
+ refreshReadableBoundary();
+ }
+
+ private void refreshReadableBoundary()
+ {
+ if (partialIndexConsumer == null)
+ return;
+ if (dataSyncPosition < partialIndexDataEnd)
+ return;
+ if (rowIndexSyncPosition < partialIndexRowEnd)
+ return;
+ if (partitionIndexSyncPosition < partialIndexPartitionEnd)
+ return;
+
+ try (FileHandle fh = fhBuilder.withLengthOverride(writer.getLastFlushOffset()).complete())
+ {
+ @SuppressWarnings({ "resource", "RedundantSuppression" })
+ PartitionIndex pi = new PartitionIndexEarly(fh, partialIndexTail.root(), partialIndexTail.count(), firstKey, partialIndexLastKey, partialIndexTail.cutoff(), partialIndexTail.tail());
+ partialIndexConsumer.accept(pi);
+ partialIndexConsumer = null;
+ }
+ finally
+ {
+ fhBuilder.withLengthOverride(-1);
+ }
+
+ }
+
+ /**
+ * @param decoratedKey the key for this record
+ * @param position the position to write with the record:
+ * - positive if position points to an index entry in the index file
+ * - negative if ~position points directly to the key in the data file
+ */
+ public void addEntry(DecoratedKey decoratedKey, long position) throws IOException
+ {
+ if (lastKey == null)
+ {
+ firstKey = decoratedKey;
+ lastDiffPoint = 0;
+ }
+ else
+ {
+ int diffPoint = ByteComparable.diffPoint(lastKey, decoratedKey, Walker.BYTE_COMPARABLE_VERSION);
+ ByteComparable prevPrefix = ByteComparable.cut(lastKey, Math.max(diffPoint, lastDiffPoint));
+ trieWriter.add(prevPrefix, lastPayload);
+ lastWrittenKey = lastKey;
+ lastDiffPoint = diffPoint;
+ }
+ lastKey = decoratedKey;
+ lastPayload = new PartitionIndex.Payload(position, decoratedKey.filterHashLowerBits());
+ }
+
+ public long complete() throws IOException
+ {
+ // Do not trigger pending partial builds.
+ partialIndexConsumer = null;
+
+ if (lastKey != lastWrittenKey)
+ {
+ ByteComparable prevPrefix = ByteComparable.cut(lastKey, lastDiffPoint);
+ trieWriter.add(prevPrefix, lastPayload);
+ }
+
+ long root = trieWriter.complete();
+ long count = trieWriter.count();
+ long firstKeyPos = writer.position();
+ if (firstKey != null)
+ {
+ ByteBufferUtil.writeWithShortLength(firstKey.getKey(), writer);
+ ByteBufferUtil.writeWithShortLength(lastKey.getKey(), writer);
+ }
+ else
+ {
+ assert lastKey == null;
+ writer.writeShort(0);
+ writer.writeShort(0);
+ }
+
+ writer.writeLong(firstKeyPos);
+ writer.writeLong(count);
+ writer.writeLong(root);
+
+ writer.sync();
+ fhBuilder.withLengthOverride(writer.getLastFlushOffset());
+
+ return root;
+ }
+
+ /**
+ * Builds a PartitionIndex representing the records written until this point without interrupting writes. Because
+ * data in buffered writers does not get immediately flushed to the file system, and we do not want to force flushing
+ * of the relevant files (which e.g. could cause a problem for compressed data files), this call cannot return
+ * immediately. Instead, it will take an index snapshot but wait with making it active (by calling the provided
+ * callback) until it registers that all relevant files (data, row index and partition index) have been flushed at
+ * least as far as the required positions.
+ *
+ * @param callWhenReady callback that is given the prepared partial index when all relevant data has been flushed
+ * @param rowIndexEnd the position in the row index file we need to be able to read to (exclusive) to read all
+ * records written so far
+ * @param dataEnd the position in the data file we need to be able to read to (exclusive) to read all records
+ * written so far
+ * @return true if the request was accepted, false if there's no point to do this at this time (e.g. another
+ * partial representation is prepared but still isn't usable).
+ */
+ public boolean buildPartial(Consumer<PartitionIndex> callWhenReady, long rowIndexEnd, long dataEnd)
+ {
+ // If we haven't advanced since the last time we prepared, there's nothing to do.
+ if (lastWrittenKey == partialIndexLastKey)
+ return false;
+
+ // Don't waste time if an index was already prepared but hasn't reached usability yet.
+ if (partialIndexConsumer != null)
+ return false;
+
+ try
+ {
+ partialIndexTail = trieWriter.makePartialRoot();
+ partialIndexDataEnd = dataEnd;
+ partialIndexRowEnd = rowIndexEnd;
+ partialIndexPartitionEnd = writer.position();
+ partialIndexLastKey = lastWrittenKey;
+ partialIndexConsumer = callWhenReady;
+ return true;
+ }
+ catch (IOException e)
+ {
+ // As writes happen on in-memory buffers, failure here is not expected.
+ throw new AssertionError(e);
+ }
+ }
+
+ // close the builder and release any associated memory
+ public void close()
+ {
+ trieWriter.close();
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java
new file mode 100644
index 0000000..3486e87
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.TailOverridingRebufferer;
+
+/**
+ * Early-opened partition index. Part of the data is already written to file, but some nodes, including the ones in the
+ * chain leading to the last entry in the index, are in the supplied byte buffer and are attached as a tail at the given
+ * position to form a view over the partially-written data.
+ */
+class PartitionIndexEarly extends PartitionIndex
+{
+ final long cutoff;
+ final ByteBuffer tail;
+
+ public PartitionIndexEarly(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last,
+ long cutoff, ByteBuffer tail)
+ {
+ super(fh, trieRoot, keyCount, first, last);
+ this.cutoff = cutoff;
+ this.tail = tail;
+ }
+
+ @Override
+ protected Rebufferer instantiateRebufferer()
+ {
+ return new TailOverridingRebufferer(super.instantiateRebufferer(), cutoff, tail);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIterator.java
new file mode 100644
index 0000000..9bd8ff9
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIterator.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.utils.FBUtilities.immutableListWithFilteredNulls;
+
+/**
+ * Partition iterator for the BTI format.
+ * <p>
+ * As the index stores prefixes of keys, the slice returned by the underlying {@link PartitionIndex.IndexPosIterator}
+ * may start and end with entries that have the same prefix as the provided bounds, but be in the wrong relationship
+ * with them. To filter these out, we start by checking the first item during initialization, and by working one item
+ * ahead, so that we can recognize the end of the slice and check the last item before we return it.
+ */
+class PartitionIterator extends PartitionIndex.IndexPosIterator implements KeyReader
+{
+ private final PartitionIndex partitionIndex;
+ private final IPartitioner partitioner;
+ private final PartitionPosition limit;
+ private final int exclusiveLimit;
+ private final FileHandle dataFile;
+ private final FileHandle rowIndexFile;
+
+ private FileDataInput dataInput;
+ private FileDataInput indexInput;
+
+ private DecoratedKey currentKey;
+ private TrieIndexEntry currentEntry;
+ private DecoratedKey nextKey;
+ private TrieIndexEntry nextEntry;
+
+ @SuppressWarnings({ "resource", "RedundantSuppression" })
+ static PartitionIterator create(PartitionIndex partitionIndex, IPartitioner partitioner, FileHandle rowIndexFile, FileHandle dataFile,
+ PartitionPosition left, int inclusiveLeft, PartitionPosition right, int exclusiveRight) throws IOException
+ {
+ PartitionIterator partitionIterator = null;
+ PartitionIndex partitionIndexCopy = null;
+ FileHandle dataFileCopy = null;
+ FileHandle rowIndexFileCopy = null;
+
+ try
+ {
+ partitionIndexCopy = partitionIndex.sharedCopy();
+ dataFileCopy = dataFile.sharedCopy();
+ rowIndexFileCopy = rowIndexFile.sharedCopy();
+
+ partitionIterator = new PartitionIterator(partitionIndexCopy, partitioner, rowIndexFileCopy, dataFileCopy, left, right, exclusiveRight);
+
+ partitionIterator.readNext();
+ // Because the index stores prefixes, the first value can be in any relationship with the left bound.
+ if (partitionIterator.nextKey != null && !(partitionIterator.nextKey.compareTo(left) > inclusiveLeft))
+ {
+ partitionIterator.readNext();
+ }
+ partitionIterator.advance();
+ return partitionIterator;
+ }
+ catch (IOException | RuntimeException ex)
+ {
+ if (partitionIterator != null)
+ {
+ partitionIterator.close();
+ }
+ else
+ {
+ Throwables.closeNonNullAndAddSuppressed(ex, rowIndexFileCopy, dataFileCopy, partitionIndexCopy);
+ }
+ throw ex;
+ }
+ }
+
+ static PartitionIterator create(PartitionIndex partitionIndex, IPartitioner partitioner, FileHandle rowIndexFile, FileHandle dataFile) throws IOException
+ {
+ return create(partitionIndex, partitioner, rowIndexFile, dataFile, partitionIndex.firstKey(), -1, partitionIndex.lastKey(), 0);
+ }
+
+ static PartitionIterator empty(PartitionIndex partitionIndex)
+ {
+ return new PartitionIterator(partitionIndex.sharedCopy());
+ }
+
+ private PartitionIterator(PartitionIndex partitionIndex, IPartitioner partitioner, FileHandle rowIndexFile, FileHandle dataFile,
+ PartitionPosition left, PartitionPosition right, int exclusiveRight)
+ {
+ super(partitionIndex, left, right);
+ this.partitionIndex = partitionIndex;
+ this.partitioner = partitioner;
+ this.limit = right;
+ this.exclusiveLimit = exclusiveRight;
+ this.rowIndexFile = rowIndexFile;
+ this.dataFile = dataFile;
+ }
+
+ private PartitionIterator(PartitionIndex partitionIndex)
+ {
+ super(partitionIndex, partitionIndex.firstKey(), partitionIndex.firstKey());
+ this.partitionIndex = partitionIndex;
+ this.partitioner = null;
+ this.limit = partitionIndex.firstKey();
+ this.exclusiveLimit = -1;
+ this.rowIndexFile = null;
+ this.dataFile = null;
+
+ this.currentEntry = null;
+ this.currentKey = null;
+ this.nextEntry = null;
+ this.nextKey = null;
+ }
+
+ @Override
+ public void close()
+ {
+ Throwable accum = null;
+ accum = Throwables.close(accum, immutableListWithFilteredNulls(partitionIndex, dataFile, rowIndexFile));
+ accum = Throwables.close(accum, immutableListWithFilteredNulls(dataInput, indexInput));
+ accum = Throwables.perform(accum, super::close);
+ Throwables.maybeFail(accum);
+ }
+
+ public DecoratedKey decoratedKey()
+ {
+ return currentKey;
+ }
+
+ public ByteBuffer key()
+ {
+ return currentKey.getKey();
+ }
+
+ @Override
+ public long dataPosition()
+ {
+ return currentEntry != null ? currentEntry.position : -1;
+ }
+
+ @Override
+ public long keyPositionForSecondaryIndex()
+ {
+ return dataPosition();
+ }
+
+ public TrieIndexEntry entry()
+ {
+ return currentEntry;
+ }
+
+ @Override
+ public boolean advance() throws IOException
+ {
+ currentKey = nextKey;
+ currentEntry = nextEntry;
+ if (currentKey != null)
+ {
+ readNext();
+ // if nextKey is null, then currentKey is the last key to be published, therefore check against any limit
+ // and suppress the partition if it is beyond the limit
+ if (nextKey == null && limit != null && currentKey.compareTo(limit) > exclusiveLimit)
+ { // exclude last partition outside range
+ currentKey = null;
+ currentEntry = null;
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private void readNext() throws IOException
+ {
+ long pos = nextIndexPos();
+ if (pos != PartitionIndex.NOT_FOUND)
+ {
+ if (pos >= 0)
+ {
+ seekIndexInput(pos);
+ nextKey = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(indexInput));
+ nextEntry = TrieIndexEntry.deserialize(indexInput, indexInput.getFilePointer());
+ }
+ else
+ {
+ pos = ~pos;
+ seekDataInput(pos);
+ nextKey = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(dataInput));
+ nextEntry = new TrieIndexEntry(pos);
+ }
+ }
+ else
+ {
+ nextKey = null;
+ nextEntry = null;
+ }
+ }
+
+ private void seekIndexInput(long pos) throws IOException
+ {
+ if (indexInput == null)
+ indexInput = rowIndexFile.createReader(pos);
+ else
+ indexInput.seek(pos);
+ }
+
+ private void seekDataInput(long pos) throws IOException
+ {
+ if (dataInput == null)
+ dataInput = dataFile.createReader(pos);
+ else
+ dataInput.seek(pos);
+ }
+
+ @Override
+ public boolean isExhausted()
+ {
+ return currentKey == null;
+ }
+
+ @Override
+ public void reset()
+ {
+ go(root);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("BTI-PartitionIterator(%s)", partitionIndex.getFileHandle().path());
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java
new file mode 100644
index 0000000..9867cff
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.io.tries.SerializationNode;
+import org.apache.cassandra.io.tries.TrieNode;
+import org.apache.cassandra.io.tries.TrieSerializer;
+import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.SizedInts;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Reader class for row index files created by {@link RowIndexWriter}.
+ * <p>
+ * Row index "tries" do not need to store whole keys, as what we need from them is to be able to tell where in the data file
+ * to start looking for a given key. Instead, we store some prefix that is greater than the greatest key of the previous
+ * index section and smaller than or equal to the smallest key of the next. So for a given key the first index section
+ * that could potentially contain it is given by the trie's floor for that key.
+ * <p>
+ * This builds upon the trie Walker class which provides basic trie walking functionality. The class is thread-unsafe
+ * and must be re-instantiated for every thread that needs access to the trie (its overhead is below that of a
+ * {@link org.apache.cassandra.io.util.RandomAccessReader}).
+ */
+@NotThreadSafe
+public class RowIndexReader extends Walker<RowIndexReader>
+{
+ private static final int FLAG_OPEN_MARKER = 8;
+
+ public static class IndexInfo
+ {
+ public final long offset;
+ public final DeletionTime openDeletion;
+
+ IndexInfo(long offset, DeletionTime openDeletion)
+ {
+ this.offset = offset;
+ this.openDeletion = openDeletion;
+ }
+ }
+
+ public RowIndexReader(FileHandle file, long root)
+ {
+ super(file.instantiateRebufferer(null), root);
+ }
+
+ public RowIndexReader(FileHandle file, TrieIndexEntry entry)
+ {
+ this(file, entry.indexTrieRoot);
+ }
+
+ /**
+ * Computes the floor for a given key.
+ */
+ public IndexInfo separatorFloor(ByteComparable key)
+ {
+ // Check for a prefix and find closest smaller branch.
+ IndexInfo res = prefixAndNeighbours(key, RowIndexReader::readPayload);
+ // If there's a prefix, in a separator trie it could be less than, equal, or greater than sought value.
+ // Sought value is still greater than max of previous section.
+ // On match the prefix must be used as a starting point.
+ if (res != null)
+ return res;
+
+ // Otherwise return the IndexInfo for the closest entry of the smaller branch (which is the max of lesserBranch).
+ // Note (see prefixAndNeighbours): since we accept prefix matches above, at this point there cannot be another
+ // prefix match that is closer than max(lesserBranch).
+ if (lesserBranch == NONE)
+ return null;
+ goMax(lesserBranch);
+ return getCurrentIndexInfo();
+ }
+
+ public IndexInfo min()
+ {
+ goMin(root);
+ return getCurrentIndexInfo();
+ }
+
+ protected IndexInfo getCurrentIndexInfo()
+ {
+ return readPayload(payloadPosition(), payloadFlags());
+ }
+
+ protected IndexInfo readPayload(int ppos, int bits)
+ {
+ return readPayload(buf, ppos, bits);
+ }
+
+ static IndexInfo readPayload(ByteBuffer buf, int ppos, int bits)
+ {
+ long dataOffset;
+ if (bits == 0)
+ return null;
+ int bytes = bits & ~FLAG_OPEN_MARKER;
+ dataOffset = SizedInts.read(buf, ppos, bytes);
+ ppos += bytes;
+ DeletionTime deletion = (bits & FLAG_OPEN_MARKER) != 0
+ ? DeletionTime.serializer.deserialize(buf, ppos)
+ : null;
+ return new IndexInfo(dataOffset, deletion);
+ }
+
+ // The trie serializer describes how the payloads are written. Placed here (instead of writer) so that reading and
+ // writing the payload are close together should they need to be changed.
+ static final TrieSerializer<IndexInfo, DataOutputPlus> trieSerializer = new TrieSerializer<IndexInfo, DataOutputPlus>()
+ {
+ @Override
+ public int sizeofNode(SerializationNode<IndexInfo> node, long nodePosition)
+ {
+ return TrieNode.typeFor(node, nodePosition).sizeofNode(node) + sizeof(node.payload());
+ }
+
+ @Override
+ public void write(DataOutputPlus dest, SerializationNode<IndexInfo> node, long nodePosition) throws IOException
+ {
+ write(dest, TrieNode.typeFor(node, nodePosition), node, nodePosition);
+ }
+
+ private int sizeof(IndexInfo payload)
+ {
+ int size = 0;
+ if (payload != null)
+ {
+ size += SizedInts.nonZeroSize(payload.offset);
+ if (!payload.openDeletion.isLive())
+ size += DeletionTime.serializer.serializedSize(payload.openDeletion);
+ }
+ return size;
+ }
+
+ private void write(DataOutputPlus dest, TrieNode type, SerializationNode<IndexInfo> node, long nodePosition) throws IOException
+ {
+ IndexInfo payload = node.payload();
+ int bytes = 0;
+ int hasOpenMarker = 0;
+ if (payload != null)
+ {
+ bytes = SizedInts.nonZeroSize(payload.offset);
+ assert bytes < 8 : "Row index does not support rows larger than 32 PiB";
+ if (!payload.openDeletion.isLive())
+ hasOpenMarker = FLAG_OPEN_MARKER;
+ }
+ type.serialize(dest, node, bytes | hasOpenMarker, nodePosition);
+ if (payload != null)
+ {
+ SizedInts.write(dest, payload.offset, bytes);
+
+ if (hasOpenMarker == FLAG_OPEN_MARKER)
+ DeletionTime.serializer.serialize(payload.openDeletion, dest);
+ }
+ }
+
+ };
+
+ // debug/test code
+ @SuppressWarnings("unused")
+ public void dumpTrie(PrintStream out)
+ {
+ dumpTrie(out, RowIndexReader::dumpRowIndexEntry);
+ }
+
+ static String dumpRowIndexEntry(ByteBuffer buf, int ppos, int bits)
+ {
+ IndexInfo ii = readPayload(buf, ppos, bits);
+
+ return ii != null
+ ? String.format("pos %x %s", ii.offset, ii.openDeletion == null ? "" : ii.openDeletion)
+ : "pos null";
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java
new file mode 100644
index 0000000..ceb667e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.PrintStream;
+
+import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
+import org.apache.cassandra.io.tries.ReverseValueIterator;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Reverse iterator over the row index. Needed to get previous index blocks for reverse iteration.
+ */
+class RowIndexReverseIterator extends ReverseValueIterator<RowIndexReverseIterator>
+{
+ private long currentNode = -1;
+
+ public RowIndexReverseIterator(FileHandle file, long root, ByteComparable start, ByteComparable end)
+ {
+ super(file.instantiateRebufferer(null), root, start, end, true);
+ }
+
+ public RowIndexReverseIterator(FileHandle file, TrieIndexEntry entry, ByteComparable end)
+ {
+ this(file, entry.indexTrieRoot, ByteComparable.EMPTY, end);
+ }
+
+ public IndexInfo nextIndexInfo()
+ {
+ if (currentNode == -1)
+ {
+ currentNode = nextPayloadedNode();
+ if (currentNode == -1)
+ return null;
+ }
+
+ go(currentNode);
+ IndexInfo info = RowIndexReader.readPayload(buf, payloadPosition(), payloadFlags());
+
+ currentNode = -1;
+ return info;
+ }
+
+ // debug/test code
+ @SuppressWarnings("unused")
+ public void dumpTrie(PrintStream out)
+ {
+ dumpTrie(out, RowIndexReader::dumpRowIndexEntry);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java
new file mode 100644
index 0000000..1ebaf14
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
+import org.apache.cassandra.io.tries.IncrementalTrieWriter;
+import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Preparer / writer of row index tries that can be read by {@link RowIndexReader} and iterated by
+ * {@link RowIndexReverseIterator}.
+ * <p>
+ * Uses {@link IncrementalTrieWriter} to build a trie of index section separators of the shortest possible length such
+ * that {@code prevMax < separator <= nextMin}.
+ */
+class RowIndexWriter implements AutoCloseable
+{
+ private final ClusteringComparator comparator;
+ private final IncrementalTrieWriter<IndexInfo> trie;
+ private ByteComparable prevMax = null;
+ private ByteComparable prevSep = null;
+
+ RowIndexWriter(ClusteringComparator comparator, DataOutputPlus out)
+ {
+ this.comparator = comparator;
+ this.trie = IncrementalTrieWriter.open(RowIndexReader.trieSerializer, out);
+ }
+
+ void reset()
+ {
+ prevMax = null;
+ prevSep = null;
+ trie.reset();
+ }
+
+ @Override
+ public void close()
+ {
+ trie.close();
+ }
+
+ void add(ClusteringPrefix<?> firstName, ClusteringPrefix<?> lastName, IndexInfo info) throws IOException
+ {
+ assert info.openDeletion != null;
+ ByteComparable sep = prevMax == null
+ ? ByteComparable.EMPTY
+ : ByteComparable.separatorGt(prevMax, comparator.asByteComparable(firstName));
+ trie.add(sep, info);
+ prevSep = sep;
+ prevMax = comparator.asByteComparable(lastName);
+ }
+
+ public long complete(long endPos) throws IOException
+ {
+ // Add a separator after the last section, so that greater inputs can be quickly rejected.
+ // To maximize its efficiency we add it with the length of the last added separator.
+ int i = 0;
+ ByteSource max = prevMax.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+ ByteSource sep = prevSep.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+ int c;
+ while ((c = max.next()) == sep.next() && c != ByteSource.END_OF_STREAM)
+ ++i;
+ assert c != ByteSource.END_OF_STREAM : "Corrupted row order, max=" + prevMax;
+
+ trie.add(nudge(prevMax, i), new IndexInfo(endPos, DeletionTime.LIVE));
+
+ return trie.complete();
+ }
+
+ /**
+ * Produces a source that is slightly greater than argument with length at least nudgeAt.
+ */
+ private ByteComparable nudge(ByteComparable value, int nudgeAt)
+ {
+ return version -> new ByteSource()
+ {
+ private final ByteSource v = value.asComparableBytes(version);
+ private int cur = 0;
+
+ @Override
+ public int next()
+ {
+ int b = ByteSource.END_OF_STREAM;
+ if (cur <= nudgeAt)
+ {
+ b = v.next();
+ if (cur == nudgeAt)
+ {
+ if (b < 0xFF)
+ ++b;
+ else
+ return b; // can't nudge here, increase next instead (eventually will be -1)
+ }
+ }
+ ++cur;
+ return b;
+ }
+ };
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/SSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/SSTableIterator.java
new file mode 100644
index 0000000..551375c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/SSTableIterator.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.AbstractSSTableIterator;
+import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+
+/**
+ * Unfiltered row iterator over a BTI SSTable.
+ */
+class SSTableIterator extends AbstractSSTableIterator<AbstractRowIndexEntry>
+{
+ /**
+ * The index of the slice being processed.
+ */
+ private int slice;
+
+ public SSTableIterator(BtiTableReader sstable,
+ FileDataInput file,
+ DecoratedKey key,
+ AbstractRowIndexEntry indexEntry,
+ Slices slices,
+ ColumnFilter columns,
+ FileHandle ifile)
+ {
+ super(sstable, file, key, indexEntry, slices, columns, ifile);
+ }
+
+ protected Reader createReaderInternal(AbstractRowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ if (indexEntry.isIndexed())
+ return new ForwardIndexedReader(indexEntry, file, shouldCloseFile);
+ else
+ return new ForwardReader(file, shouldCloseFile);
+ }
+
+ protected int nextSliceIndex()
+ {
+ int next = slice;
+ slice++;
+ return next;
+ }
+
+ protected boolean hasMoreSlices()
+ {
+ return slice < slices.size();
+ }
+
+ public boolean isReverseOrder()
+ {
+ return false;
+ }
+
+ private class ForwardIndexedReader extends ForwardReader
+ {
+ private final RowIndexReader indexReader;
+ private final long basePosition;
+
+ private ForwardIndexedReader(AbstractRowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ basePosition = indexEntry.position;
+ indexReader = new RowIndexReader(ifile, (TrieIndexEntry) indexEntry);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ indexReader.close();
+ super.close();
+ }
+
+ @Override
+ public void setForSlice(Slice slice) throws IOException
+ {
+ super.setForSlice(slice);
+ IndexInfo indexInfo = indexReader.separatorFloor(metadata.comparator.asByteComparable(slice.start()));
+ assert indexInfo != null;
+ long position = basePosition + indexInfo.offset;
+ if (file == null || position > file.getFilePointer())
+ {
+ openMarker = indexInfo.openDeletion;
+ seekToPosition(position);
+ }
+ // Otherwise we are already in the relevant index block, there is no point to go back to its beginning.
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/SSTableReversedIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/SSTableReversedIterator.java
new file mode 100644
index 0000000..911d72e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/SSTableReversedIterator.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+
+import com.carrotsearch.hppc.LongStack;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.AbstractSSTableIterator;
+import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+
+/**
+ * Unfiltered row iterator over a BTI SSTable that returns rows in reverse order.
+ */
+class SSTableReversedIterator extends AbstractSSTableIterator<TrieIndexEntry>
+{
+ /**
+ * The index of the slice being processed.
+ */
+ private int slice;
+
+ public SSTableReversedIterator(BtiTableReader sstable,
+ FileDataInput file,
+ DecoratedKey key,
+ TrieIndexEntry indexEntry,
+ Slices slices,
+ ColumnFilter columns,
+ FileHandle ifile)
+ {
+ super(sstable, file, key, indexEntry, slices, columns, ifile);
+ }
+
+ protected Reader createReaderInternal(TrieIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ if (indexEntry.isIndexed())
+ return new ReverseIndexedReader(indexEntry, file, shouldCloseFile);
+ else
+ return new ReverseReader(file, shouldCloseFile);
+ }
+
+ public boolean isReverseOrder()
+ {
+ return true;
+ }
+
+ protected int nextSliceIndex()
+ {
+ int next = slice;
+ slice++;
+ return slices.size() - (next + 1);
+ }
+
+ protected boolean hasMoreSlices()
+ {
+ return slice < slices.size();
+ }
+
+ /**
+ * Reverse iteration is performed by going through an index block (or the whole partition if not indexed) forwards
+ * and storing the positions of each entry that falls within the slice in a stack. Reverse iteration then pops out
+ * positions and reads the entries.
+ * <p>
+ * Note: The earlier version of this was constructing an in-memory view of the block instead, which gives better
+ * performance on bigger queries and index blocks (due to not having to read disk again). With the lower
+ * granularity of the tries it makes better sense to store as little as possible as the beginning of the block
+ * should very rarely be in other page/chunk cache locations. This has the benefit of being able to answer small
+ * queries (esp. LIMIT 1) faster and with less GC churn.
+ */
+ private class ReverseReader extends AbstractReader
+ {
+ final LongStack rowOffsets = new LongStack();
+ RangeTombstoneMarker blockOpenMarker, blockCloseMarker;
+ private Unfiltered next = null;
+ private boolean foundLessThan;
+ private long startPos = -1;
+
+ private ReverseReader(FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ }
+
+ @Override
+ public void setForSlice(Slice slice) throws IOException
+ {
+ // read full row and filter
+ if (startPos == -1)
+ startPos = file.getFilePointer();
+ else
+ seekToPosition(startPos);
+
+ fillOffsets(slice, true, true, Long.MAX_VALUE);
+ }
+
+ @Override
+ protected boolean hasNextInternal() throws IOException
+ {
+ if (next != null)
+ return true;
+ next = computeNext();
+ return next != null;
+ }
+
+ @Override
+ protected Unfiltered nextInternal() throws IOException
+ {
+ if (!hasNextInternal())
+ throw new NoSuchElementException();
+
+ Unfiltered toReturn = next;
+ next = null;
+ return toReturn;
+ }
+
+ private Unfiltered computeNext() throws IOException
+ {
+ Unfiltered toReturn;
+ do
+ {
+ if (blockCloseMarker != null)
+ {
+ toReturn = blockCloseMarker;
+ blockCloseMarker = null;
+ return toReturn;
+ }
+ while (!rowOffsets.isEmpty())
+ {
+ seekToPosition(rowOffsets.pop());
+ boolean hasNext = deserializer.hasNext();
+ assert hasNext : "Data file changed after offset collection pass";
+ toReturn = deserializer.readNext();
+ UnfilteredValidation.maybeValidateUnfiltered(toReturn, metadata(), key, sstable);
+ // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne.
+ if (!toReturn.isEmpty())
+ return toReturn;
+ }
+ }
+ while (!foundLessThan && advanceIndexBlock());
+
+ // open marker to be output only as slice is finished
+ if (blockOpenMarker != null)
+ {
+ toReturn = blockOpenMarker;
+ blockOpenMarker = null;
+ return toReturn;
+ }
+ return null;
+ }
+
+ protected boolean advanceIndexBlock() throws IOException
+ {
+ return false;
+ }
+
+ void fillOffsets(Slice slice, boolean filterStart, boolean filterEnd, long stopPosition) throws IOException
+ {
+ filterStart &= !slice.start().equals(ClusteringBound.BOTTOM);
+ filterEnd &= !slice.end().equals(ClusteringBound.TOP);
+
+ ClusteringBound<?> start = slice.start();
+ long currentPosition = file.getFilePointer();
+ foundLessThan = false;
+ // This is a copy of handlePreSliceData which also checks currentPosition < stopPosition.
+ // Not extracted to method as we need both marker and currentPosition.
+ if (filterStart)
+ {
+ while (currentPosition < stopPosition && deserializer.hasNext() && deserializer.compareNextTo(start) <= 0)
+ {
+ if (deserializer.nextIsRow())
+ deserializer.skipNext();
+ else
+ updateOpenMarker((RangeTombstoneMarker) deserializer.readNext());
+
+ currentPosition = file.getFilePointer();
+ foundLessThan = true;
+ }
+ }
+
+ // We've reached the beginning of our queried slice. If we have an open marker
+ // we should return that at the end of the slice to close the deletion.
+ if (openMarker != null)
+ blockOpenMarker = new RangeTombstoneBoundMarker(start, openMarker);
+
+
+ // Now deserialize everything until we reach our requested end (if we have one)
+ // See SSTableIterator.ForwardRead.computeNext() for why this is a strict inequality below: this is the same
+ // reasoning here.
+ while (currentPosition < stopPosition && deserializer.hasNext()
+ && (!filterEnd || deserializer.compareNextTo(slice.end()) < 0))
+ {
+ rowOffsets.push(currentPosition);
+ if (deserializer.nextIsRow())
+ deserializer.skipNext();
+ else
+ updateOpenMarker((RangeTombstoneMarker) deserializer.readNext());
+
+ currentPosition = file.getFilePointer();
+ }
+
+ // If we have an open marker, we should output that first, unless end is not being filtered
+ // (i.e. it's either top (where a marker can't be open) or we placed that marker during previous block).
+ if (openMarker != null && filterEnd)
+ {
+ // If we have no end and still an openMarker, this means we're indexed and the marker is closed in a following block.
+ blockCloseMarker = new RangeTombstoneBoundMarker(slice.end(), openMarker);
+ openMarker = null;
+ }
+ }
+ }
+
+ private class ReverseIndexedReader extends ReverseReader
+ {
+ private RowIndexReverseIterator indexReader;
+ private final TrieIndexEntry indexEntry;
+ private final long basePosition;
+ private Slice currentSlice;
+ private long currentBlockStart;
+
+ public ReverseIndexedReader(AbstractRowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile)
+ {
+ super(file, shouldCloseFile);
+ basePosition = indexEntry.position;
+ this.indexEntry = (TrieIndexEntry) indexEntry;
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ if (indexReader != null)
+ indexReader.close();
+ super.close();
+ }
+
+ @Override
+ public void setForSlice(Slice slice) throws IOException
+ {
+ currentSlice = slice;
+ ClusteringComparator comparator = metadata.comparator;
+ if (indexReader != null)
+ indexReader.close();
+ indexReader = new RowIndexReverseIterator(ifile,
+ indexEntry,
+ comparator.asByteComparable(slice.end()));
+ gotoBlock(indexReader.nextIndexInfo(), true, Long.MAX_VALUE);
+ }
+
+ boolean gotoBlock(IndexInfo indexInfo, boolean filterEnd, long blockEnd) throws IOException
+ {
+ blockOpenMarker = null;
+ blockCloseMarker = null;
+ rowOffsets.clear();
+ if (indexInfo == null)
+ return false;
+ currentBlockStart = basePosition + indexInfo.offset;
+ openMarker = indexInfo.openDeletion;
+
+ seekToPosition(currentBlockStart);
+ fillOffsets(currentSlice, true, filterEnd, blockEnd);
+ return !rowOffsets.isEmpty();
+ }
+
+ @Override
+ protected boolean advanceIndexBlock() throws IOException
+ {
+ return gotoBlock(indexReader.nextIndexInfo(), false, currentBlockStart);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/ScrubIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/ScrubIterator.java
new file mode 100644
index 0000000..2591a6f
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/ScrubIterator.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class ScrubIterator extends PartitionIndex.IndexPosIterator implements ScrubPartitionIterator
+{
+ public static final int EXHAUSTED = -1;
+ ByteBuffer key;
+ long dataPosition;
+ final FileHandle rowIndexFile;
+
+ ScrubIterator(PartitionIndex partitionIndex, FileHandle rowIndexFile) throws IOException
+ {
+ super(partitionIndex);
+ this.rowIndexFile = rowIndexFile.sharedCopy();
+ advance();
+ }
+
+ @Override
+ public void close()
+ {
+ super.close();
+ rowIndexFile.close();
+ }
+
+ @Override
+ public ByteBuffer key()
+ {
+ return key;
+ }
+
+ @Override
+ public long dataPosition()
+ {
+ return dataPosition;
+ }
+
+ @Override
+ public void advance() throws IOException
+ {
+ long pos = nextIndexPos();
+ if (pos != PartitionIndex.NOT_FOUND)
+ {
+ if (pos >= 0) // row index position
+ {
+ try (FileDataInput in = rowIndexFile.createReader(pos))
+ {
+ key = ByteBufferUtil.readWithShortLength(in);
+ dataPosition = TrieIndexEntry.deserialize(in, in.getFilePointer()).position;
+ }
+ }
+ else
+ {
+ key = null;
+ dataPosition = ~pos;
+ }
+ }
+ else
+ {
+ key = null;
+ dataPosition = EXHAUSTED;
+ }
+ }
+
+ @Override
+ public boolean isExhausted()
+ {
+ return dataPosition == EXHAUSTED;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/ScrubPartitionIterator.java b/src/java/org/apache/cassandra/io/sstable/format/bti/ScrubPartitionIterator.java
new file mode 100644
index 0000000..0bf88ee
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/ScrubPartitionIterator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Iterator over the partitions of an sstable used for scrubbing.
+ * <p>
+ * The difference between this and {@link PartitionIterator} is that this only uses information present in the index file
+ * and does not try to read keys of the data file (for the trie index format), thus {@link #key()} can be null.
+ * <p>
+ * Starts advanced to a position, {@link #advance()} is to be used to go to next, and iteration completes when
+ * {@link #dataPosition()} == -1.
+ */
+public interface ScrubPartitionIterator extends Closeable
+{
+ /**
+ * Serialized partition key or {@code null} if the iterator reached the end of the index or if the key may not
+ * be fully retrieved from the index file.
+ */
+ ByteBuffer key();
+
+ /**
+ * Key position in data file or -1 if the iterator reached the end of the index.
+ */
+ long dataPosition();
+
+ /**
+ * Move to the next position in the index file.
+ */
+ void advance() throws IOException;
+
+ boolean isExhausted();
+
+ void close();
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/TrieIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/format/bti/TrieIndexEntry.java
new file mode 100644
index 0000000..cdb743c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/TrieIndexEntry.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Index entry for the BTI partition index. This can be a simple position in the data file, or an entry in the row
+ * index file where the rows are indexed in blocks (see {@link RowIndexReader}).
+ */
+final class TrieIndexEntry extends AbstractRowIndexEntry
+{
+ final long indexTrieRoot;
+ private final int rowIndexBlockCount;
+ private final DeletionTime deletionTime;
+
+ TrieIndexEntry(long dataFilePosition, long indexTrieRoot, int rowIndexBlockCount, DeletionTime deletionTime)
+ {
+ super(dataFilePosition);
+ this.indexTrieRoot = indexTrieRoot;
+ this.rowIndexBlockCount = rowIndexBlockCount;
+ this.deletionTime = deletionTime;
+ }
+
+ public TrieIndexEntry(long position)
+ {
+ super(position);
+ this.indexTrieRoot = -1;
+ this.rowIndexBlockCount = 0;
+ this.deletionTime = null;
+ }
+
+ @Override
+ public int blockCount()
+ {
+ return rowIndexBlockCount;
+ }
+
+ @Override
+ public SSTableFormat<?, ?> getSSTableFormat()
+ {
+ throw noKeyCacheError();
+ }
+
+ @Override
+ public void serializeForCache(DataOutputPlus out)
+ {
+ throw noKeyCacheError();
+ }
+
+ private static AssertionError noKeyCacheError()
+ {
+ return new AssertionError("BTI SSTables should not use key cache");
+ }
+
+ @Override
+ public DeletionTime deletionTime()
+ {
+ return deletionTime;
+ }
+
+ @Override
+ public long unsharedHeapSize()
+ {
+ throw new AssertionError("BTI SSTables index entries should not be persisted in any in-memory structure");
+ }
+
+ public void serialize(DataOutputPlus indexFile, long basePosition) throws IOException
+ {
+ assert indexTrieRoot != -1 && rowIndexBlockCount > 0 && deletionTime != null;
+ indexFile.writeUnsignedVInt(position);
+ indexFile.writeVInt(indexTrieRoot - basePosition);
+ indexFile.writeUnsignedVInt32(rowIndexBlockCount);
+ DeletionTime.serializer.serialize(deletionTime, indexFile);
+ }
+
+ /**
+ * Create an index entry. The row index trie must already have been written (by RowIndexWriter) to the row index
+ * file and its root position must be specified in trieRoot.
+ */
+ public static TrieIndexEntry create(long dataStartPosition,
+ long trieRoot,
+ DeletionTime partitionLevelDeletion,
+ int rowIndexBlockCount)
+ {
+ return new TrieIndexEntry(dataStartPosition, trieRoot, trieRoot == -1 ? 0 : rowIndexBlockCount, partitionLevelDeletion);
+ }
+
+ public static TrieIndexEntry deserialize(DataInputPlus in, long basePosition) throws IOException
+ {
+ long dataFilePosition = in.readUnsignedVInt();
+ long indexTrieRoot = in.readVInt() + basePosition;
+ int rowIndexBlockCount = in.readUnsignedVInt32();
+ DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
+ return new TrieIndexEntry(dataFilePosition, indexTrieRoot, rowIndexBlockCount, deletionTime);
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java
new file mode 100644
index 0000000..105d235
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.io.util.MemoryOutputStream;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+
+/*
+ * Layout of Memory for index summaries:
+ *
+ * There are two sections:
+ * 1. A "header" containing the offset into `bytes` of entries in the summary summary data, consisting of
+ * one four byte position for each entry in the summary. This allows us do simple math in getIndex()
+ * to find the position in the Memory to start reading the actual index summary entry.
+ * (This is necessary because keys can have different lengths.)
+ * 2. A sequence of (DecoratedKey, position) pairs, where position is the offset into the actual index file.
+ */
+public class IndexSummary extends WrappedSharedCloseable
+{
+ private static final Logger logger = LoggerFactory.getLogger(IndexSummary.class);
+ public static final IndexSummarySerializer serializer = new IndexSummarySerializer();
+
+ /**
+ * A lower bound for the average number of partitions in between each index summary entry. A lower value means
+ * that more partitions will have an entry in the index summary when at the full sampling level.
+ */
+ private final int minIndexInterval;
+
+ private final IPartitioner partitioner;
+ private final int sizeAtFullSampling;
+ // we permit the memory to span a range larger than we use,
+ // so we have an accompanying count and length for each part
+ // we split our data into two ranges: offsets (indexing into entries),
+ // and entries containing the summary data
+ private final Memory offsets;
+ private final int offsetCount;
+ // entries is a list of (partition key, index file offset) pairs
+ private final Memory entries;
+ private final long entriesLength;
+
+ /**
+ * A value between 1 and BASE_SAMPLING_LEVEL that represents how many of the original
+ * index summary entries ((1 / indexInterval) * numKeys) have been retained.
+ *
+ * Thus, this summary contains (samplingLevel / BASE_SAMPLING_LEVEL) * ((1 / indexInterval) * numKeys)) entries.
+ */
+ private final int samplingLevel;
+
+ public IndexSummary(IPartitioner partitioner, Memory offsets, int offsetCount, Memory entries, long entriesLength,
+ int sizeAtFullSampling, int minIndexInterval, int samplingLevel)
+ {
+ super(new Memory[] { offsets, entries });
+ assert offsets.getInt(0) == 0;
+ this.partitioner = partitioner;
+ this.minIndexInterval = minIndexInterval;
+ this.offsetCount = offsetCount;
+ this.entriesLength = entriesLength;
+ this.sizeAtFullSampling = sizeAtFullSampling;
+ this.offsets = offsets;
+ this.entries = entries;
+ this.samplingLevel = samplingLevel;
+ assert samplingLevel > 0;
+ }
+
+ private IndexSummary(IndexSummary copy)
+ {
+ super(copy);
+ this.partitioner = copy.partitioner;
+ this.minIndexInterval = copy.minIndexInterval;
+ this.offsetCount = copy.offsetCount;
+ this.entriesLength = copy.entriesLength;
+ this.sizeAtFullSampling = copy.sizeAtFullSampling;
+ this.offsets = copy.offsets;
+ this.entries = copy.entries;
+ this.samplingLevel = copy.samplingLevel;
+ }
+
+ // binary search is notoriously more difficult to get right than it looks; this is lifted from
+ // Harmony's Collections implementation
+ public int binarySearch(PartitionPosition key)
+ {
+ // We will be comparing non-native Keys, so use a buffer with appropriate byte order
+ ByteBuffer hollow = MemoryUtil.getHollowDirectByteBuffer().order(ByteOrder.BIG_ENDIAN);
+ int low = 0, mid = offsetCount, high = mid - 1, result = -1;
+ while (low <= high)
+ {
+ mid = (low + high) >> 1;
+ fillTemporaryKey(mid, hollow);
+ result = -DecoratedKey.compareTo(partitioner, hollow, key);
+ if (result > 0)
+ {
+ low = mid + 1;
+ }
+ else if (result == 0)
+ {
+ return mid;
+ }
+ else
+ {
+ high = mid - 1;
+ }
+ }
+
+ return -mid - (result < 0 ? 1 : 2);
+ }
+
+ /**
+ * Gets the position of the actual index summary entry in our Memory attribute, 'bytes'.
+ * @param index The index of the entry or key to get the position for
+ * @return an offset into our Memory attribute where the actual entry resides
+ */
+ public int getPositionInSummary(int index)
+ {
+ // The first section of bytes holds a four-byte position for each entry in the summary, so just multiply by 4.
+ return offsets.getInt(index << 2);
+ }
+
+ public byte[] getKey(int index)
+ {
+ long start = getPositionInSummary(index);
+ int keySize = (int) (calculateEnd(index) - start - 8L);
+ byte[] key = new byte[keySize];
+ entries.getBytes(start, key, 0, keySize);
+ return key;
+ }
+
+ private void fillTemporaryKey(int index, ByteBuffer buffer)
+ {
+ long start = getPositionInSummary(index);
+ int keySize = (int) (calculateEnd(index) - start - 8L);
+ entries.setByteBuffer(buffer, start, keySize);
+ }
+
+ public void addTo(Ref.IdentityCollection identities)
+ {
+ super.addTo(identities);
+ identities.add(offsets);
+ identities.add(entries);
+ }
+
+ public long getPosition(int index)
+ {
+ return entries.getLong(calculateEnd(index) - 8);
+ }
+
+ public long getEndInSummary(int index)
+ {
+ return calculateEnd(index);
+ }
+
+ private long calculateEnd(int index)
+ {
+ return index == (offsetCount - 1) ? entriesLength : getPositionInSummary(index + 1);
+ }
+
+ public int getMinIndexInterval()
+ {
+ return minIndexInterval;
+ }
+
+ public double getEffectiveIndexInterval()
+ {
+ return (BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval;
+ }
+
+ /**
+ * Returns an estimate of the total number of keys in the SSTable.
+ */
+ public long getEstimatedKeyCount()
+ {
+ return ((long) getMaxNumberOfEntries() + 1) * minIndexInterval;
+ }
+
+ public int size()
+ {
+ return offsetCount;
+ }
+
+ public int getSamplingLevel()
+ {
+ return samplingLevel;
+ }
+
+ /**
+ * Returns the number of entries this summary would have if it were at the full sampling level, which is equal
+ * to the number of entries in the primary on-disk index divided by the min index interval.
+ */
+ public int getMaxNumberOfEntries()
+ {
+ return sizeAtFullSampling;
+ }
+
+ /**
+ * Returns the amount of off-heap memory used for the entries portion of this summary.
+ * @return size in bytes
+ */
+ long getEntriesLength()
+ {
+ return entriesLength;
+ }
+
+ Memory getOffsets()
+ {
+ return offsets;
+ }
+
+ Memory getEntries()
+ {
+ return entries;
+ }
+
+ public long getOffHeapSize()
+ {
+ return offsetCount * 4 + entriesLength;
+ }
+
+ /**
+ * Returns the number of primary (on-disk) index entries between the index summary entry at `index` and the next
+ * index summary entry (assuming there is one). Without any downsampling, this will always be equivalent to
+ * the index interval.
+ *
+ * @param index the index of an index summary entry (between zero and the index entry size)
+ *
+ * @return the number of partitions after `index` until the next partition with a summary entry
+ */
+ public int getEffectiveIndexIntervalAfterIndex(int index)
+ {
+ return Downsampling.getEffectiveIndexIntervalAfterIndex(index, samplingLevel, minIndexInterval);
+ }
+
+ public List<SSTableReader.IndexesBounds> getSampleIndexesForRanges(Collection<Range<Token>> ranges)
+ {
+ // use the index to determine a minimal section for each range
+ List<SSTableReader.IndexesBounds> positions = new ArrayList<>();
+
+ for (Range<Token> range : Range.normalize(ranges))
+ {
+ PartitionPosition leftPosition = range.left.maxKeyBound();
+ PartitionPosition rightPosition = range.right.maxKeyBound();
+
+ int left = binarySearch(leftPosition);
+ if (left < 0)
+ left = (left + 1) * -1;
+ else
+ // left range are start exclusive
+ left = left + 1;
+ if (left == size())
+ // left is past the end of the sampling
+ continue;
+
+ int right = Range.isWrapAround(range.left, range.right)
+ ? size() - 1
+ : binarySearch(rightPosition);
+ if (right < 0)
+ {
+ // range are end inclusive so we use the previous index from what binarySearch give us
+ // since that will be the last index we will return
+ right = (right + 1) * -1;
+ if (right == 0)
+ // Means the first key is already stricly greater that the right bound
+ continue;
+ right--;
+ }
+
+ if (left > right)
+ // empty range
+ continue;
+ positions.add(new SSTableReader.IndexesBounds(left, right));
+ }
+ return positions;
+ }
+
+ public Iterable<byte[]> getKeySamples(final Range<Token> range)
+ {
+ final List<SSTableReader.IndexesBounds> indexRanges = getSampleIndexesForRanges(Collections.singletonList(range));
+
+ if (indexRanges.isEmpty())
+ return Collections.emptyList();
+
+ return () -> new Iterator<byte[]>()
+ {
+ private Iterator<SSTableReader.IndexesBounds> rangeIter = indexRanges.iterator();
+ private SSTableReader.IndexesBounds current;
+ private int idx;
+
+ public boolean hasNext()
+ {
+ if (current == null || idx > current.upperPosition)
+ {
+ if (rangeIter.hasNext())
+ {
+ current = rangeIter.next();
+ idx = current.lowerPosition;
+ return true;
+ }
+ return false;
+ }
+
+ return true;
+ }
+
+ public byte[] next()
+ {
+ return getKey(idx++);
+ }
+
+ public void remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ public long getScanPosition(PartitionPosition key)
+ {
+ return getScanPositionFromBinarySearchResult(binarySearch(key));
+ }
+
+ @VisibleForTesting
+ public long getScanPositionFromBinarySearchResult(int binarySearchResult)
+ {
+ if (binarySearchResult == -1)
+ return 0;
+ else
+ return getPosition(getIndexFromBinarySearchResult(binarySearchResult));
+ }
+
+ public static int getIndexFromBinarySearchResult(int binarySearchResult)
+ {
+ if (binarySearchResult < 0)
+ {
+ // binary search gives us the first index _greater_ than the key searched for,
+ // i.e., its insertion position
+ int greaterThan = (binarySearchResult + 1) * -1;
+ if (greaterThan == 0)
+ return -1;
+ return greaterThan - 1;
+ }
+ else
+ {
+ return binarySearchResult;
+ }
+ }
+
+ public IndexSummary sharedCopy()
+ {
+ return new IndexSummary(this);
+ }
+
+ public static class IndexSummarySerializer
+ {
+ public void serialize(IndexSummary t, DataOutputPlus out) throws IOException
+ {
+ out.writeInt(t.minIndexInterval);
+ out.writeInt(t.offsetCount);
+ out.writeLong(t.getOffHeapSize());
+ out.writeInt(t.samplingLevel);
+ out.writeInt(t.sizeAtFullSampling);
+ // our on-disk representation treats the offsets and the summary data as one contiguous structure,
+ // in which the offsets are based from the start of the structure. i.e., if the offsets occupy
+ // X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
+ // the summary values are indexed from zero, so we apply a correction to the offsets when de/serializing.
+ // In this case adding X to each of the offsets.
+ int baseOffset = t.offsetCount * 4;
+ for (int i = 0 ; i < t.offsetCount ; i++)
+ {
+ int offset = t.offsets.getInt(i * 4) + baseOffset;
+ // our serialization format for this file uses native byte order, so if this is different to the
+ // default Java serialization order (BIG_ENDIAN) we have to reverse our bytes
+ if (ByteOrder.nativeOrder() != ByteOrder.BIG_ENDIAN)
+ offset = Integer.reverseBytes(offset);
+ out.writeInt(offset);
+ }
+ out.write(t.entries, 0, t.entriesLength);
+ }
+
+ @SuppressWarnings("resource")
+ public <T extends InputStream & DataInputPlus> IndexSummary deserialize(T in, IPartitioner partitioner, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
+ {
+ int minIndexInterval = in.readInt();
+ if (minIndexInterval != expectedMinIndexInterval)
+ {
+ throw new IOException(String.format("Cannot read index summary because min_index_interval changed from %d to %d.",
+ minIndexInterval, expectedMinIndexInterval));
+ }
+
+ int offsetCount = in.readInt();
+ long offheapSize = in.readLong();
+ int samplingLevel = in.readInt();
+ int fullSamplingSummarySize = in.readInt();
+
+ int effectiveIndexInterval = (int) Math.ceil((BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval);
+ if (effectiveIndexInterval > maxIndexInterval)
+ {
+ throw new IOException(String.format("Rebuilding index summary because the effective index interval (%d) is higher than" +
+ " the current max index interval (%d)", effectiveIndexInterval, maxIndexInterval));
+ }
+
+ Memory offsets = Memory.allocate(offsetCount * 4);
+ Memory entries = Memory.allocate(offheapSize - offsets.size());
+ try
+ {
+ FBUtilities.copy(in, new MemoryOutputStream(offsets), offsets.size());
+ FBUtilities.copy(in, new MemoryOutputStream(entries), entries.size());
+ }
+ catch (IOException ioe)
+ {
+ offsets.free();
+ entries.free();
+ throw ioe;
+ }
+ // our on-disk representation treats the offsets and the summary data as one contiguous structure,
+ // in which the offsets are based from the start of the structure. i.e., if the offsets occupy
+ // X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
+ // the summary values are indexed from zero, so we apply a correction to the offsets when de/serializing.
+ // In this case subtracting X from each of the offsets.
+ for (int i = 0 ; i < offsets.size() ; i += 4)
+ offsets.setInt(i, (int) (offsets.getInt(i) - offsets.size()));
+ return new IndexSummary(partitioner, offsets, offsetCount, entries, entries.size(), fullSamplingSummarySize, minIndexInterval, samplingLevel);
+ }
+
+ /**
+ * Deserializes the first and last key stored in the summary
+ * <p>
+ * Only for use by offline tools like SSTableMetadataViewer, otherwise SSTable.first/last should be used.
+ */
+ public Pair<DecoratedKey, DecoratedKey> deserializeFirstLastKey(DataInputStreamPlus in, IPartitioner partitioner) throws IOException
+ {
+ in.skipBytes(4); // minIndexInterval
+ int offsetCount = in.readInt();
+ long offheapSize = in.readLong();
+ in.skipBytes(8); // samplingLevel, fullSamplingSummarySize
+
+ in.skipBytes(offsetCount * 4);
+ in.skipBytes((int) (offheapSize - offsetCount * 4));
+
+ DecoratedKey first = partitioner.decorateKey(ByteBufferUtil.readWithLength(in));
+ DecoratedKey last = partitioner.decorateKey(ByteBufferUtil.readWithLength(in));
+ return Pair.create(first, last);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
new file mode 100644
index 0000000..5f38fc9
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.io.util.SafeMemoryWriter;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.INDEX_SUMMARY_EXPECTED_KEY_SIZE;
+import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+
+public class IndexSummaryBuilder implements AutoCloseable
+{
+ private static final Logger logger = LoggerFactory.getLogger(IndexSummaryBuilder.class);
+
+ static long defaultExpectedKeySize = INDEX_SUMMARY_EXPECTED_KEY_SIZE.getLong();
+
+ // the offset in the keys memory region to look for a given summary boundary
+ private final SafeMemoryWriter offsets;
+ private final SafeMemoryWriter entries;
+
+ private final int minIndexInterval;
+ private final int samplingLevel;
+ private final int[] startPoints;
+ private long keysWritten = 0;
+ private long indexIntervalMatches = 0;
+ private long nextSamplePosition;
+
+ // for each ReadableBoundary, we map its dataLength property to itself, permitting us to lookup the
+ // last readable boundary from the perspective of the data file
+ // [data file position limit] => [ReadableBoundary]
+ private TreeMap<Long, ReadableBoundary> lastReadableByData = new TreeMap<>();
+ // for each ReadableBoundary, we map its indexLength property to itself, permitting us to lookup the
+ // last readable boundary from the perspective of the index file
+ // [index file position limit] => [ReadableBoundary]
+ private TreeMap<Long, ReadableBoundary> lastReadableByIndex = new TreeMap<>();
+ // the last synced data file position
+ private long dataSyncPosition;
+ // the last synced index file position
+ private long indexSyncPosition;
+
+ // the last summary interval boundary that is fully readable in both data and index files
+ private ReadableBoundary lastReadableBoundary;
+
+ /**
+ * Represents a boundary that is guaranteed fully readable in the summary, index file and data file.
+ * The key contained is the last key readable if the index and data files have been flushed to the
+ * stored lengths.
+ */
+ public static class ReadableBoundary
+ {
+ public final DecoratedKey lastKey;
+ public final long indexLength;
+ public final long dataLength;
+ public final int summaryCount;
+ public final long entriesLength;
+ public ReadableBoundary(DecoratedKey lastKey, long indexLength, long dataLength, int summaryCount, long entriesLength)
+ {
+ this.lastKey = lastKey;
+ this.indexLength = indexLength;
+ this.dataLength = dataLength;
+ this.summaryCount = summaryCount;
+ this.entriesLength = entriesLength;
+ }
+ }
+
+ /**
+ * Build an index summary builder.
+ *
+ * @param expectedKeys - the number of keys we expect in the sstable
+ * @param minIndexInterval - the minimum interval between entries selected for sampling
+ * @param samplingLevel - the level at which entries are sampled
+ */
+ public IndexSummaryBuilder(long expectedKeys, int minIndexInterval, int samplingLevel)
+ {
+ this.samplingLevel = samplingLevel;
+ this.startPoints = Downsampling.getStartPoints(BASE_SAMPLING_LEVEL, samplingLevel);
+
+ long expectedEntrySize = getEntrySize(defaultExpectedKeySize);
+ long maxExpectedEntries = expectedKeys / minIndexInterval;
+ long maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
+ if (maxExpectedEntriesSize > Integer.MAX_VALUE)
+ {
+ // that's a _lot_ of keys, and a very low min index interval
+ int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys * expectedEntrySize) / Integer.MAX_VALUE);
+ maxExpectedEntries = expectedKeys / effectiveMinInterval;
+ maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
+ assert maxExpectedEntriesSize <= Integer.MAX_VALUE : maxExpectedEntriesSize;
+ logger.warn("min_index_interval of {} is too low for {} expected keys of avg size {}; using interval of {} instead",
+ minIndexInterval, expectedKeys, defaultExpectedKeySize, effectiveMinInterval);
+ this.minIndexInterval = effectiveMinInterval;
+ }
+ else
+ {
+ this.minIndexInterval = minIndexInterval;
+ }
+
+ // for initializing data structures, adjust our estimates based on the sampling level
+ maxExpectedEntries = Math.max(1, (maxExpectedEntries * samplingLevel) / BASE_SAMPLING_LEVEL);
+ offsets = new SafeMemoryWriter(4 * maxExpectedEntries).order(ByteOrder.nativeOrder());
+ entries = new SafeMemoryWriter(expectedEntrySize * maxExpectedEntries).order(ByteOrder.nativeOrder());
+
+ // the summary will always contain the first index entry (downsampling will never remove it)
+ nextSamplePosition = 0;
+ indexIntervalMatches++;
+ }
+
+ /**
+ * Given a key, return how long the serialized index summary entry will be.
+ */
+ private static long getEntrySize(DecoratedKey key)
+ {
+ return getEntrySize(key.getKey().remaining());
+ }
+
+ /**
+ * Given a key size, return how long the serialized index summary entry will be, that is add 8 bytes to
+ * accomodate for the size of the position.
+ */
+ private static long getEntrySize(long keySize)
+ {
+ return keySize + TypeSizes.sizeof(0L);
+ }
+
+ // the index file has been flushed to the provided position; stash it and use that to recalculate our max readable boundary
+ public void markIndexSynced(long upToPosition)
+ {
+ indexSyncPosition = upToPosition;
+ refreshReadableBoundary();
+ }
+
+ // the data file has been flushed to the provided position; stash it and use that to recalculate our max readable boundary
+ public void markDataSynced(long upToPosition)
+ {
+ dataSyncPosition = upToPosition;
+ refreshReadableBoundary();
+ }
+
+ private void refreshReadableBoundary()
+ {
+ // grab the readable boundary prior to the given position in either the data or index file
+ Map.Entry<?, ReadableBoundary> byData = lastReadableByData.floorEntry(dataSyncPosition);
+ Map.Entry<?, ReadableBoundary> byIndex = lastReadableByIndex.floorEntry(indexSyncPosition);
+ if (byData == null || byIndex == null)
+ return;
+
+ // take the lowest of the two, and stash it
+ lastReadableBoundary = byIndex.getValue().indexLength < byData.getValue().indexLength
+ ? byIndex.getValue() : byData.getValue();
+
+ // clear our data prior to this, since we no longer need it
+ lastReadableByData.headMap(lastReadableBoundary.dataLength, false).clear();
+ lastReadableByIndex.headMap(lastReadableBoundary.indexLength, false).clear();
+ }
+
+ public ReadableBoundary getLastReadableBoundary()
+ {
+ return lastReadableBoundary;
+ }
+
+ public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart) throws IOException
+ {
+ return maybeAddEntry(decoratedKey, indexStart, 0, 0);
+ }
+
+ /**
+ *
+ * @param decoratedKey the key for this record
+ * @param indexStart the position in the index file this record begins
+ * @param indexEnd the position in the index file we need to be able to read to (exclusive) to read this record
+ * @param dataEnd the position in the data file we need to be able to read to (exclusive) to read this record
+ * a value of 0 indicates we are not tracking readable boundaries
+ */
+ public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart, long indexEnd, long dataEnd) throws IOException
+ {
+ if (keysWritten == nextSamplePosition)
+ {
+ if ((entries.length() + getEntrySize(decoratedKey)) <= Integer.MAX_VALUE)
+ {
+ offsets.writeInt((int) entries.length());
+ entries.write(decoratedKey.getKey());
+ entries.writeLong(indexStart);
+ setNextSamplePosition(keysWritten);
+ }
+ else
+ {
+ // we cannot fully sample this sstable due to too much memory in the index summary, so let's tell the user
+ logger.error("Memory capacity of index summary exceeded (2GiB), index summary will not cover full sstable, " +
+ "you should increase min_sampling_level");
+ }
+ }
+ else if (dataEnd != 0 && keysWritten + 1 == nextSamplePosition)
+ {
+ // this is the last key in this summary interval, so stash it
+ ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int) (offsets.length() / 4), entries.length());
+ lastReadableByData.put(dataEnd, boundary);
+ lastReadableByIndex.put(indexEnd, boundary);
+ }
+
+ keysWritten++;
+ return this;
+ }
+
+ // calculate the next key we will store to our summary
+ private void setNextSamplePosition(long position)
+ {
+ tryAgain: while (true)
+ {
+ position += minIndexInterval;
+ long test = indexIntervalMatches++;
+ for (int start : startPoints)
+ if ((test - start) % BASE_SAMPLING_LEVEL == 0)
+ continue tryAgain;
+
+ nextSamplePosition = position;
+ return;
+ }
+ }
+
+ public void prepareToCommit()
+ {
+ // this method should only be called when we've finished appending records, so we truncate the
+ // memory we're using to the exact amount required to represent it before building our summary
+ entries.trim();
+ offsets.trim();
+ }
+
+ public IndexSummary build(IPartitioner partitioner)
+ {
+ return build(partitioner, null);
+ }
+
+ // build the summary up to the provided boundary; this is backed by shared memory between
+ // multiple invocations of this build method
+ public IndexSummary build(IPartitioner partitioner, ReadableBoundary boundary)
+ {
+ assert entries.length() > 0;
+
+ int count = (int) (offsets.length() / 4);
+ long entriesLength = entries.length();
+ if (boundary != null)
+ {
+ count = boundary.summaryCount;
+ entriesLength = boundary.entriesLength;
+ }
+
+ int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) minIndexInterval);
+ assert count > 0;
+ return new IndexSummary(partitioner, offsets.currentBuffer().sharedCopy(),
+ count, entries.currentBuffer().sharedCopy(), entriesLength,
+ sizeAtFullSampling, minIndexInterval, samplingLevel);
+ }
+
+ // close the builder and release any associated memory
+ public void close()
+ {
+ entries.close();
+ offsets.close();
+ }
+
+ public Throwable close(Throwable accumulate)
+ {
+ accumulate = entries.close(accumulate);
+ accumulate = offsets.close(accumulate);
+ return accumulate;
+ }
+
+ static int entriesAtSamplingLevel(int samplingLevel, int maxSummarySize)
+ {
+ return (int) Math.ceil((samplingLevel * maxSummarySize) / (double) BASE_SAMPLING_LEVEL);
+ }
+
+ static int calculateSamplingLevel(int currentSamplingLevel, int currentNumEntries, long targetNumEntries, int minIndexInterval, int maxIndexInterval)
+ {
+ // effective index interval == (BASE_SAMPLING_LEVEL / samplingLevel) * minIndexInterval
+ // so we can just solve for minSamplingLevel here:
+ // maxIndexInterval == (BASE_SAMPLING_LEVEL / minSamplingLevel) * minIndexInterval
+ int effectiveMinSamplingLevel = Math.max(1, (int) Math.ceil((BASE_SAMPLING_LEVEL * minIndexInterval) / (double) maxIndexInterval));
+
+ // Algebraic explanation for calculating the new sampling level (solve for newSamplingLevel):
+ // originalNumEntries = (baseSamplingLevel / currentSamplingLevel) * currentNumEntries
+ // newSpaceUsed = (newSamplingLevel / baseSamplingLevel) * originalNumEntries
+ // newSpaceUsed = (newSamplingLevel / baseSamplingLevel) * (baseSamplingLevel / currentSamplingLevel) * currentNumEntries
+ // newSpaceUsed = (newSamplingLevel / currentSamplingLevel) * currentNumEntries
+ // (newSpaceUsed * currentSamplingLevel) / currentNumEntries = newSamplingLevel
+ int newSamplingLevel = (int) (targetNumEntries * currentSamplingLevel) / currentNumEntries;
+ return Math.min(BASE_SAMPLING_LEVEL, Math.max(effectiveMinSamplingLevel, newSamplingLevel));
+ }
+
+ /**
+ * Downsamples an existing index summary to a new sampling level.
+ * @param existing an existing IndexSummary
+ * @param newSamplingLevel the target level for the new IndexSummary. This must be less than the current sampling
+ * level for `existing`.
+ * @param partitioner the partitioner used for the index summary
+ * @return a new IndexSummary
+ */
+ @SuppressWarnings("resource")
+ public static IndexSummary downsample(IndexSummary existing, int newSamplingLevel, int minIndexInterval, IPartitioner partitioner)
+ {
+ // To downsample the old index summary, we'll go through (potentially) several rounds of downsampling.
+ // Conceptually, each round starts at position X and then removes every Nth item. The value of X follows
+ // a particular pattern to evenly space out the items that we remove. The value of N decreases by one each
+ // round.
+
+ int currentSamplingLevel = existing.getSamplingLevel();
+ assert currentSamplingLevel > newSamplingLevel;
+ assert minIndexInterval == existing.getMinIndexInterval();
+
+ // calculate starting indexes for downsampling rounds
+ int[] startPoints = Downsampling.getStartPoints(currentSamplingLevel, newSamplingLevel);
+
+ // calculate new off-heap size
+ int newKeyCount = existing.size();
+ long newEntriesLength = existing.getEntriesLength();
+ for (int start : startPoints)
+ {
+ for (int j = start; j < existing.size(); j += currentSamplingLevel)
+ {
+ newKeyCount--;
+ long length = existing.getEndInSummary(j) - existing.getPositionInSummary(j);
+ newEntriesLength -= length;
+ }
+ }
+
+ Memory oldEntries = existing.getEntries();
+ Memory newOffsets = Memory.allocate(newKeyCount * 4);
+ Memory newEntries = Memory.allocate(newEntriesLength);
+
+ // Copy old entries to our new Memory.
+ int i = 0;
+ int newEntriesOffset = 0;
+ outer:
+ for (int oldSummaryIndex = 0; oldSummaryIndex < existing.size(); oldSummaryIndex++)
+ {
+ // to determine if we can skip this entry, go through the starting points for our downsampling rounds
+ // and see if the entry's index is covered by that round
+ for (int start : startPoints)
+ {
+ if ((oldSummaryIndex - start) % currentSamplingLevel == 0)
+ continue outer;
+ }
+
+ // write the position of the actual entry in the index summary (4 bytes)
+ newOffsets.setInt(i * 4, newEntriesOffset);
+ i++;
+ long start = existing.getPositionInSummary(oldSummaryIndex);
+ long length = existing.getEndInSummary(oldSummaryIndex) - start;
+ newEntries.put(newEntriesOffset, oldEntries, start, length);
+ newEntriesOffset += length;
+ }
+ assert newEntriesOffset == newEntriesLength;
+ return new IndexSummary(partitioner, newOffsets, newKeyCount, newEntries, newEntriesLength,
+ existing.getMaxNumberOfEntries(), minIndexInterval, newSamplingLevel);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java
new file mode 100644
index 0000000..82dc3b8
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.WrappedRunnable;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+/**
+ * Manages the fixed-size memory pool for index summaries, periodically resizing them
+ * in order to give more memory to hot sstables and less memory to cold sstables.
+ */
+public class IndexSummaryManager<T extends SSTableReader & IndexSummarySupport<T>> implements IndexSummaryManagerMBean
+{
+ private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManager.class);
+ public static final String MBEAN_NAME = "org.apache.cassandra.db:type=IndexSummaries";
+ public static final IndexSummaryManager<?> instance;
+
+ private long memoryPoolBytes;
+
+ private final ScheduledExecutorPlus executor;
+
+ // our next scheduled resizing run
+ private ScheduledFuture future;
+
+ private final Supplier<List<T>> indexSummariesProvider;
+
+ private static <T extends SSTableReader & IndexSummarySupport> List<T> getAllSupportedReaders() {
+ List<T> readers = new ArrayList<>();
+ for (Keyspace keyspace : Keyspace.all())
+ for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+ for (SSTableReader sstr : cfs.getLiveSSTables())
+ if (sstr instanceof IndexSummarySupport)
+ readers.add(((T) sstr));
+ return readers;
+ }
+
+ static
+ {
+ instance = new IndexSummaryManager<>(IndexSummaryManager::getAllSupportedReaders);
+ MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME);
+ }
+
+ private IndexSummaryManager(Supplier<List<T>> indexSummariesProvider)
+ {
+ this.indexSummariesProvider = indexSummariesProvider;
+
+ executor = executorFactory().scheduled(false, "IndexSummaryManager", Thread.MIN_PRIORITY);
+
+ long indexSummarySizeInMB = DatabaseDescriptor.getIndexSummaryCapacityInMiB();
+ int interval = DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes();
+ logger.info("Initializing index summary manager with a memory pool size of {} MB and a resize interval of {} minutes",
+ indexSummarySizeInMB, interval);
+
+ setMemoryPoolCapacityInMB(DatabaseDescriptor.getIndexSummaryCapacityInMiB());
+ setResizeIntervalInMinutes(DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
+ }
+
+ public int getResizeIntervalInMinutes()
+ {
+ return DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes();
+ }
+
+ public void setResizeIntervalInMinutes(int resizeIntervalInMinutes)
+ {
+ int oldInterval = getResizeIntervalInMinutes();
+ DatabaseDescriptor.setIndexSummaryResizeIntervalInMinutes(resizeIntervalInMinutes);
+
+ long initialDelay;
+ if (future != null)
+ {
+ initialDelay = oldInterval < 0
+ ? resizeIntervalInMinutes
+ : Math.max(0, resizeIntervalInMinutes - (oldInterval - future.getDelay(TimeUnit.MINUTES)));
+ future.cancel(false);
+ }
+ else
+ {
+ initialDelay = resizeIntervalInMinutes;
+ }
+
+ if (resizeIntervalInMinutes < 0)
+ {
+ future = null;
+ return;
+ }
+
+ future = executor.scheduleWithFixedDelay(new WrappedRunnable()
+ {
+ protected void runMayThrow() throws Exception
+ {
+ redistributeSummaries();
+ }
+ }, initialDelay, resizeIntervalInMinutes, TimeUnit.MINUTES);
+ }
+
+ // for testing only
+ @VisibleForTesting
+ Long getTimeToNextResize(TimeUnit timeUnit)
+ {
+ if (future == null)
+ return null;
+
+ return future.getDelay(timeUnit);
+ }
+
+ public long getMemoryPoolCapacityInMB()
+ {
+ return memoryPoolBytes / 1024L / 1024L;
+ }
+
+ public Map<String, Integer> getIndexIntervals()
+ {
+ List<T> summaryProviders = indexSummariesProvider.get();
+ Map<String, Integer> intervals = new HashMap<>(summaryProviders.size());
+ for (T summaryProvider : summaryProviders)
+ intervals.put(summaryProvider.getFilename(), (int) Math.round(summaryProvider.getIndexSummary().getEffectiveIndexInterval()));
+
+ return intervals;
+ }
+
+ public double getAverageIndexInterval()
+ {
+ List<T> summaryProviders = indexSummariesProvider.get();
+ double total = 0.0;
+ for (IndexSummarySupport summaryProvider : summaryProviders)
+ total += summaryProvider.getIndexSummary().getEffectiveIndexInterval();
+ return total / summaryProviders.size();
+ }
+
+ public void setMemoryPoolCapacityInMB(long memoryPoolCapacityInMB)
+ {
+ this.memoryPoolBytes = memoryPoolCapacityInMB * 1024L * 1024L;
+ }
+
+ /**
+ * Returns the actual space consumed by index summaries for all sstables.
+ * @return space currently used in MB
+ */
+ public double getMemoryPoolSizeInMB()
+ {
+ long total = 0;
+ for (IndexSummarySupport summaryProvider : indexSummariesProvider.get())
+ total += summaryProvider.getIndexSummary().getOffHeapSize();
+ return total / 1024.0 / 1024.0;
+ }
+
+ /**
+ * Marks the non-compacting sstables as compacting for index summary redistribution for all keyspaces/tables.
+ *
+ * @return Pair containing:
+ * left: total size of the off heap index summaries for the sstables we were unable to mark compacting (they were involved in other compactions)
+ * right: the transactions, keyed by table id.
+ */
+ @SuppressWarnings("resource")
+ private Pair<Long, Map<TableId, LifecycleTransaction>> getRestributionTransactions()
+ {
+ List<SSTableReader> allCompacting = new ArrayList<>();
+ Map<TableId, LifecycleTransaction> allNonCompacting = new HashMap<>();
+ for (Keyspace ks : Keyspace.all())
+ {
+ for (ColumnFamilyStore cfStore: ks.getColumnFamilyStores())
+ {
+ Set<SSTableReader> nonCompacting, allSSTables;
+ LifecycleTransaction txn;
+ do
+ {
+ View view = cfStore.getTracker().getView();
+ allSSTables = ImmutableSet.copyOf(view.select(SSTableSet.CANONICAL));
+ nonCompacting = ImmutableSet.copyOf(view.getUncompacting(allSSTables));
+ }
+ while (null == (txn = cfStore.getTracker().tryModify(nonCompacting, OperationType.INDEX_SUMMARY)));
+
+ allNonCompacting.put(cfStore.metadata.id, txn);
+ allCompacting.addAll(Sets.difference(allSSTables, nonCompacting));
+ }
+ }
+ long nonRedistributingOffHeapSize = allCompacting.stream()
+ .filter(IndexSummarySupport.class::isInstance)
+ .map(IndexSummarySupport.class::cast)
+ .map(IndexSummarySupport::getIndexSummary)
+ .mapToLong(IndexSummary::getOffHeapSize)
+ .sum();
+ return Pair.create(nonRedistributingOffHeapSize, allNonCompacting);
+ }
+
+ public void redistributeSummaries() throws IOException
+ {
+ if (CompactionManager.instance.isGlobalCompactionPaused())
+ return;
+ Pair<Long, Map<TableId, LifecycleTransaction>> redistributionTransactionInfo = getRestributionTransactions();
+ Map<TableId, LifecycleTransaction> transactions = redistributionTransactionInfo.right;
+ long nonRedistributingOffHeapSize = redistributionTransactionInfo.left;
+ try (Timer.Context ctx = CompactionManager.instance.getMetrics().indexSummaryRedistributionTime.time())
+ {
+ redistributeSummaries(new IndexSummaryRedistribution(transactions,
+ nonRedistributingOffHeapSize,
+ this.memoryPoolBytes));
+ }
+ catch (Exception e)
+ {
+ if (e instanceof CompactionInterruptedException)
+ {
+ logger.info("Index summary interrupted: {}", e.getMessage());
+ }
+ else
+ {
+ logger.error("Got exception during index summary redistribution", e);
+ throw e;
+ }
+ }
+ finally
+ {
+ try
+ {
+ FBUtilities.closeAll(transactions.values());
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Attempts to fairly distribute a fixed pool of memory for index summaries across a set of SSTables based on
+ * their recent read rates.
+ * @param redistribution encapsulating the transactions containing the sstables we are to redistribute the
+ * memory pool across and a size (in bytes) that the total index summary space usage
+ * should stay close to or under, if possible
+ * @return a list of new SSTableReader instances
+ */
+ @VisibleForTesting
+ public static <T extends SSTableReader & IndexSummarySupport> List<T> redistributeSummaries(IndexSummaryRedistribution redistribution) throws IOException
+ {
+ return CompactionManager.instance.runAsActiveCompaction(redistribution, redistribution::redistributeSummaries);
+ }
+
+ @VisibleForTesting
+ public void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
+ {
+ if (future != null)
+ {
+ future.cancel(false);
+ future = null;
+ }
+ ExecutorUtils.shutdownAndWait(timeout, unit, executor);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerMBean.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerMBean.java
new file mode 100644
index 0000000..2179544
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerMBean.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.util.Map;
+
+public interface IndexSummaryManagerMBean
+{
+ public long getMemoryPoolCapacityInMB();
+ public void setMemoryPoolCapacityInMB(long memoryPoolCapacityInMB);
+
+ /**
+ * Returns the current actual off-heap memory usage of the index summaries for all non-compacting sstables.
+ * @return The amount of memory used in MiB.
+ */
+ public double getMemoryPoolSizeInMB();
+
+ /**
+ * Returns a map of SSTable filenames to their current effective index interval.
+ */
+ public Map<String, Integer> getIndexIntervals();
+
+ public double getAverageIndexInterval();
+
+ public void redistributeSummaries() throws IOException;
+
+ public int getResizeIntervalInMinutes();
+
+ /**
+ * Set resizeIntervalInMinutes = -1 for disabled; This is the equivalent of index_summary_resize_interval being
+ * set to null in cassandra.yaml
+ */
+ public void setResizeIntervalInMinutes(int resizeIntervalInMinutes);
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryMetrics.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryMetrics.java
new file mode 100644
index 0000000..8855e41
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryMetrics.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.cassandra.io.sstable.AbstractMetricsProviders;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class IndexSummaryMetrics<R extends SSTableReader & IndexSummarySupport<R>> extends AbstractMetricsProviders<R>
+{
+ public final static IndexSummaryMetrics<?> instance = new IndexSummaryMetrics<>();
+
+ @Override
+ protected R map(SSTableReader r)
+ {
+ if (r instanceof IndexSummarySupport<?>)
+ return (R) r;
+ return null;
+ }
+
+ private final GaugeProvider<Long> indexSummaryOffHeapMemoryUsed = newGaugeProvider("IndexSummaryOffHeapMemoryUsed",
+ 0L,
+ r -> r.getIndexSummary().getOffHeapSize(),
+ Long::sum);
+
+ private final List<GaugeProvider<?>> gaugeProviders = Arrays.asList(indexSummaryOffHeapMemoryUsed);
+
+ public List<GaugeProvider<?>> getGaugeProviders()
+ {
+ return gaugeProviders;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java
new file mode 100644
index 0000000..17d1f96
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInfo.Unit;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+public class IndexSummaryRedistribution extends CompactionInfo.Holder
+{
+ private static final Logger logger = LoggerFactory.getLogger(IndexSummaryRedistribution.class);
+
+ // The target (or ideal) number of index summary entries must differ from the actual number of
+ // entries by this ratio in order to trigger an upsample or downsample of the summary. Because
+ // upsampling requires reading the primary index in order to rebuild the summary, the threshold
+ // for upsampling is is higher.
+ static final double UPSAMPLE_THRESHOLD = 1.5;
+ static final double DOWNSAMPLE_THESHOLD = 0.75;
+
+ private final Map<TableId, LifecycleTransaction> transactions;
+ private final long nonRedistributingOffHeapSize;
+ private final long memoryPoolBytes;
+ private final TimeUUID compactionId;
+ private volatile long remainingSpace;
+
+ /**
+ *
+ * @param transactions the transactions for the different keyspaces/tables we are to redistribute
+ * @param nonRedistributingOffHeapSize the total index summary off heap size for all sstables we were not able to mark compacting (due to them being involved in other compactions)
+ * @param memoryPoolBytes size of the memory pool
+ */
+ public IndexSummaryRedistribution(Map<TableId, LifecycleTransaction> transactions, long nonRedistributingOffHeapSize, long memoryPoolBytes)
+ {
+ this.transactions = transactions;
+ this.nonRedistributingOffHeapSize = nonRedistributingOffHeapSize;
+ this.memoryPoolBytes = memoryPoolBytes;
+ this.compactionId = nextTimeUUID();
+ }
+
+ private static <T extends SSTableReader & IndexSummarySupport<T>> List<T> getIndexSummarySupportingAndCloseOthers(LifecycleTransaction txn)
+ {
+ List<T> filtered = new ArrayList<>();
+ List<SSTableReader> cancels = new ArrayList<>();
+ for (SSTableReader sstable : txn.originals())
+ {
+ if (sstable instanceof IndexSummarySupport<?>)
+ filtered.add((T) sstable);
+ else
+ cancels.add(sstable);
+ }
+ txn.cancel(cancels);
+ return filtered;
+ }
+
+ public <T extends SSTableReader & IndexSummarySupport<T>> List<T> redistributeSummaries() throws IOException
+ {
+ long start = nanoTime();
+ logger.info("Redistributing index summaries");
+ List<T> redistribute = new ArrayList<>();
+
+ for (LifecycleTransaction txn : transactions.values())
+ {
+ redistribute.addAll(getIndexSummarySupportingAndCloseOthers(txn));
+ }
+
+ long total = nonRedistributingOffHeapSize;
+ for (T sstable : redistribute)
+ total += sstable.getIndexSummary().getOffHeapSize();
+
+ logger.info("Beginning redistribution of index summaries for {} sstables with memory pool size {} MiB; current spaced used is {} MiB",
+ redistribute.size(), memoryPoolBytes / 1024L / 1024L, total / 1024.0 / 1024.0);
+
+ final Map<T, Double> readRates = new HashMap<>(redistribute.size());
+ double totalReadsPerSec = 0.0;
+ for (T sstable : redistribute)
+ {
+ if (isStopRequested())
+ throw new CompactionInterruptedException(getCompactionInfo());
+
+ if (sstable.getReadMeter() != null)
+ {
+ Double readRate = sstable.getReadMeter().fifteenMinuteRate();
+ totalReadsPerSec += readRate;
+ readRates.put(sstable, readRate);
+ }
+ }
+ logger.trace("Total reads/sec across all sstables in index summary resize process: {}", totalReadsPerSec);
+
+ // copy and sort by read rates (ascending)
+ List<T> sstablesByHotness = new ArrayList<>(redistribute);
+ Collections.sort(sstablesByHotness, new ReadRateComparator(readRates));
+
+ long remainingBytes = memoryPoolBytes - nonRedistributingOffHeapSize;
+
+ logger.trace("Index summaries for compacting SSTables are using {} MiB of space",
+ (memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
+ List<T> newSSTables;
+ try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
+ {
+ newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
+
+ for (LifecycleTransaction txn : transactions.values())
+ txn.finish();
+ }
+ total = nonRedistributingOffHeapSize;
+ for (T sstable : newSSTables)
+ total += sstable.getIndexSummary().getOffHeapSize();
+
+ logger.info("Completed resizing of index summaries; current approximate memory used: {} MiB, time spent: {}ms",
+ total / 1024.0 / 1024.0, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
+
+ return newSSTables;
+ }
+
+ private <T extends SSTableReader & IndexSummarySupport<T>> List<T> adjustSamplingLevels(List<T> sstables,
+ Map<TableId, LifecycleTransaction> transactions,
+ double totalReadsPerSec,
+ long memoryPoolCapacity) throws IOException
+ {
+ List<ResampleEntry<T>> toDownsample = new ArrayList<>(sstables.size() / 4);
+ List<ResampleEntry<T>> toUpsample = new ArrayList<>(sstables.size() / 4);
+ List<ResampleEntry<T>> forceResample = new ArrayList<>();
+ List<ResampleEntry<T>> forceUpsample = new ArrayList<>();
+ List<T> newSSTables = new ArrayList<>(sstables.size());
+
+ // Going from the coldest to the hottest sstables, try to give each sstable an amount of space proportional
+ // to the number of total reads/sec it handles.
+ remainingSpace = memoryPoolCapacity;
+ for (T sstable : sstables)
+ {
+ if (isStopRequested())
+ throw new CompactionInterruptedException(getCompactionInfo());
+
+ int minIndexInterval = sstable.metadata().params.minIndexInterval;
+ int maxIndexInterval = sstable.metadata().params.maxIndexInterval;
+
+ double readsPerSec = sstable.getReadMeter() == null ? 0.0 : sstable.getReadMeter().fifteenMinuteRate();
+ long idealSpace = Math.round(remainingSpace * (readsPerSec / totalReadsPerSec));
+
+ // figure out how many entries our idealSpace would buy us, and pick a new sampling level based on that
+ int currentNumEntries = sstable.getIndexSummary().size();
+ double avgEntrySize = sstable.getIndexSummary().getOffHeapSize() / (double) currentNumEntries;
+ long targetNumEntries = Math.max(1, Math.round(idealSpace / avgEntrySize));
+ int currentSamplingLevel = sstable.getIndexSummary().getSamplingLevel();
+ int maxSummarySize = sstable.getIndexSummary().getMaxNumberOfEntries();
+
+ // if the min_index_interval changed, calculate what our current sampling level would be under the new min
+ if (sstable.getIndexSummary().getMinIndexInterval() != minIndexInterval)
+ {
+ int effectiveSamplingLevel = (int) Math.round(currentSamplingLevel * (minIndexInterval / (double) sstable.getIndexSummary().getMinIndexInterval()));
+ maxSummarySize = (int) Math.round(maxSummarySize * (sstable.getIndexSummary().getMinIndexInterval() / (double) minIndexInterval));
+ logger.trace("min_index_interval changed from {} to {}, so the current sampling level for {} is effectively now {} (was {})",
+ sstable.getIndexSummary().getMinIndexInterval(), minIndexInterval, sstable, effectiveSamplingLevel, currentSamplingLevel);
+ currentSamplingLevel = effectiveSamplingLevel;
+ }
+
+ int newSamplingLevel = IndexSummaryBuilder.calculateSamplingLevel(currentSamplingLevel, currentNumEntries, targetNumEntries,
+ minIndexInterval, maxIndexInterval);
+ int numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, maxSummarySize);
+ double effectiveIndexInterval = sstable.getIndexSummary().getEffectiveIndexInterval();
+
+ if (logger.isTraceEnabled())
+ logger.trace("{} has {} reads/sec; ideal space for index summary: {} ({} entries); considering moving " +
+ "from level {} ({} entries, {}) " +
+ "to level {} ({} entries, {})",
+ sstable.getFilename(), readsPerSec, FBUtilities.prettyPrintMemory(idealSpace), targetNumEntries,
+ currentSamplingLevel, currentNumEntries, FBUtilities.prettyPrintMemory((long) (currentNumEntries * avgEntrySize)),
+ newSamplingLevel, numEntriesAtNewSamplingLevel, FBUtilities.prettyPrintMemory((long) (numEntriesAtNewSamplingLevel * avgEntrySize)));
+
+ if (effectiveIndexInterval < minIndexInterval)
+ {
+ // The min_index_interval was changed; re-sample to match it.
+ logger.trace("Forcing resample of {} because the current index interval ({}) is below min_index_interval ({})",
+ sstable, effectiveIndexInterval, minIndexInterval);
+ long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
+ forceResample.add(new ResampleEntry<T>(sstable, spaceUsed, newSamplingLevel));
+ remainingSpace -= spaceUsed;
+ }
+ else if (effectiveIndexInterval > maxIndexInterval)
+ {
+ // The max_index_interval was lowered; force an upsample to the effective minimum sampling level
+ logger.trace("Forcing upsample of {} because the current index interval ({}) is above max_index_interval ({})",
+ sstable, effectiveIndexInterval, maxIndexInterval);
+ newSamplingLevel = Math.max(1, (BASE_SAMPLING_LEVEL * minIndexInterval) / maxIndexInterval);
+ numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, sstable.getIndexSummary().getMaxNumberOfEntries());
+ long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
+ forceUpsample.add(new ResampleEntry<T>(sstable, spaceUsed, newSamplingLevel));
+ remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel;
+ }
+ else if (targetNumEntries >= currentNumEntries * UPSAMPLE_THRESHOLD && newSamplingLevel > currentSamplingLevel)
+ {
+ long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
+ toUpsample.add(new ResampleEntry<T>(sstable, spaceUsed, newSamplingLevel));
+ remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel;
+ }
+ else if (targetNumEntries < currentNumEntries * DOWNSAMPLE_THESHOLD && newSamplingLevel < currentSamplingLevel)
+ {
+ long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
+ toDownsample.add(new ResampleEntry<T>(sstable, spaceUsed, newSamplingLevel));
+ remainingSpace -= spaceUsed;
+ }
+ else
+ {
+ // keep the same sampling level
+ logger.trace("SSTable {} is within thresholds of ideal sampling", sstable);
+ remainingSpace -= sstable.getIndexSummary().getOffHeapSize();
+ newSSTables.add(sstable);
+ transactions.get(sstable.metadata().id).cancel(sstable);
+ }
+ totalReadsPerSec -= readsPerSec;
+ }
+
+ if (remainingSpace > 0)
+ {
+ Pair<List<T>, List<ResampleEntry<T>>> result = distributeRemainingSpace(toDownsample, remainingSpace);
+ toDownsample = result.right;
+ newSSTables.addAll(result.left);
+ for (T sstable : result.left)
+ transactions.get(sstable.metadata().id).cancel(sstable);
+ }
+
+ // downsample first, then upsample
+ logger.info("index summaries: downsample: {}, force resample: {}, upsample: {}, force upsample: {}", toDownsample.size(), forceResample.size(), toUpsample.size(), forceUpsample.size());
+ toDownsample.addAll(forceResample);
+ toDownsample.addAll(toUpsample);
+ toDownsample.addAll(forceUpsample);
+ for (ResampleEntry<T> entry : toDownsample)
+ {
+ if (isStopRequested())
+ throw new CompactionInterruptedException(getCompactionInfo());
+
+ T sstable = entry.sstable;
+ logger.trace("Re-sampling index summary for {} from {}/{} to {}/{} of the original number of entries",
+ sstable, sstable.getIndexSummary().getSamplingLevel(), Downsampling.BASE_SAMPLING_LEVEL,
+ entry.newSamplingLevel, Downsampling.BASE_SAMPLING_LEVEL);
+ ColumnFamilyStore cfs = Keyspace.open(sstable.metadata().keyspace).getColumnFamilyStore(sstable.metadata().id);
+ long oldSize = sstable.bytesOnDisk();
+ long oldSizeUncompressed = sstable.logicalBytesOnDisk();
+
+ T replacement = sstable.cloneWithNewSummarySamplingLevel(cfs, entry.newSamplingLevel);
+ long newSize = replacement.bytesOnDisk();
+ long newSizeUncompressed = replacement.logicalBytesOnDisk();
+
+ newSSTables.add(replacement);
+ transactions.get(sstable.metadata().id).update(replacement, true);
+ addHooks(cfs, transactions, oldSize, newSize, oldSizeUncompressed, newSizeUncompressed);
+ }
+
+ return newSSTables;
+ }
+
+ /**
+ * Add hooks to correctly update the storage load metrics once the transaction is closed/aborted
+ */
+ @SuppressWarnings("resource") // Transactions are closed in finally outside of this method
+ private void addHooks(ColumnFamilyStore cfs, Map<TableId, LifecycleTransaction> transactions, long oldSize, long newSize, long oldSizeUncompressed, long newSizeUncompressed)
+ {
+ LifecycleTransaction txn = transactions.get(cfs.metadata.id);
+ txn.runOnCommit(() -> {
+ // The new size will be added in Transactional.commit() as an updated SSTable, more details: CASSANDRA-13738
+ StorageMetrics.load.dec(oldSize);
+ StorageMetrics.uncompressedLoad.dec(oldSizeUncompressed);
+
+ cfs.metric.liveDiskSpaceUsed.dec(oldSize);
+ cfs.metric.uncompressedLiveDiskSpaceUsed.dec(oldSizeUncompressed);
+ cfs.metric.totalDiskSpaceUsed.dec(oldSize);
+ });
+ txn.runOnAbort(() -> {
+ // the local disk was modified but bookkeeping couldn't be commited, apply the delta
+ long delta = oldSize - newSize; // if new is larger this will be negative, so dec will become a inc
+ long deltaUncompressed = oldSizeUncompressed - newSizeUncompressed;
+
+ StorageMetrics.load.dec(delta);
+ StorageMetrics.uncompressedLoad.dec(deltaUncompressed);
+
+ cfs.metric.liveDiskSpaceUsed.dec(delta);
+ cfs.metric.uncompressedLiveDiskSpaceUsed.dec(deltaUncompressed);
+ cfs.metric.totalDiskSpaceUsed.dec(delta);
+ });
+ }
+
+ @VisibleForTesting
+ static <T extends SSTableReader & IndexSummarySupport<T>> Pair<List<T>, List<ResampleEntry<T>>> distributeRemainingSpace(List<ResampleEntry<T>> toDownsample, long remainingSpace)
+ {
+ // sort by the amount of space regained by doing the downsample operation; we want to try to avoid operations
+ // that will make little difference.
+ Collections.sort(toDownsample, Comparator.comparingDouble(o -> o.sstable.getIndexSummary().getOffHeapSize() - o.newSpaceUsed));
+
+ int noDownsampleCutoff = 0;
+ List<T> willNotDownsample = new ArrayList<>();
+ while (remainingSpace > 0 && noDownsampleCutoff < toDownsample.size())
+ {
+ ResampleEntry<T> entry = toDownsample.get(noDownsampleCutoff);
+
+ long extraSpaceRequired = entry.sstable.getIndexSummary().getOffHeapSize() - entry.newSpaceUsed;
+ // see if we have enough leftover space to keep the current sampling level
+ if (extraSpaceRequired <= remainingSpace)
+ {
+ logger.trace("Using leftover space to keep {} at the current sampling level ({})",
+ entry.sstable, entry.sstable.getIndexSummary().getSamplingLevel());
+ willNotDownsample.add(entry.sstable);
+ remainingSpace -= extraSpaceRequired;
+ }
+ else
+ {
+ break;
+ }
+
+ noDownsampleCutoff++;
+ }
+ return Pair.create(willNotDownsample, toDownsample.subList(noDownsampleCutoff, toDownsample.size()));
+ }
+
+ public CompactionInfo getCompactionInfo()
+ {
+ return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
+ }
+
+ public boolean isGlobal()
+ {
+ return true;
+ }
+
+ /** Utility class for sorting sstables by their read rates. */
+ private static class ReadRateComparator implements Comparator<SSTableReader>
+ {
+ private final Map<? extends SSTableReader, Double> readRates;
+
+ ReadRateComparator(Map<? extends SSTableReader, Double> readRates)
+ {
+ this.readRates = readRates;
+ }
+
+ @Override
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ Double readRate1 = readRates.get(o1);
+ Double readRate2 = readRates.get(o2);
+ if (readRate1 == null && readRate2 == null)
+ return 0;
+ else if (readRate1 == null)
+ return -1;
+ else if (readRate2 == null)
+ return 1;
+ else
+ return Double.compare(readRate1, readRate2);
+ }
+ }
+
+ private static class ResampleEntry<T extends SSTableReader & IndexSummarySupport<T>>
+ {
+ public final T sstable;
+ public final long newSpaceUsed;
+ public final int newSamplingLevel;
+
+ ResampleEntry(T sstable, long newSpaceUsed, int newSamplingLevel)
+ {
+ this.sstable = sstable;
+ this.newSpaceUsed = newSpaceUsed;
+ this.newSamplingLevel = newSamplingLevel;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java
new file mode 100644
index 0000000..ffad83c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public interface IndexSummarySupport<T extends SSTableReader & IndexSummarySupport<T>>
+{
+ IndexSummary getIndexSummary();
+
+ T cloneWithNewSummarySamplingLevel(ColumnFamilyStore cfs, int newSamplingLevel) throws IOException;
+
+ static boolean isSupportedBy(SSTableFormat<?, ?> format)
+ {
+ return IndexSummarySupport.class.isAssignableFrom(format.getReaderFactory().getReaderClass());
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCache.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCache.java
new file mode 100644
index 0000000..7ed1955
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCache.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.keycache;
+
+import java.util.concurrent.atomic.LongAdder;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cache.InstrumentingCache;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+
+/**
+ * A simple wrapper of possibly global cache with local metrics.
+ */
+public class KeyCache
+{
+ public static final KeyCache NO_CACHE = new KeyCache(null);
+
+ private final static Logger logger = LoggerFactory.getLogger(KeyCache.class);
+
+ private final InstrumentingCache<KeyCacheKey, AbstractRowIndexEntry> cache;
+ private final LongAdder hits = new LongAdder();
+ private final LongAdder requests = new LongAdder();
+
+ public KeyCache(@Nullable InstrumentingCache<KeyCacheKey, AbstractRowIndexEntry> cache)
+ {
+ this.cache = cache;
+ }
+
+ public long getHits()
+ {
+ return cache != null ? hits.sum() : 0;
+ }
+
+ public long getRequests()
+ {
+ return cache != null ? requests.sum() : 0;
+ }
+
+ public void put(@Nonnull KeyCacheKey cacheKey, @Nonnull AbstractRowIndexEntry info)
+ {
+ if (cache == null)
+ return;
+
+ logger.trace("Adding cache entry for {} -> {}", cacheKey, info);
+ cache.put(cacheKey, info);
+ }
+
+ public @Nullable AbstractRowIndexEntry get(KeyCacheKey key, boolean updateStats)
+ {
+ if (cache == null)
+ return null;
+
+ if (updateStats)
+ {
+ requests.increment();
+ AbstractRowIndexEntry r = cache.get(key);
+ if (r != null)
+ hits.increment();
+ return r;
+ }
+ else
+ {
+ return cache.getInternal(key);
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return cache != null;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheMetrics.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheMetrics.java
new file mode 100644
index 0000000..7ba1a68
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheMetrics.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.keycache;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.cassandra.io.sstable.AbstractMetricsProviders;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class KeyCacheMetrics<R extends SSTableReader & KeyCacheSupport<R>> extends AbstractMetricsProviders<R>
+{
+ public final static KeyCacheMetrics<?> instance = new KeyCacheMetrics<>();
+
+ @Override
+ protected R map(SSTableReader r)
+ {
+ if (r instanceof KeyCacheSupport<?>)
+ return (R) r;
+ return null;
+ }
+
+ /** Key cache hit rate for this CF */
+ private final GaugeProvider<Double> keyCacheHitRate = newGaugeProvider("KeyCacheHitRate", readers -> {
+ long hits = 0L;
+ long requests = 0L;
+ for (R sstable : readers)
+ {
+ hits += sstable.getKeyCache().getHits();
+ requests += sstable.getKeyCache().getRequests();
+ }
+
+ return (double) hits / (double) Math.max(1, requests);
+ });
+
+ private final List<GaugeProvider<?>> gaugeProviders = Arrays.asList(keyCacheHitRate);
+
+ public List<GaugeProvider<?>> getGaugeProviders()
+ {
+ return gaugeProviders;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java
new file mode 100644
index 0000000..9d5db7f
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.keycache;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
+
+public interface KeyCacheSupport<T extends SSTableReader & KeyCacheSupport<T>>
+{
+ @Nonnull
+ KeyCache getKeyCache();
+
+ /**
+ * Should quickly get a lower bound prefix from cache only if everything is already availabe in memory and does not
+ * need to be loaded from disk.
+ */
+ @Nullable
+ ClusteringBound<?> getLowerBoundPrefixFromCache(DecoratedKey partitionKey, boolean isReversed);
+
+ default @Nonnull KeyCacheKey getCacheKey(ByteBuffer key)
+ {
+ T reader = (T) this;
+ return new KeyCacheKey(reader.metadata(), reader.descriptor, key);
+ }
+
+ default @Nonnull KeyCacheKey getCacheKey(DecoratedKey key)
+ {
+ return getCacheKey(key.getKey());
+ }
+
+ /**
+ * Will return null if key is not found or cache is not available.
+ */
+ default @Nullable AbstractRowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
+ {
+ return getCachedPosition(getCacheKey(key), updateStats);
+ }
+
+ /**
+ * Will return null if key is not found or cache is not available.
+ */
+ default @Nullable AbstractRowIndexEntry getCachedPosition(KeyCacheKey key, boolean updateStats)
+ {
+ KeyCache keyCache = getKeyCache();
+ AbstractRowIndexEntry cachedEntry = keyCache.get(key, updateStats);
+ assert cachedEntry == null || cachedEntry.getSSTableFormat() == ((T) this).descriptor.version.format;
+
+ return cachedEntry;
+ }
+
+ /**
+ * Caches a key only if cache is available.
+ */
+ default void cacheKey(@Nonnull DecoratedKey key, @Nonnull AbstractRowIndexEntry info)
+ {
+ T reader = (T) this;
+ assert info.getSSTableFormat() == reader.descriptor.version.format;
+
+ KeyCacheKey cacheKey = getCacheKey(key);
+ getKeyCache().put(cacheKey, info);
+ }
+
+ @Nonnull
+ AbstractRowIndexEntry deserializeKeyCacheValue(@Nonnull DataInputPlus input) throws IOException;
+
+ static boolean isSupportedBy(SSTableFormat<?, ?> format)
+ {
+ return KeyCacheSupport.class.isAssignableFrom(format.getReaderFactory().getReaderClass());
+ }
+
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 1375331..d14ebb2 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -18,37 +18,44 @@
package org.apache.cassandra.io.sstable.metadata;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
-import java.util.List;
import java.util.Map;
import java.util.UUID;
-import com.google.common.base.Preconditions;
-
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundOrBoundary;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MurmurHash;
import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
import org.apache.cassandra.utils.streamhist.StreamingTombstoneHistogramBuilder;
+import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
public class MetadataCollector implements PartitionStatisticsCollector
{
public static final double NO_COMPRESSION_RATIO = -1.0;
- private static final ByteBuffer[] EMPTY_CLUSTERING = new ByteBuffer[0];
+
+ private long currentPartitionCells = 0;
static EstimatedHistogram defaultCellPerPartitionCountHistogram()
{
@@ -58,8 +65,9 @@
static EstimatedHistogram defaultPartitionSizeHistogram()
{
- // EH of 150 can track a max value of 1697806495183, i.e., > 1.5PB
- return new EstimatedHistogram(150);
+ // EH of 155 can track a max value of 3520571548412 i.e. 3.5TB
+ return new EstimatedHistogram(155);
+
}
static TombstoneHistogram defaultTombstoneDropTimeHistogram()
@@ -81,15 +89,18 @@
NO_COMPRESSION_RATIO,
defaultTombstoneDropTimeHistogram(),
0,
- Collections.<ByteBuffer>emptyList(),
- Collections.<ByteBuffer>emptyList(),
+ Collections.emptyList(),
+ Slice.ALL,
true,
ActiveRepairService.UNREPAIRED_SSTABLE,
-1,
-1,
null,
null,
- false);
+ false,
+ true,
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ ByteBufferUtil.EMPTY_BYTE_BUFFER);
}
protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
@@ -102,9 +113,25 @@
protected double compressionRatio = NO_COMPRESSION_RATIO;
protected StreamingTombstoneHistogramBuilder estimatedTombstoneDropTime = new StreamingTombstoneHistogramBuilder(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE, SSTable.TOMBSTONE_HISTOGRAM_SPOOL_SIZE, SSTable.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS);
protected int sstableLevel;
- private ClusteringPrefix<?> minClustering = null;
- private ClusteringPrefix<?> maxClustering = null;
+
+ /**
+ * The smallest clustering prefix for any {@link Unfiltered} in the sstable.
+ *
+ * <p>This is always either a Clustering, or a start bound (since for any end range tombstone bound, there should
+ * be a corresponding start bound that is smaller).
+ */
+ private ClusteringPrefix<?> minClustering = ClusteringBound.MAX_START;
+ /**
+ * The largest clustering prefix for any {@link Unfiltered} in the sstable.
+ *
+ * <p>This is always either a Clustering, or an end bound (since for any start range tombstone bound, there should
+ * be a corresponding end bound that is bigger).
+ */
+ private ClusteringPrefix<?> maxClustering = ClusteringBound.MIN_END;
+ private boolean clusteringInitialized = false;
+
protected boolean hasLegacyCounterShards = false;
+ private boolean hasPartitionLevelDeletions = false;
protected long totalColumnsSet;
protected long totalRows;
public int totalTombstones;
@@ -169,6 +196,13 @@
return this;
}
+ public MetadataCollector addCellPerPartitionCount()
+ {
+ estimatedCellPerPartitionCount.add(currentPartitionCells);
+ currentPartitionCells = 0;
+ return this;
+ }
+
/**
* Ratio is compressed/uncompressed and it is
* if you have 1.x then compression isn't helping
@@ -193,6 +227,7 @@
public void update(Cell<?> cell)
{
+ ++currentPartitionCells;
updateTimestamp(cell.timestamp());
updateTTL(cell.ttl());
updateLocalDeletionTime(cell.localDeletionTime());
@@ -200,6 +235,13 @@
updateTombstoneCount();
}
+ public void updatePartitionDeletion(DeletionTime dt)
+ {
+ if (!dt.isLive())
+ hasPartitionLevelDeletions = true;
+ update(dt);
+ }
+
public void update(DeletionTime dt)
{
if (!dt.isLive())
@@ -250,11 +292,54 @@
return this;
}
- public MetadataCollector updateClusteringValues(ClusteringPrefix<?> clustering)
+ public void updateClusteringValues(Clustering<?> clustering)
{
- minClustering = minClustering == null || comparator.compare(clustering, minClustering) < 0 ? clustering.minimize() : minClustering;
- maxClustering = maxClustering == null || comparator.compare(clustering, maxClustering) > 0 ? clustering.minimize() : maxClustering;
- return this;
+ if (clustering == Clustering.STATIC_CLUSTERING)
+ return;
+
+ // In case of monotonically growing stream of clusterings, we will usually require only one comparison
+ // because if we detected X is greater than the current MAX, then it cannot be lower than the current MIN
+ // at the same time. The only case when we need to update MIN when the current MAX was detected to be updated
+ // is the case when MIN was not yet initialized and still point the ClusteringBound.MAX_START
+ if (comparator.compare(clustering, maxClustering) > 0)
+ {
+ maxClustering = clustering;
+ if (minClustering == ClusteringBound.MAX_START)
+ minClustering = clustering;
+ }
+ else if (comparator.compare(clustering, minClustering) < 0)
+ {
+ minClustering = clustering;
+ }
+ }
+
+ public void updateClusteringValuesByBoundOrBoundary(ClusteringBoundOrBoundary<?> clusteringBoundOrBoundary)
+ {
+ // In a SSTable, every opening marker will be closed, so the start of a range tombstone marker will never be
+ // the maxClustering (the corresponding close might though) and there is no point in doing the comparison
+ // (and vice-versa for the close). By the same reasoning, a boundary will never be either the min or max
+ // clustering, and we can save on comparisons.
+ if (clusteringBoundOrBoundary.isBoundary())
+ return;
+
+ // see the comment in updateClusteringValues(Clustering)
+ if (comparator.compare(clusteringBoundOrBoundary, maxClustering) > 0)
+ {
+ if (clusteringBoundOrBoundary.kind().isEnd())
+ maxClustering = clusteringBoundOrBoundary;
+
+ // note that since we excluded boundaries above, there is no way that the provided clustering prefix is
+ // a start and en end at the same time
+ else if (minClustering == ClusteringBound.MAX_START)
+ minClustering = clusteringBoundOrBoundary;
+ }
+ else if (comparator.compare(clusteringBoundOrBoundary, minClustering) < 0)
+ {
+ if (clusteringBoundOrBoundary.kind().isStart())
+ minClustering = clusteringBoundOrBoundary;
+ else if (maxClustering == ClusteringBound.MIN_END)
+ maxClustering = clusteringBoundOrBoundary;
+ }
}
public void updateHasLegacyCounterShards(boolean hasLegacyCounterShards)
@@ -262,12 +347,11 @@
this.hasLegacyCounterShards = this.hasLegacyCounterShards || hasLegacyCounterShards;
}
- public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header)
+ public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey)
{
- Preconditions.checkState((minClustering == null && maxClustering == null)
- || comparator.compare(maxClustering, minClustering) >= 0);
- ByteBuffer[] minValues = minClustering != null ? minClustering.getBufferArray() : EMPTY_CLUSTERING;
- ByteBuffer[] maxValues = maxClustering != null ? maxClustering.getBufferArray() : EMPTY_CLUSTERING;
+ assert minClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || minClustering.kind().isStart();
+ assert maxClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || maxClustering.kind().isEnd();
+
Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
@@ -282,15 +366,18 @@
compressionRatio,
estimatedTombstoneDropTime.build(),
sstableLevel,
- makeList(minValues),
- makeList(maxValues),
+ comparator.subtypes(),
+ Slice.make(minClustering.retainable().asStartBound(), maxClustering.retainable().asEndBound()),
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
pendingRepair,
- isTransient));
+ isTransient,
+ hasPartitionLevelDeletions,
+ firstKey,
+ lastKey));
components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality));
components.put(MetadataType.HEADER, header.toComponent());
return components;
@@ -304,18 +391,6 @@
estimatedTombstoneDropTime.releaseBuffers();
}
- private static List<ByteBuffer> makeList(ByteBuffer[] values)
- {
- // In most case, l will be the same size than values, but it's possible for it to be smaller
- List<ByteBuffer> l = new ArrayList<ByteBuffer>(values.length);
- for (int i = 0; i < values.length; i++)
- if (values[i] == null)
- break;
- else
- l.add(values[i]);
- return l;
- }
-
public static class MinMaxLongTracker
{
private final long defaultMin;
@@ -409,4 +484,4 @@
return isSet ? max : defaultMax;
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index f13daa4..5ecb582 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -17,25 +17,33 @@
*/
package org.apache.cassandra.io.sstable.metadata;
-import org.apache.cassandra.io.util.*;
-
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.*;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
import java.util.function.UnaryOperator;
import java.util.zip.CRC32;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
@@ -113,7 +121,7 @@
{
Map<MetadataType, MetadataComponent> components;
logger.trace("Load metadata for {}", descriptor);
- File statsFile = new File(descriptor.filenameFor(Component.STATS));
+ File statsFile = descriptor.fileFor(Components.STATS);
if (!statsFile.exists())
{
logger.trace("No sstable stats for {}", descriptor);
@@ -213,8 +221,8 @@
if (actualChecksum != expectedChecksum)
{
- String filename = descriptor.filenameFor(Component.STATS);
- throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename);
+ File file = descriptor.fileFor(Components.STATS);
+ throw new CorruptSSTableException(new IOException("Checksums do not match for " + file), file);
}
}
@@ -222,7 +230,7 @@
public void mutate(Descriptor descriptor, String description, UnaryOperator<StatsMetadata> transform) throws IOException
{
if (logger.isTraceEnabled() )
- logger.trace("Mutating {} to {}", descriptor.filenameFor(Component.STATS), description);
+ logger.trace("Mutating {} to {}", descriptor.fileFor(Components.STATS), description);
mutate(descriptor, transform);
}
@@ -231,7 +239,7 @@
public void mutateLevel(Descriptor descriptor, int newLevel) throws IOException
{
if (logger.isTraceEnabled())
- logger.trace("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel);
+ logger.trace("Mutating {} to level {}", descriptor.fileFor(Components.STATS), newLevel);
mutate(descriptor, stats -> stats.mutateLevel(newLevel));
}
@@ -241,7 +249,7 @@
{
if (logger.isTraceEnabled())
logger.trace("Mutating {} to repairedAt time {} and pendingRepair {}",
- descriptor.filenameFor(Component.STATS), newRepairedAt, newPendingRepair);
+ descriptor.fileFor(Components.STATS), newRepairedAt, newPendingRepair);
mutate(descriptor, stats -> stats.mutateRepairedMetadata(newRepairedAt, newPendingRepair, isTransient));
}
@@ -257,8 +265,8 @@
public void rewriteSSTableMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> currentComponents) throws IOException
{
- String filePath = descriptor.tmpFilenameFor(Component.STATS);
- try (DataOutputStreamPlus out = new FileOutputStreamPlus(filePath))
+ File file = descriptor.tmpFileFor(Components.STATS);
+ try (DataOutputStreamPlus out = file.newOutputStream(File.WriteMode.OVERWRITE))
{
serialize(currentComponents, out, descriptor.version);
out.flush();
@@ -266,8 +274,8 @@
catch (IOException e)
{
Throwables.throwIfInstanceOf(e, FileNotFoundException.class);
- throw new FSWriteError(e, filePath);
+ throw new FSWriteError(e, file);
}
- FileUtils.renameWithConfirm(filePath, descriptor.filenameFor(Component.STATS));
+ file.move(descriptor.fileFor(Components.STATS));
}
}
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index ee5505f..db8076d 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -19,28 +19,34 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.serializers.AbstractTypeSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
import org.apache.cassandra.utils.UUIDSerializer;
+import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
/**
* SSTable metadata that always stay on heap.
@@ -62,8 +68,7 @@
public final double compressionRatio;
public final TombstoneHistogram estimatedTombstoneDropTime;
public final int sstableLevel;
- public final List<ByteBuffer> minClusteringValues;
- public final List<ByteBuffer> maxClusteringValues;
+ public final Slice coveredClustering;
public final boolean hasLegacyCounterShards;
public final long repairedAt;
public final long totalColumnsSet;
@@ -74,6 +79,23 @@
// just holds the current encoding stats to avoid allocating - it is not serialized
public final EncodingStats encodingStats;
+ // Used to serialize min/max clustering. Can be null if the metadata was deserialized from a legacy version
+ private final List<AbstractType<?>> clusteringTypes;
+
+ /**
+ * This boolean is used as an approximation of whether a given key can be guaranteed not to have partition
+ * deletions in this sstable. Obviously, this is pretty imprecise: a single partition deletion in the sstable
+ * means we have to assume _any_ key may have a partition deletion. This is still likely useful as workloads that
+ * does not use partition level deletions, or only very rarely, are probably not that rare.
+ * TODO we could replace this by a small bloom-filter instead; the only downside being that we'd have to care about
+ * the size of this bloom filters not getting out of hands, and it's a tiny bit unclear if it's worth the added
+ * complexity.
+ */
+ public final boolean hasPartitionLevelDeletions;
+
+ public final ByteBuffer firstKey;
+ public final ByteBuffer lastKey;
+
public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
EstimatedHistogram estimatedCellPerPartitionCount,
IntervalSet<CommitLogPosition> commitLogIntervals,
@@ -86,15 +108,18 @@
double compressionRatio,
TombstoneHistogram estimatedTombstoneDropTime,
int sstableLevel,
- List<ByteBuffer> minClusteringValues,
- List<ByteBuffer> maxClusteringValues,
+ List<AbstractType<?>> clusteringTypes,
+ Slice coveredClustering,
boolean hasLegacyCounterShards,
long repairedAt,
long totalColumnsSet,
long totalRows,
UUID originatingHostId,
TimeUUID pendingRepair,
- boolean isTransient)
+ boolean isTransient,
+ boolean hasPartitionLevelDeletions,
+ ByteBuffer firstKey,
+ ByteBuffer lastKey)
{
this.estimatedPartitionSize = estimatedPartitionSize;
this.estimatedCellPerPartitionCount = estimatedCellPerPartitionCount;
@@ -108,8 +133,8 @@
this.compressionRatio = compressionRatio;
this.estimatedTombstoneDropTime = estimatedTombstoneDropTime;
this.sstableLevel = sstableLevel;
- this.minClusteringValues = minClusteringValues;
- this.maxClusteringValues = maxClusteringValues;
+ this.clusteringTypes = clusteringTypes;
+ this.coveredClustering = coveredClustering;
this.hasLegacyCounterShards = hasLegacyCounterShards;
this.repairedAt = repairedAt;
this.totalColumnsSet = totalColumnsSet;
@@ -118,6 +143,9 @@
this.pendingRepair = pendingRepair;
this.isTransient = isTransient;
this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL);
+ this.hasPartitionLevelDeletions = hasPartitionLevelDeletions;
+ this.firstKey = firstKey;
+ this.lastKey = lastKey;
}
public MetadataType getType()
@@ -163,15 +191,18 @@
compressionRatio,
estimatedTombstoneDropTime,
newLevel,
- minClusteringValues,
- maxClusteringValues,
+ clusteringTypes,
+ coveredClustering,
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
pendingRepair,
- isTransient);
+ isTransient,
+ hasPartitionLevelDeletions,
+ firstKey,
+ lastKey);
}
public StatsMetadata mutateRepairedMetadata(long newRepairedAt, TimeUUID newPendingRepair, boolean newIsTransient)
@@ -188,15 +219,18 @@
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
- minClusteringValues,
- maxClusteringValues,
+ clusteringTypes,
+ coveredClustering,
hasLegacyCounterShards,
newRepairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
newPendingRepair,
- newIsTransient);
+ newIsTransient,
+ hasPartitionLevelDeletions,
+ firstKey,
+ lastKey);
}
@Override
@@ -220,13 +254,15 @@
.append(estimatedTombstoneDropTime, that.estimatedTombstoneDropTime)
.append(sstableLevel, that.sstableLevel)
.append(repairedAt, that.repairedAt)
- .append(maxClusteringValues, that.maxClusteringValues)
- .append(minClusteringValues, that.minClusteringValues)
+ .append(coveredClustering, that.coveredClustering)
.append(hasLegacyCounterShards, that.hasLegacyCounterShards)
.append(totalColumnsSet, that.totalColumnsSet)
.append(totalRows, that.totalRows)
.append(originatingHostId, that.originatingHostId)
.append(pendingRepair, that.pendingRepair)
+ .append(hasPartitionLevelDeletions, that.hasPartitionLevelDeletions)
+ .append(firstKey, that.firstKey)
+ .append(lastKey, that.lastKey)
.build();
}
@@ -247,13 +283,15 @@
.append(estimatedTombstoneDropTime)
.append(sstableLevel)
.append(repairedAt)
- .append(maxClusteringValues)
- .append(minClusteringValues)
+ .append(coveredClustering)
.append(hasLegacyCounterShards)
.append(totalColumnsSet)
.append(totalRows)
.append(originatingHostId)
.append(pendingRepair)
+ .append(hasPartitionLevelDeletions)
+ .append(firstKey)
+ .append(lastKey)
.build();
}
@@ -261,6 +299,8 @@
{
private static final Logger logger = LoggerFactory.getLogger(StatsMetadataSerializer.class);
+ private final AbstractTypeSerializer typeSerializer = new AbstractTypeSerializer();
+
public int serializedSize(Version version, StatsMetadata component) throws IOException
{
int size = 0;
@@ -270,14 +310,23 @@
size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
size += TombstoneHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
size += TypeSizes.sizeof(component.sstableLevel);
- // min column names
- size += 4;
- for (ByteBuffer value : component.minClusteringValues)
- size += 2 + value.remaining(); // with short length
- // max column names
- size += 4;
- for (ByteBuffer value : component.maxClusteringValues)
- size += 2 + value.remaining(); // with short length
+
+ if (version.hasLegacyMinMax())
+ {
+ // min column names
+ size += 4;
+ ClusteringBound<?> minClusteringValues = component.coveredClustering.start();
+ size += minClusteringValues.size() * 2 /* short length */ + minClusteringValues.dataSize();
+ // max column names
+ size += 4;
+ ClusteringBound<?> maxClusteringValues = component.coveredClustering.end();
+ size += maxClusteringValues.size() * 2 /* short length */ + maxClusteringValues.dataSize();
+ }
+ else if (version.hasImprovedMinMax())
+ {
+ size = improvedMinMaxSize(version, component, size);
+ }
+
size += TypeSizes.sizeof(component.hasLegacyCounterShards);
size += 8 + 8; // totalColumnsSet, totalRows
if (version.hasCommitLogLowerBound())
@@ -304,6 +353,31 @@
size += UUIDSerializer.serializer.serializedSize(component.originatingHostId, version.correspondingMessagingVersion());
}
+ if (version.hasPartitionLevelDeletionsPresenceMarker())
+ {
+ size += TypeSizes.sizeof(component.hasPartitionLevelDeletions);
+ }
+
+ if (version.hasImprovedMinMax() && version.hasLegacyMinMax())
+ {
+ size = improvedMinMaxSize(version, component, size);
+ }
+
+ if (version.hasKeyRange())
+ {
+ size += ByteBufferUtil.serializedSizeWithVIntLength(component.firstKey);
+ size += ByteBufferUtil.serializedSizeWithVIntLength(component.lastKey);
+ }
+
+ return size;
+ }
+
+ private int improvedMinMaxSize(Version version, StatsMetadata component, int size)
+ {
+ size += typeSerializer.serializedListSize(component.clusteringTypes);
+ size += Slice.serializer.serializedSize(component.coveredClustering,
+ version.correspondingMessagingVersion(),
+ component.clusteringTypes);
return size;
}
@@ -322,12 +396,31 @@
TombstoneHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
out.writeInt(component.sstableLevel);
out.writeLong(component.repairedAt);
- out.writeInt(component.minClusteringValues.size());
- for (ByteBuffer value : component.minClusteringValues)
- ByteBufferUtil.writeWithShortLength(value, out);
- out.writeInt(component.maxClusteringValues.size());
- for (ByteBuffer value : component.maxClusteringValues)
- ByteBufferUtil.writeWithShortLength(value, out);
+
+ if (version.hasLegacyMinMax())
+ {
+ ClusteringBound<?> minClusteringValues = component.coveredClustering.start();
+ out.writeInt(countUntilNull(minClusteringValues.getBufferArray()));
+ for (ByteBuffer value : minClusteringValues.getBufferArray())
+ {
+ if (value == null)
+ break;
+ ByteBufferUtil.writeWithShortLength(value, out);
+ }
+ ClusteringBound<?> maxClusteringValues = component.coveredClustering.end();
+ out.writeInt(countUntilNull(maxClusteringValues.getBufferArray()));
+ for (ByteBuffer value : maxClusteringValues.getBufferArray())
+ {
+ if (value == null)
+ break;
+ ByteBufferUtil.writeWithShortLength(value, out);
+ }
+ }
+ else if (version.hasImprovedMinMax())
+ {
+ serializeImprovedMinMax(version, component, out);
+ }
+
out.writeBoolean(component.hasLegacyCounterShards);
out.writeLong(component.totalColumnsSet);
@@ -368,6 +461,32 @@
out.writeByte(0);
}
}
+
+ if (version.hasPartitionLevelDeletionsPresenceMarker())
+ {
+ out.writeBoolean(component.hasPartitionLevelDeletions);
+ }
+
+ if (version.hasImprovedMinMax() && version.hasLegacyMinMax())
+ {
+ serializeImprovedMinMax(version, component, out);
+ }
+
+ if (version.hasKeyRange())
+ {
+ ByteBufferUtil.writeWithVIntLength(component.firstKey, out);
+ ByteBufferUtil.writeWithVIntLength(component.lastKey, out);
+ }
+ }
+
+ private void serializeImprovedMinMax(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
+ {
+ assert component.clusteringTypes != null;
+ typeSerializer.serializeList(component.clusteringTypes, out);
+ Slice.serializer.serialize(component.coveredClustering,
+ out,
+ version.correspondingMessagingVersion(),
+ component.clusteringTypes);
}
public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException
@@ -407,24 +526,31 @@
int sstableLevel = in.readInt();
long repairedAt = in.readLong();
- // for legacy sstables, we skip deserializing the min and max clustering value
- // to prevent erroneously excluding sstables from reads (see CASSANDRA-14861)
- int colCount = in.readInt();
- List<ByteBuffer> minClusteringValues = new ArrayList<>(colCount);
- for (int i = 0; i < colCount; i++)
+ List<AbstractType<?>> clusteringTypes = null;
+ Slice coveredClustering = Slice.ALL;
+ if (version.hasLegacyMinMax())
{
- ByteBuffer val = ByteBufferUtil.readWithShortLength(in);
- if (version.hasAccurateMinMax())
- minClusteringValues.add(val);
- }
+ // We always deserialize the min/max clustering values if they are there, but we ignore them for
+ // legacy sstables where !hasAccurateMinMax due to CASSANDRA-14861.
+ int colCount = in.readInt();
+ ByteBuffer[] minClusteringValues = new ByteBuffer[colCount];
+ for (int i = 0; i < colCount; i++)
+ minClusteringValues[i] = ByteBufferUtil.readWithShortLength(in);
- colCount = in.readInt();
- List<ByteBuffer> maxClusteringValues = new ArrayList<>(colCount);
- for (int i = 0; i < colCount; i++)
- {
- ByteBuffer val = ByteBufferUtil.readWithShortLength(in);
+ colCount = in.readInt();
+ ByteBuffer[] maxClusteringValues = new ByteBuffer[colCount];
+ for (int i = 0; i < colCount; i++)
+ maxClusteringValues[i] = ByteBufferUtil.readWithShortLength(in);
+
if (version.hasAccurateMinMax())
- maxClusteringValues.add(val);
+ coveredClustering = Slice.make(BufferClusteringBound.inclusiveStartOf(minClusteringValues),
+ BufferClusteringBound.inclusiveEndOf(maxClusteringValues));
+ }
+ else if (version.hasImprovedMinMax())
+ {
+ // improvedMinMax will be in this place when legacyMinMax is removed
+ clusteringTypes = typeSerializer.deserializeList(in);
+ coveredClustering = Slice.serializer.deserialize(in, version.correspondingMessagingVersion(), clusteringTypes);
}
boolean hasLegacyCounterShards = in.readBoolean();
@@ -438,7 +564,7 @@
if (version.hasCommitLogIntervals())
commitLogIntervals = commitLogPositionSetSerializer.deserialize(in);
else
- commitLogIntervals = new IntervalSet<CommitLogPosition>(commitLogLowerBound, commitLogUpperBound);
+ commitLogIntervals = new IntervalSet<>(commitLogLowerBound, commitLogUpperBound);
TimeUUID pendingRepair = null;
if (version.hasPendingRepair() && in.readByte() != 0)
@@ -452,6 +578,29 @@
if (version.hasOriginatingHostId() && in.readByte() != 0)
originatingHostId = UUIDSerializer.serializer.deserialize(in, 0);
+ // If not recorded, the only time we can guarantee there is no partition level deletion is if there is no
+ // deletion at all. Otherwise, we have to assume there may be some.
+ boolean hasPartitionLevelDeletions = minLocalDeletionTime != Cell.NO_DELETION_TIME;
+ if (version.hasPartitionLevelDeletionsPresenceMarker())
+ {
+ hasPartitionLevelDeletions = in.readBoolean();
+ }
+
+ if (version.hasImprovedMinMax() && version.hasLegacyMinMax())
+ {
+ // improvedMinMax will be in this place until legacyMinMax is removed
+ clusteringTypes = typeSerializer.deserializeList(in);
+ coveredClustering = Slice.serializer.deserialize(in, version.correspondingMessagingVersion(), clusteringTypes);
+ }
+
+ ByteBuffer firstKey = null;
+ ByteBuffer lastKey = null;
+ if (version.hasKeyRange())
+ {
+ firstKey = ByteBufferUtil.readWithVIntLength(in);
+ lastKey = ByteBufferUtil.readWithVIntLength(in);
+ }
+
return new StatsMetadata(partitionSizes,
columnCounts,
commitLogIntervals,
@@ -464,15 +613,25 @@
compressionRatio,
tombstoneHistogram,
sstableLevel,
- minClusteringValues,
- maxClusteringValues,
+ clusteringTypes,
+ coveredClustering,
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
pendingRepair,
- isTransient);
+ isTransient,
+ hasPartitionLevelDeletions,
+ firstKey,
+ lastKey);
}
+
+ private int countUntilNull(ByteBuffer[] bufferArray)
+ {
+ int i = ArrayUtils.indexOf(bufferArray, null);
+ return i < 0 ? bufferArray.length : i;
+ }
+
}
}
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java b/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java
new file mode 100644
index 0000000..c4b550c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalDeepTrieWriterPageAware.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * This class is a variant of {@link IncrementalTrieWriterPageAware} which is able to build even very deep
+ * tries. While the parent class uses recursion for clarity, it may end up with stack overflow for tries with
+ * very long keys. This implementation can switch processing from stack to heap at a certain depth (provided
+ * as a constructor param).
+ * <p>
+ * This class intentionally repeats code present in the parent class, both in the in-stack and on-heap versions
+ * of each of the three implemented recursive operations. Removing this repetition can cause higher stack usage
+ * and thus stack overflow failures.
+ */
+@NotThreadSafe
+public class IncrementalDeepTrieWriterPageAware<VALUE> extends IncrementalTrieWriterPageAware<VALUE>
+{
+ private final int maxRecursionDepth;
+
+ public IncrementalDeepTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest, int maxRecursionDepth)
+ {
+ super(trieSerializer, dest);
+ this.maxRecursionDepth = maxRecursionDepth;
+ }
+
+ public IncrementalDeepTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+ {
+ this(trieSerializer, dest, 64);
+ }
+
+ /**
+ * Simple framework for executing recursion using on-heap linked trace to avoid stack overruns.
+ */
+ static abstract class Recursion<NODE>
+ {
+ final Recursion<NODE> parent;
+ final NODE node;
+ final Iterator<NODE> childIterator;
+
+ Recursion(NODE node, Iterator<NODE> childIterator, Recursion<NODE> parent)
+ {
+ this.parent = parent;
+ this.node = node;
+ this.childIterator = childIterator;
+ }
+
+ /**
+ * Make a child Recursion object for the given node and initialize it as necessary to continue processing
+ * with it.
+ * <p>
+ * May return null if the recursion does not need to continue inside the child branch.
+ */
+ abstract Recursion<NODE> makeChild(NODE child);
+
+ /**
+ * Complete the processing this Recursion object.
+ * <p>
+ * Note: this method is not called for the nodes for which makeChild() returns null.
+ */
+ abstract void complete() throws IOException;
+
+ /**
+ * Complete processing of the given child (possibly retrieve data to apply to any accumulation performed
+ * in this Recursion object).
+ * <p>
+ * This is called when processing a child completes, including when recursion inside the child branch
+ * is skipped by makeChild() returning null.
+ */
+ void completeChild(NODE child)
+ {}
+
+ /**
+ * Recursive process, in depth-first order, the branch rooted at this recursion node.
+ * <p>
+ * Returns this.
+ */
+ Recursion<NODE> process() throws IOException
+ {
+ Recursion<NODE> curr = this;
+
+ while (true)
+ {
+ if (curr.childIterator.hasNext())
+ {
+ NODE child = curr.childIterator.next();
+ Recursion<NODE> childRec = curr.makeChild(child);
+ if (childRec != null)
+ curr = childRec;
+ else
+ curr.completeChild(child);
+ }
+ else
+ {
+ curr.complete();
+ Recursion<NODE> currParent = curr.parent;
+ if (currParent == null)
+ return curr;
+ currParent.completeChild(curr.node);
+ curr = currParent;
+ }
+ }
+ }
+ }
+
+ @Override
+ protected int recalcTotalSize(Node<VALUE> node, long nodePosition) throws IOException
+ {
+ return recalcTotalSizeRecursiveOnStack(node, nodePosition, 0);
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicates IncrementalTrieWriterPageAware
+ private int recalcTotalSizeRecursiveOnStack(Node<VALUE> node, long nodePosition, int depth) throws IOException
+ {
+ if (node.hasOutOfPageInBranch)
+ {
+ int sz = 0;
+ for (Node<VALUE> child : node.children)
+ {
+ if (depth < maxRecursionDepth)
+ sz += recalcTotalSizeRecursiveOnStack(child, nodePosition + sz, depth + 1);
+ else
+ sz += recalcTotalSizeRecursiveOnHeap(child, nodePosition + sz);
+ }
+ node.branchSize = sz;
+ }
+
+ // The sizing below will use the branch size calculated above. Since that can change on out-of-page in branch,
+ // we need to recalculate the size if either flag is set.
+ if (node.hasOutOfPageChildren || node.hasOutOfPageInBranch)
+ node.nodeSize = serializer.sizeofNode(node, nodePosition + node.branchSize);
+
+ return node.branchSize + node.nodeSize;
+ }
+
+ private int recalcTotalSizeRecursiveOnHeap(Node<VALUE> node, long nodePosition) throws IOException
+ {
+ if (node.hasOutOfPageInBranch)
+ new RecalcTotalSizeRecursion(node, null, nodePosition).process();
+
+ if (node.hasOutOfPageChildren || node.hasOutOfPageInBranch)
+ node.nodeSize = serializer.sizeofNode(node, nodePosition + node.branchSize);
+
+ return node.branchSize + node.nodeSize;
+ }
+
+ class RecalcTotalSizeRecursion extends Recursion<Node<VALUE>>
+ {
+ final long nodePosition;
+ int sz;
+
+ RecalcTotalSizeRecursion(Node<VALUE> node, Recursion<Node<VALUE>> parent, long nodePosition)
+ {
+ super(node, node.children.iterator(), parent);
+ sz = 0;
+ this.nodePosition = nodePosition;
+ }
+
+ @Override
+ Recursion<Node<VALUE>> makeChild(Node<VALUE> child)
+ {
+ if (child.hasOutOfPageInBranch)
+ return new RecalcTotalSizeRecursion(child, this, nodePosition + sz);
+ else
+ return null;
+ }
+
+ @Override
+ void complete()
+ {
+ node.branchSize = sz;
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicates IncrementalTrieWriterPageAware and onStack code
+ @Override
+ void completeChild(Node<VALUE> child)
+ {
+ // This will be called for nodes that were recursively processed as well as the ones that weren't.
+
+ // The sizing below will use the branch size calculated above. Since that can change on out-of-page in branch,
+ // we need to recalculate the size if either flag is set.
+ if (child.hasOutOfPageChildren || child.hasOutOfPageInBranch)
+ {
+ long childPosition = this.nodePosition + sz;
+ child.nodeSize = serializer.sizeofNode(child, childPosition + child.branchSize);
+ }
+
+ sz += child.branchSize + child.nodeSize;
+ }
+ }
+
+ @Override
+ protected long write(Node<VALUE> node) throws IOException
+ {
+ return writeRecursiveOnStack(node, 0);
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicates IncrementalTrieWriterPageAware
+ private long writeRecursiveOnStack(Node<VALUE> node, int depth) throws IOException
+ {
+ long nodePosition = dest.position();
+ for (Node<VALUE> child : node.children)
+ if (child.filePos == -1)
+ {
+ if (depth < maxRecursionDepth)
+ child.filePos = writeRecursiveOnStack(child, depth + 1);
+ else
+ child.filePos = writeRecursiveOnHeap(child);
+ }
+
+ nodePosition += node.branchSize;
+ assert dest.position() == nodePosition
+ : "Expected node position to be " + nodePosition + " but got " + dest.position() + " after writing children.\n" + dumpNode(node, dest.position());
+
+ serializer.write(dest, node, nodePosition);
+
+ assert dest.position() == nodePosition + node.nodeSize
+ || dest.paddedPosition() == dest.position() // For PartitionIndexTest.testPointerGrowth where position may jump on page boundaries.
+ : "Expected node position to be " + (nodePosition + node.nodeSize) + " but got " + dest.position() + " after writing node, nodeSize " + node.nodeSize + ".\n" + dumpNode(node, nodePosition);
+ return nodePosition;
+ }
+
+ private long writeRecursiveOnHeap(Node<VALUE> node) throws IOException
+ {
+ return new WriteRecursion(node, null).process().node.filePos;
+ }
+
+ class WriteRecursion extends Recursion<Node<VALUE>>
+ {
+ long nodePosition;
+
+ WriteRecursion(Node<VALUE> node, Recursion<Node<VALUE>> parent)
+ {
+ super(node, node.children.iterator(), parent);
+ nodePosition = dest.position();
+ }
+
+ @Override
+ Recursion<Node<VALUE>> makeChild(Node<VALUE> child)
+ {
+ if (child.filePos == -1)
+ return new WriteRecursion(child, this);
+ else
+ return null;
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicates IncrementalTrieWriterPageAware and onStack code
+ @Override
+ void complete() throws IOException
+ {
+ nodePosition = nodePosition + node.branchSize;
+ assert dest.position() == nodePosition
+ : "Expected node position to be " + nodePosition + " but got " + dest.position() + " after writing children.\n" + dumpNode(node, dest.position());
+
+ serializer.write(dest, node, nodePosition);
+
+ assert dest.position() == nodePosition + node.nodeSize
+ || dest.paddedPosition() == dest.position() // For PartitionIndexTest.testPointerGrowth where position may jump on page boundaries.
+ : "Expected node position to be " + (nodePosition + node.nodeSize) + " but got " + dest.position() + " after writing node, nodeSize " + node.nodeSize + ".\n" + dumpNode(node, nodePosition);
+
+ node.filePos = nodePosition;
+ }
+ }
+
+ @Override
+ protected long writePartial(Node<VALUE> node, DataOutputPlus dest, long baseOffset) throws IOException
+ {
+ return writePartialRecursiveOnStack(node, dest, baseOffset, 0);
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicates IncrementalTrieWriterPageAware
+ private long writePartialRecursiveOnStack(Node<VALUE> node, DataOutputPlus dest, long baseOffset, int depth) throws IOException
+ {
+ long startPosition = dest.position() + baseOffset;
+
+ List<Node<VALUE>> childrenToClear = new ArrayList<>();
+ for (Node<VALUE> child : node.children)
+ {
+ if (child.filePos == -1)
+ {
+ childrenToClear.add(child);
+ if (depth < maxRecursionDepth)
+ child.filePos = writePartialRecursiveOnStack(child, dest, baseOffset, depth + 1);
+ else
+ child.filePos = writePartialRecursiveOnHeap(child, dest, baseOffset);
+ }
+ }
+
+ long nodePosition = dest.position() + baseOffset;
+
+ if (node.hasOutOfPageInBranch)
+ {
+ // Update the branch size with the size of what we have just written. This may be used by the node's
+ // maxPositionDelta, and it's a better approximation for later fitting calculations.
+ node.branchSize = (int) (nodePosition - startPosition);
+ }
+
+ serializer.write(dest, node, nodePosition);
+
+ if (node.hasOutOfPageChildren || node.hasOutOfPageInBranch)
+ {
+ // Update the node size with what we have just seen. It's a better approximation for later fitting
+ // calculations.
+ long endPosition = dest.position() + baseOffset;
+ node.nodeSize = (int) (endPosition - nodePosition);
+ }
+
+ for (Node<VALUE> child : childrenToClear)
+ child.filePos = -1;
+ return nodePosition;
+ }
+
+ private long writePartialRecursiveOnHeap(Node<VALUE> node, DataOutputPlus dest, long baseOffset) throws IOException
+ {
+ new WritePartialRecursion(node, dest, baseOffset).process();
+ long pos = node.filePos;
+ node.filePos = -1;
+ return pos;
+ }
+
+ class WritePartialRecursion extends Recursion<Node<VALUE>>
+ {
+ final DataOutputPlus dest;
+ final long baseOffset;
+ final long startPosition;
+ final List<Node<VALUE>> childrenToClear;
+
+ WritePartialRecursion(Node<VALUE> node, WritePartialRecursion parent)
+ {
+ super(node, node.children.iterator(), parent);
+ this.dest = parent.dest;
+ this.baseOffset = parent.baseOffset;
+ this.startPosition = dest.position() + baseOffset;
+ childrenToClear = new ArrayList<>();
+ }
+
+ WritePartialRecursion(Node<VALUE> node, DataOutputPlus dest, long baseOffset)
+ {
+ super(node, node.children.iterator(), null);
+ this.dest = dest;
+ this.baseOffset = baseOffset;
+ this.startPosition = dest.position() + baseOffset;
+ childrenToClear = new ArrayList<>();
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicates IncrementalTrieWriterPageAware and onStack code
+ @Override
+ Recursion<Node<VALUE>> makeChild(Node<VALUE> child)
+ {
+ if (child.filePos == -1)
+ {
+ childrenToClear.add(child);
+ return new WritePartialRecursion(child, this);
+ }
+ else
+ return null;
+ }
+
+ @Override
+ void complete() throws IOException
+ {
+ long nodePosition = dest.position() + baseOffset;
+
+ if (node.hasOutOfPageInBranch)
+ {
+ // Update the branch size with the size of what we have just written. This may be used by the node's
+ // maxPositionDelta, and it's a better approximation for later fitting calculations.
+ node.branchSize = (int) (nodePosition - startPosition);
+ }
+
+ serializer.write(dest, node, nodePosition);
+
+ if (node.hasOutOfPageChildren || node.hasOutOfPageInBranch)
+ {
+ // Update the node size with what we have just seen. It's a better approximation for later fitting
+ // calculations.
+ long endPosition = dest.position() + baseOffset;
+ node.nodeSize = (int) (endPosition - nodePosition);
+ }
+
+ for (Node<VALUE> child : childrenToClear)
+ child.filePos = -1;
+
+ node.filePos = nodePosition;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java
new file mode 100644
index 0000000..e2c1e4c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Common interface for incremental trie writers. Incremental writers take sorted input to construct a trie file while
+ * buffering only limited amount of data.
+ * The writing itself is done by some node serializer passed on construction time.
+ * <p>
+ * See {@code org/apache/cassandra/io/sstable/format/bti/BtiFormat.md} for a description of the mechanisms of writing
+ * and reading an on-disk trie.
+ */
+public interface IncrementalTrieWriter<VALUE> extends AutoCloseable
+{
+ /**
+ * Add an entry to the trie with the associated value.
+ */
+ void add(ByteComparable next, VALUE value) throws IOException;
+
+ /**
+ * Return the number of added entries.
+ */
+ long count();
+
+ /**
+ * Complete the process and return the position in the file of the root node.
+ */
+ long complete() throws IOException;
+
+ void reset();
+
+ void close();
+
+ /**
+ * Make a temporary in-memory representation of the unwritten nodes that covers everything added to the trie until
+ * this point. The object returned represents a "tail" for the file that needs to be attached at the "cutoff" point
+ * to the file (using e.g. TailOverridingRebufferer).
+ */
+ PartialTail makePartialRoot() throws IOException;
+
+
+ interface PartialTail
+ {
+ /** Position of the root of the partial representation. Resides in the tail buffer. */
+ long root();
+ /** Number of keys written */
+ long count();
+ /** Cutoff point. Positions lower that this are to be read from the file; higher ones from the tail buffer. */
+ long cutoff();
+ /** Buffer containing in-memory representation of the tail. */
+ ByteBuffer tail();
+ }
+
+ /**
+ * Construct a suitable trie writer.
+ */
+ static <VALUE> IncrementalTrieWriter<VALUE> open(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+ {
+ return new IncrementalDeepTrieWriterPageAware<>(trieSerializer, dest);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java
new file mode 100644
index 0000000..c46099a
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterBase.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.concurrent.LightweightRecycler;
+import org.apache.cassandra.utils.concurrent.ThreadLocals;
+
+/**
+ * Helper base class for incremental trie builders.
+ */
+public abstract class IncrementalTrieWriterBase<VALUE, DEST, NODE extends IncrementalTrieWriterBase.BaseNode<VALUE, NODE>>
+implements IncrementalTrieWriter<VALUE>
+{
+ protected final Deque<NODE> stack = new ArrayDeque<>();
+ protected final TrieSerializer<VALUE, ? super DEST> serializer;
+ protected final DEST dest;
+ protected ByteComparable prev = null;
+ long count = 0;
+
+ protected IncrementalTrieWriterBase(TrieSerializer<VALUE, ? super DEST> serializer, DEST dest, NODE root)
+ {
+ this.serializer = serializer;
+ this.dest = dest;
+ this.stack.addLast(root);
+ }
+
+ protected void reset(NODE root)
+ {
+ this.prev = null;
+ this.count = 0;
+ this.stack.clear();
+ this.stack.addLast(root);
+ }
+
+
+ @Override
+ public void close()
+ {
+ this.prev = null;
+ this.count = 0;
+ this.stack.clear();
+ }
+
+ @Override
+ public void add(ByteComparable next, VALUE value) throws IOException
+ {
+ ++count;
+ int stackpos = 0;
+ ByteSource sn = next.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+ int n = sn.next();
+
+ if (prev != null)
+ {
+ ByteSource sp = prev.asComparableBytes(Walker.BYTE_COMPARABLE_VERSION);
+ int p = sp.next();
+ while ( n == p )
+ {
+ assert n != ByteSource.END_OF_STREAM : String.format("Incremental trie requires unique sorted keys, got equal %s(%s) after %s(%s).",
+ next,
+ next.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION),
+ prev,
+ prev.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION));
+
+ ++stackpos;
+ n = sn.next();
+ p = sp.next();
+ }
+ assert p < n : String.format("Incremental trie requires sorted keys, got %s(%s) after %s(%s).",
+ next,
+ next.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION),
+ prev,
+ prev.byteComparableAsString(Walker.BYTE_COMPARABLE_VERSION));
+ }
+ prev = next;
+
+ while (stack.size() > stackpos + 1)
+ completeLast();
+
+ NODE node = stack.getLast();
+ while (n != ByteSource.END_OF_STREAM)
+ {
+ node = node.addChild((byte) n);
+ stack.addLast(node);
+ ++stackpos;
+ n = sn.next();
+ }
+
+ VALUE existingPayload = node.setPayload(value);
+ assert existingPayload == null;
+ }
+
+ public long complete() throws IOException
+ {
+ NODE root = stack.getFirst();
+ if (root.filePos != -1)
+ return root.filePos;
+
+ return performCompletion().filePos;
+ }
+
+ NODE performCompletion() throws IOException
+ {
+ NODE root = null;
+ while (!stack.isEmpty())
+ root = completeLast();
+ stack.addLast(root);
+ return root;
+ }
+
+ public long count()
+ {
+ return count;
+ }
+
+ protected NODE completeLast() throws IOException
+ {
+ NODE node = stack.removeLast();
+ complete(node);
+ return node;
+ }
+
+ abstract void complete(NODE value) throws IOException;
+ abstract public PartialTail makePartialRoot() throws IOException;
+
+ static class PTail implements PartialTail
+ {
+ long root;
+ long cutoff;
+ long count;
+ ByteBuffer tail;
+
+ @Override
+ public long root()
+ {
+ return root;
+ }
+
+ @Override
+ public long cutoff()
+ {
+ return cutoff;
+ }
+
+ @Override
+ public ByteBuffer tail()
+ {
+ return tail;
+ }
+
+ @Override
+ public long count()
+ {
+ return count;
+ }
+ }
+
+ static abstract class BaseNode<VALUE, NODE extends BaseNode<VALUE, NODE>> implements SerializationNode<VALUE>
+ {
+ private static final int CHILDREN_LIST_RECYCLER_LIMIT = 1024;
+ @SuppressWarnings("rawtypes")
+ private static final LightweightRecycler<ArrayList> CHILDREN_LIST_RECYCLER = ThreadLocals.createLightweightRecycler(CHILDREN_LIST_RECYCLER_LIMIT);
+ @SuppressWarnings("rawtypes")
+ private static final ArrayList EMPTY_LIST = new ArrayList<>(0);
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private static <NODE> ArrayList<NODE> allocateChildrenList()
+ {
+ return CHILDREN_LIST_RECYCLER.reuseOrAllocate(() -> new ArrayList(4));
+ }
+
+ private static <NODE> void recycleChildrenList(ArrayList<NODE> children)
+ {
+ CHILDREN_LIST_RECYCLER.tryRecycle(children);
+ }
+
+ VALUE payload;
+ ArrayList<NODE> children;
+ final int transition;
+ long filePos = -1;
+
+ @SuppressWarnings("unchecked")
+ BaseNode(int transition)
+ {
+ children = EMPTY_LIST;
+ this.transition = transition;
+ }
+
+ public VALUE payload()
+ {
+ return payload;
+ }
+
+ public VALUE setPayload(VALUE newPayload)
+ {
+ VALUE p = payload;
+ payload = newPayload;
+ return p;
+ }
+
+ public NODE addChild(byte b)
+ {
+ assert children.isEmpty() || (children.get(children.size() - 1).transition & 0xFF) < (b & 0xFF);
+ NODE node = newNode(b);
+ if (children == EMPTY_LIST)
+ children = allocateChildrenList();
+
+ children.add(node);
+ return node;
+ }
+
+ public int childCount()
+ {
+ return children.size();
+ }
+
+ void finalizeWithPosition(long position)
+ {
+ this.filePos = position;
+
+ // Make sure we are not holding on to pointers to data we no longer need
+ // (otherwise we keep the whole trie in memory).
+ if (children != EMPTY_LIST)
+ // the recycler will also clear the collection before adding it to the pool
+ recycleChildrenList(children);
+
+ children = null;
+ payload = null;
+ }
+
+ public int transition(int i)
+ {
+ return children.get(i).transition;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%02x", transition);
+ }
+
+ abstract NODE newNode(byte transition);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java
new file mode 100644
index 0000000..2274975
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterPageAware.java
@@ -0,0 +1,467 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Incremental builders of on-disk tries which packs trie stages into disk cache pages.
+ *
+ * The incremental core is as in {@link IncrementalTrieWriterSimple}, which this augments by:
+ * <ul>
+ * <li> calculating branch sizes reflecting the amount of data that needs to be written to store the trie
+ * branch rooted at each node
+ * <li> delaying writing any part of a completed node until its branch size is above the page size
+ * <li> laying out (some of) its children branches (each smaller than a page) to be contained within a page
+ * <li> adjusting the branch size to reflect the fact that the children are now written (i.e. removing their size)
+ * </ul>
+ * <p>
+ * The process is bottom-up, i.e. pages are packed at the bottom and the root page is usually smaller.
+ * This may appear less efficient than a top-down process which puts more information in the top pages that
+ * tend to stay in cache, but in both cases performing a search will usually require an additional disk read
+ * for the leaf page. When we maximize the amount of relevant data that read brings by using the bottom-up
+ * process, we have practically the same efficiency with smaller intermediate page footprint, i.e. fewer data
+ * to keep in cache.
+ * <p>
+ * As an example, taking a sample page size fitting 4 nodes, a simple trie would be split like this:
+ * <pre>
+ * Node 0 |
+ * -a-> | Node 1
+ * | -s-> Node 2
+ * | -k-> Node 3 (payload 1)
+ * | -s-> Node 4 (payload 2)
+ * -----------------------------------
+ * -b-> Node 5 |
+ * -a-> |Node 6
+ * | -n-> Node 7
+ * | -k-> Node 8 (payload 3)
+ * | -s-> Node 9 (payload 4)
+ * </pre>
+ * where lines denote page boundaries.
+ * <p>
+ * The process itself will start by adding "ask" which adds three nodes after the root to the stack. Adding "ass"
+ * completes Node 3, setting its branch a size of 1 and replaces it on the stack with Node 4.
+ * The step of adding "bank" starts by completing Node 4 (size 1), Node 2 (size 3), Node 1 (size 4), then adds 4 more
+ * nodes to the stack. Adding "banks" descends one more node.
+ * <p>
+ * The trie completion step completes nodes 9 (size 1), 8 (size 2), 7 (size 3), 6 (size 4), 5 (size 5). Since the size
+ * of node 5 is above the page size, the algorithm lays out its children. Nodes 6, 7, 8, 9 are written in order. The
+ * size of node 5 is now just the size of it individually, 1. The process continues with completing Node 0 (size 6).
+ * This is bigger than the page size, so some of its children need to be written. The algorithm takes the largest,
+ * Node 1, and lays it out with its children in the file. Node 0 now has an adjusted size of 2 which is below the
+ * page size, and we can continue the process.
+ * <p>
+ * Since this was the root of the trie, the current page is padded and the remaining nodes 0, 5 are written.
+ */
+@NotThreadSafe
+public class IncrementalTrieWriterPageAware<VALUE>
+extends IncrementalTrieWriterBase<VALUE, DataOutputPlus, IncrementalTrieWriterPageAware.Node<VALUE>>
+implements IncrementalTrieWriter<VALUE>
+{
+ final int maxBytesPerPage;
+
+ private final static Comparator<Node<?>> BRANCH_SIZE_COMPARATOR = (l, r) ->
+ {
+ // Smaller branches first.
+ int c = Integer.compare(l.branchSize + l.nodeSize, r.branchSize + r.nodeSize);
+ if (c != 0)
+ return c;
+
+ // Then order by character, which serves several purposes:
+ // - enforces inequality to make sure equal sizes aren't treated as duplicates,
+ // - makes sure the item we use for comparison key comes greater than all equal-sized nodes,
+ // - orders equal sized items so that most recently processed (and potentially having closer children) comes
+ // last and is thus the first one picked for layout.
+ c = Integer.compare(l.transition, r.transition);
+
+ assert c != 0 || l == r;
+ return c;
+ };
+
+ IncrementalTrieWriterPageAware(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+ {
+ super(trieSerializer, dest, new Node<>((byte) 0));
+ this.maxBytesPerPage = dest.maxBytesInPage();
+ }
+
+ @Override
+ public void reset()
+ {
+ reset(new Node<>((byte) 0));
+ }
+
+ @Override
+ Node<VALUE> performCompletion() throws IOException
+ {
+ Node<VALUE> root = super.performCompletion();
+
+ int actualSize = recalcTotalSize(root, dest.position());
+ int bytesLeft = dest.bytesLeftInPage();
+ if (actualSize > bytesLeft)
+ {
+ if (actualSize <= maxBytesPerPage)
+ {
+ dest.padToPageBoundary();
+ bytesLeft = maxBytesPerPage;
+ // position changed, recalculate again
+ actualSize = recalcTotalSize(root, dest.position());
+ }
+
+ if (actualSize > bytesLeft)
+ {
+ // Still greater. Lay out children separately.
+ layoutChildren(root);
+
+ // Pad if needed and place.
+ if (root.nodeSize > dest.bytesLeftInPage())
+ {
+ dest.padToPageBoundary();
+ // Recalculate again as pointer size may have changed, triggering assertion in writeRecursive.
+ recalcTotalSize(root, dest.position());
+ }
+ }
+ }
+
+
+ root.finalizeWithPosition(write(root));
+ return root;
+ }
+
+ @Override
+ void complete(Node<VALUE> node) throws IOException
+ {
+ assert node.filePos == -1;
+
+ int branchSize = 0;
+ for (Node<VALUE> child : node.children)
+ branchSize += child.branchSize + child.nodeSize;
+
+ node.branchSize = branchSize;
+
+ int nodeSize = serializer.sizeofNode(node, dest.position());
+ if (nodeSize + branchSize < maxBytesPerPage)
+ {
+ // Good. This node and all children will (most probably) fit page.
+ node.nodeSize = nodeSize;
+ node.hasOutOfPageChildren = false;
+ node.hasOutOfPageInBranch = false;
+
+ for (Node<VALUE> child : node.children)
+ if (child.filePos != -1)
+ node.hasOutOfPageChildren = true;
+ else if (child.hasOutOfPageChildren || child.hasOutOfPageInBranch)
+ node.hasOutOfPageInBranch = true;
+
+ return;
+ }
+
+ // Cannot fit. Lay out children; The current node will be marked as one with out-of-page children.
+ layoutChildren(node);
+ }
+
+ private void layoutChildren(Node<VALUE> node) throws IOException
+ {
+ assert node.filePos == -1;
+
+ NavigableSet<Node<VALUE>> children = node.getChildrenWithUnsetPosition();
+
+ int bytesLeft = dest.bytesLeftInPage();
+ Node<VALUE> cmp = new Node<>(256); // goes after all equal-sized unplaced nodes (whose transition character is 0-255)
+ cmp.nodeSize = 0;
+ while (!children.isEmpty())
+ {
+ cmp.branchSize = bytesLeft;
+ Node<VALUE> child = children.headSet(cmp, true).pollLast(); // grab biggest that could fit
+ if (child == null)
+ {
+ dest.padToPageBoundary();
+ bytesLeft = maxBytesPerPage;
+ child = children.pollLast(); // just biggest
+ }
+
+ assert child != null;
+ if (child.hasOutOfPageChildren || child.hasOutOfPageInBranch)
+ {
+ // We didn't know what size this branch will actually need to be, node's children may be far.
+ // We now know where we would place it, so let's reevaluate size.
+ int actualSize = recalcTotalSize(child, dest.position());
+ if (actualSize > bytesLeft)
+ {
+ if (bytesLeft == maxBytesPerPage)
+ {
+ // Branch doesn't even fit in a page.
+
+ // Note: In this situation we aren't actually making the best choice as the layout should have
+ // taken place at the child (which could have made the current parent small enough to fit).
+ // This is not trivial to fix but should be very rare.
+
+ layoutChildren(child);
+ bytesLeft = dest.bytesLeftInPage();
+
+ assert (child.filePos == -1);
+ }
+
+ // Doesn't fit, but that's probably because we don't have a full page. Put it back with the new
+ // size and retry when we do have enough space.
+ children.add(child);
+ continue;
+ }
+ }
+
+ child.finalizeWithPosition(write(child));
+ bytesLeft = dest.bytesLeftInPage();
+ }
+
+ // The sizing below will use the branch size, so make sure it's set.
+ node.branchSize = 0;
+ node.hasOutOfPageChildren = true;
+ node.hasOutOfPageInBranch = false;
+ node.nodeSize = serializer.sizeofNode(node, dest.position());
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicated in IncrementalDeepTrieWriterPageAware
+ protected int recalcTotalSize(Node<VALUE> node, long nodePosition) throws IOException
+ {
+ if (node.hasOutOfPageInBranch)
+ {
+ int sz = 0;
+ for (Node<VALUE> child : node.children)
+ sz += recalcTotalSize(child, nodePosition + sz);
+ node.branchSize = sz;
+ }
+
+ // The sizing below will use the branch size calculated above. Since that can change on out-of-page in branch,
+ // we need to recalculate the size if either flag is set.
+ if (node.hasOutOfPageChildren || node.hasOutOfPageInBranch)
+ node.nodeSize = serializer.sizeofNode(node, nodePosition + node.branchSize);
+
+ return node.branchSize + node.nodeSize;
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicated in IncrementalDeepTrieWriterPageAware
+ protected long write(Node<VALUE> node) throws IOException
+ {
+ long nodePosition = dest.position();
+ for (Node<VALUE> child : node.children)
+ if (child.filePos == -1)
+ child.filePos = write(child);
+
+ nodePosition += node.branchSize;
+ assert dest.position() == nodePosition
+ : "Expected node position to be " + nodePosition + " but got " + dest.position() + " after writing children.\n" + dumpNode(node, dest.position());
+
+ serializer.write(dest, node, nodePosition);
+
+ assert dest.position() == nodePosition + node.nodeSize
+ || dest.paddedPosition() == dest.position() // For PartitionIndexTest.testPointerGrowth where position may jump on page boundaries.
+ : "Expected node position to be " + (nodePosition + node.nodeSize) + " but got " + dest.position() + " after writing node, nodeSize " + node.nodeSize + ".\n" + dumpNode(node, nodePosition);
+ return nodePosition;
+ }
+
+ protected String dumpNode(Node<VALUE> node, long nodePosition)
+ {
+ StringBuilder res = new StringBuilder(String.format("At %,d(%x) type %s child count %s nodeSize %,d branchSize %,d %s%s%n",
+ nodePosition, nodePosition,
+ TrieNode.typeFor(node, nodePosition), node.childCount(), node.nodeSize, node.branchSize,
+ node.hasOutOfPageChildren ? "C" : "",
+ node.hasOutOfPageInBranch ? "B" : ""));
+ for (Node<VALUE> child : node.children)
+ res.append(String.format("Child %2x at %,d(%x) type %s child count %s size %s nodeSize %,d branchSize %,d %s%s%n",
+ child.transition & 0xFF,
+ child.filePos,
+ child.filePos,
+ child.children != null ? TrieNode.typeFor(child, child.filePos) : "n/a",
+ child.children != null ? child.childCount() : "n/a",
+ child.children != null ? serializer.sizeofNode(child, child.filePos) : "n/a",
+ child.nodeSize,
+ child.branchSize,
+ child.hasOutOfPageChildren ? "C" : "",
+ child.hasOutOfPageInBranch ? "B" : ""));
+
+ return res.toString();
+ }
+
+ @Override
+ public PartialTail makePartialRoot() throws IOException
+ {
+ // The expectation is that the partial tail will be in memory, so we don't bother with page-fitting.
+ // We could also send some completed children to disk, but that could make suboptimal layout choices, so we'd
+ // rather not. Just write anything not written yet to a buffer, from bottom to top, and we're done.
+ try (DataOutputBuffer buf = new DataOutputBuffer())
+ {
+ PTail tail = new PTail();
+ // Readers ask rebufferers for page-aligned positions, so make sure tail starts at one.
+ // "Padding" of the cutoff point may leave some unaddressable space in the constructed file view.
+ // Nothing will point to it, though, so that's fine.
+ tail.cutoff = dest.paddedPosition();
+ tail.count = count;
+ tail.root = writePartial(stack.getFirst(), buf, tail.cutoff);
+ tail.tail = buf.asNewBuffer();
+ return tail;
+ }
+ }
+
+ @SuppressWarnings("DuplicatedCode") // intentionally duplicated in IncrementalDeepTrieWriterPageAware
+ protected long writePartial(Node<VALUE> node, DataOutputPlus dest, long baseOffset) throws IOException
+ {
+ long startPosition = dest.position() + baseOffset;
+
+ List<Node<VALUE>> childrenToClear = new ArrayList<>();
+ for (Node<VALUE> child : node.children)
+ {
+ if (child.filePos == -1)
+ {
+ childrenToClear.add(child);
+ child.filePos = writePartial(child, dest, baseOffset);
+ }
+ }
+
+ long nodePosition = dest.position() + baseOffset;
+
+ if (node.hasOutOfPageInBranch)
+ {
+ // Update the branch size with the size of what we have just written. This may be used by the node's
+ // maxPositionDelta, and it's a better approximation for later fitting calculations.
+ node.branchSize = (int) (nodePosition - startPosition);
+ }
+
+ serializer.write(dest, node, nodePosition);
+
+ if (node.hasOutOfPageChildren || node.hasOutOfPageInBranch)
+ {
+ // Update the node size with what we have just seen. It's a better approximation for later fitting
+ // calculations.
+ long endPosition = dest.position() + baseOffset;
+ node.nodeSize = (int) (endPosition - nodePosition);
+ }
+
+ for (Node<VALUE> child : childrenToClear)
+ child.filePos = -1;
+ return nodePosition;
+ }
+
+ static class Node<Value> extends IncrementalTrieWriterBase.BaseNode<Value, Node<Value>>
+ {
+ /**
+ * Currently calculated size of the branch below this node, not including the node itself.
+ * If hasOutOfPageInBranch is true, this may be underestimated as the size
+ * depends on the position the branch is written.
+ */
+ int branchSize = -1;
+ /**
+ * Currently calculated node size. If hasOutOfPageChildren is true, this may be underestimated as the size
+ * depends on the position the node is written.
+ */
+ int nodeSize = -1;
+
+ /**
+ * Whether there is an out-of-page, already written node in the branches below the immediate children of the
+ * node.
+ */
+ boolean hasOutOfPageInBranch = false;
+ /**
+ * Whether a child of the node is out of page, already written.
+ * Forced to true before being set to make sure maxPositionDelta performs its evaluation on non-completed
+ * nodes for makePartialRoot.
+ */
+ boolean hasOutOfPageChildren = true;
+
+ Node(int transition)
+ {
+ super(transition);
+ }
+
+ @Override
+ Node<Value> newNode(byte transition)
+ {
+ return new Node<>(transition & 0xFF);
+ }
+
+ public long serializedPositionDelta(int i, long nodePosition)
+ {
+ assert (children.get(i).filePos != -1);
+ return children.get(i).filePos - nodePosition;
+ }
+
+ /**
+ * The max delta is the delta with either:
+ * - the position where the first child not-yet-placed child will be laid out.
+ * - the position of the furthest child that is already placed.
+ *
+ * This method assumes all children's branch and node sizes, as well as this node's branchSize, are already
+ * calculated.
+ */
+ public long maxPositionDelta(long nodePosition)
+ {
+ // The max delta is the position the first child would be laid out.
+ assert (childCount() > 0);
+
+ if (!hasOutOfPageChildren)
+ // We need to be able to address the first child. We don't need to cover its branch, though.
+ return -(branchSize - children.get(0).branchSize);
+
+ long minPlaced = 0;
+ long minUnplaced = 1;
+ for (Node<Value> child : children)
+ {
+ if (child.filePos != -1)
+ minPlaced = Math.min(minPlaced, child.filePos - nodePosition);
+ else if (minUnplaced > 0) // triggers once
+ minUnplaced = -(branchSize - child.branchSize);
+ }
+
+ return Math.min(minPlaced, minUnplaced);
+ }
+
+ NavigableSet<Node<Value>> getChildrenWithUnsetPosition()
+ {
+ NavigableSet<Node<Value>> result = new TreeSet<>(BRANCH_SIZE_COMPARATOR);
+ for (Node<Value> child : children)
+ if (child.filePos == -1)
+ result.add(child);
+
+ return result;
+ }
+
+ @Override
+ void finalizeWithPosition(long position)
+ {
+ this.branchSize = 0; // takes no space in current page
+ this.nodeSize = 0;
+ this.hasOutOfPageInBranch = false; // its size no longer needs to be recalculated
+ this.hasOutOfPageChildren = false;
+ super.finalizeWithPosition(position);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%02x branchSize=%04x nodeSize=%04x %s%s", transition, branchSize, nodeSize, hasOutOfPageInBranch ? "B" : "", hasOutOfPageChildren ? "C" : "");
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java
new file mode 100644
index 0000000..6620b2e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/IncrementalTrieWriterSimple.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Incremental builder of on-disk tries. Takes sorted input.
+ * <p>
+ * Incremental building is done by maintaining a stack of nodes in progress which follows the path to reach the last
+ * added entry. When a new entry is needed, comparison with the previous can tell us how much of the parents stack
+ * remains the same. The rest of the stack is complete as no new entry can affect them due to the input sorting.
+ * The completed nodes can be written to disk and discarded, keeping only a pointer to their location in the file
+ * (this pointer will be discarded too when the parent node is completed). This ensures that a very limited amount of
+ * data is kept in memory at all times.
+ * <p>
+ * Note: This class is currently unused (but tested) and stands only as form of documentation for
+ * {@link IncrementalTrieWriterPageAware}.
+ */
+@NotThreadSafe
+public class IncrementalTrieWriterSimple<VALUE>
+ extends IncrementalTrieWriterBase<VALUE, DataOutputPlus, IncrementalTrieWriterSimple.Node<VALUE>>
+ implements IncrementalTrieWriter<VALUE>
+{
+ private long position = 0;
+
+ public IncrementalTrieWriterSimple(TrieSerializer<VALUE, ? super DataOutputPlus> trieSerializer, DataOutputPlus dest)
+ {
+ super(trieSerializer, dest, new Node<>((byte) 0));
+ }
+
+ @Override
+ protected void complete(Node<VALUE> node) throws IOException
+ {
+ long nodePos = position;
+ position += write(node, dest, position);
+ node.finalizeWithPosition(nodePos);
+ }
+
+ @Override
+ public void reset()
+ {
+ reset(new Node<>((byte) 0));
+ position = 0;
+ }
+
+ @Override
+ public PartialTail makePartialRoot() throws IOException
+ {
+ try (DataOutputBuffer buf = new DataOutputBuffer())
+ {
+ PTail tail = new PTail();
+ tail.cutoff = position;
+ tail.count = count;
+ long nodePos = position;
+ for (Node<VALUE> node : (Iterable<Node<VALUE>>) stack::descendingIterator)
+ {
+ node.filePos = nodePos;
+ nodePos += write(node, buf, nodePos);
+ // Hacky but works: temporarily write node's position. Will be overwritten when we finalize node.
+ }
+
+ tail.tail = buf.asNewBuffer();
+ tail.root = stack.getFirst().filePos;
+
+ for (Node<VALUE> node : (Iterable<Node<VALUE>>) stack::descendingIterator)
+ node.filePos = -1;
+
+ return tail;
+ }
+ }
+
+ private long write(Node<VALUE> node, DataOutputPlus dest, long nodePosition) throws IOException
+ {
+ long size = serializer.sizeofNode(node, nodePosition);
+ serializer.write(dest, node, nodePosition);
+ return size;
+ }
+
+ static class Node<Value> extends IncrementalTrieWriterBase.BaseNode<Value, Node<Value>>
+ {
+ Node(int transition)
+ {
+ super(transition);
+ }
+
+ @Override
+ Node<Value> newNode(byte transition)
+ {
+ return new Node<>(transition & 0xFF);
+ }
+
+ public long serializedPositionDelta(int i, long nodePosition)
+ {
+ assert children.get(i).filePos != -1;
+ return children.get(i).filePos - nodePosition;
+ }
+
+ public long maxPositionDelta(long nodePosition)
+ {
+ long min = 0;
+ for (Node<Value> child : children)
+ {
+ if (child.filePos != -1)
+ min = Math.min(min, child.filePos - nodePosition);
+ }
+ return min;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java b/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java
new file mode 100644
index 0000000..27c199a
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Thread-unsafe reverse value iterator for on-disk tries. Uses the assumptions of {@link Walker}.
+ * <p>
+ * The main utility of this class is the {@link #nextPayloadedNode()} method, which lists all nodes that contain a
+ * payload within the requested bounds. The treatment of the bounds is non-standard (see
+ * {@link #ReverseValueIterator(Rebufferer, long, ByteComparable, ByteComparable, boolean)}), necessary to properly walk
+ * tries of prefixes and separators.
+ */
+@NotThreadSafe
+public class ReverseValueIterator<Concrete extends ReverseValueIterator<Concrete>> extends Walker<Concrete>
+{
+ static final int NOT_AT_LIMIT = Integer.MIN_VALUE;
+ private final ByteSource limit;
+ private IterationPosition stack;
+ private long next;
+ private boolean reportingPrefixes;
+
+ static class IterationPosition
+ {
+ final long node;
+ final int limit;
+ final IterationPosition prev;
+ int childIndex;
+
+ public IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
+ {
+ super();
+ this.node = node;
+ this.childIndex = childIndex;
+ this.limit = limit;
+ this.prev = prev;
+ }
+ }
+
+ protected ReverseValueIterator(Rebufferer source, long root)
+ {
+ super(source, root);
+ limit = null;
+ initializeNoRightBound(root, NOT_AT_LIMIT, false);
+ }
+
+ /**
+ * Constrained iterator. The end position is always treated as inclusive, and we have two possible treatments for
+ * the start:
+ * <ul>
+ * <li> When {@code admitPrefix=false}, exact matches and any prefixes of the start are excluded.
+ * <li> When {@code admitPrefix=true}, the longest prefix of the start present in the trie is also included,
+ * provided that there is no entry in the trie between that prefix and the start. An exact match also
+ * satisfies this and is included.
+ * </ul>
+ * This behaviour is shared with the forward counterpart {@link ValueIterator}.
+ */
+ protected ReverseValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, boolean admitPrefix)
+ {
+ super(source, root);
+ limit = start != null ? start.asComparableBytes(BYTE_COMPARABLE_VERSION) : null;
+
+ if (end != null)
+ initializeWithRightBound(root, end.asComparableBytes(BYTE_COMPARABLE_VERSION), admitPrefix, limit != null);
+ else
+ initializeNoRightBound(root, limit != null ? limit.next() : NOT_AT_LIMIT, admitPrefix);
+ }
+
+ void initializeWithRightBound(long root, ByteSource endStream, boolean admitPrefix, boolean hasLimit)
+ {
+ IterationPosition prev = null;
+ boolean atLimit = hasLimit;
+ int childIndex;
+ int limitByte;
+ reportingPrefixes = admitPrefix;
+
+ // Follow end position while we still have a prefix, stacking path.
+ go(root);
+ while (true)
+ {
+ int s = endStream.next();
+ childIndex = search(s);
+
+ limitByte = NOT_AT_LIMIT;
+ if (atLimit)
+ {
+ limitByte = limit.next();
+ if (s > limitByte)
+ atLimit = false;
+ }
+ if (childIndex < 0)
+ break;
+
+ prev = new IterationPosition(position, childIndex, limitByte, prev);
+ go(transition(childIndex)); // childIndex is positive, this transition must exist
+ }
+
+ // Advancing now gives us first match.
+ childIndex = -1 - childIndex;
+ stack = new IterationPosition(position, childIndex, limitByte, prev);
+ next = advanceNode();
+ }
+
+ private void initializeNoRightBound(long root, int limitByte, boolean admitPrefix)
+ {
+ go(root);
+ stack = new IterationPosition(root, -1 - search(256), limitByte, null);
+ next = advanceNode();
+ reportingPrefixes = admitPrefix;
+ }
+
+
+
+ /**
+ * Returns the position of the next node with payload contained in the iterated span.
+ */
+ protected long nextPayloadedNode()
+ {
+ long toReturn = next;
+ if (next != -1)
+ next = advanceNode();
+ return toReturn;
+ }
+
+ long advanceNode()
+ {
+ if (stack == null)
+ return -1;
+
+ long child;
+ int transitionByte;
+
+ go(stack.node);
+ while (true)
+ {
+ // advance position in node
+ int childIdx = stack.childIndex - 1;
+ boolean beyondLimit = true;
+ if (childIdx >= 0)
+ {
+ transitionByte = transitionByte(childIdx);
+ beyondLimit = transitionByte < stack.limit;
+ if (beyondLimit)
+ {
+ assert stack.limit >= 0; // we are at a limit position (not in a node that's completely within the span)
+ reportingPrefixes = false; // there exists a smaller child than limit, no longer should report prefixes
+ }
+ }
+ else
+ transitionByte = Integer.MIN_VALUE;
+
+ if (beyondLimit)
+ {
+ // ascend to parent, remove from stack
+ IterationPosition stackTop = stack;
+ stack = stack.prev;
+
+ // Report payloads on the way up
+ // unless we are at limit and there has been a smaller child
+ if (payloadFlags() != 0)
+ {
+ // If we are fully inside the covered space, report.
+ // Note that on the exact match of the limit, stackTop.limit would be END_OF_STREAM.
+ // This comparison rejects the exact match; if we wanted to include it, we could test < 0 instead.
+ if (stackTop.limit == NOT_AT_LIMIT)
+ return stackTop.node;
+ else if (reportingPrefixes)
+ {
+ reportingPrefixes = false; // if we are at limit position only report one prefix, the closest
+ return stackTop.node;
+ }
+ // else skip this payload
+ }
+
+ if (stack == null) // exhausted whole trie
+ return NONE;
+ go(stack.node);
+ continue;
+ }
+
+ child = transition(childIdx);
+ if (child != NONE)
+ {
+ go(child);
+
+ stack.childIndex = childIdx;
+
+ // descend, stack up position
+ int l = NOT_AT_LIMIT;
+ if (transitionByte == stack.limit)
+ l = limit.next();
+
+ stack = new IterationPosition(child, transitionRange(), l, stack);
+ }
+ else
+ {
+ stack.childIndex = childIdx;
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/SerializationNode.java b/src/java/org/apache/cassandra/io/tries/SerializationNode.java
new file mode 100644
index 0000000..aae7a4f
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/SerializationNode.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+/**
+ * An abstraction of a node given to the trie serializer to write.
+ */
+public interface SerializationNode<VALUE>
+{
+ /**
+ * The number of children of the node.
+ */
+ int childCount();
+
+ /**
+ * The payload of the node if the node has any associated, otherwise null.
+ */
+ VALUE payload();
+
+ /**
+ * The transition character for the child at position i. Must be an integer between 0 and 255.
+ */
+ int transition(int i);
+
+ /**
+ * Returns the distance between this node's position and the child at index i.
+ * Given as a difference calculation to be able to handle two different types of calls:
+ * - writing nodes where all children's positions are already completely determined
+ * - sizing and writing branches within a page where we don't know where we'll actually place
+ * the nodes, but we know how far backward the child nodes will end up
+ */
+ long serializedPositionDelta(int i, long nodePosition);
+
+ /**
+ * Returns the furthest distance that needs to be written to store this node, i.e.
+ * min(serializedPositionDelta(i, nodePosition) for 0 <= i < childCount())
+ * Given separately as the loop above can be inefficient (e.g. when children are not yet written).
+ */
+ long maxPositionDelta(long nodePosition);
+}
diff --git a/src/java/org/apache/cassandra/io/tries/TrieNode.java b/src/java/org/apache/cassandra/io/tries/TrieNode.java
new file mode 100644
index 0000000..7da4ee3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/TrieNode.java
@@ -0,0 +1,987 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.SizedInts;
+
+/**
+ * Trie node types and manipulation mechanisms. The main purpose of this is to allow for handling tries directly as
+ * they are on disk without any serialization, and to enable the creation of such files.
+ * <p>
+ * The serialization methods take as argument a generic {@code SerializationNode} and provide a method {@code typeFor}
+ * for choosing a suitable type to represent it, which can then be used to calculate size and write the node.
+ * <p>
+ * To read a file containing trie nodes, one would use {@code at} to identify the node type and then the various
+ * read methods to retrieve the data. They all take a buffer (usually memory-mapped) containing the data, and a position
+ * in it that identifies the node.
+ * <p>
+ * These node types do not specify any treatment of payloads. They are only concerned with providing 4 bits of
+ * space for {@code payloadFlags}, and a way of calculating the position after the node. Users of this class by convention
+ * use non-zero payloadFlags to indicate a payload exists, write it (possibly in flag-dependent format) at serialization
+ * time after the node itself is written, and read it using the {@code payloadPosition} value.
+ * <p>
+ * To improve efficiency, multiple node types depending on the number of transitions are provided:
+ * -- payload only, which has no outgoing transitions
+ * -- single outgoing transition
+ * -- sparse, which provides a list of transition bytes with corresponding targets
+ * -- dense, where the transitions span a range of values and having the list (and the search in it) can be avoided
+ * <p>
+ * For each of the transition-carrying types we also have "in-page" versions where transition targets are the 4, 8 or 12
+ * lowest bits of the position within the same page. To save one further byte, the single in-page versions using 4 or 12
+ * bits cannot carry a payload.
+ * <p>
+ * This class is effectively an enumeration; abstract class permits instances to extend each other and reuse code.
+ * <p>
+ * See {@code org/apache/cassandra/io/sstable/format/bti/BtiFormat.md} for a description of the mechanisms of writing
+ * and reading an on-disk trie.
+ */
+@SuppressWarnings({ "SameParameterValue" })
+public abstract class TrieNode
+{
+ /** Value used to indicate a branch (e.g. for transition and lastTransition) does not exist. */
+ public static int NONE = -1;
+
+ // Consumption (read) methods
+
+ /**
+ * Returns the type of node stored at this position. It can then be used to call the methods below.
+ */
+ public static TrieNode at(ByteBuffer src, int position)
+ {
+ return Types.values[(src.get(position) >> 4) & 0xF];
+ }
+
+ /**
+ * Returns the 4 payload flag bits. Node types that cannot carry a payload return 0.
+ */
+ public int payloadFlags(ByteBuffer src, int position)
+ {
+ return src.get(position) & 0x0F;
+ }
+
+ /**
+ * Return the position just after the node, where the payload is usually stored.
+ */
+ abstract public int payloadPosition(ByteBuffer src, int position);
+
+ /**
+ * Returns search index for the given byte in the node. If exact match is present, this is >= 0, otherwise as in
+ * binary search.
+ */
+ abstract public int search(ByteBuffer src, int position, int transitionByte); // returns as binarySearch
+
+ /**
+ * Returns the upper childIndex limit. Calling transition with values 0...transitionRange - 1 is valid.
+ */
+ abstract public int transitionRange(ByteBuffer src, int position);
+
+ /**
+ * Returns the byte value for this child index, or Integer.MAX_VALUE if there are no transitions with this index or
+ * higher to permit listing the children without needing to call transitionRange.
+ *
+ * @param childIndex must be >= 0, though it is allowed to pass a value greater than {@code transitionRange - 1}
+ */
+ abstract public int transitionByte(ByteBuffer src, int position, int childIndex);
+
+ /**
+ * Returns the delta between the position of this node and the position of the target of the specified transition.
+ * This is always a negative number. Dense nodes use 0 to specify "no transition".
+ *
+ * @param childIndex must be >= 0 and < {@link #transitionRange(ByteBuffer, int)} - note that this is not validated
+ * and behaviour of this method is undefined for values outside of that range
+ */
+ abstract long transitionDelta(ByteBuffer src, int position, int childIndex);
+
+ /**
+ * Returns position of node to transition to for the given search index. Argument must be positive. May return NONE
+ * if a transition with that index does not exist (DENSE nodes).
+ * Position is the offset of the node within the ByteBuffer. positionLong is its global placement, which is the
+ * base for any offset calculations.
+ *
+ * @param positionLong although it seems to be obvious, this argument must be "real", that is, each child must have
+ * the calculated absolute position >= 0, otherwise the behaviour of this method is undefined
+ * @param childIndex must be >= 0 and < {@link #transitionRange(ByteBuffer, int)} - note that this is not validated
+ * and behaviour of this method is undefined for values outside of that range
+ */
+ public long transition(ByteBuffer src, int position, long positionLong, int childIndex)
+ {
+ // note: this is not valid for dense nodes
+ return positionLong + transitionDelta(src, position, childIndex);
+ }
+
+ /**
+ * Returns the highest transition for this node, or NONE if none exist (PAYLOAD_ONLY nodes).
+ */
+ public long lastTransition(ByteBuffer src, int position, long positionLong)
+ {
+ return transition(src, position, positionLong, transitionRange(src, position) - 1);
+ }
+
+ /**
+ * Returns a transition that is higher than the index returned by {@code search}. This may not exist (if the
+ * argument was higher than the last transition byte), in which case this returns the given {@code defaultValue}.
+ */
+ abstract public long greaterTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue);
+
+ /**
+ * Returns a transition that is lower than the index returned by {@code search}. Returns {@code defaultValue} for
+ * {@code searchIndex} equals 0 or -1 as lesser transition for those indexes does not exist.
+ */
+ abstract public long lesserTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue);
+
+ // Construction (serialization) methods
+
+ /**
+ * Returns a node type that is suitable to store the node.
+ */
+ public static TrieNode typeFor(SerializationNode<?> node, long nodePosition)
+ {
+ int c = node.childCount();
+ if (c == 0)
+ return Types.PAYLOAD_ONLY;
+
+ int bitsPerPointerIndex = 0;
+ long delta = node.maxPositionDelta(nodePosition);
+ assert delta < 0;
+ while (!Types.singles[bitsPerPointerIndex].fits(-delta))
+ ++bitsPerPointerIndex;
+
+ if (c == 1)
+ {
+ if (node.payload() != null && Types.singles[bitsPerPointerIndex].bytesPerPointer == FRACTIONAL_BYTES)
+ ++bitsPerPointerIndex; // next index will permit payload
+
+ return Types.singles[bitsPerPointerIndex];
+ }
+
+ TrieNode sparse = Types.sparses[bitsPerPointerIndex];
+ TrieNode dense = Types.denses[bitsPerPointerIndex];
+ return (sparse.sizeofNode(node) < dense.sizeofNode(node)) ? sparse : dense;
+ }
+
+ /**
+ * Returns the size needed to serialize this node.
+ */
+ abstract public int sizeofNode(SerializationNode<?> node);
+
+ /**
+ * Serializes the node. All transition target positions must already have been defined. {@code payloadBits} must
+ * be four bits.
+ */
+ abstract public void serialize(DataOutputPlus out, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException;
+
+ // Implementations
+
+ final int bytesPerPointer;
+ static final int FRACTIONAL_BYTES = 0;
+
+ TrieNode(int ordinal, int bytesPerPointer)
+ {
+ this.ordinal = ordinal;
+ this.bytesPerPointer = bytesPerPointer;
+ }
+
+ final int ordinal;
+
+ static private class PayloadOnly extends TrieNode
+ {
+ // byte flags
+ // var payload
+ PayloadOnly(int ordinal)
+ {
+ super(ordinal, FRACTIONAL_BYTES);
+ }
+
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ return position + 1;
+ }
+
+ @Override
+ public int search(ByteBuffer src, int position, int transitionByte)
+ {
+ return -1;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return 0;
+ }
+
+ @Override
+ public long transition(ByteBuffer src, int position, long positionLong, int childIndex)
+ {
+ return NONE;
+ }
+
+ @Override
+ public long lastTransition(ByteBuffer src, int position, long positionLong)
+ {
+ return NONE;
+ }
+
+ @Override
+ public long greaterTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ return defaultValue;
+ }
+
+ @Override
+ public long lesserTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ return defaultValue;
+ }
+
+ @Override
+ public int transitionByte(ByteBuffer src, int position, int childIndex)
+ {
+ return Integer.MAX_VALUE;
+ }
+
+ @Override
+ public int transitionRange(ByteBuffer src, int position)
+ {
+ return 0;
+ }
+
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ return 1;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ dest.writeByte((ordinal << 4) + (payloadBits & 0x0F));
+ }
+ }
+
+ static private class Single extends TrieNode
+ {
+ // byte flags
+ // byte transition
+ // bytesPerPointer bytes transition target
+ // var payload
+
+ Single(int ordinal, int bytesPerPointer)
+ {
+ super(ordinal, bytesPerPointer);
+ }
+
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ return position + 2 + bytesPerPointer;
+ }
+
+ @Override
+ public int search(ByteBuffer src, int position, int transitionByte)
+ {
+ int c = src.get(position + 1) & 0xFF;
+ if (transitionByte == c)
+ return 0;
+ return transitionByte < c ? -1 : -2;
+ }
+
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -readBytes(src, position + 2);
+ }
+
+ @Override
+ public long lastTransition(ByteBuffer src, int position, long positionLong)
+ {
+ return transition(src, position, positionLong, 0);
+ }
+
+ @Override
+ public long greaterTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ return (searchIndex == -1) ? transition(src, position, positionLong, 0) : defaultValue;
+ }
+
+ @Override
+ public long lesserTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ return searchIndex == 0 || searchIndex == -1 ? defaultValue : transition(src, position, positionLong, 0);
+ }
+
+ @Override
+ public int transitionByte(ByteBuffer src, int position, int childIndex)
+ {
+ return childIndex == 0 ? src.get(position + 1) & 0xFF : Integer.MAX_VALUE;
+ }
+
+ @Override
+ public int transitionRange(ByteBuffer src, int position)
+ {
+ return 1;
+ }
+
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ return 2 + bytesPerPointer;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ int childCount = node.childCount();
+ assert childCount == 1;
+ dest.writeByte((ordinal << 4) + (payloadBits & 0x0F));
+
+ dest.writeByte(node.transition(0));
+ writeBytes(dest, -node.serializedPositionDelta(0, nodePosition));
+ }
+ }
+
+ static private class SingleNoPayload4 extends Single
+ {
+ // 4-bit type ordinal
+ // 4-bit target delta
+ // byte transition
+ // no payload!
+ SingleNoPayload4(int ordinal)
+ {
+ super(ordinal, FRACTIONAL_BYTES);
+ }
+
+ @Override
+ public int payloadFlags(ByteBuffer src, int position)
+ {
+ return 0;
+ }
+
+ // Although we don't have a payload position, provide one for calculating the size of the node.
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ return position + 2;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -(src.get(position) & 0xF);
+ }
+
+ @Override
+ boolean fits(long delta)
+ {
+ return 0 <= delta && delta <= 0xF;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ assert payloadBits == 0;
+ int childCount = node.childCount();
+ assert childCount == 1;
+ long pd = -node.serializedPositionDelta(0, nodePosition);
+ assert pd > 0 && pd < 0x10;
+ dest.writeByte((ordinal << 4) + (int) (pd & 0x0F));
+ dest.writeByte(node.transition(0));
+ }
+
+ @Override
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ return 2;
+ }
+ }
+
+ static private class SingleNoPayload12 extends Single
+ {
+ // 4-bit type ordinal
+ // 12-bit target delta
+ // byte transition
+ // no payload!
+ SingleNoPayload12(int ordinal)
+ {
+ super(ordinal, FRACTIONAL_BYTES);
+ }
+
+ @Override
+ public int payloadFlags(ByteBuffer src, int position)
+ {
+ return 0;
+ }
+
+ // Although we don't have a payload position, provide one for calculating the size of the node.
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ return position + 3;
+ }
+
+ @Override
+ public int search(ByteBuffer src, int position, int transitionByte)
+ {
+ int c = src.get(position + 2) & 0xFF;
+ if (transitionByte == c)
+ return 0;
+ return transitionByte < c ? -1 : -2;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -(src.getShort(position) & 0xFFF);
+ }
+
+ @Override
+ public int transitionByte(ByteBuffer src, int position, int childIndex)
+ {
+ return childIndex == 0 ? src.get(position + 2) & 0xFF : Integer.MAX_VALUE;
+ }
+
+ @Override
+ boolean fits(long delta)
+ {
+ return 0 <= delta && delta <= 0xFFF;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ assert payloadBits == 0;
+ int childCount = node.childCount();
+ assert childCount == 1;
+ long pd = -node.serializedPositionDelta(0, nodePosition);
+ assert pd > 0 && pd < 0x1000;
+ dest.writeByte((ordinal << 4) + (int) ((pd >> 8) & 0x0F));
+ dest.writeByte((byte) pd);
+ dest.writeByte(node.transition(0));
+ }
+
+ @Override
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ return 3;
+ }
+ }
+
+ static private class Sparse extends TrieNode
+ {
+ // byte flags
+ // byte count (<= 255)
+ // count bytes transitions
+ // count ints transition targets
+ // var payload
+
+ Sparse(int ordinal, int bytesPerPointer)
+ {
+ super(ordinal, bytesPerPointer);
+ }
+
+ @Override
+ public int transitionRange(ByteBuffer src, int position)
+ {
+ return src.get(position + 1) & 0xFF;
+ }
+
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ int count = transitionRange(src, position);
+ return position + 2 + (bytesPerPointer + 1) * count;
+ }
+
+ @Override
+ public int search(ByteBuffer src, int position, int key)
+ {
+ int l = -1; // known < key
+ int r = transitionRange(src, position); // known > key
+ position += 2;
+
+ while (l + 1 < r)
+ {
+ int m = (l + r + 1) / 2;
+ int childTransition = src.get(position + m) & 0xFF;
+ int cmp = Integer.compare(key, childTransition);
+ if (cmp < 0)
+ r = m;
+ else if (cmp > 0)
+ l = m;
+ else
+ return m;
+ }
+
+ return -r - 1;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ assert childIndex >= 0;
+ int range = transitionRange(src, position);
+ assert childIndex < range;
+ return -readBytes(src, position + 2 + range + bytesPerPointer * childIndex);
+ }
+
+ @Override
+ public long greaterTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ if (searchIndex < 0)
+ searchIndex = -1 - searchIndex;
+ else
+ ++searchIndex;
+ if (searchIndex >= transitionRange(src, position))
+ return defaultValue;
+ return transition(src, position, positionLong, searchIndex);
+ }
+
+ public long lesserTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ if (searchIndex == 0 || searchIndex == -1)
+ return defaultValue;
+ if (searchIndex < 0)
+ searchIndex = -2 - searchIndex;
+ else
+ --searchIndex;
+ return transition(src, position, positionLong, searchIndex);
+ }
+
+ @Override
+ public int transitionByte(ByteBuffer src, int position, int childIndex)
+ {
+ return childIndex < transitionRange(src, position) ? src.get(position + 2 + childIndex) & 0xFF : Integer.MAX_VALUE;
+ }
+
+ @Override
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ return 2 + node.childCount() * (1 + bytesPerPointer);
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ int childCount = node.childCount();
+ assert childCount > 0;
+ assert childCount < 256;
+ dest.writeByte((ordinal << 4) + (payloadBits & 0x0F));
+ dest.writeByte(childCount);
+
+ for (int i = 0; i < childCount; ++i)
+ dest.writeByte(node.transition(i));
+ for (int i = 0; i < childCount; ++i)
+ writeBytes(dest, -node.serializedPositionDelta(i, nodePosition));
+ }
+ }
+
+ static private class Sparse12 extends Sparse
+ {
+ // byte flags
+ // byte count (<= 255)
+ // count bytes transitions
+ // count 12-bits transition targets
+ // var payload
+ Sparse12(int ordinal)
+ {
+ super(ordinal, FRACTIONAL_BYTES);
+ }
+
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ int count = transitionRange(src, position);
+ return position + 2 + (5 * count + 1) / 2;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -read12Bits(src, position + 2 + transitionRange(src, position), childIndex);
+ }
+
+ @Override
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ return 2 + (node.childCount() * 5 + 1) / 2;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ int childCount = node.childCount();
+ assert childCount < 256;
+ dest.writeByte((ordinal << 4) + (payloadBits & 0x0F));
+ dest.writeByte(childCount);
+
+ for (int i = 0; i < childCount; ++i)
+ dest.writeByte(node.transition(i));
+ int i;
+ for (i = 0; i + 2 <= childCount; i += 2)
+ {
+ int p0 = (int) -node.serializedPositionDelta(i, nodePosition);
+ int p1 = (int) -node.serializedPositionDelta(i + 1, nodePosition);
+ assert p0 > 0 && p0 < (1 << 12);
+ assert p1 > 0 && p1 < (1 << 12);
+ dest.writeByte(p0 >> 4);
+ dest.writeByte((p0 << 4) | (p1 >> 8));
+ dest.writeByte(p1);
+ }
+ if (i < childCount)
+ {
+ long pd = -node.serializedPositionDelta(i, nodePosition);
+ assert pd > 0 && pd < (1 << 12);
+ dest.writeShort((short) (pd << 4));
+ }
+ }
+
+ @Override
+ boolean fits(long delta)
+ {
+ return 0 <= delta && delta <= 0xFFF;
+ }
+ }
+
+ static private class Dense extends TrieNode
+ {
+ // byte flags
+ // byte start
+ // byte length-1
+ // length ints transition targets (-1 for not present)
+ // var payload
+
+ static final int NULL_VALUE = 0;
+
+ Dense(int ordinal, int bytesPerPointer)
+ {
+ super(ordinal, bytesPerPointer);
+ }
+
+ @Override
+ public int transitionRange(ByteBuffer src, int position)
+ {
+ return 1 + (src.get(position + 2) & 0xFF);
+ }
+
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ return position + 3 + transitionRange(src, position) * bytesPerPointer;
+ }
+
+ @Override
+ public int search(ByteBuffer src, int position, int transitionByte)
+ {
+ int l = src.get(position + 1) & 0xFF;
+ int i = transitionByte - l;
+ if (i < 0)
+ return -1;
+ int len = transitionRange(src, position);
+ if (i >= len)
+ return -len - 1;
+ long t = transition(src, position, 0L, i);
+ return t != -1 ? i : -i - 1;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -readBytes(src, position + 3 + childIndex * bytesPerPointer);
+ }
+
+ @Override
+ public long transition(ByteBuffer src, int position, long positionLong, int childIndex)
+ {
+ long v = transitionDelta(src, position, childIndex);
+ return v != NULL_VALUE ? v + positionLong : NONE;
+ }
+
+ @Override
+ public long greaterTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ if (searchIndex < 0)
+ searchIndex = -1 - searchIndex;
+ else
+ ++searchIndex;
+ int len = transitionRange(src, position);
+ for (; searchIndex < len; ++searchIndex)
+ {
+ long t = transition(src, position, positionLong, searchIndex);
+ if (t != NONE)
+ return t;
+ }
+ return defaultValue;
+ }
+
+ @Override
+ public long lesserTransition(ByteBuffer src, int position, long positionLong, int searchIndex, long defaultValue)
+ {
+ if (searchIndex == 0 || searchIndex == -1)
+ return defaultValue;
+
+ if (searchIndex < 0)
+ searchIndex = -2 - searchIndex;
+ else
+ --searchIndex;
+ for (; searchIndex >= 0; --searchIndex)
+ {
+ long t = transition(src, position, positionLong, searchIndex);
+ if (t != -1)
+ return t;
+ }
+ assert false : "transition must always exist at 0, and we should not be called for less of that";
+ return defaultValue;
+ }
+
+ @Override
+ public int transitionByte(ByteBuffer src, int position, int childIndex)
+ {
+ if (childIndex >= transitionRange(src, position))
+ return Integer.MAX_VALUE;
+ int l = src.get(position + 1) & 0xFF;
+ return l + childIndex;
+ }
+
+ @Override
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ int l = node.transition(0);
+ int r = node.transition(node.childCount() - 1);
+ return 3 + (r - l + 1) * bytesPerPointer;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ int childCount = node.childCount();
+ dest.writeByte((ordinal << 4) + (payloadBits & 0x0F));
+ int l = node.transition(0);
+ int r = node.transition(childCount - 1);
+ assert 0 <= l && l <= r && r <= 255;
+ dest.writeByte(l);
+ dest.writeByte(r - l); // r is included, i.e. this is len - 1
+
+ for (int i = 0; i < childCount; ++i)
+ {
+ int next = node.transition(i);
+ while (l < next)
+ {
+ writeBytes(dest, NULL_VALUE);
+ ++l;
+ }
+ writeBytes(dest, -node.serializedPositionDelta(i, nodePosition));
+ ++l;
+ }
+ }
+ }
+
+ static private class Dense12 extends Dense
+ {
+ // byte flags
+ // byte start
+ // byte length-1
+ // length 12-bits transition targets (-1 for not present)
+ // var payload
+
+ Dense12(int ordinal)
+ {
+ super(ordinal, FRACTIONAL_BYTES);
+ }
+
+ @Override
+ public int payloadPosition(ByteBuffer src, int position)
+ {
+ return position + 3 + (transitionRange(src, position) * 3 + 1) / 2;
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -read12Bits(src, position + 3, childIndex);
+ }
+
+ @Override
+ public int sizeofNode(SerializationNode<?> node)
+ {
+ int l = node.transition(0);
+ int r = node.transition(node.childCount() - 1);
+ return 3 + ((r - l + 1) * 3 + 1) / 2;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus dest, SerializationNode<?> node, int payloadBits, long nodePosition) throws IOException
+ {
+ int childCount = node.childCount();
+ dest.writeByte((ordinal << 4) + (payloadBits & 0x0F));
+ int l = node.transition(0);
+ int r = node.transition(childCount - 1);
+ assert 0 <= l && l <= r && r <= 255;
+ dest.writeByte(l);
+ dest.writeByte(r - l); // r is included, i.e. this is len - 1
+
+ int carry = 0;
+ int start = l;
+ for (int i = 0; i < childCount; ++i)
+ {
+ int next = node.transition(i);
+ while (l < next)
+ {
+ carry = write12Bits(dest, NULL_VALUE, l - start, carry);
+ ++l;
+ }
+ long pd = node.serializedPositionDelta(i, nodePosition);
+ carry = write12Bits(dest, (int) -pd, l - start, carry);
+ ++l;
+ }
+ if (((l - start) & 1) == 1)
+ dest.writeByte(carry);
+ }
+
+ @Override
+ boolean fits(long delta)
+ {
+ return 0 <= delta && delta <= 0xFFF;
+ }
+ }
+
+ static private class LongDense extends Dense
+ {
+ // byte flags
+ // byte start
+ // byte length-1
+ // length long transition targets (-1 for not present)
+ // var payload
+ LongDense(int ordinal)
+ {
+ super(ordinal, 8);
+ }
+
+ @Override
+ public long transitionDelta(ByteBuffer src, int position, int childIndex)
+ {
+ return -src.getLong(position + 3 + childIndex * 8);
+ }
+
+ @Override
+ public void writeBytes(DataOutputPlus dest, long ofs) throws IOException
+ {
+ dest.writeLong(ofs);
+ }
+
+ @Override
+ boolean fits(long delta)
+ {
+ return true;
+ }
+ }
+
+
+ static int read12Bits(ByteBuffer src, int base, int searchIndex)
+ {
+ int word = src.getShort(base + (3 * searchIndex) / 2);
+ if ((searchIndex & 1) == 0)
+ word = (word >> 4);
+ return word & 0xFFF;
+ }
+
+ static int write12Bits(DataOutput dest, int value, int index, int carry) throws IOException
+ {
+ assert 0 <= value && value <= 0xFFF;
+ if ((index & 1) == 0)
+ {
+ dest.writeByte(value >> 4);
+ return value << 4;
+ }
+ else
+ {
+ dest.writeByte(carry | (value >> 8));
+ dest.writeByte(value);
+ return 0;
+ }
+ }
+
+ long readBytes(ByteBuffer src, int position)
+ {
+ return SizedInts.readUnsigned(src, position, bytesPerPointer);
+ }
+
+ void writeBytes(DataOutputPlus dest, long ofs) throws IOException
+ {
+ assert fits(ofs);
+ SizedInts.write(dest, ofs, bytesPerPointer);
+ }
+
+ boolean fits(long delta)
+ {
+ return 0 <= delta && delta < (1L << (bytesPerPointer * 8));
+ }
+
+ @Override
+ public String toString()
+ {
+ String res = getClass().getSimpleName();
+ if (bytesPerPointer >= 1)
+ res += (bytesPerPointer * 8);
+ return res;
+ }
+
+ static class Types
+ {
+ static final TrieNode PAYLOAD_ONLY = new PayloadOnly(0);
+ static final TrieNode SINGLE_NOPAYLOAD_4 = new SingleNoPayload4(1);
+ static final TrieNode SINGLE_8 = new Single(2, 1);
+ static final TrieNode SINGLE_NOPAYLOAD_12 = new SingleNoPayload12(3);
+ static final TrieNode SINGLE_16 = new Single(4, 2);
+ static final TrieNode SPARSE_8 = new Sparse(5, 1);
+ static final TrieNode SPARSE_12 = new Sparse12(6);
+ static final TrieNode SPARSE_16 = new Sparse(7, 2);
+ static final TrieNode SPARSE_24 = new Sparse(8, 3);
+ static final TrieNode SPARSE_40 = new Sparse(9, 5);
+ static final TrieNode DENSE_12 = new Dense12(10);
+ static final TrieNode DENSE_16 = new Dense(11, 2);
+ static final TrieNode DENSE_24 = new Dense(12, 3);
+ static final TrieNode DENSE_32 = new Dense(13, 4);
+ static final TrieNode DENSE_40 = new Dense(14, 5);
+ static final TrieNode LONG_DENSE = new LongDense(15);
+
+ // The position of each type in this list must match its ordinal value. Checked by the static block below.
+ static final TrieNode[] values = new TrieNode[]{ PAYLOAD_ONLY,
+ SINGLE_NOPAYLOAD_4, SINGLE_8, SINGLE_NOPAYLOAD_12, SINGLE_16,
+ SPARSE_8, SPARSE_12, SPARSE_16, SPARSE_24, SPARSE_40,
+ DENSE_12, DENSE_16, DENSE_24, DENSE_32, DENSE_40,
+ LONG_DENSE }; // Catch-all
+
+ // We can't fit all types * all sizes in 4 bits, so we use a selection. When we don't have a matching instance
+ // we just use something more general that can do its job.
+ // The arrays below must have corresponding types for all sizes specified by the singles row.
+ // Note: 12 bit sizes are important, because that size will fit any pointer within a page-packed branch.
+ static final TrieNode[] singles = new TrieNode[]{ SINGLE_NOPAYLOAD_4, SINGLE_8, SINGLE_NOPAYLOAD_12, SINGLE_16, DENSE_24, DENSE_32, DENSE_40, LONG_DENSE };
+ static final TrieNode[] sparses = new TrieNode[]{ SPARSE_8, SPARSE_8, SPARSE_12, SPARSE_16, SPARSE_24, SPARSE_40, SPARSE_40, LONG_DENSE };
+ static final TrieNode[] denses = new TrieNode[]{ DENSE_12, DENSE_12, DENSE_12, DENSE_16, DENSE_24, DENSE_32, DENSE_40, LONG_DENSE };
+
+ static
+ {
+ //noinspection ConstantConditions
+ assert sparses.length == singles.length && denses.length == singles.length && values.length <= 16;
+ for (int i = 0; i < values.length; ++i)
+ assert values[i].ordinal == i;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/TrieSerializer.java b/src/java/org/apache/cassandra/io/tries/TrieSerializer.java
new file mode 100644
index 0000000..e0010cd
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/TrieSerializer.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+
+public interface TrieSerializer<VALUE, DEST>
+{
+ int sizeofNode(SerializationNode<VALUE> node, long nodePosition);
+
+ // Only called after all children's serializedPositions have been set.
+ void write(DEST dest, SerializationNode<VALUE> node, long nodePosition) throws IOException;
+}
diff --git a/src/java/org/apache/cassandra/io/tries/ValueIterator.java b/src/java/org/apache/cassandra/io/tries/ValueIterator.java
new file mode 100644
index 0000000..a6e0cd7
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/ValueIterator.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Thread-unsafe value iterator for on-disk tries. Uses the assumptions of {@link Walker}.
+ * <p>
+ * The main utility of this class is the {@link #nextPayloadedNode()} method, which lists all nodes that contain a
+ * payload within the requested bounds. The treatment of the bounds is non-standard (see
+ * {@link #ValueIterator(Rebufferer, long, ByteComparable, ByteComparable, boolean)}), necessary to properly walk
+ * tries of prefixes and separators.
+ */
+@NotThreadSafe
+public class ValueIterator<CONCRETE extends ValueIterator<CONCRETE>> extends Walker<CONCRETE>
+{
+ private final ByteSource limit;
+ private IterationPosition stack;
+ private long next;
+
+ static class IterationPosition
+ {
+ final long node;
+ final int limit;
+ final IterationPosition prev;
+ int childIndex;
+
+ IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
+ {
+ super();
+ this.node = node;
+ this.childIndex = childIndex;
+ this.limit = limit;
+ this.prev = prev;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("[Node %d, child %d, limit %d]", node, childIndex, limit);
+ }
+ }
+
+ protected ValueIterator(Rebufferer source, long root)
+ {
+ super(source, root);
+ limit = null;
+ initializeNoLeftBound(root, 256);
+ }
+
+ /**
+ * Constrained iterator. The end position is always treated as inclusive, and we have two possible treatments for
+ * the start:
+ * <ul>
+ * <li> When {@code admitPrefix=false}, exact matches and any prefixes of the start are excluded.
+ * <li> When {@code admitPrefix=true}, the longest prefix of the start present in the trie is also included,
+ * provided that there is no entry in the trie between that prefix and the start. An exact match also
+ * satisfies this and is included.
+ * </ul>
+ * This behaviour is shared with the reverse counterpart {@link ReverseValueIterator}.
+ */
+ protected ValueIterator(Rebufferer source, long root, ByteComparable start, ByteComparable end, boolean admitPrefix)
+ {
+ super(source, root);
+ limit = end != null ? end.asComparableBytes(BYTE_COMPARABLE_VERSION) : null;
+
+ if (start != null)
+ initializeWithLeftBound(root, start.asComparableBytes(BYTE_COMPARABLE_VERSION), admitPrefix, limit != null);
+ else
+ initializeNoLeftBound(root, limit != null ? limit.next() : 256);
+ }
+
+ private void initializeWithLeftBound(long root, ByteSource startStream, boolean admitPrefix, boolean atLimit)
+ {
+ IterationPosition prev = null;
+ int childIndex;
+ int limitByte;
+ long payloadedNode = -1;
+
+ try
+ {
+ // Follow start position while we still have a prefix, stacking path and saving prefixes.
+ go(root);
+ while (true)
+ {
+ int s = startStream.next();
+ childIndex = search(s);
+
+ // For a separator trie the latest payload met along the prefix is a potential match for start
+ if (admitPrefix)
+ {
+ if (childIndex == 0 || childIndex == -1)
+ {
+ if (payloadFlags() != 0)
+ payloadedNode = position;
+ }
+ else
+ {
+ payloadedNode = -1;
+ }
+ }
+
+ limitByte = 256;
+ if (atLimit)
+ {
+ limitByte = limit.next();
+ if (s < limitByte)
+ atLimit = false;
+ }
+ if (childIndex < 0)
+ break;
+
+ prev = new IterationPosition(position, childIndex, limitByte, prev);
+ go(transition(childIndex)); // child index is positive, transition must exist
+ }
+
+ childIndex = -1 - childIndex - 1;
+ stack = new IterationPosition(position, childIndex, limitByte, prev);
+
+ // Advancing now gives us first match if we didn't find one already.
+ if (payloadedNode != -1)
+ next = payloadedNode;
+ else
+ next = advanceNode();
+ }
+ catch (Throwable t)
+ {
+ super.close();
+ throw t;
+ }
+ }
+
+ private void initializeNoLeftBound(long root, int limitByte)
+ {
+ stack = new IterationPosition(root, -1, limitByte, null);
+
+ try
+ {
+ go(root);
+ if (payloadFlags() != 0)
+ next = root;
+ else
+ next = advanceNode();
+ }
+ catch (Throwable t)
+ {
+ super.close();
+ throw t;
+ }
+ }
+
+ /**
+ * Returns the position of the next node with payload contained in the iterated span.
+ */
+ protected long nextPayloadedNode()
+ {
+ long toReturn = next;
+ if (next != -1)
+ next = advanceNode();
+ return toReturn;
+ }
+
+ private long advanceNode()
+ {
+ long child;
+ int transitionByte;
+
+ go(stack.node);
+ while (true)
+ {
+ int childIndex = stack.childIndex + 1;
+ transitionByte = transitionByte(childIndex);
+
+ if (transitionByte > stack.limit)
+ {
+ // ascend
+ stack = stack.prev;
+ if (stack == null) // exhausted whole trie
+ return -1;
+ go(stack.node);
+ continue;
+ }
+
+ child = transition(childIndex);
+
+ if (child != NONE)
+ {
+ assert child >= 0 : String.format("Expected value >= 0 but got %d - %s", child, this);
+
+ // descend
+ go(child);
+
+ int l = 256;
+ if (transitionByte == stack.limit)
+ l = limit.next();
+
+ stack.childIndex = childIndex;
+ stack = new IterationPosition(child, -1, l, stack);
+
+ if (payloadFlags() != 0)
+ return child;
+ }
+ else
+ {
+ stack.childIndex = childIndex;
+ }
+ }
+
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/tries/Walker.java b/src/java/org/apache/cassandra/io/tries/Walker.java
new file mode 100644
index 0000000..9c70cec
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/tries/Walker.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.PrintStream;
+import java.nio.ByteBuffer;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.Rebufferer.BufferHolder;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Thread-unsafe trie walking helper. This is analogous to {@link org.apache.cassandra.io.util.RandomAccessReader} for
+ * tries -- takes an on-disk trie accessible via a supplied Rebufferer and lets user seek to nodes and work with them.
+ * <p>
+ * Assumes data was written using page-aware builder and thus no node crosses a page and thus a buffer boundary.
+ * <p>
+ * See {@code org/apache/cassandra/io/sstable/format/bti/BtiFormat.md} for a description of the mechanisms of writing
+ * and reading an on-disk trie.
+ */
+@NotThreadSafe
+public class Walker<CONCRETE extends Walker<CONCRETE>> implements AutoCloseable
+{
+ /** Value used to indicate a branch (e.g. lesser/greaterBranch) does not exist. */
+ public static int NONE = TrieNode.NONE;
+
+ private final Rebufferer source;
+ protected final long root;
+
+ // State relating to current node.
+ private BufferHolder bh; // from Rebufferer
+ private int offset; // offset of current node within buf
+ protected TrieNode nodeType; // type of current node
+ protected ByteBuffer buf; // buffer containing the data
+ protected long position; // file position of current node
+
+ // State relating to searches.
+ protected long greaterBranch;
+ protected long lesserBranch;
+
+ // Version of the byte comparable conversion to use
+ public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS50;
+
+ /**
+ * Creates a walker. Rebufferer must be aligned and with a buffer size that is at least 4k.
+ */
+ public Walker(Rebufferer source, long root)
+ {
+ this.source = source;
+ this.root = root;
+ try
+ {
+ bh = source.rebuffer(root);
+ buf = bh.buffer();
+ }
+ catch (RuntimeException ex)
+ {
+ if (bh != null) bh.release();
+ source.closeReader();
+ throw ex;
+ }
+ }
+
+ public void close()
+ {
+ bh.release();
+ source.closeReader();
+ }
+
+ protected final void go(long position)
+ {
+ long curOffset = position - bh.offset();
+ if (curOffset < 0 || curOffset >= buf.limit())
+ {
+ bh.release();
+ bh = Rebufferer.EMPTY; // prevents double release if the call below fails
+ bh = source.rebuffer(position);
+ buf = bh.buffer();
+ curOffset = position - bh.offset();
+ assert curOffset >= 0 && curOffset < buf.limit() : String.format("Invalid offset: %d, buf: %s, bh: %s", curOffset, buf, bh);
+ }
+ this.offset = (int) curOffset;
+ this.position = position;
+ nodeType = TrieNode.at(buf, (int) curOffset);
+ }
+
+ protected final int payloadFlags()
+ {
+ return nodeType.payloadFlags(buf, offset);
+ }
+
+ protected final int payloadPosition()
+ {
+ return nodeType.payloadPosition(buf, offset);
+ }
+
+ protected final int search(int transitionByte)
+ {
+ return nodeType.search(buf, offset, transitionByte);
+ }
+
+ protected final long transition(int childIndex)
+ {
+ return nodeType.transition(buf, offset, position, childIndex);
+ }
+
+ protected final long lastTransition()
+ {
+ return nodeType.lastTransition(buf, offset, position);
+ }
+
+ protected final long greaterTransition(int searchIndex, long defaultValue)
+ {
+ return nodeType.greaterTransition(buf, offset, position, searchIndex, defaultValue);
+ }
+
+ protected final long lesserTransition(int searchIndex, long defaultValue)
+ {
+ return nodeType.lesserTransition(buf, offset, position, searchIndex, defaultValue);
+ }
+
+ protected final int transitionByte(int childIndex)
+ {
+ return nodeType.transitionByte(buf, offset, childIndex);
+ }
+
+ protected final int transitionRange()
+ {
+ return nodeType.transitionRange(buf, offset);
+ }
+
+ protected final boolean hasChildren()
+ {
+ return transitionRange() > 0;
+ }
+
+ protected final void goMax(long pos)
+ {
+ go(pos);
+ while (true)
+ {
+ long lastChild = lastTransition();
+ if (lastChild == NONE)
+ return;
+ go(lastChild);
+ }
+ }
+
+ protected final void goMin(long pos)
+ {
+ go(pos);
+ while (true)
+ {
+ int payloadBits = payloadFlags();
+ if (payloadBits > 0)
+ return;
+
+ long firstChild = transition(0);
+ if (firstChild == NONE)
+ return;
+ go(firstChild);
+ }
+ }
+
+ public interface Extractor<RESULT, VALUE>
+ {
+ RESULT extract(VALUE walker, int payloadPosition, int payloadFlags);
+ }
+
+ /**
+ * Follows the given key while there are transitions in the trie for it.
+ *
+ * @return the first unmatched byte of the key, may be {@link ByteSource#END_OF_STREAM}
+ */
+ public int follow(ByteComparable key)
+ {
+ ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ go(root);
+ while (true)
+ {
+ int b = stream.next();
+ int childIndex = search(b);
+
+ if (childIndex < 0)
+ return b;
+
+ go(transition(childIndex));
+ }
+ }
+
+ /**
+ * Follows the trie for a given key, remembering the closest greater branch.
+ * On return the walker is positioned at the longest prefix that matches the input (with or without payload), and
+ * min(greaterBranch) is the immediate greater neighbour.
+ *
+ * @return the first unmatched byte of the key, may be {@link ByteSource#END_OF_STREAM}
+ */
+ public int followWithGreater(ByteComparable key)
+ {
+ greaterBranch = NONE;
+
+ ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ go(root);
+ while (true)
+ {
+ int b = stream.next();
+ int searchIndex = search(b);
+
+ greaterBranch = greaterTransition(searchIndex, greaterBranch);
+ if (searchIndex < 0)
+ return b;
+
+ go(transition(searchIndex));
+ }
+ }
+
+ /**
+ * Follows the trie for a given key, remembering the closest lesser branch.
+ * On return the walker is positioned at the longest prefix that matches the input (with or without payload), and
+ * max(lesserBranch) is the immediate lesser neighbour.
+ *
+ * @return the first unmatched byte of the key, may be {@link ByteSource#END_OF_STREAM}
+ */
+ public int followWithLesser(ByteComparable key)
+ {
+ lesserBranch = NONE;
+
+ ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ go(root);
+ while (true)
+ {
+ int b = stream.next();
+ int searchIndex = search(b);
+
+ lesserBranch = lesserTransition(searchIndex, lesserBranch);
+
+ if (searchIndex < 0)
+ return b;
+
+ go(transition(searchIndex));
+ }
+ }
+
+
+ /**
+ * Takes a prefix of the given key. The prefix is in the sense of a separator key match, i.e. it is only
+ * understood as valid if there are no greater entries in the trie (e.g. data at 'a' is ignored if 'ab' or 'abba'
+ * is in the trie when looking for 'abc' or 'ac', but accepted when looking for 'aa').
+ * In order to not have to go back to data that may have exited cache, payloads are extracted when the node is
+ * visited (instead of saving the node's position), which requires an extractor to be passed as parameter.
+ */
+ @SuppressWarnings("unchecked")
+ public <RESULT> RESULT prefix(ByteComparable key, Extractor<RESULT, CONCRETE> extractor)
+ {
+ RESULT payload = null;
+
+ ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ go(root);
+ while (true)
+ {
+ int b = stream.next();
+ int childIndex = search(b);
+
+ if (childIndex > 0)
+ payload = null;
+ else
+ {
+ int payloadBits = payloadFlags();
+ if (payloadBits > 0)
+ payload = extractor.extract((CONCRETE) this, payloadPosition(), payloadBits);
+ if (childIndex < 0)
+ return payload;
+ }
+
+ go(transition(childIndex));
+ }
+ }
+
+ /**
+ * Follows the trie for a given key, taking a prefix (in the sense above) and searching for neighboring values.
+ * On return min(greaterBranch) and max(lesserBranch) are the immediate non-prefix neighbours for the sought value.
+ * <p>
+ * Note: in a separator trie the closest smaller neighbour can be another prefix of the given key. This method
+ * does not take that into account. E.g. if trie contains "abba", "as" and "ask", looking for "asking" will find
+ * "ask" as the match, but max(lesserBranch) will point to "abba" instead of the correct "as". This problem can
+ * only occur if there is a valid prefix match.
+ */
+ @SuppressWarnings("unchecked")
+ public <RESULT> RESULT prefixAndNeighbours(ByteComparable key, Extractor<RESULT, CONCRETE> extractor)
+ {
+ RESULT payload = null;
+ greaterBranch = NONE;
+ lesserBranch = NONE;
+
+ ByteSource stream = key.asComparableBytes(BYTE_COMPARABLE_VERSION);
+ go(root);
+ while (true)
+ {
+ int b = stream.next();
+ int searchIndex = search(b);
+
+ greaterBranch = greaterTransition(searchIndex, greaterBranch);
+
+ if (searchIndex == -1 || searchIndex == 0)
+ {
+ int payloadBits = payloadFlags();
+ if (payloadBits > 0)
+ payload = extractor.extract((CONCRETE) this, payloadPosition(), payloadBits);
+ }
+ else
+ {
+ lesserBranch = lesserTransition(searchIndex, lesserBranch);
+ payload = null;
+ }
+
+ if (searchIndex < 0)
+ return payload;
+
+ go(transition(searchIndex));
+ }
+ }
+
+ /**
+ * To be used only in analysis.
+ */
+ protected int nodeTypeOrdinal()
+ {
+ return nodeType.ordinal;
+ }
+
+ /**
+ * To be used only in analysis.
+ */
+ protected int nodeSize()
+ {
+ return payloadPosition() - offset;
+ }
+
+ public interface PayloadToString
+ {
+ String payloadAsString(ByteBuffer buf, int payloadPos, int payloadFlags);
+ }
+
+ public void dumpTrie(PrintStream out, PayloadToString payloadReader)
+ {
+ out.print("ROOT");
+ dumpTrie(out, payloadReader, root, "");
+ }
+
+ private void dumpTrie(PrintStream out, PayloadToString payloadReader, long node, String indent)
+ {
+ go(node);
+ int bits = payloadFlags();
+ out.format(" %s@%x %s%n", nodeType.toString(), node, bits == 0 ? "" : payloadReader.payloadAsString(buf, payloadPosition(), bits));
+ int range = transitionRange();
+ for (int i = 0; i < range; ++i)
+ {
+ long child = transition(i);
+ if (child == NONE)
+ continue;
+ out.format("%s%02x %s>", indent, transitionByte(i), PageAware.pageStart(position) == PageAware.pageStart(child) ? "--" : "==");
+ dumpTrie(out, payloadReader, child, indent + " ");
+ go(node);
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("[Trie Walker - NodeType: %s, source: %s, buffer: %s, buffer file offset: %d, Node buffer offset: %d, Node file position: %d]",
+ nodeType, source, buf, bh.offset(), offset, position);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
index 4e9bbb5..a712ba6 100644
--- a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
@@ -26,9 +26,10 @@
import com.google.common.base.Preconditions;
import net.nicoulaj.compilecommand.annotations.DontInline;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.utils.FastByteOperations;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NIO_DATA_OUTPUT_STREAM_PLUS_BUFFER_SIZE;
+
/**
* An implementation of the DataOutputStreamPlus interface using a ByteBuffer to stage writes
* before flushing them to an underlying channel.
@@ -37,7 +38,7 @@
*/
public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus
{
- private static final int DEFAULT_BUFFER_SIZE = Integer.getInteger(Config.PROPERTY_PREFIX + "nio_data_output_stream_plus_buffer_size", 1024 * 32);
+ private static final int DEFAULT_BUFFER_SIZE = NIO_DATA_OUTPUT_STREAM_PLUS_BUFFER_SIZE.getInt();
protected ByteBuffer buffer;
@@ -66,6 +67,16 @@
this.buffer = buffer;
}
+ protected BufferedDataOutputStreamPlus(int size)
+ {
+ this.buffer = allocate(size);
+ }
+
+ protected ByteBuffer allocate(int size)
+ {
+ return ByteBuffer.allocate(size);
+ }
+
@Override
public void write(byte[] b) throws IOException
{
@@ -75,6 +86,7 @@
@Override
public void write(byte[] b, int off, int len) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (b == null)
throw new NullPointerException();
@@ -111,6 +123,7 @@
@Override
public void write(ByteBuffer src) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
int srcPos = src.position();
int srcCount;
int trgAvailable;
@@ -128,6 +141,7 @@
@Override
public void write(int b) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (!buffer.hasRemaining())
doFlush(1);
buffer.put((byte) (b & 0xFF));
@@ -136,6 +150,7 @@
@Override
public void writeBoolean(boolean v) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (!buffer.hasRemaining())
doFlush(1);
buffer.put(v ? (byte)1 : (byte)0);
@@ -148,11 +163,12 @@
}
@Override
- public void writeBytes(long register, int bytes) throws IOException
+ public void writeMostSignificantBytes(long register, int bytes) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (buffer.remaining() < Long.BYTES)
{
- super.writeBytes(register, bytes);
+ super.writeMostSignificantBytes(register, bytes);
}
else
{
@@ -171,6 +187,7 @@
@Override
public void writeChar(int v) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (buffer.remaining() < 2)
writeSlow(v, 2);
else
@@ -180,6 +197,7 @@
@Override
public void writeInt(int v) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (buffer.remaining() < 4)
writeSlow(v, 4);
else
@@ -189,6 +207,7 @@
@Override
public void writeLong(long v) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
if (buffer.remaining() < 8)
writeSlow(v, 8);
else
@@ -210,6 +229,7 @@
@DontInline
private void writeSlow(long bytes, int count) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
int origCount = count;
if (ByteOrder.BIG_ENDIAN == buffer.order())
while (count > 0) writeByte((int) (bytes >>> (8 * --count)));
@@ -234,6 +254,7 @@
@Override
public void writeUTF(String s) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
UnbufferedDataOutputStreamPlus.writeUTF(s, this);
}
@@ -243,6 +264,7 @@
@DontInline
protected void doFlush(int count) throws IOException
{
+ assert buffer != null : "Attempt to use a closed data output";
buffer.flip();
while (buffer.hasRemaining())
@@ -271,7 +293,8 @@
public BufferedDataOutputStreamPlus order(ByteOrder order)
{
+ assert buffer != null : "Attempt to use a closed data output";
this.buffer.order(order);
return this;
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/ChannelProxy.java b/src/java/org/apache/cassandra/io/util/ChannelProxy.java
index 717def7..81665be 100644
--- a/src/java/org/apache/cassandra/io/util/ChannelProxy.java
+++ b/src/java/org/apache/cassandra/io/util/ChannelProxy.java
@@ -40,6 +40,7 @@
*/
public final class ChannelProxy extends SharedCloseableImpl
{
+ private final File file;
private final String filePath;
private final FileChannel channel;
@@ -62,14 +63,15 @@
public ChannelProxy(File file)
{
- this(file.path(), openChannel(file));
+ this(file, openChannel(file));
}
- public ChannelProxy(String filePath, FileChannel channel)
+ public ChannelProxy(File file, FileChannel channel)
{
- super(new Cleanup(filePath, channel));
+ super(new Cleanup(file.path(), channel));
- this.filePath = filePath;
+ this.file = file;
+ this.filePath = file.path();
this.channel = channel;
}
@@ -77,6 +79,7 @@
{
super(copy);
+ this.file = copy.file;
this.filePath = copy.filePath;
this.channel = copy.channel;
}
@@ -130,6 +133,11 @@
return filePath;
}
+ public File file()
+ {
+ return file;
+ }
+
public int read(ByteBuffer buffer, long position)
{
try
diff --git a/src/java/org/apache/cassandra/io/util/DataInputPlus.java b/src/java/org/apache/cassandra/io/util/DataInputPlus.java
index bda8461..d117c7f 100644
--- a/src/java/org/apache/cassandra/io/util/DataInputPlus.java
+++ b/src/java/org/apache/cassandra/io/util/DataInputPlus.java
@@ -17,10 +17,14 @@
*/
package org.apache.cassandra.io.util;
-import java.io.*;
+import java.io.DataInput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
import org.apache.cassandra.utils.Shared;
import org.apache.cassandra.utils.vint.VIntCoding;
+import org.apache.cassandra.utils.vint.VIntCoding.VIntOutOfRangeException;
import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
@@ -30,12 +34,31 @@
@Shared(scope = SIMULATION)
public interface DataInputPlus extends DataInput
{
+ /**
+ * Read a 64-bit integer back.
+ *
+ * This method assumes it was originally written using
+ * {@link DataOutputPlus#writeVInt(long)} or similar that zigzag encodes the vint.
+ */
default long readVInt() throws IOException
{
return VIntCoding.readVInt(this);
}
/**
+ * Read up to a 32-bit integer back.
+ *
+ * This method assumes the integer was originally written using
+ * {@link DataOutputPlus#writeVInt32(int)} or similar that zigzag encodes the vint.
+ *
+ * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer
+ */
+ default int readVInt32() throws IOException
+ {
+ return VIntCoding.readVInt32(this);
+ }
+
+ /**
* Think hard before opting for an unsigned encoding. Is this going to bite someone because some day
* they might need to pass in a sentinel value using negative numbers? Is the risk worth it
* to save a few bytes?
@@ -48,6 +71,19 @@
}
/**
+ * Read up to a 32-bit integer back.
+ *
+ * This method assumes the original integer was written using {@link DataOutputPlus#writeUnsignedVInt32(int)}
+ * or similar that doesn't zigzag encodes the vint.
+ *
+ * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer
+ */
+ default int readUnsignedVInt32() throws IOException
+ {
+ return VIntCoding.readUnsignedVInt32(this);
+ }
+
+ /**
* Always skips the requested number of bytes, unless EOF is reached
*
* @param n number of bytes to skip
@@ -64,14 +100,8 @@
/**
* Wrapper around an InputStream that provides no buffering but can decode varints
- *
- * TODO: probably shouldn't use DataInputStream as a parent
*/
- public class DataInputStreamPlus extends DataInputStream implements DataInputPlus
+ abstract class DataInputStreamPlus extends InputStream implements DataInputPlus
{
- public DataInputStreamPlus(InputStream is)
- {
- super(is);
- }
}
}
diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
index 65d4e58..aef3614 100644
--- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
+++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
@@ -23,51 +23,30 @@
import java.util.zip.CheckedInputStream;
import java.util.zip.Checksum;
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.utils.ChecksumType;
-import org.apache.cassandra.utils.Throwables;
public class DataIntegrityMetadata
{
- public static ChecksumValidator checksumValidator(Descriptor desc) throws IOException
- {
- return new ChecksumValidator(desc);
- }
-
public static class ChecksumValidator implements Closeable
{
private final ChecksumType checksumType;
private final RandomAccessReader reader;
public final int chunkSize;
- private final String dataFilename;
- public ChecksumValidator(Descriptor descriptor) throws IOException
+ public ChecksumValidator(File dataFile, File crcFile) throws IOException
{
this(ChecksumType.CRC32,
- RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC))),
- descriptor.filenameFor(Component.DATA));
+ RandomAccessReader.open(crcFile),
+ dataFile.absolutePath());
}
public ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, String dataFilename) throws IOException
{
this.checksumType = checksumType;
this.reader = reader;
- this.dataFilename = dataFilename;
chunkSize = reader.readInt();
}
- @VisibleForTesting
- protected ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, int chunkSize)
- {
- this.checksumType = checksumType;
- this.reader = reader;
- this.dataFilename = null;
- this.chunkSize = chunkSize;
- }
-
public void seek(long offset)
{
long start = chunkStart(offset);
@@ -82,10 +61,10 @@
public void validate(byte[] bytes, int start, int end) throws IOException
{
- int current = (int) checksumType.of(bytes, start, end);
- int actual = reader.readInt();
- if (current != actual)
- throw new IOException("Corrupted File : " + dataFilename);
+ int calculatedValue = (int) checksumType.of(bytes, start, end);
+ int storedValue = reader.readInt();
+ if (calculatedValue != storedValue)
+ throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), reader.getPath(), storedValue, calculatedValue));
}
/**
@@ -96,10 +75,10 @@
*/
public void validate(ByteBuffer buffer) throws IOException
{
- int current = (int) checksumType.of(buffer);
- int actual = reader.readInt();
- if (current != actual)
- throw new IOException("Corrupted File : " + dataFilename);
+ int calculatedValue = (int) checksumType.of(buffer);
+ int storedValue = reader.readInt();
+ if (calculatedValue != storedValue)
+ throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), reader.getPath(), storedValue, calculatedValue));
}
public void close()
@@ -108,55 +87,33 @@
}
}
- public static FileDigestValidator fileDigestValidator(Descriptor desc) throws IOException
- {
- return new FileDigestValidator(desc);
- }
-
- public static class FileDigestValidator implements Closeable
+ public static class FileDigestValidator
{
private final Checksum checksum;
- private final RandomAccessReader digestReader;
- private final RandomAccessReader dataReader;
- private final Descriptor descriptor;
- private long storedDigestValue;
+ private final File dataFile;
+ private final File digestFile;
- public FileDigestValidator(Descriptor descriptor) throws IOException
+ public FileDigestValidator(File dataFile, File digestFile) throws IOException
{
- this.descriptor = descriptor;
- checksum = ChecksumType.CRC32.newInstance();
- digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DIGEST)));
- dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA)));
- try
- {
- storedDigestValue = Long.parseLong(digestReader.readLine());
- }
- catch (Exception e)
- {
- close();
- // Attempting to create a FileDigestValidator without a DIGEST file will fail
- throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA));
- }
+ this.dataFile = dataFile;
+ this.digestFile = digestFile;
+ this.checksum = ChecksumType.CRC32.newInstance();
}
// Validate the entire file
public void validate() throws IOException
{
- CheckedInputStream checkedInputStream = new CheckedInputStream(dataReader, checksum);
- byte[] chunk = new byte[64 * 1024];
-
- while( checkedInputStream.read(chunk) > 0 ) { }
- long calculatedDigestValue = checkedInputStream.getChecksum().getValue();
- if (storedDigestValue != calculatedDigestValue)
+ try (RandomAccessReader digestReader = RandomAccessReader.open(digestFile);
+ RandomAccessReader dataReader = RandomAccessReader.open(dataFile);
+ CheckedInputStream checkedInputStream = new CheckedInputStream(dataReader, checksum);)
{
- throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA));
+ long storedDigestValue = Long.parseLong(digestReader.readLine());
+ byte[] chunk = new byte[64 * 1024];
+ while (checkedInputStream.read(chunk) > 0) ;
+ long calculatedDigestValue = checkedInputStream.getChecksum().getValue();
+ if (storedDigestValue != calculatedDigestValue)
+ throw new IOException(String.format("Corrupted file: integrity check (digest) failed for %s: %d != %d", dataFile, storedDigestValue, calculatedDigestValue));
}
}
-
- public void close()
- {
- Throwables.perform(digestReader::close,
- dataReader::close);
- }
}
}
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java b/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java
index d6f3a4a..d27a935 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java
@@ -26,7 +26,10 @@
import com.google.common.base.Preconditions;
import io.netty.util.concurrent.FastThreadLocal;
-import org.apache.cassandra.config.Config;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.DATA_OUTPUT_BUFFER_ALLOCATE_TYPE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DOB_DOUBLING_THRESHOLD_MB;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DOB_MAX_RECYCLE_BYTES;
/**
* An implementation of the DataOutputStream interface using a FastByteArrayOutputStream and exposing
@@ -39,17 +42,19 @@
/*
* Threshold at which resizing transitions from doubling to increasing by 50%
*/
- static final long DOUBLING_THRESHOLD = Long.getLong(Config.PROPERTY_PREFIX + "DOB_DOUBLING_THRESHOLD_MB", 64);
+ static final long DOUBLING_THRESHOLD = DOB_DOUBLING_THRESHOLD_MB.getLong();
/*
* Only recycle OutputBuffers up to 1Mb. Larger buffers will be trimmed back to this size.
*/
- private static final int MAX_RECYCLE_BUFFER_SIZE = Integer.getInteger(Config.PROPERTY_PREFIX + "dob_max_recycle_bytes", 1024 * 1024);
+ private static final int MAX_RECYCLE_BUFFER_SIZE = DOB_MAX_RECYCLE_BYTES.getInt();
+ private enum AllocationType { DIRECT, ONHEAP }
+ private static final AllocationType ALLOCATION_TYPE = DATA_OUTPUT_BUFFER_ALLOCATE_TYPE.getEnum(AllocationType.DIRECT);
private static final int DEFAULT_INITIAL_BUFFER_SIZE = 128;
/**
- * Scratch buffers used mostly for serializing in memory. It's important to call #recycle() when finished
+ * Scratch buffers used mostly for serializing in memory. It's important to call #close() when finished
* to keep the memory overhead from being too large in the system.
*/
public static final FastThreadLocal<DataOutputBuffer> scratchBuffer = new FastThreadLocal<DataOutputBuffer>()
@@ -59,29 +64,38 @@
{
return new DataOutputBuffer()
{
+ @Override
public void close()
{
- if (buffer.capacity() <= MAX_RECYCLE_BUFFER_SIZE)
+ if (buffer != null && buffer.capacity() <= MAX_RECYCLE_BUFFER_SIZE)
{
buffer.clear();
}
else
{
- buffer = ByteBuffer.allocate(DEFAULT_INITIAL_BUFFER_SIZE);
+ setBuffer(allocate(DEFAULT_INITIAL_BUFFER_SIZE));
}
}
+
+ @Override
+ protected ByteBuffer allocate(int size)
+ {
+ return ALLOCATION_TYPE == AllocationType.DIRECT ?
+ ByteBuffer.allocateDirect(size) :
+ ByteBuffer.allocate(size);
+ }
};
}
};
public DataOutputBuffer()
{
- this(DEFAULT_INITIAL_BUFFER_SIZE);
+ super(DEFAULT_INITIAL_BUFFER_SIZE);
}
public DataOutputBuffer(int size)
{
- super(ByteBuffer.allocate(size));
+ super(size);
}
public DataOutputBuffer(ByteBuffer buffer)
@@ -158,9 +172,15 @@
{
if (count <= 0)
return;
- ByteBuffer newBuffer = ByteBuffer.allocate(checkedArraySizeCast(calculateNewSize(count)));
+ ByteBuffer newBuffer = allocate(checkedArraySizeCast(calculateNewSize(count)));
buffer.flip();
newBuffer.put(buffer);
+ setBuffer(newBuffer);
+ }
+
+ protected void setBuffer(ByteBuffer newBuffer)
+ {
+ FileUtils.clean(buffer); // free if direct
buffer = newBuffer;
}
@@ -221,6 +241,18 @@
return result;
}
+ /**
+ * Gets the underlying ByteBuffer and calls {@link ByteBuffer#flip()}. This method is "unsafe" in the sense that
+ * it returns the underlying buffer, which may be modified by other methods after calling this method (or cleared on
+ * {@link #close()}). If the calling logic knows that no new calls to this object will happen after calling this
+ * method, then this method can avoid the copying done in {@link #asNewBuffer()}, and {@link #buffer()}.
+ */
+ public ByteBuffer unsafeGetBufferAndFlip()
+ {
+ buffer.flip();
+ return buffer;
+ }
+
public byte[] getData()
{
assert buffer.arrayOffset() == 0;
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
index 205dab7..8b3b49f 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java
@@ -17,13 +17,13 @@
*/
package org.apache.cassandra.io.util;
+import org.apache.cassandra.utils.Shared;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.cassandra.utils.Shared;
-import org.apache.cassandra.utils.vint.VIntCoding;
-
import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
/**
@@ -47,6 +47,17 @@
VIntCoding.writeVInt(i, this);
}
+ @Deprecated
+ default void writeVInt(int i)
+ {
+ throw new UnsupportedOperationException("Use writeVInt32/readVInt32");
+ }
+
+ default void writeVInt32(int i) throws IOException
+ {
+ VIntCoding.writeVInt32(i, this);
+ }
+
/**
* This is more efficient for storing unsigned values, both in storage and CPU burden.
*
@@ -59,6 +70,17 @@
VIntCoding.writeUnsignedVInt(i, this);
}
+ @Deprecated
+ default void writeUnsignedVInt(int i)
+ {
+ throw new UnsupportedOperationException("Use writeUnsignedVInt32/readUnsignedVInt32");
+ }
+
+ default void writeUnsignedVInt32(int i) throws IOException
+ {
+ VIntCoding.writeUnsignedVInt32(i, this);
+ }
+
/**
* An efficient way to write the type {@code bytes} of a long
*
@@ -66,7 +88,7 @@
* @param bytes - the number of bytes the register occupies. Valid values are between 1 and 8 inclusive.
* @throws IOException
*/
- default void writeBytes(long register, int bytes) throws IOException
+ default void writeMostSignificantBytes(long register, int bytes) throws IOException
{
switch (bytes)
{
@@ -129,4 +151,40 @@
{
return false;
}
-}
+
+ // The methods below support page-aware layout for writing. These would only be implemented if position() is
+ // also supported.
+
+ /**
+ * Returns the number of bytes that a page can take at maximum.
+ */
+ default int maxBytesInPage()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Pad this page with 0s to move on to the next.
+ */
+ default void padToPageBoundary() throws IOException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns how many bytes are left in the page.
+ */
+ default int bytesLeftInPage()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns the next padded position. This is either the current position (if already padded), or the start of next
+ * page.
+ */
+ default long paddedPosition()
+ {
+ throw new UnsupportedOperationException();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java
index e931899..7dd2341 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputStreamPlus.java
@@ -23,9 +23,10 @@
import java.nio.channels.WritableByteChannel;
import io.netty.util.concurrent.FastThreadLocal;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DATA_OUTPUT_STREAM_PLUS_TEMP_BUFFER_SIZE;
+
/**
* Abstract base class for DataOutputStreams that accept writes from ByteBuffer or Memory and also provide
* access to the underlying WritableByteChannel associated with their output stream.
@@ -47,8 +48,7 @@
this.channel = channel;
}
- private static int MAX_BUFFER_SIZE =
- Integer.getInteger(Config.PROPERTY_PREFIX + "data_output_stream_plus_temp_buffer_size", 8192);
+ private static final int MAX_BUFFER_SIZE = DATA_OUTPUT_STREAM_PLUS_TEMP_BUFFER_SIZE.getInt();
/*
* Factored out into separate method to create more flexibility around inlining
diff --git a/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java b/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java
new file mode 100644
index 0000000..aa8e7e0
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+public class EmptyRebufferer implements Rebufferer, RebuffererFactory
+{
+ private final ChannelProxy channel;
+
+ public EmptyRebufferer(ChannelProxy channel)
+ {
+ this.channel = channel;
+ }
+
+ @Override
+ public void close()
+ {
+
+ }
+
+ @Override
+ public ChannelProxy channel()
+ {
+ return channel;
+ }
+
+ @Override
+ public long fileLength()
+ {
+ return 0;
+ }
+
+ @Override
+ public double getCrcCheckChance()
+ {
+ return 0;
+ }
+
+ @Override
+ public BufferHolder rebuffer(long position)
+ {
+ return EMPTY;
+ }
+
+ @Override
+ public void closeReader()
+ {
+
+ }
+
+ @Override
+ public Rebufferer instantiateRebufferer()
+ {
+ return this;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/util/File.java b/src/java/org/apache/cassandra/io/util/File.java
index 1f48707..de41538 100644
--- a/src/java/org/apache/cassandra/io/util/File.java
+++ b/src/java/org/apache/cassandra/io/util/File.java
@@ -23,7 +23,12 @@
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.channels.FileChannel;
-import java.nio.file.*; // checkstyle: permit this import
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths; // checkstyle: permit this import
import java.util.Objects;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
@@ -31,9 +36,9 @@
import java.util.function.IntFunction;
import java.util.function.Predicate;
import java.util.stream.Stream;
-
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.RateLimiter;
import net.openhft.chronicle.core.util.ThrowingFunction;
@@ -122,17 +127,28 @@
}
/**
+ * Unsafe constructor that allows a File to use a differet {@link FileSystem} than {@link File#filesystem}.
+ *
+ * The main caller of such a method are cases such as JVM Dtest functions that need access to the logging framwork
+ * files, which exists on in {@link FileSystems#getDefault()}.
+ */
+ @VisibleForTesting
+ public File(FileSystem fs, String first, String... more)
+ {
+ this.path = fs.getPath(first, more);
+ }
+
+ /**
* @param path the path to wrap
*/
public File(Path path)
{
if (path != null && path.getFileSystem() != filesystem)
- throw new IllegalArgumentException("Incompatible file system");
+ throw new IllegalArgumentException("Incompatible file system; path FileSystem (" + path.getFileSystem() + ") is not the same reference (" + filesystem + ")");
this.path = path;
}
-
public static Path getPath(String first, String... more)
{
return filesystem.getPath(first, more);
@@ -753,6 +769,13 @@
return new FileInputStreamPlus(this);
}
+ public File withSuffix(String suffix)
+ {
+ if (path == null)
+ throw new IllegalStateException("Cannot suffix an empty path");
+ return new File(path.getParent().resolve(path.getFileName().toString() + suffix));
+ }
+
private Path toPathForWrite()
{
if (path == null)
@@ -767,6 +790,12 @@
return path;
}
+ @VisibleForTesting
+ public static FileSystem unsafeGetFilesystem()
+ {
+ return filesystem;
+ }
+
public static void unsafeSetFilesystem(FileSystem fs)
{
filesystem = fs;
diff --git a/src/java/org/apache/cassandra/io/util/FileHandle.java b/src/java/org/apache/cassandra/io/util/FileHandle.java
index 6bab460..5c4bd72 100644
--- a/src/java/org/apache/cassandra/io/util/FileHandle.java
+++ b/src/java/org/apache/cassandra/io/util/FileHandle.java
@@ -17,31 +17,29 @@
*/
package org.apache.cassandra.io.util;
-import java.util.Objects;
import java.util.Optional;
+import java.util.function.Function;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.RateLimiter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.Config;
import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.utils.NativeLibrary;
+import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.concurrent.RefCounted;
import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
-import static org.apache.cassandra.utils.Throwables.maybeFail;
-import org.apache.cassandra.utils.Throwables;
-
/**
* {@link FileHandle} provides access to a file for reading, including the ones written by various {@link SequentialWriter}
* instances, and it is typically used by {@link org.apache.cassandra.io.sstable.format.SSTableReader}.
- *
+ * <p>
* Use {@link FileHandle.Builder} to create an instance, and call {@link #createReader()} (and its variants) to
* access the readers for the underlying file.
- *
+ * <p>
* You can use {@link Builder#complete()} several times during its lifecycle with different {@code overrideLength}(i.e. early opening file).
* For that reason, the builder keeps a reference to the file channel and makes a copy for each {@link Builder#complete()} call.
* Therefore, it is important to close the {@link Builder} when it is no longer needed, as well as any {@link FileHandle}
@@ -49,8 +47,6 @@
*/
public class FileHandle extends SharedCloseableImpl
{
- private static final Logger logger = LoggerFactory.getLogger(FileHandle.class);
-
public final ChannelProxy channel;
public final long onDiskLength;
@@ -88,8 +84,13 @@
}
/**
- * @return Path to the file this factory is referencing
+ * @return file this factory is referencing
*/
+ public File file()
+ {
+ return new File(channel.filePath());
+ }
+
public String path()
{
return channel.filePath();
@@ -114,7 +115,6 @@
public void addTo(Ref.IdentityCollection identities)
{
super.addTo(identities);
- compressionMetadata.ifPresent(metadata -> metadata.addTo(identities));
}
@Override
@@ -155,7 +155,14 @@
}
catch (Throwable t)
{
- try { reader.close(); } catch (Throwable t2) { t.addSuppressed(t2); }
+ try
+ {
+ reader.close();
+ }
+ catch (Throwable t2)
+ {
+ t.addSuppressed(t2);
+ }
throw t;
}
}
@@ -173,10 +180,10 @@
else
return metadata.chunkFor(before).offset;
}).orElse(before);
- NativeLibrary.trySkipCache(channel.getFileDescriptor(), 0, position, path());
+ NativeLibrary.trySkipCache(channel.getFileDescriptor(), 0, position, file().absolutePath());
}
- private Rebufferer instantiateRebufferer(RateLimiter limiter)
+ public Rebufferer instantiateRebufferer(RateLimiter limiter)
{
Rebufferer rebufferer = rebuffererFactory.instantiateRebufferer();
@@ -238,35 +245,23 @@
/**
* Configures how the file will be read (compressed, mmapped, use cache etc.)
*/
- public static class Builder implements AutoCloseable
+ public static class Builder
{
- private final String path;
+ public static final long NO_LENGTH_OVERRIDE = -1;
- private ChannelProxy channel;
+ public final File file;
+
private CompressionMetadata compressionMetadata;
- private MmappedRegions regions;
private ChunkCache chunkCache;
private int bufferSize = RandomAccessReader.DEFAULT_BUFFER_SIZE;
private BufferType bufferType = BufferType.OFF_HEAP;
-
private boolean mmapped = false;
- private boolean compressed = false;
+ private long lengthOverride = -1;
+ private MmappedRegionsCache mmappedRegionsCache;
- public Builder(String path)
+ public Builder(File file)
{
- this.path = path;
- }
-
- public Builder(ChannelProxy channel)
- {
- this.channel = channel;
- this.path = channel.filePath();
- }
-
- public Builder compressed(boolean compressed)
- {
- this.compressed = compressed;
- return this;
+ this.file = file;
}
/**
@@ -283,13 +278,15 @@
/**
* Provide {@link CompressionMetadata} to use when reading compressed file.
+ * Upon completion, builder will create a shared copy of this object and that copy will be used in the created
+ * instance of {@link FileHandle}. Therefore, the caller is responsible for closing the instance of
+ * {@link CompressionMetadata} passed to this method after builder completion.
*
- * @param metadata CompressionMetadata to use
+ * @param metadata CompressionMetadata to use, can be {@code null} if no compression is used
* @return this object
*/
public Builder withCompressionMetadata(CompressionMetadata metadata)
{
- this.compressed = Objects.nonNull(metadata);
this.compressionMetadata = metadata;
return this;
}
@@ -306,6 +303,18 @@
return this;
}
+ public Builder mmapped(Config.DiskAccessMode diskAccessMode)
+ {
+ this.mmapped = diskAccessMode == Config.DiskAccessMode.mmap;
+ return this;
+ }
+
+ public Builder withMmappedRegionsCache(MmappedRegionsCache mmappedRegionsCache)
+ {
+ this.mmappedRegionsCache = mmappedRegionsCache;
+ return this;
+ }
+
/**
* Set the buffer size to use (if appropriate).
*
@@ -331,128 +340,97 @@
}
/**
- * Complete building {@link FileHandle} without overriding file length.
+ * Override the file length.
*
- * @see #complete(long)
- */
- public FileHandle complete()
- {
- return complete(-1L);
- }
-
- /**
- * Complete building {@link FileHandle} with the given length, which overrides the file length.
- *
- * @param overrideLength Override file length (in bytes) so that read cannot go further than this value.
+ * @param lengthOverride Override file length (in bytes) so that read cannot go further than this value.
* If the value is less than or equal to 0, then the value is ignored.
* @return Built file
*/
- @SuppressWarnings("resource")
- public FileHandle complete(long overrideLength)
+ public Builder withLengthOverride(long lengthOverride)
{
- boolean channelOpened = false;
- if (channel == null)
- {
- channel = new ChannelProxy(path);
- channelOpened = true;
- }
+ this.lengthOverride = lengthOverride;
+ return this;
+ }
- ChannelProxy channelCopy = channel.sharedCopy();
+ /**
+ * Complete building {@link FileHandle}.
+ */
+ public FileHandle complete()
+ {
+ return complete(ChannelProxy::new);
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("resource")
+ public FileHandle complete(Function<File, ChannelProxy> channelProxyFactory)
+ {
+ ChannelProxy channel = null;
+ MmappedRegions regions = null;
+ CompressionMetadata compressionMetadata = null;
try
{
- if (compressed && compressionMetadata == null)
- compressionMetadata = CompressionMetadata.create(channelCopy.filePath());
+ compressionMetadata = this.compressionMetadata != null ? this.compressionMetadata.sharedCopy() : null;
+ channel = channelProxyFactory.apply(file);
- long length = overrideLength > 0 ? overrideLength : compressed ? compressionMetadata.compressedFileLength : channelCopy.size();
+ long fileLength = (compressionMetadata != null) ? compressionMetadata.compressedFileLength : channel.size();
+ long length = lengthOverride > 0 ? lengthOverride : fileLength;
RebuffererFactory rebuffererFactory;
- if (mmapped)
+ if (length == 0)
{
- if (compressed)
+ rebuffererFactory = new EmptyRebufferer(channel);
+ }
+ else if (mmapped)
+ {
+ if (compressionMetadata != null)
{
- regions = MmappedRegions.map(channelCopy, compressionMetadata);
- rebuffererFactory = maybeCached(new CompressedChunkReader.Mmap(channelCopy, compressionMetadata,
- regions));
+ regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, compressionMetadata)
+ : MmappedRegions.map(channel, compressionMetadata);
+ rebuffererFactory = maybeCached(new CompressedChunkReader.Mmap(channel, compressionMetadata, regions));
}
else
{
- updateRegions(channelCopy, length);
- rebuffererFactory = new MmapRebufferer(channelCopy, length, regions.sharedCopy());
+ regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, length)
+ : MmappedRegions.map(channel, length);
+ rebuffererFactory = new MmapRebufferer(channel, length, regions);
}
}
else
{
- regions = null;
- if (compressed)
+ if (compressionMetadata != null)
{
- rebuffererFactory = maybeCached(new CompressedChunkReader.Standard(channelCopy, compressionMetadata));
+ rebuffererFactory = maybeCached(new CompressedChunkReader.Standard(channel, compressionMetadata));
}
else
{
int chunkSize = DiskOptimizationStrategy.roundForCaching(bufferSize, ChunkCache.roundUp);
- rebuffererFactory = maybeCached(new SimpleChunkReader(channelCopy, length, bufferType, chunkSize));
+ rebuffererFactory = maybeCached(new SimpleChunkReader(channel, length, bufferType, chunkSize));
}
}
- Cleanup cleanup = new Cleanup(channelCopy, rebuffererFactory, compressionMetadata, chunkCache);
- return new FileHandle(cleanup, channelCopy, rebuffererFactory, compressionMetadata, length);
+ Cleanup cleanup = new Cleanup(channel, rebuffererFactory, compressionMetadata, chunkCache);
+
+ FileHandle fileHandle = new FileHandle(cleanup, channel, rebuffererFactory, compressionMetadata, length);
+ return fileHandle;
}
catch (Throwable t)
{
- channelCopy.close();
- if (channelOpened)
- {
- ChannelProxy c = channel;
- channel = null;
- throw Throwables.cleaned(c.close(t));
- }
+ Throwables.closeNonNullAndAddSuppressed(t, regions, channel, compressionMetadata);
throw t;
}
}
- public Throwable close(Throwable accumulate)
- {
- if (!compressed && regions != null)
- accumulate = regions.close(accumulate);
- if (channel != null)
- return channel.close(accumulate);
-
- return accumulate;
- }
-
- public void close()
- {
- maybeFail(close(null));
- }
-
private RebuffererFactory maybeCached(ChunkReader reader)
{
if (chunkCache != null && chunkCache.capacity() > 0)
- return chunkCache.maybeWrap(reader);
+ return chunkCache.wrap(reader);
return reader;
}
-
- private void updateRegions(ChannelProxy channel, long length)
- {
- if (regions != null && !regions.isValid(channel))
- {
- Throwable err = regions.close(null);
- if (err != null)
- logger.error("Failed to close mapped regions", err);
-
- regions = null;
- }
-
- if (regions == null)
- regions = MmappedRegions.map(channel, length);
- else
- regions.extend(length);
- }
}
@Override
public String toString()
{
- return getClass().getSimpleName() + "(path='" + path() + '\'' +
+ return getClass().getSimpleName() + "(path='" + file() + '\'' +
", length=" + rebuffererFactory.fileLength() +
')';
}
diff --git a/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java b/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java
index 79e8438..2bd57a9 100644
--- a/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java
+++ b/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java
@@ -27,37 +27,29 @@
public class FileInputStreamPlus extends RebufferingInputStream
{
final FileChannel channel;
+ public final File file;
public FileInputStreamPlus(String file) throws NoSuchFileException
{
this(new File(file));
}
- public FileInputStreamPlus(File file) throws NoSuchFileException
- {
- this(file.newReadChannel());
- }
-
public FileInputStreamPlus(Path path) throws NoSuchFileException
{
- this(PathUtils.newReadChannel(path));
+ this(new File(path));
}
- public FileInputStreamPlus(Path path, int bufferSize) throws NoSuchFileException
+ public FileInputStreamPlus(File file) throws NoSuchFileException
{
- this(PathUtils.newReadChannel(path), bufferSize);
+ this(file, 1 << 14);
}
- private FileInputStreamPlus(FileChannel channel)
- {
- this(channel, 1 << 14);
- }
-
- private FileInputStreamPlus(FileChannel channel, int bufferSize)
+ public FileInputStreamPlus(File file, int bufferSize) throws NoSuchFileException
{
super(ByteBuffer.allocateDirect(bufferSize));
- this.channel = channel;
+ this.channel = file.newReadChannel();
this.buffer.limit(0);
+ this.file = file;
}
@Override
diff --git a/src/java/org/apache/cassandra/io/util/FileStoreUtils.java b/src/java/org/apache/cassandra/io/util/FileStoreUtils.java
new file mode 100644
index 0000000..39455bb
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/FileStoreUtils.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.file.FileStore;
+import java.util.function.Consumer;
+
+public class FileStoreUtils
+{
+ /**
+ * Try and get the total space of the given filestore
+ * @return long value of available space if no errors
+ * Long.MAX_VALUE if on a large file system that overflows
+ * 0 on exception during IOToLongFunction
+ */
+ public static long tryGetSpace(FileStore filestore, PathUtils.IOToLongFunction<FileStore> getSpace)
+ {
+ return tryGetSpace(filestore, getSpace, ignore -> {});
+ }
+
+ public static long tryGetSpace(FileStore filestore, PathUtils.IOToLongFunction<FileStore> getSpace, Consumer<IOException> orElse)
+ {
+ try
+ {
+ return handleLargeFileSystem(getSpace.apply(filestore));
+ }
+ catch (IOException e)
+ {
+ orElse.accept(e);
+ return 0L;
+ }
+ }
+
+ /**
+ * Private constructor as the class contains only static methods.
+ */
+ private FileStoreUtils()
+ {
+ }
+
+ /**
+ * Handle large file system by returning {@code Long.MAX_VALUE} when the size overflows.
+ * @param size returned by the Java's FileStore methods
+ * @return the size or {@code Long.MAX_VALUE} if the size was bigger than {@code Long.MAX_VALUE}
+ */
+ private static long handleLargeFileSystem(long size)
+ {
+ return size < 0 ? Long.MAX_VALUE : size;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 01f1f18..9d8efc7 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -51,9 +51,8 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import com.google.common.util.concurrent.RateLimiter;
import com.google.common.base.Preconditions;
-
+import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +63,6 @@
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.SyncUtil;
-import static com.google.common.base.Throwables.propagate;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
import static org.apache.cassandra.utils.Throwables.maybeFail;
@@ -210,9 +208,14 @@
public static void createHardLinkWithoutConfirm(String from, String to)
{
+ createHardLinkWithoutConfirm(new File(from), new File(to));
+ }
+
+ public static void createHardLinkWithoutConfirm(File from, File to)
+ {
try
{
- createHardLink(new File(from), new File(to));
+ createHardLink(from, to);
}
catch (FSWriteError fse)
{
@@ -223,9 +226,14 @@
public static void copyWithOutConfirm(String from, String to)
{
+ copyWithOutConfirm(new File(from), new File(to));
+ }
+
+ public static void copyWithOutConfirm(File from, File to)
+ {
try
{
- Files.copy(File.getPath(from), File.getPath(to));
+ Files.copy(from.toPath(), to.toPath());
}
catch (IOException e)
{
@@ -478,7 +486,7 @@
public static void handleFSErrorAndPropagate(FSError e)
{
JVMStabilityInspector.inspectThrowable(e);
- throw propagate(e);
+ throw e;
}
/**
@@ -790,4 +798,4 @@
}
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
index b5c7f34..bcbf2ef 100644
--- a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
+++ b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java
@@ -20,30 +20,28 @@
*/
package org.apache.cassandra.io.util;
-import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.RateLimiter;
/**
* Rebufferer wrapper that applies rate limiting.
- *
+ * <p>
* Instantiated once per RandomAccessReader, thread-unsafe.
* The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call.
+ * Only one BufferHolder can be active at a time. Calling {@link #rebuffer(long)} before the previously obtained
+ * buffer holder is released will throw {@link AssertionError}.
*/
-public class LimitingRebufferer implements Rebufferer, Rebufferer.BufferHolder
+@NotThreadSafe
+public class LimitingRebufferer extends WrappingRebufferer
{
- final private Rebufferer wrapped;
final private RateLimiter limiter;
final private int limitQuant;
- private BufferHolder bufferHolder;
- private ByteBuffer buffer;
- private long offset;
-
public LimitingRebufferer(Rebufferer wrapped, RateLimiter limiter, int limitQuant)
{
- this.wrapped = wrapped;
+ super(wrapped);
this.limiter = limiter;
this.limitQuant = limitQuant;
}
@@ -51,9 +49,7 @@
@Override
public BufferHolder rebuffer(long position)
{
- bufferHolder = wrapped.rebuffer(position);
- buffer = bufferHolder.buffer();
- offset = bufferHolder.offset();
+ super.rebuffer(position);
int posInBuffer = Ints.checkedCast(position - offset);
int remaining = buffer.limit() - posInBuffer;
if (remaining == 0)
@@ -69,58 +65,8 @@
}
@Override
- public ChannelProxy channel()
- {
- return wrapped.channel();
- }
-
- @Override
- public long fileLength()
- {
- return wrapped.fileLength();
- }
-
- @Override
- public double getCrcCheckChance()
- {
- return wrapped.getCrcCheckChance();
- }
-
- @Override
- public void close()
- {
- wrapped.close();
- }
-
- @Override
- public void closeReader()
- {
- wrapped.closeReader();
- }
-
- @Override
public String toString()
{
return "LimitingRebufferer[" + limiter + "]:" + wrapped;
}
-
- // BufferHolder methods
-
- @Override
- public ByteBuffer buffer()
- {
- return buffer;
- }
-
- @Override
- public long offset()
- {
- return offset;
- }
-
- @Override
- public void release()
- {
- bufferHolder.release();
- }
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegions.java b/src/java/org/apache/cassandra/io/util/MmappedRegions.java
index 0b7dd39..be64654 100644
--- a/src/java/org/apache/cassandra/io/util/MmappedRegions.java
+++ b/src/java/org/apache/cassandra/io/util/MmappedRegions.java
@@ -36,18 +36,25 @@
public class MmappedRegions extends SharedCloseableImpl
{
- /** In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size */
+ /**
+ * In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size
+ */
public static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE;
- /** When we need to grow the arrays, we add this number of region slots */
+ /**
+ * When we need to grow the arrays, we add this number of region slots
+ */
static final int REGION_ALLOC_SIZE = 15;
- /** The original state, which is shared with the tidier and
+ /**
+ * The original state, which is shared with the tidier and
* contains all the regions mapped so far. It also
- * does the actual mapping. */
+ * does the actual mapping.
+ */
private final State state;
- /** A copy of the latest state. We update this each time the original state is
+ /**
+ * A copy of the latest state. We update this each time the original state is
* updated and we share this with copies. If we are a copy, then this
* is null. Copies can only access existing regions, they cannot create
* new ones. This is for thread safety and because MmappedRegions is
@@ -92,7 +99,7 @@
}
/**
- * @param channel file to map. the MmappedRegions instance will hold shared copy of given channel.
+ * @param channel file to map. the MmappedRegions instance will hold shared copy of given channel.
* @param metadata
* @return new instance
*/
@@ -126,7 +133,12 @@
return copy == null;
}
- public void extend(long length)
+ /**
+ * Extends this collection of mmapped regions up to the provided total length.
+ *
+ * @return {@code true} if new regions have been created
+ */
+ public boolean extend(long length)
{
if (length < 0)
throw new IllegalArgumentException("Length must not be negative");
@@ -134,12 +146,41 @@
assert !isCopy() : "Copies cannot be extended";
if (length <= state.length)
- return;
+ return false;
+ int initialRegions = state.last;
updateState(length);
copy = new State(state);
+ return state.last > initialRegions;
}
+ /**
+ * Extends this collection of mmapped regions up to the length of the compressed file described by the provided
+ * metadata.
+ *
+ * @return {@code true} if new regions have been created
+ */
+ public boolean extend(CompressionMetadata compressionMetadata)
+ {
+ assert !isCopy() : "Copies cannot be extended";
+
+ if (compressionMetadata.compressedFileLength <= state.length)
+ return false;
+
+ int initialRegions = state.last;
+ if (compressionMetadata.compressedFileLength - state.length <= MAX_SEGMENT_SIZE)
+ updateState(compressionMetadata.compressedFileLength);
+ else
+ updateState(compressionMetadata);
+
+ copy = new State(state);
+ return state.last > initialRegions;
+ }
+
+ /**
+ * Updates state by adding the remaining segments. It starts with the current state last segment end position and
+ * subsequently add new segments until all data up to the provided length are mapped.
+ */
private void updateState(long length)
{
state.length = length;
@@ -154,8 +195,8 @@
private void updateState(CompressionMetadata metadata)
{
- long offset = 0;
- long lastSegmentOffset = 0;
+ long lastSegmentOffset = state.getPosition();
+ long offset = metadata.getDataOffsetForChunkOffset(lastSegmentOffset);
long segmentSize = 0;
while (offset < metadata.dataLength)
@@ -198,7 +239,7 @@
assert !isCleanedUp() : "Attempted to use closed region";
return state.floor(position);
}
-
+
public void closeQuietly()
{
Throwable err = close(null);
@@ -245,19 +286,29 @@
private static final class State
{
- /** The file channel */
+ /**
+ * The file channel
+ */
private final ChannelProxy channel;
- /** An array of region buffers, synchronized with offsets */
+ /**
+ * An array of region buffers, synchronized with offsets
+ */
private ByteBuffer[] buffers;
- /** An array of region offsets, synchronized with buffers */
+ /**
+ * An array of region offsets, synchronized with buffers
+ */
private long[] offsets;
- /** The maximum file length we have mapped */
+ /**
+ * The maximum file length we have mapped
+ */
private long length;
- /** The index to the last region added */
+ /**
+ * The index to the last region added
+ */
private int last;
private State(ChannelProxy channel)
@@ -292,7 +343,7 @@
{
assert 0 <= position && position <= length : String.format("%d > %d", position, length);
- int idx = Arrays.binarySearch(offsets, 0, last +1, position);
+ int idx = Arrays.binarySearch(offsets, 0, last + 1, position);
assert idx != -1 : String.format("Bad position %d for regions %s, last %d in %s", position, Arrays.toString(offsets), last, channel);
if (idx < 0)
idx = -(idx + 2); // round down to entry at insertion point
@@ -362,5 +413,4 @@
}
}
}
-
}
diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java b/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java
new file mode 100644
index 0000000..9e687ac
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.io.compress.CompressionMetadata;
+
+/**
+ * It is a utility class for caching {@link MmappedRegions} primarily used by a {@link FileHandle.Builder} when a handle
+ * to the same file is created multiple times (as when an sstable is opened early).
+ */
+@NotThreadSafe
+public class MmappedRegionsCache implements AutoCloseable
+{
+ private final Map<File, MmappedRegions> cache = new HashMap<>();
+ private boolean closed = false;
+
+ /**
+ * Looks for mmapped regions in cache. If found, a shared copy is created and extended to the provided length.
+ * If mmapped regions do not exist yet for the provided key, they are created and a shared copy is returned.
+ *
+ * @param channel channel for which the mmapped regions are requested
+ * @param length length of the file
+ * @return a shared copy of the cached mmapped regions
+ */
+ @SuppressWarnings("resource")
+ public MmappedRegions getOrCreate(ChannelProxy channel, long length)
+ {
+ Preconditions.checkState(!closed);
+ MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, length));
+ Preconditions.checkArgument(regions.isValid(channel));
+ regions.extend(length);
+ return regions.sharedCopy();
+ }
+
+ /**
+ * Looks for mmapped regions in cache. If found, a shared copy is created and extended according to the provided metadata.
+ * If mmapped regions do not exist yet for the provided key, they are created and a shared copy is returned.
+ *
+ * @param channel channel for which the mmapped regions are requested
+ * @param metadata compression metadata of the file
+ * @return a shared copy of the cached mmapped regions
+ */
+ @SuppressWarnings("resource")
+ public MmappedRegions getOrCreate(ChannelProxy channel, CompressionMetadata metadata)
+ {
+ Preconditions.checkState(!closed);
+ MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, metadata));
+ Preconditions.checkArgument(regions.isValid(channel));
+ regions.extend(metadata);
+ return regions.sharedCopy();
+ }
+
+ @Override
+ @SuppressWarnings("resource")
+ public void close()
+ {
+ closed = true;
+ Iterator<MmappedRegions> it = cache.values().iterator();
+ while (it.hasNext())
+ {
+ MmappedRegions region = it.next();
+ region.closeQuietly();
+ it.remove();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/util/PageAware.java b/src/java/org/apache/cassandra/io/util/PageAware.java
new file mode 100644
index 0000000..8eba8e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/PageAware.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+
+public final class PageAware
+{
+ public static final int PAGE_SIZE = 4096; // must be a power of two
+ public static final int PAGE_SIZE_SHIFT = Integer.numberOfTrailingZeros(PAGE_SIZE);
+
+ /**
+ * Calculate the end of the page identified by the given position.
+ * Equivalent to floor(dstPos / PAGE_SIZE + 1) * PAGE_SIZE.
+ * <p>
+ * When the argument is equal to the page boundary, returns the next page boundary. E.g. pageLimit(0) == PAGE_SIZE.
+ */
+ public static long pageLimit(long dstPos)
+ {
+ return (dstPos | (PAGE_SIZE - 1)) + 1;
+ }
+
+ /**
+ * Calculate the start of the page that contains the given position.
+ * Equivalent to floor(dstPos / PAGE_SIZE) * PAGE_SIZE.
+ */
+ public static long pageStart(long dstPos)
+ {
+ return dstPos & -PAGE_SIZE;
+ }
+
+ /**
+ * Calculate the earliest page boundary for the given position.
+ * Equivalent to ceil(dstPos / PAGE_SIZE) * PAGE_SIZE.
+ * <p>
+ * When the argument is equal to a page boundary, returns the argument.
+ */
+ public static long padded(long dstPos)
+ {
+ return pageStart(dstPos + PAGE_SIZE - 1);
+ }
+
+ /**
+ * Calculate the number of bytes left in this page.
+ * Equivalent to pageLimit(position) - position.
+ * <p>
+ * When the argument is equal to a page boundary, returns PAGE_SIZE.
+ */
+ public static int bytesLeftInPage(long dstPos)
+ {
+ return PAGE_SIZE - (int) (dstPos & (PAGE_SIZE - 1));
+ }
+
+ /**
+ * Calculate the number of pages that fit in the given size, rounded up to a page if the size is not an exact multiple.
+ *
+ * @param size the size that needs to cover a number of pages
+ * @return the number of pages, rounded up
+ */
+ public static int numPages(int size)
+ {
+ return (size + PAGE_SIZE - 1) >> PAGE_SIZE_SHIFT;
+ }
+
+ /**
+ * Given a position relative to the start of a number of pages, determine the exact page number this
+ * position falls into. For example, positions from 0 to {@link #PAGE_SIZE} -1 will fall into page zero
+ * and so forth.
+ *
+ * @param dstPos the position
+ * @return the page number, indexed at zero
+ */
+ public static int pageNum(long dstPos)
+ {
+ return Math.toIntExact(dstPos >> PAGE_SIZE_SHIFT);
+ }
+
+ /**
+ * Pad the given output stream with zeroes until the next page boundary.
+ * If the destination position is already at a page boundary, do not do anything.
+ */
+ public static void pad(DataOutputPlus dest) throws IOException
+ {
+ long position = dest.position();
+ long bytesLeft = padded(position) - position;
+ // bytesLeft is provably within [0, pageSize - 1]
+ dest.write(EmptyPage.EMPTY_PAGE, 0, (int) bytesLeft);
+ }
+
+ static class EmptyPage
+ {
+ static final byte[] EMPTY_PAGE = new byte[PAGE_SIZE];
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/PathUtils.java b/src/java/org/apache/cassandra/io/util/PathUtils.java
index 4b3efdb..027742f 100644
--- a/src/java/org/apache/cassandra/io/util/PathUtils.java
+++ b/src/java/org/apache/cassandra/io/util/PathUtils.java
@@ -58,7 +58,7 @@
*/
public final class PathUtils
{
- private static final boolean consistentDirectoryListings = CassandraRelevantProperties.DETERMINISM_CONSISTENT_DIRECTORY_LISTINGS.getBoolean();
+ private static final boolean consistentDirectoryListings = CassandraRelevantProperties.CONSISTENT_DIRECTORY_LISTINGS.getBoolean();
private static final Set<StandardOpenOption> READ_OPTIONS = unmodifiableSet(EnumSet.of(READ));
private static final Set<StandardOpenOption> WRITE_OPTIONS = unmodifiableSet(EnumSet.of(WRITE, CREATE, TRUNCATE_EXISTING));
diff --git a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
index 18cabd3..b7ae205 100644
--- a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
+++ b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
@@ -21,13 +21,13 @@
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
-import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.google.common.base.Preconditions;
import net.nicoulaj.compilecommand.annotations.DontInline;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.utils.FastByteOperations;
import org.apache.cassandra.utils.vint.VIntCoding;
@@ -36,10 +36,10 @@
/**
* Rough equivalent of BufferedInputStream and DataInputStream wrapping a ByteBuffer that can be refilled
* via rebuffer. Implementations provide this buffer from various channels (socket, file, memory, etc).
- *
+ * <p>
* RebufferingInputStream is not thread safe.
*/
-public abstract class RebufferingInputStream extends InputStream implements DataInputPlus, Closeable
+public abstract class RebufferingInputStream extends DataInputStreamPlus implements DataInputPlus, Closeable
{
protected ByteBuffer buffer;
@@ -231,11 +231,19 @@
return readPrimitiveSlowly(8);
}
+ @Override
public long readVInt() throws IOException
{
return VIntCoding.decodeZigZag64(readUnsignedVInt());
}
+ @Override
+ public int readVInt32() throws IOException
+ {
+ return VIntCoding.checkedCast(VIntCoding.decodeZigZag64(readUnsignedVInt()));
+ }
+
+ @Override
public long readUnsignedVInt() throws IOException
{
//If 9 bytes aren't available use the slow path in VIntCoding
@@ -268,6 +276,12 @@
}
@Override
+ public int readUnsignedVInt32() throws IOException
+ {
+ return VIntCoding.checkedCast(readUnsignedVInt());
+ }
+
+ @Override
public float readFloat() throws IOException
{
if (buffer.remaining() >= 4)
diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
index 431ece3..cb70aac 100644
--- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
@@ -37,6 +37,7 @@
{
// absolute path to the given file
private final String filePath;
+ private final File file;
// Offset for start of buffer relative to underlying file
protected long bufferOffset;
@@ -83,16 +84,19 @@
return accumulate;
}
+ @Override
protected void doPrepare()
{
syncInternal();
}
+ @Override
protected Throwable doCommit(Throwable accumulate)
{
return accumulate;
}
+ @Override
protected Throwable doAbort(Throwable accumulate)
{
return accumulate;
@@ -162,12 +166,13 @@
this.strictFlushing = strictFlushing;
this.fchannel = (FileChannel)channel;
+ this.file = file;
this.filePath = file.absolutePath();
this.option = option;
}
- public void skipBytes(int numBytes) throws IOException
+ public void skipBytes(long numBytes) throws IOException
{
flush();
fchannel.position(fchannel.position() + numBytes);
@@ -250,16 +255,42 @@
runPostFlush.run();
}
+ @Override
public boolean hasPosition()
{
return true;
}
+ @Override
public long position()
{
return current();
}
+ @Override
+ public int maxBytesInPage()
+ {
+ return PageAware.PAGE_SIZE;
+ }
+
+ @Override
+ public void padToPageBoundary() throws IOException
+ {
+ PageAware.pad(this);
+ }
+
+ @Override
+ public int bytesLeftInPage()
+ {
+ return PageAware.bytesLeftInPage(position());
+ }
+
+ @Override
+ public long paddedPosition()
+ {
+ return PageAware.padded(position());
+ }
+
/**
* Returns the current file pointer of the underlying on-disk file.
* Note that since write works by buffering data, the value of this will increase by buffer
@@ -296,6 +327,11 @@
return filePath;
}
+ public File getFile()
+ {
+ return file;
+ }
+
protected void resetBuffer()
{
bufferOffset = current();
@@ -373,16 +409,19 @@
return channel.isOpen();
}
+ @Override
public final void prepareToCommit()
{
txnProxy.prepareToCommit();
}
+ @Override
public final Throwable commit(Throwable accumulate)
{
return txnProxy.commit(accumulate);
}
+ @Override
public final Throwable abort(Throwable accumulate)
{
return txnProxy.abort(accumulate);
@@ -428,4 +467,4 @@
this.pointer = pointer;
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java b/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java
index 05fdb6b..8d00ce5 100644
--- a/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java
+++ b/src/java/org/apache/cassandra/io/util/SimpleChunkReader.java
@@ -57,7 +57,10 @@
@Override
public Rebufferer instantiateRebufferer()
{
- return new BufferManagingRebufferer.Unaligned(this);
+ if (Integer.bitCount(bufferSize) == 1)
+ return new BufferManagingRebufferer.Aligned(this);
+ else
+ return new BufferManagingRebufferer.Unaligned(this);
}
@Override
@@ -69,4 +72,4 @@
bufferSize,
fileLength());
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/SizedInts.java b/src/java/org/apache/cassandra/io/util/SizedInts.java
new file mode 100644
index 0000000..d18e1f0
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/SizedInts.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Utility class for sizing, writing and reading ints with length stored separately.
+ * Used for trie payloads.
+ */
+public class SizedInts
+{
+ /**
+ * Returns the number of bytes we need to store the given position.
+ * This method understands 0 to need 1 byte.
+ * <p>
+ * If your use case permits 0 to be encoded in 0 length, use {@link #sizeAllowingZero} below.
+ */
+ public static int nonZeroSize(long value)
+ {
+ if (value < 0)
+ value = ~value;
+ int lz = Long.numberOfLeadingZeros(value); // 1 <= lz <= 64
+ return (64 - lz + 1 + 7) / 8; // significant bits, +1 for sign, rounded up. At least 1, at most 8.
+ }
+
+ /**
+ * Returns the number of bytes we need to store the given position. Returns 0 for 0 argument.
+ */
+ public static int sizeAllowingZero(long value)
+ {
+ if (value == 0)
+ return 0;
+ return nonZeroSize(value);
+ }
+
+ public static long read(ByteBuffer src, int startPos, int bytes)
+ {
+ switch (bytes)
+ {
+ case 0:
+ return 0;
+ case 1:
+ return src.get(startPos);
+ case 2:
+ return src.getShort(startPos);
+ case 3:
+ {
+ long high = src.get(startPos);
+ return (high << 16L) | (src.getShort(startPos + 1) & 0xFFFFL);
+ }
+ case 4:
+ return src.getInt(startPos);
+ case 5:
+ {
+ long high = src.get(startPos);
+ return (high << 32L) | (src.getInt(startPos + 1) & 0xFFFFFFFFL);
+ }
+ case 6:
+ {
+ long high = src.getShort(startPos);
+ return (high << 32L) | (src.getInt(startPos + 2) & 0xFFFFFFFFL);
+ }
+ case 7:
+ {
+ long high = src.get(startPos);
+ high = (high << 16L) | (src.getShort(startPos + 1) & 0xFFFFL);
+ return (high << 32L) | (src.getInt(startPos + 3) & 0xFFFFFFFFL);
+ }
+ case 8:
+ return src.getLong(startPos);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ public static long readUnsigned(ByteBuffer src, int startPos, int bytes)
+ {
+ if (bytes == 8)
+ return src.getLong(startPos);
+ else
+ return read(src, startPos, bytes) & ((1L << (bytes * 8)) - 1);
+ }
+
+ public static void write(DataOutputPlus dest, long value, int size) throws IOException
+ {
+ dest.writeMostSignificantBytes(value << ((8 - size) * 8), size);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/TailOverridingRebufferer.java b/src/java/org/apache/cassandra/io/util/TailOverridingRebufferer.java
new file mode 100644
index 0000000..e6b7ed2
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/TailOverridingRebufferer.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.util;
+
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Special rebufferer that replaces the tail of the file (from the specified cutoff point) with the given buffer.
+ * <p>
+ * Instantiated once per RandomAccessReader, thread-unsafe.
+ * The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call.
+ * Only one BufferHolder can be active at a time. Calling {@link #rebuffer(long)} before the previously obtained
+ * buffer holder is released will throw {@link AssertionError}.
+ */
+@NotThreadSafe
+public class TailOverridingRebufferer extends WrappingRebufferer
+{
+ private final long cutoff;
+ private final ByteBuffer tail;
+
+ public TailOverridingRebufferer(Rebufferer source, long cutoff, ByteBuffer tail)
+ {
+ super(source);
+ this.cutoff = cutoff;
+ this.tail = tail;
+ }
+
+ @Override
+ public Rebufferer.BufferHolder rebuffer(long position)
+ {
+ assert buffer == null : "Buffer holder has been already acquired and has been not released yet";
+ if (position < cutoff)
+ {
+ super.rebuffer(position);
+ if (offset + buffer.limit() > cutoff)
+ buffer.limit((int) (cutoff - offset));
+ }
+ else
+ {
+ buffer = tail.duplicate();
+ offset = cutoff;
+ }
+ return this;
+ }
+
+ @Override
+ public long fileLength()
+ {
+ return cutoff + tail.limit();
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[+%d@%d]:%s", getClass().getSimpleName(), tail.limit(), cutoff, wrapped.toString());
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/util/TeeDataInputPlus.java b/src/java/org/apache/cassandra/io/util/TeeDataInputPlus.java
new file mode 100644
index 0000000..b16c1cc
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/TeeDataInputPlus.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.utils.Throwables;
+
+/**
+ * DataInput that also stores the raw inputs into an output buffer
+ * This is useful for storing serialized buffers as they are deserialized.
+ *
+ * Note: If a non-zero limit is included it is important to for callers to check {@link #isLimitReached()}
+ * before using the tee buffer as it could be cropped.
+ */
+public class TeeDataInputPlus implements DataInputPlus
+{
+ private final DataInputPlus source;
+ private final DataOutputPlus teeBuffer;
+
+ private final long limit;
+ private boolean limitReached;
+
+ public TeeDataInputPlus(DataInputPlus source, DataOutputPlus teeBuffer)
+ {
+ this(source, teeBuffer, 0);
+ }
+
+ public TeeDataInputPlus(DataInputPlus source, DataOutputPlus teeBuffer, long limit)
+ {
+ assert source != null && teeBuffer != null;
+ this.source = source;
+ this.teeBuffer = teeBuffer;
+ this.limit = limit;
+ this.limitReached = false;
+ }
+
+ private void maybeWrite(int length, Throwables.DiscreteAction<IOException> writeAction) throws IOException
+ {
+ if (limit <= 0 || (!limitReached && (teeBuffer.position() + length) < limit))
+ writeAction.perform();
+ else
+ limitReached = true;
+ }
+
+ @Override
+ public void readFully(byte[] bytes) throws IOException
+ {
+ source.readFully(bytes);
+ maybeWrite(bytes.length, () -> teeBuffer.write(bytes));
+ }
+
+ @Override
+ public void readFully(byte[] bytes, int offset, int length) throws IOException
+ {
+ source.readFully(bytes, offset, length);
+ maybeWrite(length, () -> teeBuffer.write(bytes, offset, length));
+ }
+
+ @Override
+ public int skipBytes(int n) throws IOException
+ {
+ for (int i = 0; i < n; i++)
+ {
+ try
+ {
+ byte v = source.readByte();
+ maybeWrite(TypeSizes.BYTE_SIZE, () -> teeBuffer.writeByte(v));
+ }
+ catch (EOFException eof)
+ {
+ return i;
+ }
+ }
+ return n;
+ }
+
+ @Override
+ public boolean readBoolean() throws IOException
+ {
+ boolean v = source.readBoolean();
+ maybeWrite(TypeSizes.BOOL_SIZE, () -> teeBuffer.writeBoolean(v));
+ return v;
+ }
+
+ @Override
+ public byte readByte() throws IOException
+ {
+ byte v = source.readByte();
+ maybeWrite(TypeSizes.BYTE_SIZE, () -> teeBuffer.writeByte(v));
+ return v;
+ }
+
+ @Override
+ public int readUnsignedByte() throws IOException
+ {
+ int v = source.readUnsignedByte();
+ maybeWrite(TypeSizes.BYTE_SIZE, () -> teeBuffer.writeByte(v));
+ return v;
+ }
+
+ @Override
+ public short readShort() throws IOException
+ {
+ short v = source.readShort();
+ maybeWrite(TypeSizes.SHORT_SIZE, () -> teeBuffer.writeShort(v));
+ return v;
+ }
+
+ @Override
+ public int readUnsignedShort() throws IOException
+ {
+ int v = source.readUnsignedShort();
+ maybeWrite(TypeSizes.SHORT_SIZE, () -> teeBuffer.writeShort(v));
+ return v;
+ }
+
+ @Override
+ public char readChar() throws IOException
+ {
+ char v = source.readChar();
+ maybeWrite(TypeSizes.BYTE_SIZE, () -> teeBuffer.writeChar(v));
+ return v;
+ }
+
+ @Override
+ public int readInt() throws IOException
+ {
+ int v = source.readInt();
+ maybeWrite(TypeSizes.INT_SIZE, () -> teeBuffer.writeInt(v));
+ return v;
+ }
+
+ @Override
+ public long readLong() throws IOException
+ {
+ long v = source.readLong();
+ maybeWrite(TypeSizes.LONG_SIZE, () -> teeBuffer.writeLong(v));
+ return v;
+ }
+
+ @Override
+ public float readFloat() throws IOException
+ {
+ float v = source.readFloat();
+ maybeWrite(TypeSizes.FLOAT_SIZE, () -> teeBuffer.writeFloat(v));
+ return v;
+ }
+
+ @Override
+ public double readDouble() throws IOException
+ {
+ double v = source.readDouble();
+ maybeWrite(TypeSizes.DOUBLE_SIZE, () -> teeBuffer.writeDouble(v));
+ return v;
+ }
+
+ @Override
+ public String readLine() throws IOException
+ {
+ //This one isn't safe since we know the actual line termination type
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String readUTF() throws IOException
+ {
+ String v = source.readUTF();
+ maybeWrite(TypeSizes.sizeof(v), () -> teeBuffer.writeUTF(v));
+ return v;
+ }
+
+ @Override
+ public long readVInt() throws IOException
+ {
+ long v = source.readVInt();
+ maybeWrite(TypeSizes.sizeofVInt(v), () -> teeBuffer.writeVInt(v));
+ return v;
+ }
+
+ @Override
+ public long readUnsignedVInt() throws IOException
+ {
+ long v = source.readUnsignedVInt();
+ maybeWrite(TypeSizes.sizeofUnsignedVInt(v), () -> teeBuffer.writeUnsignedVInt(v));
+ return v;
+ }
+
+ @Override
+ public void skipBytesFully(int n) throws IOException
+ {
+ source.skipBytesFully(n);
+ maybeWrite(n, () -> {
+ for (int i = 0; i < n; i++)
+ teeBuffer.writeByte(0);
+ });
+ }
+
+ /**
+ * Used to detect if the teeBuffer hit the supplied limit.
+ * If true this means the teeBuffer does not contain the full input.
+ */
+ public boolean isLimitReached()
+ {
+ return limitReached;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java b/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java
new file mode 100644
index 0000000..5fbe5ea
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.util;
+
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Instantiated once per RandomAccessReader, thread-unsafe.
+ * The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call.
+ * Only one buffer holder can be active at a time. Calling {@link #rebuffer(long)} before the previously obtained
+ * buffer holder is released will throw {@link AssertionError}. We will get that exception also in case we try to close
+ * the rebufferer without closing the recently obtained buffer holder.
+ * <p>
+ * Calling methods of {@link BufferHolder} will also produce {@link AssertionError} if buffer holder is not acquired.
+ * <p>
+ * The overriding classes must conform to the aforementioned rules.
+ */
+@NotThreadSafe
+public abstract class WrappingRebufferer implements Rebufferer, Rebufferer.BufferHolder
+{
+ protected final Rebufferer wrapped;
+
+ protected BufferHolder bufferHolder;
+ protected ByteBuffer buffer;
+ protected long offset;
+
+ public WrappingRebufferer(Rebufferer wrapped)
+ {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public BufferHolder rebuffer(long position)
+ {
+ assert buffer == null;
+ bufferHolder = wrapped.rebuffer(position);
+ buffer = bufferHolder.buffer();
+ offset = bufferHolder.offset();
+
+ return this;
+ }
+
+ @Override
+ public ChannelProxy channel()
+ {
+ return wrapped.channel();
+ }
+
+ @Override
+ public long fileLength()
+ {
+ return wrapped.fileLength();
+ }
+
+ @Override
+ public double getCrcCheckChance()
+ {
+ return wrapped.getCrcCheckChance();
+ }
+
+ @Override
+ public void close()
+ {
+ assert buffer == null : "Rebufferer is attempted to be closed but the buffer holder has not been released";
+ wrapped.close();
+ }
+
+ @Override
+ public void closeReader()
+ {
+ wrapped.closeReader();
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s[]:%s", getClass().getSimpleName(), wrapped.toString());
+ }
+
+ @Override
+ public ByteBuffer buffer()
+ {
+ assert buffer != null : "Buffer holder has not been acquired";
+ return buffer;
+ }
+
+ @Override
+ public long offset()
+ {
+ assert buffer != null : "Buffer holder has not been acquired";
+ return offset;
+ }
+
+ @Override
+ public void release()
+ {
+ assert buffer != null;
+ if (bufferHolder != null)
+ {
+ bufferHolder.release();
+ bufferHolder = null;
+ }
+ buffer = null;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
index 2248248..77e04e6 100644
--- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
+++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java
@@ -26,6 +26,8 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import com.google.common.annotations.VisibleForTesting;
+
import com.codahale.metrics.ExponentiallyDecayingReservoir;
import com.codahale.metrics.Snapshot;
@@ -41,12 +43,14 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
+import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_DYNAMIC_SNITCH_SEVERITY;
+
/**
* A dynamic snitch that sorts endpoints by latency with an adapted phi failure detector
*/
public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements LatencySubscribers.Subscriber, DynamicEndpointSnitchMBean
{
- private static final boolean USE_SEVERITY = !Boolean.getBoolean("cassandra.ignore_dynamic_snitch_severity");
+ private static final boolean USE_SEVERITY = !IGNORE_DYNAMIC_SNITCH_SEVERITY.getBoolean();
private static final double ALPHA = 0.75; // set to 0.75 to make EDS more biased to towards the newer values
private static final int WINDOW_SIZE = 100;
@@ -200,7 +204,7 @@
{
Double score = scores.get(replica.endpoint());
if (score == null)
- score = 0.0;
+ score = defaultStore(replica.endpoint());
subsnitchOrderedScores.add(score);
}
@@ -224,6 +228,11 @@
return replicas;
}
+ private static double defaultStore(InetAddressAndPort target)
+ {
+ return USE_SEVERITY ? getSeverity(target) : 0.0;
+ }
+
// Compare endpoints given an immutable snapshot of the scores
private int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2, Map<InetAddressAndPort, Double> scores)
{
@@ -232,12 +241,12 @@
if (scored1 == null)
{
- scored1 = 0.0;
+ scored1 = defaultStore(a1.endpoint());
}
if (scored2 == null)
{
- scored2 = 0.0;
+ scored2 = defaultStore(a2.endpoint());
}
if (scored1.equals(scored2))
@@ -269,7 +278,8 @@
sample.update(unit.toMillis(latency));
}
- private void updateScores() // this is expensive
+ @VisibleForTesting
+ public void updateScores() // this is expensive
{
if (!StorageService.instance.isInitialized())
return;
@@ -359,12 +369,19 @@
return timings;
}
+ @Override
public void setSeverity(double severity)
{
+ addSeverity(severity);
+ }
+
+ public static void addSeverity(double severity)
+ {
Gossiper.instance.addLocalApplicationState(ApplicationState.SEVERITY, StorageService.instance.valueFactory.severity(severity));
}
- private double getSeverity(InetAddressAndPort endpoint)
+ @VisibleForTesting
+ public static double getSeverity(InetAddressAndPort endpoint)
{
EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
if (state == null)
diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
index e6a920b..78627e4 100644
--- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
+++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
@@ -26,7 +26,11 @@
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.regex.Pattern;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.net.HostAndPort;
@@ -64,7 +68,8 @@
public final byte[] addressBytes;
- private InetAddressAndPort(InetAddress address, byte[] addressBytes, int port)
+ @VisibleForTesting
+ InetAddressAndPort(InetAddress address, byte[] addressBytes, int port)
{
super(address, port);
Preconditions.checkNotNull(address);
@@ -215,6 +220,31 @@
return getByNameOverrideDefaults(name, null);
}
+
+ public static List<InetAddressAndPort> getAllByName(String name) throws UnknownHostException
+ {
+ return getAllByNameOverrideDefaults(name, null);
+ }
+
+ /**
+ *
+ * @param name Hostname + optional ports string
+ * @param port Port to connect on, overridden by values in hostname string, defaults to DatabaseDescriptor default if not specified anywhere.
+ */
+ public static List<InetAddressAndPort> getAllByNameOverrideDefaults(String name, Integer port) throws UnknownHostException
+ {
+ HostAndPort hap = HostAndPort.fromString(name);
+ if (hap.hasPort())
+ {
+ port = hap.getPort();
+ }
+ Integer finalPort = port;
+
+ return Stream.of(InetAddress.getAllByName(hap.getHost()))
+ .map((address) -> getByAddressOverrideDefaults(address, finalPort))
+ .collect(Collectors.toList());
+ }
+
/**
*
* @param name Hostname + optional ports string
diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
index 1d39bbe..9d1989e 100644
--- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
+++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
@@ -350,6 +350,7 @@
String dc = e.getKey();
ReplicationFactor rf = getReplicationFactor(dc);
Guardrails.minimumReplicationFactor.guard(rf.fullReplicas, keyspaceName, false, state);
+ Guardrails.maximumReplicationFactor.guard(rf.fullReplicas, keyspaceName, false, state);
int nodeCount = dcsNodes.get(dc).size();
// nodeCount==0 on many tests
if (rf.fullReplicas > nodeCount && nodeCount != 0)
diff --git a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java
index d8ca9e4..9e5ba23 100644
--- a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java
+++ b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java
@@ -30,6 +30,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.cassandra.auth.IInternodeAuthenticator.InternodeConnectionDirection.OUTBOUND_PRECONNECT;
+
/**
* Sidekick helper for snitches that want to reconnect from one IP addr for a node to another.
* Typically, this is for situations like EC2 where a node will have a public address and a private address,
@@ -64,7 +66,8 @@
@VisibleForTesting
static void reconnect(InetAddressAndPort publicAddress, InetAddressAndPort localAddress, IEndpointSnitch snitch, String localDc)
{
- if (!new OutboundConnectionSettings(publicAddress, localAddress).withDefaults(ConnectionCategory.MESSAGING).authenticate())
+ final OutboundConnectionSettings settings = new OutboundConnectionSettings(publicAddress, localAddress).withDefaults(ConnectionCategory.MESSAGING);
+ if (!settings.authenticator().authenticate(settings.to.getAddress(), settings.to.getPort(), null, OUTBOUND_PRECONNECT))
{
logger.debug("InternodeAuthenticator said don't reconnect to {} on {}", publicAddress, localAddress);
return;
diff --git a/src/java/org/apache/cassandra/locator/Replica.java b/src/java/org/apache/cassandra/locator/Replica.java
index 4c5f7c6..4c58e64 100644
--- a/src/java/org/apache/cassandra/locator/Replica.java
+++ b/src/java/org/apache/cassandra/locator/Replica.java
@@ -191,6 +191,5 @@
{
return transientReplica(endpoint, new Range<>(start, end));
}
-
}
diff --git a/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java b/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
index fe500b4..b6ccaec 100644
--- a/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
+++ b/src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
@@ -23,8 +23,12 @@
import java.util.List;
import java.util.Map;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.FBUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,8 +36,18 @@
public class SimpleSeedProvider implements SeedProvider
{
private static final Logger logger = LoggerFactory.getLogger(SimpleSeedProvider.class);
+ @VisibleForTesting
+ public static final String SEEDS_KEY = "seeds";
+ @VisibleForTesting
+ public static final String RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY = "resolve_multiple_ip_addresses_per_dns_record";
+ private static final int SEED_COUNT_WARN_THRESHOLD = CassandraRelevantProperties.SEED_COUNT_WARN_THRESHOLD.getInt(20);
- public SimpleSeedProvider(Map<String, String> args) {}
+ private final String[] defaultSeeds;
+
+ public SimpleSeedProvider(Map<String, String> args)
+ {
+ defaultSeeds = new String[] { FBUtilities.getLocalAddressAndPort().getHostAddress(true) };
+ }
public List<InetAddressAndPort> getSeeds()
{
@@ -46,14 +60,44 @@
{
throw new AssertionError(e);
}
- String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1);
+
+ assert conf.seed_provider != null : "conf.seed_provider is null!";
+
+ boolean resolveMultipleIps;
+ String[] hosts;
+
+ Map<String, String> parameters = conf.seed_provider.parameters;
+ if (parameters == null)
+ {
+ resolveMultipleIps = false;
+ hosts = defaultSeeds;
+ }
+ else
+ {
+ hosts = parameters.getOrDefault(SEEDS_KEY, defaultSeeds[0]).split(",", -1);
+ resolveMultipleIps = Boolean.parseBoolean(parameters.getOrDefault(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, Boolean.FALSE.toString()));
+ }
+
List<InetAddressAndPort> seeds = new ArrayList<>(hosts.length);
+
for (String host : hosts)
{
try
{
- if(!host.trim().isEmpty()) {
- seeds.add(InetAddressAndPort.getByName(host.trim()));
+ if (!host.trim().isEmpty())
+ {
+ if (resolveMultipleIps)
+ {
+ List<InetAddressAndPort> resolvedSeeds = InetAddressAndPort.getAllByName(host.trim());
+ seeds.addAll(resolvedSeeds);
+ logger.debug("{} resolves to {}", host, resolvedSeeds);
+ }
+ else
+ {
+ InetAddressAndPort addressAndPort = InetAddressAndPort.getByName(host.trim());
+ seeds.add(addressAndPort);
+ logger.debug("Only resolving one IP per DNS record - {} resolves to {}", host, addressAndPort);
+ }
}
}
catch (UnknownHostException ex)
@@ -62,6 +106,12 @@
logger.warn("Seed provider couldn't lookup host {}", host);
}
}
+
+ if (seeds.size() > SEED_COUNT_WARN_THRESHOLD)
+ logger.warn("Seed provider returned more than {} seeds. " +
+ "A large seed list may impact effectiveness of the third gossip round.",
+ SEED_COUNT_WARN_THRESHOLD);
+
return Collections.unmodifiableList(seeds);
}
}
diff --git a/src/java/org/apache/cassandra/locator/SimpleStrategy.java b/src/java/org/apache/cassandra/locator/SimpleStrategy.java
index e5b9210..488b601 100644
--- a/src/java/org/apache/cassandra/locator/SimpleStrategy.java
+++ b/src/java/org/apache/cassandra/locator/SimpleStrategy.java
@@ -109,6 +109,7 @@
int nodeCount = StorageService.instance.getHostIdToEndpoint().size();
// nodeCount==0 on many tests
Guardrails.minimumReplicationFactor.guard(rf.fullReplicas, keyspaceName, false, state);
+ Guardrails.maximumReplicationFactor.guard(rf.fullReplicas, keyspaceName, false, state);
if (rf.fullReplicas > nodeCount && nodeCount != 0)
{
String msg = "Your replication factor " + rf.fullReplicas
diff --git a/src/java/org/apache/cassandra/locator/SnitchProperties.java b/src/java/org/apache/cassandra/locator/SnitchProperties.java
index afb6804..b0bfe4c 100644
--- a/src/java/org/apache/cassandra/locator/SnitchProperties.java
+++ b/src/java/org/apache/cassandra/locator/SnitchProperties.java
@@ -25,10 +25,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_RACKDC_PROPERTIES;
+
public class SnitchProperties
{
private static final Logger logger = LoggerFactory.getLogger(SnitchProperties.class);
- public static final String RACKDC_PROPERTY_FILENAME = "cassandra-rackdc.properties";
+ public static final String RACKDC_PROPERTY_FILENAME = CASSANDRA_RACKDC_PROPERTIES.getKey();
private final Properties properties;
@@ -36,7 +38,7 @@
{
properties = new Properties();
InputStream stream = null;
- String configURL = System.getProperty(RACKDC_PROPERTY_FILENAME);
+ String configURL = CASSANDRA_RACKDC_PROPERTIES.getString();
try
{
URL url;
diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestSizeMetrics.java b/src/java/org/apache/cassandra/metrics/ClientRequestSizeMetrics.java
new file mode 100644
index 0000000..4def87e
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ClientRequestSizeMetrics.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.util.Collection;
+
+import com.codahale.metrics.Counter;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.selection.Selection;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class ClientRequestSizeMetrics
+{
+ private static final String TYPE = "ClientRequestSize";
+
+ public static final Counter totalColumnsRead = Metrics.counter(DefaultNameFactory.createMetricName(TYPE, "ColumnsRead", null));
+ public static final Counter totalRowsRead = Metrics.counter(DefaultNameFactory.createMetricName(TYPE, "RowsRead", null));
+ public static final Counter totalColumnsWritten = Metrics.counter(DefaultNameFactory.createMetricName(TYPE, "ColumnsWritten", null));
+ public static final Counter totalRowsWritten = Metrics.counter(DefaultNameFactory.createMetricName(TYPE, "RowsWritten", null));
+
+ public static void recordReadResponseMetrics(ResultMessage.Rows rows, StatementRestrictions restrictions, Selection selection)
+ {
+ if (!DatabaseDescriptor.getClientRequestSizeMetricsEnabled())
+ return;
+
+ int rowCount = rows.result.size();
+ ClientRequestSizeMetrics.totalRowsRead.inc(rowCount);
+
+ int nonRestrictedColumns = selection.getColumns().size();
+
+ for (ColumnMetadata column : selection.getColumns())
+ if (restrictions.isEqualityRestricted(column))
+ nonRestrictedColumns--;
+
+ long columnCount = (long) rowCount * nonRestrictedColumns;
+ ClientRequestSizeMetrics.totalColumnsRead.inc(columnCount);
+ }
+
+ public static void recordRowAndColumnCountMetrics(Collection<? extends IMutation> mutations)
+ {
+ if (!DatabaseDescriptor.getClientRequestSizeMetricsEnabled())
+ return;
+
+ int rowCount = 0;
+ int columnCount = 0;
+
+ for (IMutation mutation : mutations)
+ {
+ for (PartitionUpdate update : mutation.getPartitionUpdates())
+ {
+ columnCount += update.affectedColumnCount();
+ rowCount += update.affectedRowCount();
+ }
+ }
+
+ ClientRequestSizeMetrics.totalColumnsWritten.inc(columnCount);
+ ClientRequestSizeMetrics.totalRowsWritten.inc(rowCount);
+ }
+
+ public static void recordRowAndColumnCountMetrics(PartitionUpdate update)
+ {
+ if (!DatabaseDescriptor.getClientRequestSizeMetricsEnabled())
+ return;
+
+ ClientRequestSizeMetrics.totalColumnsWritten.inc(update.affectedColumnCount());
+ ClientRequestSizeMetrics.totalRowsWritten.inc(update.affectedRowCount());
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
index 308023e..565e33f 100644
--- a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
+++ b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java
@@ -42,6 +42,7 @@
import static java.lang.Math.max;
import static java.lang.Math.min;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DECAYING_ESTIMATED_HISTOGRAM_RESERVOIR_STRIPE_COUNT;
/**
* A decaying histogram reservoir where values collected during each minute will be twice as significant as the values
@@ -91,7 +92,7 @@
*/
public static final int DEFAULT_BUCKET_COUNT = 164;
public static final int LOW_BUCKET_COUNT = 127;
- public static final int DEFAULT_STRIPE_COUNT = Integer.parseInt(System.getProperty("cassandra.dehr_stripe_count", "2"));
+ public static final int DEFAULT_STRIPE_COUNT = DECAYING_ESTIMATED_HISTOGRAM_RESERVOIR_STRIPE_COUNT.getInt();
public static final int MAX_BUCKET_COUNT = 237;
public static final boolean DEFAULT_ZERO_CONSIDERATION = false;
diff --git a/src/java/org/apache/cassandra/metrics/FrequencySampler.java b/src/java/org/apache/cassandra/metrics/FrequencySampler.java
index 8a8918b..d4dfe86 100644
--- a/src/java/org/apache/cassandra/metrics/FrequencySampler.java
+++ b/src/java/org/apache/cassandra/metrics/FrequencySampler.java
@@ -33,33 +33,31 @@
* <p>add("x", 10); and add("x", 20); will result in "x" = 30</p> This uses StreamSummary to only store the
* approximate cardinality (capacity) of keys. If the number of distinct keys exceed the capacity, the error of the
* sample may increase depending on distribution of keys among the total set.
+ *
+ * Note: {@link Sampler#samplerExecutor} is single threaded but we still need to synchronize as we have access
+ * from both internal and the external JMX context that can cause races.
*
* @param <T>
*/
public abstract class FrequencySampler<T> extends Sampler<T>
{
private static final Logger logger = LoggerFactory.getLogger(FrequencySampler.class);
- private long endTimeNanos = -1;
private StreamSummary<T> summary;
/**
* Start to record samples
*
- * @param capacity
- * Number of sample items to keep in memory, the lower this is
- * the less accurate results are. For best results use value
- * close to cardinality, but understand the memory trade offs.
+ * @param capacity Number of sample items to keep in memory, the lower this is
+ * the less accurate results are. For best results use value
+ * close to cardinality, but understand the memory trade offs.
*/
- public synchronized void beginSampling(int capacity, int durationMillis)
+ public synchronized void beginSampling(int capacity, long durationMillis)
{
- if (endTimeNanos == -1 || clock.now() > endTimeNanos)
- {
- summary = new StreamSummary<>(capacity);
- endTimeNanos = clock.now() + MILLISECONDS.toNanos(durationMillis);
- }
- else
+ if (isActive())
throw new RuntimeException("Sampling already in progress");
+ updateEndTime(clock.now() + MILLISECONDS.toNanos(durationMillis));
+ summary = new StreamSummary<>(capacity);
}
/**
@@ -69,12 +67,12 @@
public synchronized List<Sample<T>> finishSampling(int count)
{
List<Sample<T>> results = Collections.emptyList();
- if (endTimeNanos != -1)
+ if (isEnabled())
{
- endTimeNanos = -1;
+ disable();
results = summary.topK(count)
.stream()
- .map(c -> new Sample<T>(c.getItem(), c.getCount(), c.getError()))
+ .map(c -> new Sample<>(c.getItem(), c.getCount(), c.getError()))
.collect(Collectors.toList());
}
return results;
@@ -82,24 +80,16 @@
protected synchronized void insert(final T item, final long value)
{
- // samplerExecutor is single threaded but still need
- // synchronization against jmx calls to finishSampling
- if (value > 0 && clock.now() <= endTimeNanos)
+ if (value > 0 && isActive())
{
try
{
summary.offer(item, (int) Math.min(value, Integer.MAX_VALUE));
- } catch (Exception e)
+ }
+ catch (Exception e)
{
logger.trace("Failure to offer sample", e);
}
}
}
-
- public boolean isEnabled()
- {
- return endTimeNanos != -1 && clock.now() <= endTimeNanos;
- }
-
}
-
diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
index 776027e..707cca2 100644
--- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
@@ -20,18 +20,22 @@
import java.util.Set;
import java.util.function.ToLongFunction;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName;
import org.apache.cassandra.metrics.TableMetrics.ReleasableMetric;
-import com.google.common.collect.Sets;
-
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
/**
@@ -59,16 +63,15 @@
public final Gauge<Long> pendingFlushes;
/** Estimate of number of pending compactios for this CF */
public final Gauge<Long> pendingCompactions;
- /** Disk space used by SSTables belonging to this CF */
+ /** Disk space used by SSTables belonging to tables in this keyspace */
public final Gauge<Long> liveDiskSpaceUsed;
- /** Total disk space used by SSTables belonging to this CF, including obsolete ones waiting to be GC'd */
+ /** Disk space used by SSTables belonging to tables in this keyspace, scaled down by replication factor */
+ public final Gauge<Long> unreplicatedLiveDiskSpaceUsed;
+ /** Uncompressed/logical size of SSTables belonging to tables in this keyspace */
+ public final Gauge<Long> uncompressedLiveDiskSpaceUsed;
+ /** Uncompressed/logical size of SSTables belonging to tables in this keyspace, scaled down by replication factor */
+ public final Gauge<Long> unreplicatedUncompressedLiveDiskSpaceUsed;
public final Gauge<Long> totalDiskSpaceUsed;
- /** Disk space used by bloom filter */
- public final Gauge<Long> bloomFilterDiskSpaceUsed;
- /** Off heap memory used by bloom filter */
- public final Gauge<Long> bloomFilterOffHeapMemoryUsed;
- /** Off heap memory used by index summary */
- public final Gauge<Long> indexSummaryOffHeapMemoryUsed;
/** Off heap memory used by compression meta data*/
public final Gauge<Long> compressionMetadataOffHeapMemoryUsed;
/** (Local) read metrics */
@@ -77,8 +80,10 @@
public final LatencyMetrics rangeLatency;
/** (Local) write metrics */
public final LatencyMetrics writeLatency;
- /** Histogram of the number of sstable data files accessed per read */
+ /** Histogram of the number of sstable data files accessed per single partition read */
public final Histogram sstablesPerReadHistogram;
+ /** Histogram of the number of sstable data files accessed per partition range read */
+ public final Histogram sstablesPerRangeReadHistogram;
/** Tombstones scanned in queries on this Keyspace */
public final Histogram tombstoneScannedHistogram;
/** Live cells scanned in queries on this Keyspace */
@@ -168,8 +173,10 @@
public final Meter rowIndexSizeAborts;
public final Histogram rowIndexSize;
+ public final ImmutableMap<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> formatSpecificGauges;
+
public final MetricNameFactory factory;
- private Keyspace keyspace;
+ private final Keyspace keyspace;
/** set containing names of all the metrics stored here, for releasing later */
private Set<ReleasableMetric> allMetrics = Sets.newHashSet();
@@ -201,14 +208,15 @@
metric -> metric.memtableSwitchCount.getCount());
pendingCompactions = createKeyspaceGauge("PendingCompactions", metric -> metric.pendingCompactions.getValue());
pendingFlushes = createKeyspaceGauge("PendingFlushes", metric -> metric.pendingFlushes.getCount());
+
liveDiskSpaceUsed = createKeyspaceGauge("LiveDiskSpaceUsed", metric -> metric.liveDiskSpaceUsed.getCount());
+ uncompressedLiveDiskSpaceUsed = createKeyspaceGauge("UncompressedLiveDiskSpaceUsed", metric -> metric.uncompressedLiveDiskSpaceUsed.getCount());
+ unreplicatedLiveDiskSpaceUsed = createKeyspaceGauge("UnreplicatedLiveDiskSpaceUsed",
+ metric -> metric.liveDiskSpaceUsed.getCount() / keyspace.getReplicationStrategy().getReplicationFactor().fullReplicas);
+ unreplicatedUncompressedLiveDiskSpaceUsed = createKeyspaceGauge("UnreplicatedUncompressedLiveDiskSpaceUsed",
+ metric -> metric.uncompressedLiveDiskSpaceUsed.getCount() / keyspace.getReplicationStrategy().getReplicationFactor().fullReplicas);
totalDiskSpaceUsed = createKeyspaceGauge("TotalDiskSpaceUsed", metric -> metric.totalDiskSpaceUsed.getCount());
- bloomFilterDiskSpaceUsed = createKeyspaceGauge("BloomFilterDiskSpaceUsed",
- metric -> metric.bloomFilterDiskSpaceUsed.getValue());
- bloomFilterOffHeapMemoryUsed = createKeyspaceGauge("BloomFilterOffHeapMemoryUsed",
- metric -> metric.bloomFilterOffHeapMemoryUsed.getValue());
- indexSummaryOffHeapMemoryUsed = createKeyspaceGauge("IndexSummaryOffHeapMemoryUsed",
- metric -> metric.indexSummaryOffHeapMemoryUsed.getValue());
+
compressionMetadataOffHeapMemoryUsed = createKeyspaceGauge("CompressionMetadataOffHeapMemoryUsed",
metric -> metric.compressionMetadataOffHeapMemoryUsed.getValue());
@@ -219,6 +227,7 @@
// create histograms for TableMetrics to replicate updates to
sstablesPerReadHistogram = createKeyspaceHistogram("SSTablesPerReadHistogram", true);
+ sstablesPerRangeReadHistogram = createKeyspaceHistogram("SSTablesPerRangeReadHistogram", true);
tombstoneScannedHistogram = createKeyspaceHistogram("TombstoneScannedHistogram", false);
liveScannedHistogram = createKeyspaceHistogram("LiveScannedHistogram", false);
colUpdateTimeDeltaHistogram = createKeyspaceHistogram("ColUpdateTimeDeltaHistogram", false);
@@ -265,6 +274,8 @@
rowIndexSizeWarnings = createKeyspaceMeter("RowIndexSizeWarnings");
rowIndexSizeAborts = createKeyspaceMeter("RowIndexSizeAborts");
rowIndexSize = createKeyspaceHistogram("RowIndexSize", false);
+
+ formatSpecificGauges = createFormatSpecificGauges(keyspace);
}
/**
@@ -278,10 +289,30 @@
}
}
+ private ImmutableMap<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> createFormatSpecificGauges(Keyspace keyspace)
+ {
+ ImmutableMap.Builder<SSTableFormat<? ,?>, ImmutableMap<String, Gauge<? extends Number>>> builder = ImmutableMap.builder();
+ for (SSTableFormat<?, ?> format : DatabaseDescriptor.getSSTableFormats().values())
+ {
+ ImmutableMap.Builder<String, Gauge<? extends Number>> gauges = ImmutableMap.builder();
+ for (GaugeProvider<?> gaugeProvider : format.getFormatSpecificMetricsProviders().getGaugeProviders())
+ {
+ String finalName = gaugeProvider.name;
+ allMetrics.add(() -> releaseMetric(finalName));
+ Gauge<? extends Number> gauge = Metrics.register(factory.createMetricName(finalName), gaugeProvider.getKeyspaceGauge(keyspace));
+ gauges.put(gaugeProvider.name, gauge);
+ }
+ builder.put(format, gauges.build());
+ }
+ return builder.build();
+ }
+
/**
* Creates a gauge that will sum the current value of a metric for all column families in this keyspace
- * @param name
- * @param extractor
+ *
+ * @param name the name of the metric being created
+ * @param extractor a function that produces a specified metric value for a given table
+ *
* @return Gauge>Long> that computes sum of MetricValue.getValue()
*/
private Gauge<Long> createKeyspaceGauge(String name, final ToLongFunction<TableMetrics> extractor)
diff --git a/src/java/org/apache/cassandra/metrics/MaxSampler.java b/src/java/org/apache/cassandra/metrics/MaxSampler.java
index df24bb9..0593e34 100644
--- a/src/java/org/apache/cassandra/metrics/MaxSampler.java
+++ b/src/java/org/apache/cassandra/metrics/MaxSampler.java
@@ -26,39 +26,35 @@
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+/**
+ * Note: {@link Sampler#samplerExecutor} is single threaded but we still need to synchronize as we have access
+ * from both internal and the external JMX context that can cause races.
+ */
public abstract class MaxSampler<T> extends Sampler<T>
{
private int capacity;
private MinMaxPriorityQueue<Sample<T>> queue;
- private long endTimeNanos = -1;
private final Comparator<Sample<T>> comp = Collections.reverseOrder(Comparator.comparing(p -> p.count));
- public boolean isEnabled()
+ @Override
+ public synchronized void beginSampling(int capacity, long durationMillis)
{
- return endTimeNanos != -1 && clock.now() <= endTimeNanos;
- }
-
- public synchronized void beginSampling(int capacity, int durationMillis)
- {
- if (endTimeNanos == -1 || clock.now() > endTimeNanos)
- {
- endTimeNanos = clock.now() + MILLISECONDS.toNanos(durationMillis);
- queue = MinMaxPriorityQueue
- .orderedBy(comp)
- .maximumSize(Math.max(1, capacity))
- .create();
- this.capacity = capacity;
- }
- else
+ if (isActive())
throw new RuntimeException("Sampling already in progress");
+ updateEndTime(clock.now() + MILLISECONDS.toNanos(durationMillis));
+ queue = MinMaxPriorityQueue.orderedBy(comp)
+ .maximumSize(Math.max(1, capacity))
+ .create();
+ this.capacity = capacity;
}
+ @Override
public synchronized List<Sample<T>> finishSampling(int count)
{
List<Sample<T>> result = new ArrayList<>(count);
- if (endTimeNanos != -1)
+ if (isEnabled())
{
- endTimeNanos = -1;
+ disable();
Sample<T> next;
while ((next = queue.poll()) != null && result.size() <= count)
result.add(next);
@@ -69,9 +65,12 @@
@Override
protected synchronized void insert(T item, long value)
{
- if (value > 0 && clock.now() <= endTimeNanos
- && (queue.isEmpty() || queue.size() < capacity || queue.peekLast().count < value))
+ if (isActive() && permitsValue(value))
queue.add(new Sample<T>(item, value, 0));
}
+ private boolean permitsValue(long value)
+ {
+ return value > 0 && (queue.isEmpty() || queue.size() < capacity || queue.peekLast().count < value);
+ }
}
diff --git a/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java b/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java
new file mode 100644
index 0000000..b65f52f
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/MinMaxAvgMetric.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Gauge;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class MinMaxAvgMetric
+{
+ private final MetricNameFactory factory;
+ private final String namePrefix;
+
+ final Gauge<Long> minGauge;
+ final Gauge<Long> maxGauge;
+ final Gauge<Double> avgGauge;
+ final Gauge<Double> stddevGauge;
+ final Gauge<Integer> numSamplesGauge;
+
+ private long min;
+ private long max;
+ private long sum;
+ private long sumSquares;
+ private int numSamples;
+
+ public MinMaxAvgMetric(MetricNameFactory factory, String namePrefix)
+ {
+ this.factory = factory;
+ this.namePrefix = namePrefix;
+
+ minGauge = Metrics.register(factory.createMetricName(namePrefix + "Min"), () -> min);
+ maxGauge = Metrics.register(factory.createMetricName(namePrefix + "Max"), () -> max);
+ avgGauge = Metrics.register(factory.createMetricName(namePrefix + "Avg"), () -> numSamples > 0 ? ((double) sum) / numSamples : 0);
+ stddevGauge = Metrics.register(factory.createMetricName(namePrefix + "StdDev"), () -> stddev());
+ numSamplesGauge = Metrics.register(factory.createMetricName(namePrefix + "NumSamples"), () -> numSamples);
+ }
+
+ public void release()
+ {
+ Metrics.remove(factory.createMetricName(namePrefix + "Min"));
+ Metrics.remove(factory.createMetricName(namePrefix + "Max"));
+ Metrics.remove(factory.createMetricName(namePrefix + "Avg"));
+ Metrics.remove(factory.createMetricName(namePrefix + "StdDev"));
+ Metrics.remove(factory.createMetricName(namePrefix + "NumSamples"));
+ }
+
+ public void reset()
+ {
+ sum = 0;
+ sumSquares = 0;
+ max = Long.MIN_VALUE;
+ min = Long.MAX_VALUE;
+ numSamples = 0;
+ }
+
+ public void update(long value)
+ {
+ max = max > value ? max : value;
+ min = min < value ? min : value;
+ sum += value;
+ sumSquares += value * value;
+ numSamples++;
+ }
+
+ private Double stddev()
+ {
+ if (numSamples > 0)
+ {
+ double avgSquare = ((double) sumSquares) / numSamples;
+ double avg = ((double) sum) / numSamples;
+ return Math.sqrt(avgSquare - avg * avg);
+ }
+ else
+ {
+ return 0.0;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return "{" +
+ "min=" + min +
+ ", max=" + max +
+ ", avg=" + (sum * 1.0 / numSamples) +
+ ", stdDev=" + stddev() +
+ ", numSamples=" + numSamples +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/Sampler.java b/src/java/org/apache/cassandra/metrics/Sampler.java
index b3d0f21..6eb8508 100644
--- a/src/java/org/apache/cassandra/metrics/Sampler.java
+++ b/src/java/org/apache/cassandra/metrics/Sampler.java
@@ -17,10 +17,12 @@
*/
package org.apache.cassandra.metrics;
+import java.io.PrintStream;
import java.io.Serializable;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.function.BiFunction;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.net.MessagingService;
@@ -34,9 +36,61 @@
public abstract class Sampler<T>
{
+ private static long DISABLED = -1L;
+
+ private static final BiFunction<SamplerType, SamplingManager.ResultBuilder, SamplingManager.ResultBuilder>
+ FrequencySamplerFomatter = (type, resultBuilder) ->
+ resultBuilder.forType(type, type.description)
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("Count", "count")
+ .addColumn("+/-", "error");
+
public enum SamplerType
{
- READS, WRITES, LOCAL_READ_TIME, WRITE_SIZE, CAS_CONTENTIONS
+ READS("Frequency of reads by partition", FrequencySamplerFomatter),
+ WRITES("Frequency of writes by partition", FrequencySamplerFomatter),
+ LOCAL_READ_TIME("Longest read query times", ((samplerType, resultBuilder) ->
+ resultBuilder.forType(samplerType, samplerType.description)
+ .addColumn("Query", "value")
+ .addColumn("Microseconds", "count"))),
+ READ_ROW_COUNT("Partitions read with the most rows", ((samplerType, resultBuilder) ->
+ resultBuilder.forType(samplerType, samplerType.description)
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("Rows", "count"))),
+
+ READ_TOMBSTONE_COUNT("Partitions read with the most tombstones", ((samplerType, resultBuilder) ->
+ resultBuilder.forType(samplerType, samplerType.description)
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("Tombstones", "count"))),
+
+ READ_SSTABLE_COUNT("Partitions read with the most sstables", ((samplerType, resultBuilder) ->
+ resultBuilder.forType(samplerType, samplerType.description)
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("SSTables", "count"))),
+ WRITE_SIZE("Max mutation size by partition", ((samplerType, resultBuilder) ->
+ resultBuilder.forType(samplerType, samplerType.description)
+ .addColumn("Table", "table")
+ .addColumn("Partition", "value")
+ .addColumn("Bytes", "count"))),
+ CAS_CONTENTIONS("Frequency of CAS contention by partition", FrequencySamplerFomatter);
+
+ private final String description;
+ private final BiFunction<SamplerType, SamplingManager.ResultBuilder, SamplingManager.ResultBuilder> formatter;
+
+ SamplerType(String description, BiFunction<SamplerType, SamplingManager.ResultBuilder, SamplingManager.ResultBuilder> formatter)
+ {
+ this.description = description;
+ this.formatter = formatter;
+ }
+
+ void format(SamplingManager.ResultBuilder resultBuilder, PrintStream ps)
+ {
+ formatter.apply(this, resultBuilder).print(ps);
+ }
}
@VisibleForTesting
@@ -50,6 +104,8 @@
.withRejectedExecutionHandler((runnable, executor) -> MessagingService.instance().metrics.recordSelfDroppedMessage(Verb._SAMPLE))
.build();
+ private long endTimeNanos = -1;
+
public void addSample(final T item, final int value)
{
if (isEnabled())
@@ -58,10 +114,53 @@
protected abstract void insert(T item, long value);
- public abstract boolean isEnabled();
+ /**
+ * A sampler is enabled between {@link this#beginSampling} and {@link this#finishSampling}
+ * @return true if the sampler is enabled.
+ */
+ public boolean isEnabled()
+ {
+ return endTimeNanos != DISABLED;
+ }
- public abstract void beginSampling(int capacity, int durationMillis);
+ public void disable()
+ {
+ endTimeNanos = DISABLED;
+ }
+ /**
+ * @return true if the sampler is active.
+ * A sampler is active only if it is enabled and the current time is within the `durationMillis` when beginning sampling.
+ */
+ public boolean isActive()
+ {
+ return isEnabled() && clock.now() <= endTimeNanos;
+ }
+
+ /**
+ * Update the end time for the sampler. Implicitly, calling this method enables the sampler.
+ */
+ public void updateEndTime(long endTimeMillis)
+ {
+ this.endTimeNanos = endTimeMillis;
+ }
+
+ /**
+ * Begin sampling with the configured capacity and duration
+ * @param capacity Number of sample items to keep in memory, the lower this is
+ * the less accurate results are. For best results use value
+ * close to cardinality, but understand the memory trade offs.
+ * @param durationMillis Upperbound duration in milliseconds for sampling. The sampler
+ * stops accepting new samples after exceeding the duration
+ * even if {@link #finishSampling(int)}} is not called.
+ */
+ public abstract void beginSampling(int capacity, long durationMillis);
+
+ /**
+ * Stop sampling and return the results
+ * @param count The number of the samples requested to retrieve from the sampler
+ * @return a list of samples, the size is the minimum of the total samples and {@param count}.
+ */
public abstract List<Sample<T>> finishSampling(int count);
public abstract String toString(T value);
diff --git a/src/java/org/apache/cassandra/metrics/SamplingManager.java b/src/java/org/apache/cassandra/metrics/SamplingManager.java
new file mode 100644
index 0000000..37d8d35
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/SamplingManager.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.OpenDataException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.tools.nodetool.ProfileLoad;
+import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
+import org.apache.cassandra.utils.Pair;
+
+public class SamplingManager
+{
+ private static final Logger logger = LoggerFactory.getLogger(SamplingManager.class);
+
+ /**
+ * Tracks the active scheduled sampling tasks.
+ * The key of the map is a {@link JobId}, which is effectively a keyspace + table abstracted behind some syntactic
+ * sugar so we can use them without peppering Pairs throughout this class. Both keyspace and table are nullable,
+ * a paradigm we inherit from {@link ProfileLoad} so need to accommodate here.
+ *
+ * The value of the map is the current scheduled task.
+ */
+ private final ConcurrentHashMap<JobId, Future<?>> activeSamplingTasks = new ConcurrentHashMap<>();
+
+ /** Tasks that are actively being cancelled */
+ private final Set<JobId> cancelingTasks = ConcurrentHashMap.newKeySet();
+
+ public static String formatResult(ResultBuilder resultBuilder)
+ {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (PrintStream ps = new PrintStream(baos))
+ {
+ for (Sampler.SamplerType samplerType : Sampler.SamplerType.values())
+ {
+ samplerType.format(resultBuilder, ps);
+ }
+ return baos.toString();
+ }
+ }
+
+ public static Iterable<ColumnFamilyStore> getTables(String ks, String table)
+ {
+ // null KEYSPACE == all the tables
+ if (ks == null)
+ return ColumnFamilyStore.all();
+
+ Keyspace keyspace = Keyspace.open(ks);
+
+ // KEYSPACE defined w/null table == all the tables on that KEYSPACE
+ if (table == null)
+ return keyspace.getColumnFamilyStores();
+ // Or we just have a specific ks+table combo we're looking to profile
+ else
+ return Collections.singletonList(keyspace.getColumnFamilyStore(table));
+ }
+
+ /**
+ * Register the samplers for the keyspace and table.
+ * @param ks Keyspace. Nullable. If null, the scheduled sampling is on all keyspaces and tables
+ * @param table Nullable. If null, the scheduled sampling is on all tables of the specified keyspace
+ * @param duration Duration of each scheduled sampling job in milliseconds
+ * @param interval Interval of each scheduled sampling job in milliseconds
+ * @param capacity Capacity of the sampler, higher for more accuracy
+ * @param count Number of the top samples to list
+ * @param samplers a list of samplers to enable
+ * @return true if the scheduled sampling is started successfully. Otherwise return fasle
+ */
+ public boolean register(String ks, String table, int duration, int interval, int capacity, int count, List<String> samplers)
+ {
+ JobId jobId = new JobId(ks, table);
+ logger.info("Registering samplers {} for {}", samplers, jobId);
+
+ if (!canSchedule(jobId))
+ {
+ logger.info("Unable to register {} due to existing ongoing sampling.", jobId);
+ return false;
+ }
+
+ // 'begin' tasks are chained to finish before their paired 'finish'
+ activeSamplingTasks.put(jobId, ScheduledExecutors.optionalTasks.submit(
+ createSamplingBeginRunnable(jobId, getTables(ks, table), duration, interval, capacity, count, samplers)
+ ));
+ return true;
+ }
+
+ public boolean unregister(String ks, String table)
+ {
+ // unregister all
+ // return true when all tasks are cancelled successfully
+ if (ks == null && table == null)
+ {
+ boolean res = true;
+ for (JobId id : activeSamplingTasks.keySet())
+ {
+ res = cancelTask(id) & res;
+ }
+ return res;
+ }
+ else
+ {
+ return cancelTask(new JobId(ks, table));
+ }
+ }
+
+ public List<String> allJobs()
+ {
+ return jobIds().stream()
+ .map(JobId::toString)
+ .collect(Collectors.toList());
+ }
+
+ private Set<JobId> jobIds()
+ {
+ Set<JobId> all = new HashSet<>();
+ all.addAll(activeSamplingTasks.keySet());
+ all.addAll(cancelingTasks);
+ return all;
+ }
+
+ /**
+ * Validate if a schedule on the keyspace and table is permitted
+ * @param jobId
+ * @return true if possible, false if there are overlapping tables already being sampled
+ */
+ private boolean canSchedule(JobId jobId)
+ {
+ Set<JobId> allJobIds = jobIds();
+ // There is a schedule that works on all tables. Overlapping guaranteed.
+ if (allJobIds.contains(JobId.ALL_KS_AND_TABLES) || (!allJobIds.isEmpty() && jobId.equals(JobId.ALL_KS_AND_TABLES)))
+ return false;
+ // there is an exactly duplicated schedule
+ else if (allJobIds.contains(jobId))
+ return false;
+ else
+ // make sure has no overlapping tables under the keyspace
+ return !allJobIds.contains(JobId.createForAllTables(jobId.keyspace));
+ }
+
+ /**
+ * Cancel a task by its id. The corresponding task will be stopped once its final sampling completes.
+ * @param jobId
+ * @return true if the task exists, false if not found
+ */
+ private boolean cancelTask(JobId jobId)
+ {
+ Future<?> task = activeSamplingTasks.remove(jobId);
+ if (task != null)
+ cancelingTasks.add(jobId);
+ return task != null;
+ }
+
+ /**
+ * Begin sampling and schedule a future task to end the sampling task
+ */
+ private Runnable createSamplingBeginRunnable(JobId jobId, Iterable<ColumnFamilyStore> tables, int duration, int interval, int capacity, int count, List<String> samplers)
+ {
+ return () ->
+ {
+ if (cancelingTasks.contains(jobId))
+ {
+ logger.debug("The sampling job of {} is currently canceling. Not issuing a new run.", jobId);
+ activeSamplingTasks.remove(jobId);
+ return;
+ }
+ List<String> tableNames = StreamSupport.stream(tables.spliterator(), false)
+ .map(cfs -> String.format("%s.%s", cfs.keyspace, cfs.name))
+ .collect(Collectors.toList());
+ logger.info("Starting to sample tables {} with the samplers {} for {} ms", tableNames, samplers, duration);
+ for (String sampler : samplers)
+ {
+ for (ColumnFamilyStore cfs : tables)
+ {
+ cfs.beginLocalSampling(sampler, capacity, duration);
+ }
+ }
+ Future<?> fut = ScheduledExecutors.optionalTasks.schedule(
+ createSamplingEndRunnable(jobId, tables, duration, interval, capacity, count, samplers),
+ interval,
+ TimeUnit.MILLISECONDS);
+ // reached to the end of the current runnable
+ // update the referenced future to SamplingFinish
+ activeSamplingTasks.put(jobId, fut);
+ };
+ }
+
+ /**
+ * Finish the sampling and begin a new one immediately after.
+ *
+ * NOTE: Do not call this outside the context of {@link this#createSamplingBeginRunnable}, as we need to preserve
+ * ordering between a "start" and "end" runnable
+ */
+ private Runnable createSamplingEndRunnable(JobId jobId, Iterable<ColumnFamilyStore> tables, int duration, int interval, int capacity, int count, List<String> samplers)
+ {
+ return () ->
+ {
+ Map<String, List<CompositeData>> results = new HashMap<>();
+ for (String sampler : samplers)
+ {
+ List<CompositeData> topk = new ArrayList<>();
+ for (ColumnFamilyStore cfs : tables)
+ {
+ try
+ {
+ topk.addAll(cfs.finishLocalSampling(sampler, count));
+ }
+ catch (OpenDataException e)
+ {
+ logger.warn("Failed to retrieve the sampled data. Abort the background sampling job: {}.", jobId, e);
+ activeSamplingTasks.remove(jobId);
+ cancelingTasks.remove(jobId);
+ return;
+ }
+ }
+
+ topk.sort((left, right) -> Long.compare((long) right.get("count"), (long) left.get("count")));
+ // sublist is not serializable for jmx
+ topk = new ArrayList<>(topk.subList(0, Math.min(topk.size(), count)));
+ results.put(sampler, topk);
+ }
+ AtomicBoolean first = new AtomicBoolean(false);
+ ResultBuilder rb = new ResultBuilder(first, results, samplers);
+ logger.info(formatResult(rb));
+
+ // If nobody has canceled us, we ping-pong back to a "begin" runnable to run another profile load
+ if (!cancelingTasks.contains(jobId))
+ {
+ Future<?> fut = ScheduledExecutors.optionalTasks.submit(
+ createSamplingBeginRunnable(jobId, tables, duration, interval, capacity, count, samplers));
+ activeSamplingTasks.put(jobId, fut);
+ }
+ // If someone *has* canceled us, we need to remove the runnable from activeSampling and also remove the
+ // cancellation sentinel so subsequent re-submits of profiling don't get blocked immediately
+ else
+ {
+ logger.info("The sampling job {} has been cancelled.", jobId);
+ activeSamplingTasks.remove(jobId);
+ cancelingTasks.remove(jobId);
+ }
+ };
+ }
+
+ private static class JobId
+ {
+ public static final JobId ALL_KS_AND_TABLES = new JobId(null, null);
+
+ public final String keyspace;
+ public final String table;
+
+ public JobId(String ks, String tb)
+ {
+ keyspace = ks;
+ table = tb;
+ }
+
+ public static JobId createForAllTables(String keyspace)
+ {
+ return new JobId(keyspace, null);
+ }
+
+ @Override
+ public String toString()
+ {
+ return maybeWildCard(keyspace) + '.' + maybeWildCard(table);
+ }
+
+ private String maybeWildCard(String input)
+ {
+ return input == null ? "*" : input;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ JobId jobId = (JobId) o;
+ return Objects.equals(keyspace, jobId.keyspace) && Objects.equals(table, jobId.table);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(keyspace, table);
+ }
+ }
+
+ public static class ResultBuilder
+ {
+ protected Sampler.SamplerType type;
+ protected String description;
+ protected AtomicBoolean first;
+ protected Map<String, List<CompositeData>> results;
+ protected List<String> targets;
+ protected List<Pair<String, String>> dataKeys;
+
+ public ResultBuilder(AtomicBoolean first, Map<String, List<CompositeData>> results, List<String> targets)
+ {
+ this.first = first;
+ this.results = results;
+ this.targets = targets;
+ this.dataKeys = new ArrayList<>();
+ this.dataKeys.add(Pair.create(" ", " "));
+ }
+
+ public SamplingManager.ResultBuilder forType(Sampler.SamplerType type, String description)
+ {
+ SamplingManager.ResultBuilder rb = new SamplingManager.ResultBuilder(first, results, targets);
+ rb.type = type;
+ rb.description = description;
+ return rb;
+ }
+
+ public SamplingManager.ResultBuilder addColumn(String title, String key)
+ {
+ this.dataKeys.add(Pair.create(title, key));
+ return this;
+ }
+
+ protected String get(CompositeData cd, String key)
+ {
+ if (cd.containsKey(key))
+ return cd.get(key).toString();
+ return key;
+ }
+
+ public void print(PrintStream ps)
+ {
+ if (targets.contains(type.toString()))
+ {
+ if (!first.get())
+ ps.println();
+ first.set(false);
+ ps.println(description + ':');
+ TableBuilder out = new TableBuilder();
+ out.add(dataKeys.stream().map(p -> p.left).collect(Collectors.toList()).toArray(new String[] {}));
+ List<CompositeData> topk = results.get(type.toString());
+ for (CompositeData cd : topk)
+ {
+ out.add(dataKeys.stream().map(p -> get(cd, p.right)).collect(Collectors.toList()).toArray(new String[] {}));
+ }
+ if (topk.size() == 0)
+ {
+ ps.println(" Nothing recorded during sampling period...");
+ }
+ else
+ {
+ out.printTo(ps);
+ }
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/StorageMetrics.java b/src/java/org/apache/cassandra/metrics/StorageMetrics.java
index 9399ba6..d86a214 100644
--- a/src/java/org/apache/cassandra/metrics/StorageMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/StorageMetrics.java
@@ -17,7 +17,12 @@
*/
package org.apache.cassandra.metrics;
+import java.util.function.ToLongFunction;
+import java.util.stream.StreamSupport;
+
import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.db.Keyspace;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -29,8 +34,23 @@
private static final MetricNameFactory factory = new DefaultNameFactory("Storage");
public static final Counter load = Metrics.counter(factory.createMetricName("Load"));
+ public static final Counter uncompressedLoad = Metrics.counter(factory.createMetricName("UncompressedLoad"));
+
+ public static final Gauge<Long> unreplicatedLoad =
+ createSummingGauge("UnreplicatedLoad", metric -> metric.unreplicatedLiveDiskSpaceUsed.getValue());
+ public static final Gauge<Long> unreplicatedUncompressedLoad =
+ createSummingGauge("UnreplicatedUncompressedLoad", metric -> metric.unreplicatedUncompressedLiveDiskSpaceUsed.getValue());
+
public static final Counter uncaughtExceptions = Metrics.counter(factory.createMetricName("Exceptions"));
public static final Counter totalHintsInProgress = Metrics.counter(factory.createMetricName("TotalHintsInProgress"));
public static final Counter totalHints = Metrics.counter(factory.createMetricName("TotalHints"));
public static final Counter repairExceptions = Metrics.counter(factory.createMetricName("RepairExceptions"));
+
+ private static Gauge<Long> createSummingGauge(String name, ToLongFunction<KeyspaceMetrics> extractor)
+ {
+ return Metrics.register(factory.createMetricName(name),
+ () -> StreamSupport.stream(Keyspace.all().spliterator(), false)
+ .mapToLong(keyspace -> extractor.applyAsLong(keyspace.metric))
+ .sum());
+ }
}
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index 5e7ab78..f4b9221 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -17,33 +17,41 @@
*/
package org.apache.cassandra.metrics;
-import static java.util.concurrent.TimeUnit.MICROSECONDS;
-import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import com.codahale.metrics.Timer;
-
-import com.google.common.annotations.VisibleForTesting;
-
import org.apache.commons.lang3.ArrayUtils;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.GaugeProvider;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.Sampler.SamplerType;
@@ -52,12 +60,9 @@
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.Pair;
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.RatioGauge;
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
/**
* Metrics for {@link ColumnFamilyStore}.
@@ -100,8 +105,10 @@
public final Gauge<Long> estimatedPartitionCount;
/** Histogram of estimated number of columns. */
public final Gauge<long[]> estimatedColumnCountHistogram;
- /** Histogram of the number of sstable data files accessed per read */
+ /** Histogram of the number of sstable data files accessed per single partition read */
public final TableHistogram sstablesPerReadHistogram;
+ /** Histogram of the number of sstable data files accessed per partition range read */
+ public final TableHistogram sstablesPerRangeReadHistogram;
/** (Local) read metrics */
public final LatencyMetrics readLatency;
/** (Local) range slice metrics */
@@ -120,8 +127,14 @@
public final Gauge<Integer> liveSSTableCount;
/** Number of SSTables with old version on disk for this CF */
public final Gauge<Integer> oldVersionSSTableCount;
+ /** Maximum duration of an SSTable for this table, computed as maxTimestamp - minTimestamp*/
+ public final Gauge<Long> maxSSTableDuration;
+ /** Maximum size of SSTable of this table - the physical size on disk of all components for such SSTable in bytes*/
+ public final Gauge<Long> maxSSTableSize;
/** Disk space used by SSTables belonging to this table */
public final Counter liveDiskSpaceUsed;
+ /** Uncompressed/logical disk space used by SSTables belonging to this table */
+ public final Counter uncompressedLiveDiskSpaceUsed;
/** Total disk space used by SSTables belonging to this table, including obsolete ones waiting to be GC'd */
public final Counter totalDiskSpaceUsed;
/** Size of the smallest compacted partition */
@@ -130,24 +143,8 @@
public final Gauge<Long> maxPartitionSize;
/** Size of the smallest compacted partition */
public final Gauge<Long> meanPartitionSize;
- /** Number of false positives in bloom filter */
- public final Gauge<Long> bloomFilterFalsePositives;
- /** Number of false positives in bloom filter from last read */
- public final Gauge<Long> recentBloomFilterFalsePositives;
- /** False positive ratio of bloom filter */
- public final Gauge<Double> bloomFilterFalseRatio;
- /** False positive ratio of bloom filter from last read */
- public final Gauge<Double> recentBloomFilterFalseRatio;
- /** Disk space used by bloom filter */
- public final Gauge<Long> bloomFilterDiskSpaceUsed;
- /** Off heap memory used by bloom filter */
- public final Gauge<Long> bloomFilterOffHeapMemoryUsed;
- /** Off heap memory used by index summary */
- public final Gauge<Long> indexSummaryOffHeapMemoryUsed;
/** Off heap memory used by compression meta data*/
public final Gauge<Long> compressionMetadataOffHeapMemoryUsed;
- /** Key cache hit rate for this CF */
- public final Gauge<Double> keyCacheHitRate;
/** Tombstones scanned in queries on this CF */
public final TableHistogram tombstoneScannedHistogram;
/** Live rows scanned in queries on this CF */
@@ -260,6 +257,12 @@
public final Sampler<ByteBuffer> topCasPartitionContention;
/** When sampler activated, will track the slowest local reads **/
public final Sampler<String> topLocalReadQueryTime;
+ /** When sampler activated, will track partitions read with the most rows **/
+ public final Sampler<ByteBuffer> topReadPartitionRowCount;
+ /** When sampler activated, will track partitions read with the most tombstones **/
+ public final Sampler<ByteBuffer> topReadPartitionTombstoneCount;
+ /** When sample activated, will track partitions read with the most merged sstables **/
+ public final Sampler<ByteBuffer> topReadPartitionSSTableCount;
public final TableMeter clientTombstoneWarnings;
public final TableMeter clientTombstoneAborts;
@@ -276,6 +279,8 @@
public final TableMeter rowIndexSizeAborts;
public final TableHistogram rowIndexSize;
+ public final ImmutableMap<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> formatSpecificGauges;
+
private static Pair<Long, Long> totalNonSystemTablesSize(Predicate<SSTableReader> predicate)
{
long total = 0;
@@ -440,11 +445,39 @@
}
};
+ topReadPartitionRowCount = new MaxSampler<ByteBuffer>()
+ {
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+
+ topReadPartitionTombstoneCount = new MaxSampler<ByteBuffer>()
+ {
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+
+ topReadPartitionSSTableCount = new MaxSampler<ByteBuffer>()
+ {
+ @Override
+ public String toString(ByteBuffer value)
+ {
+ return cfs.metadata().partitionKeyType.getString(value);
+ }
+ };
+
samplers.put(SamplerType.READS, topReadPartitionFrequency);
samplers.put(SamplerType.WRITES, topWritePartitionFrequency);
samplers.put(SamplerType.WRITE_SIZE, topWritePartitionSize);
samplers.put(SamplerType.CAS_CONTENTIONS, topCasPartitionContention);
samplers.put(SamplerType.LOCAL_READ_TIME, topLocalReadQueryTime);
+ samplers.put(SamplerType.READ_ROW_COUNT, topReadPartitionRowCount);
+ samplers.put(SamplerType.READ_TOMBSTONE_COUNT, topReadPartitionTombstoneCount);
+ samplers.put(SamplerType.READ_SSTABLE_COUNT, topReadPartitionSSTableCount);
memtableColumnsCount = createTableGauge("MemtableColumnsCount",
() -> cfs.getTracker().getView().getCurrentMemtable().operationCount());
@@ -512,6 +545,7 @@
SSTableReader::getEstimatedCellPerPartitionCount), null);
sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram, true);
+ sstablesPerRangeReadHistogram = createTableHistogram("SSTablesPerRangeReadHistogram", cfs.keyspace.metric.sstablesPerRangeReadHistogram, true);
compressionRatio = createTableGauge("CompressionRatio", new Gauge<Double>()
{
public Double getValue()
@@ -604,7 +638,37 @@
return count;
}
});
+ maxSSTableDuration = createTableGauge("MaxSSTableDuration", new Gauge<Long>()
+ {
+ @Override
+ public Long getValue()
+ {
+ return cfs.getTracker()
+ .getView()
+ .liveSSTables()
+ .stream()
+ .filter(sstable -> sstable.getMinTimestamp() != Long.MAX_VALUE && sstable.getMaxTimestamp() != Long.MAX_VALUE)
+ .map(ssTableReader -> ssTableReader.getMaxTimestamp() - ssTableReader.getMinTimestamp())
+ .max(Long::compare)
+ .orElse(0L) / 1000;
+ }
+ });
+ maxSSTableSize = createTableGauge("MaxSSTableSize", new Gauge<Long>()
+ {
+ @Override
+ public Long getValue()
+ {
+ return cfs.getTracker()
+ .getView()
+ .liveSSTables()
+ .stream()
+ .map(SSTableReader::bytesOnDisk)
+ .max(Long::compare)
+ .orElse(0L);
+ }
+ });
liveDiskSpaceUsed = createTableCounter("LiveDiskSpaceUsed");
+ uncompressedLiveDiskSpaceUsed = createTableCounter("UncompressedLiveDiskSpaceUsed");
totalDiskSpaceUsed = createTableCounter("TotalDiskSpaceUsed");
minPartitionSize = createTableGauge("MinPartitionSize", "MinRowSize", new Gauge<Long>()
{
@@ -686,132 +750,6 @@
return count > 0 ? sum / count : 0;
}
});
- bloomFilterFalsePositives = createTableGauge("BloomFilterFalsePositives", new Gauge<Long>()
- {
- public Long getValue()
- {
- long count = 0L;
- for (SSTableReader sstable: cfs.getSSTables(SSTableSet.LIVE))
- count += sstable.getBloomFilterFalsePositiveCount();
- return count;
- }
- });
- recentBloomFilterFalsePositives = createTableGauge("RecentBloomFilterFalsePositives", new Gauge<Long>()
- {
- public Long getValue()
- {
- long count = 0L;
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE))
- count += sstable.getRecentBloomFilterFalsePositiveCount();
- return count;
- }
- });
- bloomFilterFalseRatio = createTableGauge("BloomFilterFalseRatio", new Gauge<Double>()
- {
- public Double getValue()
- {
- long falsePositiveCount = 0L;
- long truePositiveCount = 0L;
- long trueNegativeCount = 0L;
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE))
- {
- falsePositiveCount += sstable.getBloomFilterFalsePositiveCount();
- truePositiveCount += sstable.getBloomFilterTruePositiveCount();
- trueNegativeCount += sstable.getBloomFilterTrueNegativeCount();
- }
- if (falsePositiveCount == 0L && truePositiveCount == 0L)
- return 0d;
- return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
- }
- }, new Gauge<Double>() // global gauge
- {
- public Double getValue()
- {
- long falsePositiveCount = 0L;
- long truePositiveCount = 0L;
- long trueNegativeCount = 0L;
- for (Keyspace keyspace : Keyspace.all())
- {
- for (SSTableReader sstable : keyspace.getAllSSTables(SSTableSet.LIVE))
- {
- falsePositiveCount += sstable.getBloomFilterFalsePositiveCount();
- truePositiveCount += sstable.getBloomFilterTruePositiveCount();
- trueNegativeCount += sstable.getBloomFilterTrueNegativeCount();
- }
- }
- if (falsePositiveCount == 0L && truePositiveCount == 0L)
- return 0d;
- return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
- }
- });
- recentBloomFilterFalseRatio = createTableGauge("RecentBloomFilterFalseRatio", new Gauge<Double>()
- {
- public Double getValue()
- {
- long falsePositiveCount = 0L;
- long truePositiveCount = 0L;
- long trueNegativeCount = 0L;
- for (SSTableReader sstable: cfs.getSSTables(SSTableSet.LIVE))
- {
- falsePositiveCount += sstable.getRecentBloomFilterFalsePositiveCount();
- truePositiveCount += sstable.getRecentBloomFilterTruePositiveCount();
- trueNegativeCount += sstable.getRecentBloomFilterTrueNegativeCount();
- }
- if (falsePositiveCount == 0L && truePositiveCount == 0L)
- return 0d;
- return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
- }
- }, new Gauge<Double>() // global gauge
- {
- public Double getValue()
- {
- long falsePositiveCount = 0L;
- long truePositiveCount = 0L;
- long trueNegativeCount = 0L;
- for (Keyspace keyspace : Keyspace.all())
- {
- for (SSTableReader sstable : keyspace.getAllSSTables(SSTableSet.LIVE))
- {
- falsePositiveCount += sstable.getRecentBloomFilterFalsePositiveCount();
- truePositiveCount += sstable.getRecentBloomFilterTruePositiveCount();
- trueNegativeCount += sstable.getRecentBloomFilterTrueNegativeCount();
- }
- }
- if (falsePositiveCount == 0L && truePositiveCount == 0L)
- return 0d;
- return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount);
- }
- });
- bloomFilterDiskSpaceUsed = createTableGauge("BloomFilterDiskSpaceUsed", new Gauge<Long>()
- {
- public Long getValue()
- {
- long total = 0;
- for (SSTableReader sst : cfs.getSSTables(SSTableSet.CANONICAL))
- total += sst.getBloomFilterSerializedSize();
- return total;
- }
- });
- bloomFilterOffHeapMemoryUsed = createTableGauge("BloomFilterOffHeapMemoryUsed", new Gauge<Long>()
- {
- public Long getValue()
- {
- long total = 0;
- for (SSTableReader sst : cfs.getSSTables(SSTableSet.LIVE))
- total += sst.getBloomFilterOffHeapSize();
- return total;
- }
- });
- indexSummaryOffHeapMemoryUsed = createTableGauge("IndexSummaryOffHeapMemoryUsed", new Gauge<Long>()
- {
- public Long getValue()
- {
- long total = 0;
- for (SSTableReader sst : cfs.getSSTables(SSTableSet.LIVE))
- total += sst.getIndexSummaryOffHeapSize();
- return total;
- }
- });
compressionMetadataOffHeapMemoryUsed = createTableGauge("CompressionMetadataOffHeapMemoryUsed", new Gauge<Long>()
{
public Long getValue()
@@ -830,30 +768,6 @@
additionalWrites = createTableCounter("AdditionalWrites");
additionalWriteLatencyNanos = createTableGauge("AdditionalWriteLatencyNanos", () -> MICROSECONDS.toNanos(cfs.additionalWriteLatencyMicros));
- keyCacheHitRate = createTableGauge("KeyCacheHitRate", "KeyCacheHitRate", new RatioGauge()
- {
- @Override
- public Ratio getRatio()
- {
- return Ratio.of(getNumerator(), getDenominator());
- }
-
- protected double getNumerator()
- {
- long hits = 0L;
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE))
- hits += sstable.getKeyCacheHit();
- return hits;
- }
-
- protected double getDenominator()
- {
- long requests = 0L;
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE))
- requests += sstable.getKeyCacheRequest();
- return Math.max(requests, 1); // to avoid NaN.
- }
- }, null);
tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.keyspace.metric.tombstoneScannedHistogram, false);
liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram, false);
colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram", cfs.keyspace.metric.colUpdateTimeDeltaHistogram, false);
@@ -944,6 +858,8 @@
rowIndexSizeWarnings = createTableMeter("RowIndexSizeWarnings", cfs.keyspace.metric.rowIndexSizeWarnings);
rowIndexSizeAborts = createTableMeter("RowIndexSizeAborts", cfs.keyspace.metric.rowIndexSizeAborts);
rowIndexSize = createTableHistogram("RowIndexSize", cfs.keyspace.metric.rowIndexSize, false);
+
+ formatSpecificGauges = createFormatSpecificGauges(cfs);
}
private Memtable.MemoryUsage getMemoryUsageWithIndexes(ColumnFamilyStore cfs)
@@ -960,6 +876,11 @@
sstablesPerReadHistogram.update(count);
}
+ public void updateSSTableIteratedInRangeRead(int count)
+ {
+ sstablesPerRangeReadHistogram.update(count);
+ }
+
/**
* Release all associated metrics.
*/
@@ -971,6 +892,22 @@
}
}
+ private ImmutableMap<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> createFormatSpecificGauges(ColumnFamilyStore cfs)
+ {
+ ImmutableMap.Builder<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> builder = ImmutableMap.builder();
+ for (SSTableFormat<?, ?> format : DatabaseDescriptor.getSSTableFormats().values())
+ {
+ ImmutableMap.Builder<String, Gauge<? extends Number>> gauges = ImmutableMap.builder();
+ for (GaugeProvider<?> gaugeProvider : format.getFormatSpecificMetricsProviders().getGaugeProviders())
+ {
+ Gauge<? extends Number> gauge = createTableGauge(gaugeProvider.name, gaugeProvider.getTableGauge(cfs), gaugeProvider.getGlobalGauge());
+ gauges.put(gaugeProvider.name, gauge);
+ }
+ builder.put(format, gauges.build());
+ }
+ return builder.build();
+ }
+
/**
* Create a gauge that will be part of a merged version of all column families. The global gauge
* will merge each CF gauge by adding their values
@@ -1097,6 +1034,7 @@
// using SSTableSet.CANONICAL.
assert sstable.openReason != SSTableReader.OpenReason.EARLY;
+ @SuppressWarnings("resource")
CompressionMetadata compressionMetadata = sstable.getCompressionMetadata();
compressedLengthSum += compressionMetadata.compressedFileLength;
dataLengthSum += compressionMetadata.dataLength;
diff --git a/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java b/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java
index 56a860a..6ec0c41 100644
--- a/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java
+++ b/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java
@@ -45,7 +45,6 @@
import org.apache.cassandra.db.transform.Transformation;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
@@ -238,7 +237,7 @@
return;
if (top.size() < maxTopPartitionCount || value > currentMinValue)
- track(new TopPartition(SSTable.getMinimalKey(key), value));
+ track(new TopPartition(key.retainable(), value));
}
private void track(TopPartition tp)
@@ -404,4 +403,4 @@
this.lastUpdated = lastUpdated;
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java b/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java
new file mode 100644
index 0000000..9343503
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/TrieMemtableMetricsView.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Counter;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class TrieMemtableMetricsView
+{
+ private static final String UNCONTENDED_PUTS = "Uncontended memtable puts";
+ private static final String CONTENDED_PUTS = "Contended memtable puts";
+ private static final String CONTENTION_TIME = "Contention time";
+ private static final String LAST_FLUSH_SHARD_SIZES = "Shard sizes during last flush";
+
+ // the number of memtable puts that did not need to wait on write lock
+ public final Counter uncontendedPuts;
+
+ // the number of memtable puts that needed to wait on write lock
+ public final Counter contendedPuts;
+
+ // shard put contention measurements
+ public final LatencyMetrics contentionTime;
+
+ // shard sizes distribution
+ public final MinMaxAvgMetric lastFlushShardDataSizes;
+
+ private final TrieMemtableMetricNameFactory factory;
+
+ public TrieMemtableMetricsView(String keyspace, String table)
+ {
+ factory = new TrieMemtableMetricNameFactory(keyspace, table);
+
+ uncontendedPuts = Metrics.counter(factory.createMetricName(UNCONTENDED_PUTS));
+ contendedPuts = Metrics.counter(factory.createMetricName(CONTENDED_PUTS));
+ contentionTime = new LatencyMetrics(factory, CONTENTION_TIME);
+ lastFlushShardDataSizes = new MinMaxAvgMetric(factory, LAST_FLUSH_SHARD_SIZES);
+ }
+
+ public void release()
+ {
+ Metrics.remove(factory.createMetricName(UNCONTENDED_PUTS));
+ Metrics.remove(factory.createMetricName(CONTENDED_PUTS));
+ contentionTime.release();
+ lastFlushShardDataSizes.release();
+ }
+
+ static class TrieMemtableMetricNameFactory implements MetricNameFactory
+ {
+ private final String keyspace;
+ private final String table;
+
+ TrieMemtableMetricNameFactory(String keyspace, String table)
+ {
+ this.keyspace = keyspace;
+ this.table = table;
+ }
+
+ public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
+ {
+ String groupName = TableMetrics.class.getPackage().getName();
+ String type = "TrieMemtable";
+
+ StringBuilder mbeanName = new StringBuilder();
+ mbeanName.append(groupName).append(":");
+ mbeanName.append("type=").append(type);
+ mbeanName.append(",keyspace=").append(keyspace);
+ mbeanName.append(",scope=").append(table);
+ mbeanName.append(",name=").append(metricName);
+
+ return new CassandraMetricsRegistry.MetricName(groupName, type, metricName, keyspace + "." + table, mbeanName.toString());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/CustomParamsSerializer.java b/src/java/org/apache/cassandra/net/CustomParamsSerializer.java
index a866651..c6c72fe 100644
--- a/src/java/org/apache/cassandra/net/CustomParamsSerializer.java
+++ b/src/java/org/apache/cassandra/net/CustomParamsSerializer.java
@@ -38,7 +38,7 @@
@Override
public void serialize(Map<String, byte[]> t, DataOutputPlus out, int version) throws IOException
{
- out.writeUnsignedVInt(t.size());
+ out.writeUnsignedVInt32(t.size());
for (Map.Entry<String, byte[]> e : t.entrySet())
{
out.writeUTF(e.getKey());
@@ -61,7 +61,7 @@
@Override
public Map<String, byte[]> deserialize(DataInputPlus in, int version) throws IOException
{
- int entries = (int) in.readUnsignedVInt();
+ int entries = in.readUnsignedVInt32();
Map<String, byte[]> customParams = Maps.newHashMapWithExpectedSize(entries);
for (int i = 0 ; i < entries ; ++i)
diff --git a/src/java/org/apache/cassandra/net/ForwardingInfo.java b/src/java/org/apache/cassandra/net/ForwardingInfo.java
index 76e2a75..2ee199a 100644
--- a/src/java/org/apache/cassandra/net/ForwardingInfo.java
+++ b/src/java/org/apache/cassandra/net/ForwardingInfo.java
@@ -17,21 +17,20 @@
*/
package org.apache.cassandra.net;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.BiConsumer;
-
import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
-
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiConsumer;
+
import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
import static org.apache.cassandra.net.MessagingService.VERSION_40;
import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize;
@@ -85,7 +84,7 @@
int count = ids.length;
if (version >= VERSION_40)
- out.writeUnsignedVInt(count);
+ out.writeUnsignedVInt32(count);
else
out.writeInt(count);
@@ -118,7 +117,7 @@
public ForwardingInfo deserialize(DataInputPlus in, int version) throws IOException
{
- int count = version >= VERSION_40 ? Ints.checkedCast(in.readUnsignedVInt()) : in.readInt();
+ int count = version >= VERSION_40 ? in.readUnsignedVInt32() : in.readInt();
long[] ids = new long[count];
List<InetAddressAndPort> targets = new ArrayList<>(count);
@@ -126,7 +125,7 @@
for (int i = 0; i < count; i++)
{
targets.add(inetAddressAndPortSerializer.deserialize(in, version));
- ids[i] = version >= VERSION_40 ? Ints.checkedCast(in.readUnsignedVInt()) : in.readInt();
+ ids[i] = version >= VERSION_40 ? in.readUnsignedVInt32() : in.readInt();
}
return new ForwardingInfo(targets, ids);
diff --git a/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java b/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java
index 807d026..e4de527 100644
--- a/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java
+++ b/src/java/org/apache/cassandra/net/InboundConnectionInitiator.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
+import java.security.cert.Certificate;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.Future;
@@ -46,6 +47,7 @@
import io.netty.handler.logging.LoggingHandler;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslHandler;
+import org.apache.cassandra.auth.IInternodeAuthenticator;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -60,7 +62,11 @@
import static java.lang.Math.*;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.auth.IInternodeAuthenticator.InternodeConnectionDirection.INBOUND;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.net.InternodeConnectionUtils.DISCARD_HANDLER_NAME;
+import static org.apache.cassandra.net.InternodeConnectionUtils.SSL_HANDLER_NAME;
+import static org.apache.cassandra.net.InternodeConnectionUtils.certificates;
import static org.apache.cassandra.net.MessagingService.*;
import static org.apache.cassandra.net.SocketFactory.WIRETRACE;
import static org.apache.cassandra.net.SocketFactory.newSslHandler;
@@ -102,7 +108,7 @@
pipelineInjector.accept(pipeline);
- // order of handlers: ssl -> logger -> handshakeHandler
+ // order of handlers: ssl -> client-authentication -> logger -> handshakeHandler
// For either unencrypted or transitional modes, allow Ssl optionally.
switch(settings.encryption.tlsEncryptionPolicy())
{
@@ -111,14 +117,17 @@
pipeline.addAfter(PIPELINE_INTERNODE_ERROR_EXCLUSIONS, "rejectssl", new RejectSslHandler());
break;
case OPTIONAL:
- pipeline.addAfter(PIPELINE_INTERNODE_ERROR_EXCLUSIONS, "ssl", new OptionalSslHandler(settings.encryption));
+ pipeline.addAfter(PIPELINE_INTERNODE_ERROR_EXCLUSIONS, SSL_HANDLER_NAME, new OptionalSslHandler(settings.encryption));
break;
case ENCRYPTED:
SslHandler sslHandler = getSslHandler("creating", channel, settings.encryption);
- pipeline.addAfter(PIPELINE_INTERNODE_ERROR_EXCLUSIONS, "ssl", sslHandler);
+ pipeline.addAfter(PIPELINE_INTERNODE_ERROR_EXCLUSIONS, SSL_HANDLER_NAME, sslHandler);
break;
}
+ // Pipeline for performing client authentication
+ pipeline.addLast("client-authentication", new ClientAuthenticationHandler(settings.authenticator));
+
if (WIRETRACE)
pipeline.addLast("logger", new LoggingHandler(LogLevel.INFO));
@@ -199,6 +208,61 @@
}
/**
+ * Handler to perform authentication for internode inbound connections.
+ * This handler is called even before messaging handshake starts.
+ */
+ private static class ClientAuthenticationHandler extends ByteToMessageDecoder
+ {
+ private final IInternodeAuthenticator authenticator;
+
+ public ClientAuthenticationHandler(IInternodeAuthenticator authenticator)
+ {
+ this.authenticator = authenticator;
+ }
+
+ @Override
+ protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list) throws Exception
+ {
+ // Extract certificates from SSL handler(handler with name "ssl").
+ final Certificate[] certificates = certificates(channelHandlerContext.channel());
+ if (!authenticate(channelHandlerContext.channel().remoteAddress(), certificates))
+ {
+ logger.error("Unable to authenticate peer {} for internode authentication", channelHandlerContext.channel());
+
+ // To release all the pending buffered data, replace authentication handler with discard handler.
+ // This avoids pending inbound data to be fired through the pipeline
+ channelHandlerContext.pipeline().replace(this, DISCARD_HANDLER_NAME, new InternodeConnectionUtils.ByteBufDiscardHandler());
+ channelHandlerContext.pipeline().close();
+ }
+ else
+ {
+ channelHandlerContext.pipeline().remove(this);
+ }
+ }
+
+ private boolean authenticate(SocketAddress socketAddress, final Certificate[] certificates) throws IOException
+ {
+ if (socketAddress.getClass().getSimpleName().equals("EmbeddedSocketAddress"))
+ return true;
+
+ if (!(socketAddress instanceof InetSocketAddress))
+ throw new IOException(String.format("Unexpected SocketAddress type: %s, %s", socketAddress.getClass(), socketAddress));
+
+ InetSocketAddress addr = (InetSocketAddress) socketAddress;
+ if (!authenticator.authenticate(addr.getAddress(), addr.getPort(), certificates, INBOUND))
+ {
+ // Log at info level as anything that can reach the inbound port could hit this
+ // and trigger a log of noise. Failed outbound connections to known cluster endpoints
+ // still fail with an ERROR message and exception to alert operators that aren't watching logs closely.
+ logger.info("Authenticate rejected inbound internode connection from {}", addr);
+ return false;
+ }
+ return true;
+ }
+
+ }
+
+ /**
* 'Server-side' component that negotiates the internode handshake when establishing a new connection.
* This handler will be the first in the netty channel for each incoming connection (secure socket (TLS) notwithstanding),
* and once the handshake is successful, it will configure the proper handlers ({@link InboundMessageHandler}
@@ -223,8 +287,7 @@
}
/**
- * On registration, immediately schedule a timeout to kill this connection if it does not handshake promptly,
- * and authenticate the remote address.
+ * On registration, immediately schedule a timeout to kill this connection if it does not handshake promptly.
*/
public void handlerAdded(ChannelHandlerContext ctx) throws Exception
{
@@ -232,31 +295,6 @@
logger.error("Timeout handshaking with {} (on {})", SocketFactory.addressId(initiate.from, (InetSocketAddress) ctx.channel().remoteAddress()), settings.bindAddress);
failHandshake(ctx);
}, HandshakeProtocol.TIMEOUT_MILLIS, MILLISECONDS);
-
- if (!authenticate(ctx.channel().remoteAddress()))
- {
- failHandshake(ctx);
- }
- }
-
- private boolean authenticate(SocketAddress socketAddress) throws IOException
- {
- if (socketAddress.getClass().getSimpleName().equals("EmbeddedSocketAddress"))
- return true;
-
- if (!(socketAddress instanceof InetSocketAddress))
- throw new IOException(String.format("Unexpected SocketAddress type: %s, %s", socketAddress.getClass(), socketAddress));
-
- InetSocketAddress addr = (InetSocketAddress)socketAddress;
- if (!settings.authenticate(addr.getAddress(), addr.getPort()))
- {
- // Log at info level as anything that can reach the inbound port could hit this
- // and trigger a log of noise. Failed outbound connections to known cluster endpoints
- // still fail with an ERROR message and exception to alert operators that aren't watching logs closely.
- logger.info("Authenticate rejected inbound internode connection from {}", addr);
- return false;
- }
- return true;
}
@Override
@@ -578,7 +616,7 @@
{
// Connection uses SSL/TLS, replace the detection handler with a SslHandler and so use encryption.
SslHandler sslHandler = getSslHandler("replacing optional", ctx.channel(), encryptionOptions);
- ctx.pipeline().replace(this, "ssl", sslHandler);
+ ctx.pipeline().replace(this, SSL_HANDLER_NAME, sslHandler);
}
else
{
diff --git a/src/java/org/apache/cassandra/net/InboundConnectionSettings.java b/src/java/org/apache/cassandra/net/InboundConnectionSettings.java
index 2eab9bc..448da62 100644
--- a/src/java/org/apache/cassandra/net/InboundConnectionSettings.java
+++ b/src/java/org/apache/cassandra/net/InboundConnectionSettings.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.net;
-import java.net.InetAddress;
import java.util.function.Function;
import com.google.common.base.Preconditions;
@@ -71,16 +70,6 @@
this(null, null, null, null, null, null, null, null, null);
}
- public boolean authenticate(InetAddressAndPort endpoint)
- {
- return authenticator.authenticate(endpoint.getAddress(), endpoint.getPort());
- }
-
- public boolean authenticate(InetAddress address, int port)
- {
- return authenticator.authenticate(address, port);
- }
-
public String toString()
{
return format("address: (%s), nic: %s, encryption: %s",
diff --git a/src/java/org/apache/cassandra/net/InternodeConnectionUtils.java b/src/java/org/apache/cassandra/net/InternodeConnectionUtils.java
new file mode 100644
index 0000000..fd3d1bd
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/InternodeConnectionUtils.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.nio.channels.ClosedChannelException;
+import java.security.cert.Certificate;
+import javax.net.ssl.SSLPeerUnverifiedException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.ssl.SslHandler;
+
+/**
+ * Class that contains certificate utility methods.
+ */
+public class InternodeConnectionUtils
+{
+ public static String SSL_HANDLER_NAME = "ssl";
+ public static String DISCARD_HANDLER_NAME = "discard";
+ private static final Logger logger = LoggerFactory.getLogger(InternodeConnectionUtils.class);
+
+ public static Certificate[] certificates(Channel channel)
+ {
+ final SslHandler sslHandler = (SslHandler) channel.pipeline().get(SSL_HANDLER_NAME);
+ Certificate[] certificates = null;
+ if (sslHandler != null)
+ {
+ try
+ {
+ certificates = sslHandler.engine()
+ .getSession()
+ .getPeerCertificates();
+ }
+ catch (SSLPeerUnverifiedException e)
+ {
+ logger.debug("Failed to get peer certificates for peer {}", channel.remoteAddress(), e);
+ }
+ }
+ return certificates;
+ }
+
+ public static boolean isSSLError(final Throwable cause)
+ {
+ return (cause instanceof ClosedChannelException)
+ && cause.getCause() == null
+ && cause.getStackTrace()[0].getClassName().contains("SslHandler")
+ && cause.getStackTrace()[0].getMethodName().contains("channelInactive");
+ }
+
+ /**
+ * Discard handler releases the received data silently. when internode authentication fails, the channel is closed,
+ * but the pending buffered data may still be fired through the pipeline. To avoid that, authentication handler is
+ * replaced with this DiscardHandler to release all the buffered data, to avoid handling unauthenticated data in the
+ * following handlers.
+ */
+ public static class ByteBufDiscardHandler extends ChannelInboundHandlerAdapter
+ {
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg)
+ {
+ if (msg instanceof ByteBuf)
+ {
+ ((ByteBuf) msg).release();
+ }
+ else
+ {
+ ctx.fireChannelRead(msg);
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java
index 09e4ba3..fa14134 100644
--- a/src/java/org/apache/cassandra/net/Message.java
+++ b/src/java/org/apache/cassandra/net/Message.java
@@ -26,14 +26,12 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import javax.annotation.Nullable;
-
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Ints;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
@@ -50,19 +48,13 @@
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
-
import static org.apache.cassandra.db.TypeSizes.sizeof;
import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
-import static org.apache.cassandra.net.MessagingService.VERSION_3014;
-import static org.apache.cassandra.net.MessagingService.VERSION_30;
-import static org.apache.cassandra.net.MessagingService.VERSION_40;
-import static org.apache.cassandra.net.MessagingService.instance;
+import static org.apache.cassandra.net.MessagingService.*;
import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
-import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize;
-import static org.apache.cassandra.utils.vint.VIntCoding.getUnsignedVInt;
-import static org.apache.cassandra.utils.vint.VIntCoding.skipUnsignedVInt;
+import static org.apache.cassandra.utils.vint.VIntCoding.*;
/**
* Immutable main unit of internode communication - what used to be {@code MessageIn} and {@code MessageOut} fused
@@ -785,8 +777,8 @@
// the same between now and when the recipient reconstructs it.
out.writeInt((int) approxTime.translate().toMillisSinceEpoch(header.createdAtNanos));
out.writeUnsignedVInt(NANOSECONDS.toMillis(header.expiresAtNanos - header.createdAtNanos));
- out.writeUnsignedVInt(header.verb.id);
- out.writeUnsignedVInt(header.flags);
+ out.writeUnsignedVInt32(header.verb.id);
+ out.writeUnsignedVInt32(header.flags);
serializeParams(header.params, out, version);
}
@@ -797,8 +789,8 @@
MonotonicClockTranslation timeSnapshot = approxTime.translate();
long creationTimeNanos = calculateCreationTimeNanos(in.readInt(), timeSnapshot, currentTimeNanos);
long expiresAtNanos = getExpiresAtNanos(creationTimeNanos, currentTimeNanos, TimeUnit.MILLISECONDS.toNanos(in.readUnsignedVInt()));
- Verb verb = Verb.fromId(Ints.checkedCast(in.readUnsignedVInt()));
- int flags = Ints.checkedCast(in.readUnsignedVInt());
+ Verb verb = Verb.fromId(in.readUnsignedVInt32());
+ int flags = in.readUnsignedVInt32();
Map<ParamType, Object> params = deserializeParams(in, version);
return new Header(id, verb, peer, creationTimeNanos, expiresAtNanos, flags, params);
}
@@ -840,10 +832,10 @@
long expiresInMillis = getUnsignedVInt(buf, index);
index += computeUnsignedVIntSize(expiresInMillis);
- Verb verb = Verb.fromId(Ints.checkedCast(getUnsignedVInt(buf, index)));
+ Verb verb = Verb.fromId(getUnsignedVInt32(buf, index));
index += computeUnsignedVIntSize(verb.id);
- int flags = Ints.checkedCast(getUnsignedVInt(buf, index));
+ int flags = getUnsignedVInt32(buf, index);
index += computeUnsignedVIntSize(flags);
Map<ParamType, Object> params = extractParams(buf, index, version);
@@ -857,7 +849,7 @@
private <T> void serializePost40(Message<T> message, DataOutputPlus out, int version) throws IOException
{
serializeHeaderPost40(message.header, out, version);
- out.writeUnsignedVInt(message.payloadSize(version));
+ out.writeUnsignedVInt32(message.payloadSize(version));
message.verb().serializer().serialize(message.payload, out, version);
}
@@ -1221,7 +1213,7 @@
private void serializeParams(Map<ParamType, Object> params, DataOutputPlus out, int version) throws IOException
{
if (version >= VERSION_40)
- out.writeUnsignedVInt(params.size());
+ out.writeUnsignedVInt32(params.size());
else
out.writeInt(params.size());
@@ -1229,7 +1221,7 @@
{
ParamType type = kv.getKey();
if (version >= VERSION_40)
- out.writeUnsignedVInt(type.id);
+ out.writeUnsignedVInt32(type.id);
else
out.writeUTF(type.legacyAlias);
@@ -1238,7 +1230,7 @@
int length = Ints.checkedCast(serializer.serializedSize(value, version));
if (version >= VERSION_40)
- out.writeUnsignedVInt(length);
+ out.writeUnsignedVInt32(length);
else
out.writeInt(length);
@@ -1248,7 +1240,7 @@
private Map<ParamType, Object> deserializeParams(DataInputPlus in, int version) throws IOException
{
- int count = version >= VERSION_40 ? Ints.checkedCast(in.readUnsignedVInt()) : in.readInt();
+ int count = version >= VERSION_40 ? in.readUnsignedVInt32() : in.readInt();
if (count == 0)
return NO_PARAMS;
@@ -1258,11 +1250,11 @@
for (int i = 0; i < count; i++)
{
ParamType type = version >= VERSION_40
- ? ParamType.lookUpById(Ints.checkedCast(in.readUnsignedVInt()))
+ ? ParamType.lookUpById(in.readUnsignedVInt32())
: ParamType.lookUpByAlias(in.readUTF());
int length = version >= VERSION_40
- ? Ints.checkedCast(in.readUnsignedVInt())
+ ? in.readUnsignedVInt32()
: in.readInt();
if (null != type)
@@ -1311,12 +1303,12 @@
private void skipParamsPost40(DataInputPlus in) throws IOException
{
- int count = Ints.checkedCast(in.readUnsignedVInt());
+ int count = in.readUnsignedVInt32();
for (int i = 0; i < count; i++)
{
skipUnsignedVInt(in);
- in.skipBytesFully(Ints.checkedCast(in.readUnsignedVInt()));
+ in.skipBytesFully(in.readUnsignedVInt32());
}
}
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java
index bc39056..c23c6af 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -20,10 +20,13 @@
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
@@ -208,11 +211,27 @@
public static final int VERSION_30 = 10;
public static final int VERSION_3014 = 11;
public static final int VERSION_40 = 12;
- public static final int VERSION_41 = 13;
public static final int minimum_version = VERSION_30;
public static final int current_version = VERSION_40;
static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version);
static AcceptVersions accept_streaming = new AcceptVersions(current_version, current_version);
+ static Map<Integer, Integer> versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, v -> v.ordinal()));
+
+ /**
+ * This is an optimisation to speed up the translation of the serialization
+ * version to the {@link Version} enum ordinal.
+ *
+ * @param version the serialization version
+ * @return a {@link Version} ordinal value
+ */
+ public static int getVersionOrdinal(int version)
+ {
+ Integer ordinal = versionOrdinalMap.get(version);
+ if (ordinal == null)
+ throw new IllegalStateException("Unkown serialization version: " + version);
+
+ return ordinal;
+ }
public enum Version
{
@@ -483,7 +502,10 @@
{
OutboundConnections pool = channelManagers.get(to);
if (pool != null)
+ {
pool.interrupt();
+ logger.info("Interrupted outbound connections to {}", to);
+ }
}
/**
diff --git a/src/java/org/apache/cassandra/net/OutboundConnection.java b/src/java/org/apache/cassandra/net/OutboundConnection.java
index 821521b..2af6d3b 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnection.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnection.java
@@ -61,6 +61,7 @@
import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.net.InternodeConnectionUtils.isSSLError;
import static org.apache.cassandra.net.MessagingService.current_version;
import static org.apache.cassandra.net.OutboundConnectionInitiator.*;
import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD;
@@ -1100,8 +1101,9 @@
if (hasPending())
{
+ boolean isSSLFailure = isSSLError(cause);
Promise<Result<MessagingSuccess>> result = AsyncPromise.withExecutor(eventLoop);
- state = new Connecting(state.disconnected(), result, eventLoop.schedule(() -> attempt(result), max(100, retryRateMillis), MILLISECONDS));
+ state = new Connecting(state.disconnected(), result, eventLoop.schedule(() -> attempt(result, isSSLFailure), max(100, retryRateMillis), MILLISECONDS));
retryRateMillis = min(1000, retryRateMillis * 2);
}
else
@@ -1189,7 +1191,7 @@
*
* Note: this should only be invoked on the event loop.
*/
- private void attempt(Promise<Result<MessagingSuccess>> result)
+ private void attempt(Promise<Result<MessagingSuccess>> result, boolean sslFallbackEnabled)
{
++connectionAttempts;
@@ -1216,7 +1218,20 @@
// ensure we connect to the correct SSL port
settings = settings.withLegacyPortIfNecessary(messagingVersion);
- initiateMessaging(eventLoop, type, settings, messagingVersion, result)
+ // In mixed mode operation, some nodes might be configured to use SSL for internode connections and
+ // others might be configured to not use SSL. When a node is configured in optional SSL mode, It should
+ // be able to handle SSL and Non-SSL internode connections. We take care of this when accepting NON-SSL
+ // connection in Inbound connection by having optional SSL handler for inbound connections.
+ // For outbound connections, if the authentication fails, we should fall back to other SSL strategies
+ // while talking to older nodes in the cluster which are configured to make NON-SSL connections
+ SslFallbackConnectionType[] fallBackSslFallbackConnectionTypes = SslFallbackConnectionType.values();
+ int index = sslFallbackEnabled && settings.withEncryption() && settings.encryption.getOptional() ?
+ (int) (connectionAttempts - 1) % fallBackSslFallbackConnectionTypes.length : 0;
+ if (fallBackSslFallbackConnectionTypes[index] != SslFallbackConnectionType.SERVER_CONFIG)
+ {
+ logger.info("ConnectionId {} is falling back to {} reconnect strategy for retry", id(), fallBackSslFallbackConnectionTypes[index]);
+ }
+ initiateMessaging(eventLoop, type, fallBackSslFallbackConnectionTypes[index], settings, messagingVersion, result)
.addListener(future -> {
if (future.isCancelled())
return;
@@ -1231,7 +1246,7 @@
{
Promise<Result<MessagingSuccess>> result = AsyncPromise.withExecutor(eventLoop);
state = new Connecting(state.disconnected(), result);
- attempt(result);
+ attempt(result, false);
return result;
}
}
diff --git a/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java b/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java
index a187068..ebd30f5 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnectionInitiator.java
@@ -21,13 +21,16 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.channels.ClosedChannelException;
+import java.security.cert.Certificate;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.annotations.VisibleForTesting;
+
import io.netty.util.concurrent.Future; //checkstyle: permit this import
import io.netty.util.concurrent.Promise; //checkstyle: permit this import
import org.apache.cassandra.utils.concurrent.AsyncPromise;
-import org.apache.cassandra.utils.concurrent.ImmediateFuture;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +59,15 @@
import org.apache.cassandra.security.ISslContextFactory;
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import org.apache.cassandra.utils.memory.BufferPools;
import static java.util.concurrent.TimeUnit.*;
+import static org.apache.cassandra.auth.IInternodeAuthenticator.InternodeConnectionDirection.OUTBOUND;
+import static org.apache.cassandra.auth.IInternodeAuthenticator.InternodeConnectionDirection.OUTBOUND_PRECONNECT;
+import static org.apache.cassandra.net.InternodeConnectionUtils.DISCARD_HANDLER_NAME;
+import static org.apache.cassandra.net.InternodeConnectionUtils.SSL_HANDLER_NAME;
+import static org.apache.cassandra.net.InternodeConnectionUtils.certificates;
import static org.apache.cassandra.net.MessagingService.VERSION_40;
import static org.apache.cassandra.net.HandshakeProtocol.*;
import static org.apache.cassandra.net.ConnectionType.STREAMING;
@@ -85,15 +94,17 @@
private static final Logger logger = LoggerFactory.getLogger(OutboundConnectionInitiator.class);
private final ConnectionType type;
+ private final SslFallbackConnectionType sslConnectionType;
private final OutboundConnectionSettings settings;
private final int requestMessagingVersion; // for pre40 nodes
private final Promise<Result<SuccessType>> resultPromise;
private boolean isClosed;
- private OutboundConnectionInitiator(ConnectionType type, OutboundConnectionSettings settings,
+ private OutboundConnectionInitiator(ConnectionType type, SslFallbackConnectionType sslConnectionType, OutboundConnectionSettings settings,
int requestMessagingVersion, Promise<Result<SuccessType>> resultPromise)
{
this.type = type;
+ this.sslConnectionType = sslConnectionType;
this.requestMessagingVersion = requestMessagingVersion;
this.settings = settings;
this.resultPromise = resultPromise;
@@ -106,9 +117,10 @@
*
* The returned {@code Future} is guaranteed to be completed on the supplied eventLoop.
*/
- public static Future<Result<StreamingSuccess>> initiateStreaming(EventLoop eventLoop, OutboundConnectionSettings settings, int requestMessagingVersion)
+ public static Future<Result<StreamingSuccess>> initiateStreaming(EventLoop eventLoop, OutboundConnectionSettings settings,
+ SslFallbackConnectionType sslConnectionType, int requestMessagingVersion)
{
- return new OutboundConnectionInitiator<StreamingSuccess>(STREAMING, settings, requestMessagingVersion, AsyncPromise.withExecutor(eventLoop))
+ return new OutboundConnectionInitiator<StreamingSuccess>(STREAMING, sslConnectionType, settings, requestMessagingVersion, AsyncPromise.withExecutor(eventLoop))
.initiate(eventLoop);
}
@@ -119,9 +131,10 @@
*
* The returned {@code Future} is guaranteed to be completed on the supplied eventLoop.
*/
- static Future<Result<MessagingSuccess>> initiateMessaging(EventLoop eventLoop, ConnectionType type, OutboundConnectionSettings settings, int requestMessagingVersion, Promise<Result<MessagingSuccess>> result)
+ static Future<Result<MessagingSuccess>> initiateMessaging(EventLoop eventLoop, ConnectionType type, SslFallbackConnectionType sslConnectionType,
+ OutboundConnectionSettings settings, int requestMessagingVersion, Promise<Result<MessagingSuccess>> result)
{
- return new OutboundConnectionInitiator<>(type, settings, requestMessagingVersion, result)
+ return new OutboundConnectionInitiator<>(type, sslConnectionType, settings, requestMessagingVersion, result)
.initiate(eventLoop);
}
@@ -130,13 +143,15 @@
if (logger.isTraceEnabled())
logger.trace("creating outbound bootstrap to {}, requestVersion: {}", settings, requestMessagingVersion);
- if (!settings.authenticate())
+ if (!settings.authenticator.authenticate(settings.to.getAddress(), settings.to.getPort(), null, OUTBOUND_PRECONNECT))
{
// interrupt other connections, so they must attempt to re-authenticate
MessagingService.instance().interruptOutbound(settings.to);
- return ImmediateFuture.failure(new IOException("authentication failed to " + settings.connectToId()));
+ logger.error("Authentication failed to " + settings.connectToId());
+ return ImmediateFuture.failure(new IOException("Authentication failed to " + settings.connectToId()));
}
+
// this is a bit ugly, but is the easiest way to ensure that if we timeout we can propagate a suitable error message
// and still guarantee that, if on timing out we raced with success, the successfully created channel is handled
AtomicBoolean timedout = new AtomicBoolean();
@@ -192,25 +207,33 @@
return bootstrap;
}
+ public enum SslFallbackConnectionType
+ {
+ SERVER_CONFIG, // Original configuration of the server
+ MTLS,
+ SSL,
+ NO_SSL
+ }
+
private class Initializer extends ChannelInitializer<SocketChannel>
{
public void initChannel(SocketChannel channel) throws Exception
{
ChannelPipeline pipeline = channel.pipeline();
- // order of handlers: ssl -> logger -> handshakeHandler
- if (settings.withEncryption())
+ // order of handlers: ssl -> server-authentication -> logger -> handshakeHandler
+ if ((sslConnectionType == SslFallbackConnectionType.SERVER_CONFIG && settings.withEncryption())
+ || sslConnectionType == SslFallbackConnectionType.SSL || sslConnectionType == SslFallbackConnectionType.MTLS)
{
- // check if we should actually encrypt this connection
- SslContext sslContext = SSLFactory.getOrCreateSslContext(settings.encryption, true,
- ISslContextFactory.SocketType.CLIENT);
+ SslContext sslContext = getSslContext(sslConnectionType);
// for some reason channel.remoteAddress() will return null
InetAddressAndPort address = settings.to;
InetSocketAddress peer = settings.encryption.require_endpoint_verification ? new InetSocketAddress(address.getAddress(), address.getPort()) : null;
SslHandler sslHandler = newSslHandler(channel, sslContext, peer);
logger.trace("creating outbound netty SslContext: context={}, engine={}", sslContext.getClass().getName(), sslHandler.engine().getClass().getName());
- pipeline.addFirst("ssl", sslHandler);
+ pipeline.addFirst(SSL_HANDLER_NAME, sslHandler);
}
+ pipeline.addLast("server-authentication", new ServerAuthenticationHandler(settings));
if (WIRETRACE)
pipeline.addLast("logger", new LoggingHandler(LogLevel.INFO));
@@ -218,6 +241,59 @@
pipeline.addLast("handshake", new Handler());
}
+ private SslContext getSslContext(SslFallbackConnectionType connectionType) throws IOException
+ {
+ boolean requireClientAuth = false;
+ if (connectionType == SslFallbackConnectionType.MTLS || connectionType == SslFallbackConnectionType.SSL)
+ {
+ requireClientAuth = true;
+ }
+ else if (connectionType == SslFallbackConnectionType.SERVER_CONFIG)
+ {
+ requireClientAuth = settings.withEncryption();
+ }
+ return SSLFactory.getOrCreateSslContext(settings.encryption, requireClientAuth, ISslContextFactory.SocketType.CLIENT);
+ }
+
+ }
+
+ /**
+ * Authenticates the server before an outbound connection is established. If a connection is SSL based connection
+ * Server's identity is verified during ssl handshake using root certificate in truststore. One may choose to ignore
+ * outbound authentication or perform required authentication for outbound connections in the implementation
+ * of IInternodeAuthenticator interface.
+ */
+ @VisibleForTesting
+ static class ServerAuthenticationHandler extends ByteToMessageDecoder
+ {
+ final OutboundConnectionSettings settings;
+
+ ServerAuthenticationHandler(OutboundConnectionSettings settings)
+ {
+ this.settings = settings;
+ }
+
+ @Override
+ protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list) throws Exception
+ {
+ // Extract certificates from SSL handler(handler with name "ssl").
+ final Certificate[] certificates = certificates(channelHandlerContext.channel());
+ if (!settings.authenticator.authenticate(settings.to.getAddress(), settings.to.getPort(), certificates, OUTBOUND))
+ {
+ // interrupt other connections, so they must attempt to re-authenticate
+ MessagingService.instance().interruptOutbound(settings.to);
+ logger.error("Authentication failed to " + settings.connectToId());
+
+ // To release all the pending buffered data, replace authentication handler with discard handler.
+ // This avoids pending inbound data to be fired through the pipeline
+ channelHandlerContext.pipeline().replace(this, DISCARD_HANDLER_NAME, new InternodeConnectionUtils.ByteBufDiscardHandler());
+ channelHandlerContext.pipeline().close();
+ }
+ else
+ {
+ channelHandlerContext.pipeline().remove(this);
+ }
+ }
}
private class Handler extends ByteToMessageDecoder
diff --git a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
index 5b246e3..4721e61 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java
@@ -23,9 +23,9 @@
import io.netty.channel.WriteBufferWaterMark;
import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.locator.IEndpointSnitch;
@@ -43,11 +43,10 @@
@SuppressWarnings({ "WeakerAccess", "unused" })
public class OutboundConnectionSettings
{
- private static final String INTRADC_TCP_NODELAY_PROPERTY = Config.PROPERTY_PREFIX + "otc_intradc_tcp_nodelay";
/**
* Enabled/disable TCP_NODELAY for intradc connections. Defaults to enabled.
*/
- private static final boolean INTRADC_TCP_NODELAY = Boolean.parseBoolean(System.getProperty(INTRADC_TCP_NODELAY_PROPERTY, "true"));
+ private static final boolean INTRADC_TCP_NODELAY = CassandraRelevantProperties.OTC_INTRADC_TCP_NODELAY.getBoolean();
public enum Framing
{
@@ -82,7 +81,7 @@
public final IInternodeAuthenticator authenticator;
public final InetAddressAndPort to;
public final InetAddressAndPort connectTo; // may be represented by a different IP address on this node's local network
- public final EncryptionOptions encryption;
+ public final ServerEncryptionOptions encryption;
public final Framing framing;
public final Integer socketSendBufferSizeInBytes;
public final Integer applicationSendQueueCapacityInBytes;
@@ -112,7 +111,7 @@
private OutboundConnectionSettings(IInternodeAuthenticator authenticator,
InetAddressAndPort to,
InetAddressAndPort connectTo,
- EncryptionOptions encryption,
+ ServerEncryptionOptions encryption,
Framing framing,
Integer socketSendBufferSizeInBytes,
Integer applicationSendQueueCapacityInBytes,
@@ -157,11 +156,6 @@
this.endpointToVersion = endpointToVersion;
}
- public boolean authenticate()
- {
- return authenticator.authenticate(to.getAddress(), to.getPort());
- }
-
public boolean withEncryption()
{
return encryption != null;
@@ -365,7 +359,7 @@
return debug != null ? debug : OutboundDebugCallbacks.NONE;
}
- public EncryptionOptions encryption()
+ public ServerEncryptionOptions encryption()
{
return encryption != null ? encryption : defaultEncryptionOptions(to);
}
@@ -501,7 +495,7 @@
}
@VisibleForTesting
- static EncryptionOptions defaultEncryptionOptions(InetAddressAndPort endpoint)
+ static ServerEncryptionOptions defaultEncryptionOptions(InetAddressAndPort endpoint)
{
ServerEncryptionOptions options = DatabaseDescriptor.getInternodeMessagingEncyptionOptions();
return options.shouldEncrypt(endpoint) ? options : null;
diff --git a/src/java/org/apache/cassandra/net/OutboundConnections.java b/src/java/org/apache/cassandra/net/OutboundConnections.java
index ad87ec5..0729194 100644
--- a/src/java/org/apache/cassandra/net/OutboundConnections.java
+++ b/src/java/org/apache/cassandra/net/OutboundConnections.java
@@ -39,9 +39,8 @@
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
-import static java.lang.Integer.getInteger;
import static java.lang.Math.max;
-import static org.apache.cassandra.config.Config.PROPERTY_PREFIX;
+import static org.apache.cassandra.config.CassandraRelevantProperties.OTCP_LARGE_MESSAGE_THRESHOLD;
import static org.apache.cassandra.gms.Gossiper.instance;
import static org.apache.cassandra.net.FrameEncoderCrc.HEADER_AND_TRAILER_LENGTH;
import static org.apache.cassandra.net.LegacyLZ4Constants.HEADER_LENGTH;
@@ -62,7 +61,7 @@
private static final Logger logger = LoggerFactory.getLogger(OutboundConnections.class);
@VisibleForTesting
- public static final int LARGE_MESSAGE_THRESHOLD = getInteger(PROPERTY_PREFIX + "otcp_large_message_threshold", 1024 * 64)
+ public static final int LARGE_MESSAGE_THRESHOLD = OTCP_LARGE_MESSAGE_THRESHOLD.getInt()
- max(max(HEADER_LENGTH, HEADER_AND_TRAILER_LENGTH), FrameEncoderLZ4.HEADER_AND_TRAILER_LENGTH);
private final Condition metricsReady = newOneTimeCondition();
diff --git a/src/java/org/apache/cassandra/net/ParamType.java b/src/java/org/apache/cassandra/net/ParamType.java
index 37f4bf8..6cb8fb4 100644
--- a/src/java/org/apache/cassandra/net/ParamType.java
+++ b/src/java/org/apache/cassandra/net/ParamType.java
@@ -26,6 +26,7 @@
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.Int32Serializer;
import org.apache.cassandra.utils.Int64Serializer;
+import org.apache.cassandra.utils.RangesSerializer;
import org.apache.cassandra.utils.TimeUUID;
import static java.lang.Math.max;
@@ -42,30 +43,30 @@
*/
public enum ParamType
{
- FORWARD_TO (0, "FWD_TO", ForwardingInfo.serializer),
- RESPOND_TO (1, "FWD_FRM", fwdFrmSerializer),
+ FORWARD_TO (0, "FWD_TO", ForwardingInfo.serializer),
+ RESPOND_TO (1, "FWD_FRM", fwdFrmSerializer),
@Deprecated
- FAILURE_RESPONSE (2, "FAIL", LegacyFlag.serializer),
+ FAILURE_RESPONSE (2, "FAIL", LegacyFlag.serializer),
@Deprecated
- FAILURE_REASON (3, "FAIL_REASON", RequestFailureReason.serializer),
+ FAILURE_REASON (3, "FAIL_REASON", RequestFailureReason.serializer),
@Deprecated
- FAILURE_CALLBACK (4, "CAL_BAC", LegacyFlag.serializer),
+ FAILURE_CALLBACK (4, "CAL_BAC", LegacyFlag.serializer),
- TRACE_SESSION (5, "TraceSession", TimeUUID.Serializer.instance),
- TRACE_TYPE (6, "TraceType", Tracing.traceTypeSerializer),
+ TRACE_SESSION (5, "TraceSession", TimeUUID.Serializer.instance),
+ TRACE_TYPE (6, "TraceType", Tracing.traceTypeSerializer),
@Deprecated
- TRACK_REPAIRED_DATA (7, "TrackRepaired", LegacyFlag.serializer),
+ TRACK_REPAIRED_DATA (7, "TrackRepaired", LegacyFlag.serializer),
- TOMBSTONE_FAIL(8, "TSF", Int32Serializer.serializer),
- TOMBSTONE_WARNING(9, "TSW", Int32Serializer.serializer),
- LOCAL_READ_SIZE_FAIL(10, "LRSF", Int64Serializer.serializer),
- LOCAL_READ_SIZE_WARN(11, "LRSW", Int64Serializer.serializer),
- ROW_INDEX_READ_SIZE_FAIL(12, "RIRSF", Int64Serializer.serializer),
- ROW_INDEX_READ_SIZE_WARN(13, "RIRSW", Int64Serializer.serializer),
-
- CUSTOM_MAP (14, "CUSTOM", CustomParamsSerializer.serializer);
+ TOMBSTONE_FAIL (8, "TSF", Int32Serializer.serializer),
+ TOMBSTONE_WARNING (9, "TSW", Int32Serializer.serializer),
+ LOCAL_READ_SIZE_FAIL (10, "LRSF", Int64Serializer.serializer),
+ LOCAL_READ_SIZE_WARN (11, "LRSW", Int64Serializer.serializer),
+ ROW_INDEX_READ_SIZE_FAIL (12, "RIRSF", Int64Serializer.serializer),
+ ROW_INDEX_READ_SIZE_WARN (13, "RIRSW", Int64Serializer.serializer),
+ CUSTOM_MAP (14, "CUSTOM", CustomParamsSerializer.serializer),
+ SNAPSHOT_RANGES (15, "SNAPSHOT_RANGES", RangesSerializer.serializer);
final int id;
@Deprecated final String legacyAlias; // pre-4.0 we used to serialize entire param name string
diff --git a/src/java/org/apache/cassandra/net/RequestCallback.java b/src/java/org/apache/cassandra/net/RequestCallback.java
index bd14cae..14e0169 100644
--- a/src/java/org/apache/cassandra/net/RequestCallback.java
+++ b/src/java/org/apache/cassandra/net/RequestCallback.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.net;
+import java.util.Map;
+
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -63,4 +65,19 @@
return false;
}
+ static boolean isTimeout(Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
+ {
+ // The reason that all must be timeout to be called a timeout is as follows
+ // Assume RF=6, QUORUM, and failureReasonByEndpoint.size() == 3
+ // R1 -> TIMEOUT
+ // R2 -> TIMEOUT
+ // R3 -> READ_TOO_MANY_TOMBSTONES
+ // Since we got a reply back, and that was a failure, we should return a failure letting the user know.
+ // When all failures are a timeout, then this is a race condition with
+ // org.apache.cassandra.utils.concurrent.Awaitable.await(long, java.util.concurrent.TimeUnit)
+ // The race is that the message expire path runs and expires all messages, this then casues the condition
+ // to signal telling the caller "got all replies!".
+ return failureReasonByEndpoint.values().stream().allMatch(RequestFailureReason.TIMEOUT::equals);
+ }
+
}
diff --git a/src/java/org/apache/cassandra/net/SocketFactory.java b/src/java/org/apache/cassandra/net/SocketFactory.java
index 33fff6b..7825626 100644
--- a/src/java/org/apache/cassandra/net/SocketFactory.java
+++ b/src/java/org/apache/cassandra/net/SocketFactory.java
@@ -61,7 +61,6 @@
import io.netty.util.internal.logging.InternalLoggerFactory;
import io.netty.util.internal.logging.Slf4JLoggerFactory;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.security.SSLFactory;
@@ -73,6 +72,7 @@
import static io.netty.channel.unix.Errors.ERROR_ECONNREFUSED_NEGATIVE;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.INTERNODE_EVENT_THREADS;
import static org.apache.cassandra.utils.Throwables.isCausedBy;
/**
@@ -83,7 +83,7 @@
{
private static final Logger logger = LoggerFactory.getLogger(SocketFactory.class);
- private static final int EVENT_THREADS = Integer.getInteger(Config.PROPERTY_PREFIX + "internode-event-threads", FBUtilities.getAvailableProcessors());
+ private static final int EVENT_THREADS = INTERNODE_EVENT_THREADS.getInt(FBUtilities.getAvailableProcessors());
/**
* The default task queue used by {@code NioEventLoop} and {@code EpollEventLoop} is {@code MpscUnboundedArrayQueue},
@@ -215,7 +215,7 @@
* Creates a new {@link SslHandler} from provided SslContext.
* @param peer enables endpoint verification for remote address when not null
*/
- static SslHandler newSslHandler(Channel channel, SslContext sslContext, @Nullable InetSocketAddress peer)
+ public static SslHandler newSslHandler(Channel channel, SslContext sslContext, @Nullable InetSocketAddress peer)
{
if (peer == null)
return sslContext.newHandler(channel.alloc());
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
index d50a187..d52f14c 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -96,6 +96,7 @@
import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
import org.apache.cassandra.streaming.ReplicationDoneVerbHandler;
+import org.apache.cassandra.utils.ReflectionUtils;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.UUIDSerializer;
@@ -351,7 +352,7 @@
Supplier<? extends IVerbHandler<?>> original = this.handler;
Field field = Verb.class.getDeclaredField("handler");
field.setAccessible(true);
- Field modifiers = Field.class.getDeclaredField("modifiers");
+ Field modifiers = ReflectionUtils.getModifiersField();
modifiers.setAccessible(true);
modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(this, handler);
@@ -364,7 +365,7 @@
Supplier<? extends IVersionedAsymmetricSerializer<?, ?>> original = this.serializer;
Field field = Verb.class.getDeclaredField("serializer");
field.setAccessible(true);
- Field modifiers = Field.class.getDeclaredField("modifiers");
+ Field modifiers = ReflectionUtils.getModifiersField();
modifiers.setAccessible(true);
modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(this, serializer);
@@ -377,7 +378,7 @@
ToLongFunction<TimeUnit> original = this.expiration;
Field field = Verb.class.getDeclaredField("expiration");
field.setAccessible(true);
- Field modifiers = Field.class.getDeclaredField("modifiers");
+ Field modifiers = ReflectionUtils.getModifiersField();
modifiers.setAccessible(true);
modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(this, expiration);
diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
index 28a6acc..5df05c7 100644
--- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
@@ -149,7 +149,7 @@
state.trace("{}/{} ({}%) {} idx:{}{}",
new Object[] { FBUtilities.prettyPrintMemory(pi.currentBytes),
FBUtilities.prettyPrintMemory(pi.totalBytes),
- pi.currentBytes * 100 / pi.totalBytes,
+ pi.progressPercentage(),
pi.direction == ProgressInfo.Direction.OUT ? "sent to" : "received from",
pi.sessionIndex,
pi.peer });
@@ -201,4 +201,4 @@
plan.getCoordinator().getAllStreamSessions().forEach(StreamSession::abort);
});
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
index 7ce7d1f..728a813 100644
--- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
@@ -27,6 +27,8 @@
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.RepairMetrics;
import org.apache.cassandra.repair.consistent.SyncStatSummary;
@@ -104,14 +106,18 @@
{
Set<String> mismatchingTables = new HashSet<>();
Set<InetAddressAndPort> nodes = new HashSet<>();
+ Set<Range<Token>> ranges = new HashSet<>();
for (RepairSessionResult sessionResult : results)
{
for (RepairResult repairResult : emptyIfNull(sessionResult.repairJobResults))
{
for (SyncStat stat : emptyIfNull(repairResult.stats))
{
- if (stat.numberOfDifferences > 0)
+ if (!stat.differences.isEmpty())
+ {
mismatchingTables.add(repairResult.desc.columnFamily);
+ ranges.addAll(stat.differences);
+ }
// snapshot all replicas, even if they don't have any differences
nodes.add(stat.nodes.coordinator);
nodes.add(stat.nodes.peer);
@@ -125,10 +131,13 @@
// we can just check snapshot existence locally since the repair coordinator is always a replica (unlike in the read case)
if (!Keyspace.open(keyspace).getColumnFamilyStore(table).snapshotExists(snapshotName))
{
- logger.info("{} Snapshotting {}.{} for preview repair mismatch with tag {} on instances {}",
+ List<Range<Token>> normalizedRanges = Range.normalize(ranges);
+ logger.info("{} Snapshotting {}.{} for preview repair mismatch for ranges {} with tag {} on instances {}",
options.getPreviewKind().logPrefix(parentSession),
- keyspace, table, snapshotName, nodes);
- DiagnosticSnapshotService.repairedDataMismatch(Keyspace.open(keyspace).getColumnFamilyStore(table).metadata(), nodes);
+ keyspace, table, normalizedRanges, snapshotName, nodes);
+ DiagnosticSnapshotService.repairedDataMismatch(Keyspace.open(keyspace).getColumnFamilyStore(table).metadata(),
+ nodes,
+ normalizedRanges);
}
else
{
diff --git a/src/java/org/apache/cassandra/repair/SyncStat.java b/src/java/org/apache/cassandra/repair/SyncStat.java
index 7bb503f..2241915 100644
--- a/src/java/org/apache/cassandra/repair/SyncStat.java
+++ b/src/java/org/apache/cassandra/repair/SyncStat.java
@@ -17,8 +17,11 @@
*/
package org.apache.cassandra.repair;
+import java.util.Collection;
import java.util.List;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.streaming.SessionSummary;
/**
@@ -27,23 +30,23 @@
public class SyncStat
{
public final SyncNodePair nodes;
- public final long numberOfDifferences; // TODO: revert to Range<Token>
+ public final Collection<Range<Token>> differences;
public final List<SessionSummary> summaries;
- public SyncStat(SyncNodePair nodes, long numberOfDifferences)
+ public SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences)
{
- this(nodes, numberOfDifferences, null);
+ this(nodes, differences, null);
}
- public SyncStat(SyncNodePair nodes, long numberOfDifferences, List<SessionSummary> summaries)
+ public SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries)
{
this.nodes = nodes;
- this.numberOfDifferences = numberOfDifferences;
this.summaries = summaries;
+ this.differences = differences;
}
public SyncStat withSummaries(List<SessionSummary> summaries)
{
- return new SyncStat(nodes, numberOfDifferences, summaries);
+ return new SyncStat(nodes, differences, summaries);
}
}
diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java
index b325eb4..7393eff 100644
--- a/src/java/org/apache/cassandra/repair/SyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SyncTask.java
@@ -60,7 +60,7 @@
this.rangesToSync = rangesToSync;
this.nodePair = new SyncNodePair(primaryEndpoint, peer);
this.previewKind = previewKind;
- this.stat = new SyncStat(nodePair, rangesToSync.size());
+ this.stat = new SyncStat(nodePair, rangesToSync);
}
protected abstract void startSync();
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
index ed2bf0b..ed1e37a 100644
--- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
@@ -98,6 +98,10 @@
import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_CLEANUP_INTERVAL_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_DELETE_TIMEOUT_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_FAIL_TIMEOUT_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_STATUS_CHECK_TIMEOUT_SECONDS;
import static org.apache.cassandra.net.Verb.FAILED_SESSION_MSG;
import static org.apache.cassandra.net.Verb.FINALIZE_PROMISE_MSG;
import static org.apache.cassandra.net.Verb.PREPARE_CONSISTENT_RSP;
@@ -121,26 +125,22 @@
* Amount of time a session can go without any activity before we start checking the status of other
* participants to see if we've missed a message
*/
- static final int CHECK_STATUS_TIMEOUT = Integer.getInteger("cassandra.repair_status_check_timeout_seconds",
- Ints.checkedCast(TimeUnit.HOURS.toSeconds(1)));
+ static final int CHECK_STATUS_TIMEOUT = REPAIR_STATUS_CHECK_TIMEOUT_SECONDS.getInt();
/**
* Amount of time a session can go without any activity before being automatically set to FAILED
*/
- static final int AUTO_FAIL_TIMEOUT = Integer.getInteger("cassandra.repair_fail_timeout_seconds",
- Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)));
+ static final int AUTO_FAIL_TIMEOUT = REPAIR_FAIL_TIMEOUT_SECONDS.getInt();
/**
* Amount of time a completed session is kept around after completion before being deleted, this gives
* compaction plenty of time to move sstables from successful sessions into the repaired bucket
*/
- static final int AUTO_DELETE_TIMEOUT = Integer.getInteger("cassandra.repair_delete_timeout_seconds",
- Ints.checkedCast(TimeUnit.DAYS.toSeconds(1)));
+ static final int AUTO_DELETE_TIMEOUT = REPAIR_DELETE_TIMEOUT_SECONDS.getInt();
/**
* How often LocalSessions.cleanup is run
*/
- public static final int CLEANUP_INTERVAL = Integer.getInteger("cassandra.repair_cleanup_interval_seconds",
- Ints.checkedCast(TimeUnit.MINUTES.toSeconds(10)));
+ public static final int CLEANUP_INTERVAL = REPAIR_CLEANUP_INTERVAL_SECONDS.getInt();
private static Set<TableId> uuidToTableId(Set<UUID> src)
{
diff --git a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
index 3d21702..855ad4b 100644
--- a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
+++ b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
@@ -21,13 +21,18 @@
import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import com.google.common.collect.Lists;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.repair.RepairResult;
import org.apache.cassandra.repair.RepairSessionResult;
import org.apache.cassandra.repair.SyncStat;
@@ -50,7 +55,7 @@
int files = 0;
long bytes = 0;
- long ranges = 0;
+ Set<Range<Token>> ranges = new HashSet<>();
Session(InetSocketAddress src, InetSocketAddress dst)
{
@@ -64,15 +69,15 @@
bytes += summary.totalSize;
}
- void consumeSummaries(Collection<StreamSummary> summaries, long numRanges)
+ void consumeSummaries(Collection<StreamSummary> summaries, Collection<Range<Token>> ranges)
{
summaries.forEach(this::consumeSummary);
- ranges += numRanges;
+ this.ranges.addAll(ranges);
}
public String toString()
{
- return String.format("%s -> %s: %s ranges, %s sstables, %s bytes", src, dst, ranges, files, FBUtilities.prettyPrintMemory(bytes));
+ return String.format("%s -> %s: %s ranges, %s sstables, %s bytes", src, dst, ranges.size(), files, FBUtilities.prettyPrintMemory(bytes));
}
}
@@ -84,7 +89,7 @@
int files = -1;
long bytes = -1;
- int ranges = -1;
+ Collection<Range<Token>> ranges = new HashSet<>();
boolean totalsCalculated = false;
final Map<Pair<InetSocketAddress, InetSocketAddress>, Session> sessions = new HashMap<>();
@@ -109,8 +114,8 @@
{
for (SessionSummary summary: stat.summaries)
{
- getOrCreate(summary.coordinator, summary.peer).consumeSummaries(summary.sendingSummaries, stat.numberOfDifferences);
- getOrCreate(summary.peer, summary.coordinator).consumeSummaries(summary.receivingSummaries, stat.numberOfDifferences);
+ getOrCreate(summary.coordinator, summary.peer).consumeSummaries(summary.sendingSummaries, stat.differences);
+ getOrCreate(summary.peer, summary.coordinator).consumeSummaries(summary.receivingSummaries, stat.differences);
}
}
@@ -123,12 +128,12 @@
{
files = 0;
bytes = 0;
- ranges = 0;
+ ranges = new HashSet<>();
for (Session session: sessions.values())
{
files += session.files;
bytes += session.bytes;
- ranges += session.ranges;
+ ranges.addAll(session.ranges);
}
totalsCalculated = true;
}
@@ -147,21 +152,36 @@
}
StringBuilder output = new StringBuilder();
- output.append(String.format("%s.%s - %s ranges, %s sstables, %s bytes\n", keyspace, table, ranges, files, FBUtilities.prettyPrintMemory(bytes)));
+ output.append(String.format("%s.%s - %s ranges, %s sstables, %s bytes\n", keyspace, table, ranges.size(), files, FBUtilities.prettyPrintMemory(bytes)));
+ if (ranges.size() > 0)
+ {
+ output.append(" Mismatching ranges: ");
+ int i = 0;
+ Iterator<Range<Token>> rangeIterator = ranges.iterator();
+ while (rangeIterator.hasNext() && i < 30)
+ {
+ Range<Token> r = rangeIterator.next();
+ output.append('(').append(r.left).append(',').append(r.right).append("],");
+ i++;
+ }
+ if (i == 30)
+ output.append("...");
+ output.append(System.lineSeparator());
+ }
for (Session session: sessions.values())
{
- output.append(" ").append(session.toString()).append('\n');
+ output.append(" ").append(session.toString()).append(System.lineSeparator());
}
return output.toString();
}
}
- private Map<Pair<String, String>, Table> summaries = new HashMap<>();
+ private final Map<Pair<String, String>, Table> summaries = new HashMap<>();
private final boolean isEstimate;
private int files = -1;
private long bytes = -1;
- private int ranges = -1;
+ private Set<Range<Token>> ranges = new HashSet<>();
private boolean totalsCalculated = false;
public SyncStatSummary(boolean isEstimate)
@@ -190,14 +210,14 @@
public boolean isEmpty()
{
calculateTotals();
- return files == 0 && bytes == 0 && ranges == 0;
+ return files == 0 && bytes == 0 && ranges.isEmpty();
}
private void calculateTotals()
{
files = 0;
bytes = 0;
- ranges = 0;
+ ranges = new HashSet<>();
summaries.values().forEach(Table::calculateTotals);
for (Table table: summaries.values())
{
@@ -208,7 +228,7 @@
table.calculateTotals();
files += table.files;
bytes += table.bytes;
- ranges += table.ranges;
+ ranges.addAll(table.ranges);
}
totalsCalculated = true;
}
@@ -228,11 +248,11 @@
if (isEstimate)
{
- output.append(String.format("Total estimated streaming: %s ranges, %s sstables, %s bytes\n", ranges, files, FBUtilities.prettyPrintMemory(bytes)));
+ output.append(String.format("Total estimated streaming: %s ranges, %s sstables, %s bytes\n", ranges.size(), files, FBUtilities.prettyPrintMemory(bytes)));
}
else
{
- output.append(String.format("Total streaming: %s ranges, %s sstables, %s bytes\n", ranges, files, FBUtilities.prettyPrintMemory(bytes)));
+ output.append(String.format("Total streaming: %s ranges, %s sstables, %s bytes\n", ranges.size(), files, FBUtilities.prettyPrintMemory(bytes)));
}
for (Pair<String, String> tableName: tables)
diff --git a/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java b/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java
index 2d21deb..89b1eec 100644
--- a/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java
+++ b/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java
@@ -32,7 +32,6 @@
import javax.management.openmbean.SimpleType;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
import com.google.common.collect.Sets;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -57,7 +56,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -126,7 +125,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/repair/consistent/admin/PendingStat.java b/src/java/org/apache/cassandra/repair/consistent/admin/PendingStat.java
index b7d4fea..0f50e0e 100644
--- a/src/java/org/apache/cassandra/repair/consistent/admin/PendingStat.java
+++ b/src/java/org/apache/cassandra/repair/consistent/admin/PendingStat.java
@@ -32,7 +32,6 @@
import javax.management.openmbean.SimpleType;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -55,7 +54,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -94,7 +93,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/repair/consistent/admin/PendingStats.java b/src/java/org/apache/cassandra/repair/consistent/admin/PendingStats.java
index f1b515e..e253a0e 100644
--- a/src/java/org/apache/cassandra/repair/consistent/admin/PendingStats.java
+++ b/src/java/org/apache/cassandra/repair/consistent/admin/PendingStats.java
@@ -28,7 +28,6 @@
import javax.management.openmbean.SimpleType;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
public class PendingStats
{
@@ -51,7 +50,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -87,7 +86,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/repair/consistent/admin/RepairStats.java b/src/java/org/apache/cassandra/repair/consistent/admin/RepairStats.java
index bbb4778..6d3e424 100644
--- a/src/java/org/apache/cassandra/repair/consistent/admin/RepairStats.java
+++ b/src/java/org/apache/cassandra/repair/consistent/admin/RepairStats.java
@@ -25,7 +25,6 @@
import javax.management.openmbean.*;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
import org.apache.cassandra.repair.consistent.RepairedState;
@@ -47,7 +46,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -75,7 +74,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -112,7 +111,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -165,7 +164,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/repair/messages/RepairOption.java b/src/java/org/apache/cassandra/repair/messages/RepairOption.java
index 6bb7fdb..f0508a3 100644
--- a/src/java/org/apache/cassandra/repair/messages/RepairOption.java
+++ b/src/java/org/apache/cassandra/repair/messages/RepairOption.java
@@ -395,22 +395,20 @@
public boolean optimiseStreams()
{
- if(optimiseStreams)
- return true;
-
- if (isPullRepair() || isForcedRepair())
+ if (isPullRepair())
return false;
- if (isIncremental() && DatabaseDescriptor.autoOptimiseIncRepairStreams())
+ if (isPreview())
+ {
+ if (DatabaseDescriptor.autoOptimisePreviewRepairStreams())
+ return true;
+ }
+ else if (isIncremental() && DatabaseDescriptor.autoOptimiseIncRepairStreams())
+ return true;
+ else if (!isIncremental() && DatabaseDescriptor.autoOptimiseFullRepairStreams())
return true;
- if (isPreview() && DatabaseDescriptor.autoOptimisePreviewRepairStreams())
- return true;
-
- if (!isIncremental() && DatabaseDescriptor.autoOptimiseFullRepairStreams())
- return true;
-
- return false;
+ return optimiseStreams;
}
public boolean ignoreUnreplicatedKeyspaces()
diff --git a/src/java/org/apache/cassandra/schema/ColumnMetadata.java b/src/java/org/apache/cassandra/schema/ColumnMetadata.java
index fdbd166..f68a7b5 100644
--- a/src/java/org/apache/cassandra/schema/ColumnMetadata.java
+++ b/src/java/org/apache/cassandra/schema/ColumnMetadata.java
@@ -21,11 +21,14 @@
import java.util.*;
import java.util.function.Predicate;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Collections2;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
import org.apache.cassandra.cql3.selection.Selectable;
import org.apache.cassandra.cql3.selection.Selector;
import org.apache.cassandra.cql3.selection.SimpleSelector;
@@ -95,6 +98,12 @@
*/
private final long comparisonOrder;
+ /**
+ * Masking function used to dynamically mask the contents of this column.
+ */
+ @Nullable
+ private final ColumnMask mask;
+
private static long comparisonOrder(Kind kind, boolean isComplex, long position, ColumnIdentifier name)
{
assert position >= 0 && position < 1 << 12;
@@ -106,52 +115,58 @@
public static ColumnMetadata partitionKeyColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type, int position)
{
- return new ColumnMetadata(table, name, type, position, Kind.PARTITION_KEY);
+ return new ColumnMetadata(table, name, type, position, Kind.PARTITION_KEY, null);
}
public static ColumnMetadata partitionKeyColumn(String keyspace, String table, String name, AbstractType<?> type, int position)
{
- return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, position, Kind.PARTITION_KEY);
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, position, Kind.PARTITION_KEY, null);
}
public static ColumnMetadata clusteringColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type, int position)
{
- return new ColumnMetadata(table, name, type, position, Kind.CLUSTERING);
+ return new ColumnMetadata(table, name, type, position, Kind.CLUSTERING, null);
}
public static ColumnMetadata clusteringColumn(String keyspace, String table, String name, AbstractType<?> type, int position)
{
- return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, position, Kind.CLUSTERING);
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, position, Kind.CLUSTERING, null);
}
public static ColumnMetadata regularColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type)
{
- return new ColumnMetadata(table, name, type, NO_POSITION, Kind.REGULAR);
+ return new ColumnMetadata(table, name, type, NO_POSITION, Kind.REGULAR, null);
}
public static ColumnMetadata regularColumn(String keyspace, String table, String name, AbstractType<?> type)
{
- return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.REGULAR);
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.REGULAR, null);
}
public static ColumnMetadata staticColumn(TableMetadata table, ByteBuffer name, AbstractType<?> type)
{
- return new ColumnMetadata(table, name, type, NO_POSITION, Kind.STATIC);
+ return new ColumnMetadata(table, name, type, NO_POSITION, Kind.STATIC, null);
}
public static ColumnMetadata staticColumn(String keyspace, String table, String name, AbstractType<?> type)
{
- return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.STATIC);
+ return new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, NO_POSITION, Kind.STATIC, null);
}
- public ColumnMetadata(TableMetadata table, ByteBuffer name, AbstractType<?> type, int position, Kind kind)
+ public ColumnMetadata(TableMetadata table,
+ ByteBuffer name,
+ AbstractType<?> type,
+ int position,
+ Kind kind,
+ @Nullable ColumnMask mask)
{
this(table.keyspace,
table.name,
ColumnIdentifier.getInterned(name, UTF8Type.instance),
type,
position,
- kind);
+ kind,
+ mask);
}
@VisibleForTesting
@@ -160,18 +175,26 @@
ColumnIdentifier name,
AbstractType<?> type,
int position,
- Kind kind)
+ Kind kind,
+ @Nullable ColumnMask mask)
{
super(ksName, cfName, name, type);
assert name != null && type != null && kind != null;
assert (position == NO_POSITION) == !kind.isPrimaryKeyKind(); // The position really only make sense for partition and clustering columns (and those must have one),
// so make sure we don't sneak it for something else since it'd breaks equals()
+
+ // The propagation of system distributed keyspaces at startup can be problematic for old nodes without DDM,
+ // since those won't know what to do with the mask mutations. Thus, we don't support DDM on those keyspaces.
+ if (mask != null && SchemaConstants.isReplicatedSystemKeyspace(ksName))
+ throw new AssertionError("DDM is not supported on system distributed keyspaces");
+
this.kind = kind;
this.position = position;
this.cellPathComparator = makeCellPathComparator(kind, type);
this.cellComparator = cellPathComparator == null ? ColumnData.comparator : (a, b) -> cellPathComparator.compare(a.path(), b.path());
this.asymmetricCellPathComparator = cellPathComparator == null ? null : (a, b) -> cellPathComparator.compare(((Cell<?>)a).path(), (CellPath) b);
this.comparisonOrder = comparisonOrder(kind, isComplex(), Math.max(0, position), name);
+ this.mask = mask;
}
private static Comparator<CellPath> makeCellPathComparator(Kind kind, AbstractType<?> type)
@@ -203,17 +226,22 @@
public ColumnMetadata copy()
{
- return new ColumnMetadata(ksName, cfName, name, type, position, kind);
+ return new ColumnMetadata(ksName, cfName, name, type, position, kind, mask);
}
public ColumnMetadata withNewName(ColumnIdentifier newName)
{
- return new ColumnMetadata(ksName, cfName, newName, type, position, kind);
+ return new ColumnMetadata(ksName, cfName, newName, type, position, kind, mask);
}
public ColumnMetadata withNewType(AbstractType<?> newType)
{
- return new ColumnMetadata(ksName, cfName, name, newType, position, kind);
+ return new ColumnMetadata(ksName, cfName, name, newType, position, kind, mask);
+ }
+
+ public ColumnMetadata withNewMask(@Nullable ColumnMask newMask)
+ {
+ return new ColumnMetadata(ksName, cfName, name, type, position, kind, newMask);
}
public boolean isPartitionKey()
@@ -231,6 +259,11 @@
return kind == Kind.STATIC;
}
+ public boolean isMasked()
+ {
+ return mask != null;
+ }
+
public boolean isRegular()
{
return kind == Kind.REGULAR;
@@ -249,6 +282,12 @@
return position;
}
+ @Nullable
+ public ColumnMask getMask()
+ {
+ return mask;
+ }
+
@Override
public boolean equals(Object o)
{
@@ -269,7 +308,8 @@
&& kind == other.kind
&& position == other.position
&& ksName.equals(other.ksName)
- && cfName.equals(other.cfName);
+ && cfName.equals(other.cfName)
+ && Objects.equals(mask, other.mask);
}
Optional<Difference> compare(ColumnMetadata other)
@@ -299,6 +339,7 @@
result = 31 * result + (type == null ? 0 : type.hashCode());
result = 31 * result + (kind == null ? 0 : kind.hashCode());
result = 31 * result + position;
+ result = 31 * result + (mask == null ? 0 : mask.hashCode());
hash = result;
}
return result;
@@ -334,7 +375,7 @@
@Override
public boolean processesSelection()
{
- return false;
+ return isMasked();
}
/**
@@ -433,6 +474,9 @@
if (isStatic())
builder.append(" static");
+
+ if (isMasked())
+ mask.appendCqlTo(builder);
}
public static String toCQLString(Iterable<ColumnMetadata> defs)
@@ -488,7 +532,7 @@
public Selector.Factory newSelectorFactory(TableMetadata table, AbstractType<?> expectedType, List<ColumnMetadata> defs, VariableSpecifications boundNames) throws InvalidRequestException
{
- return SimpleSelector.newFactory(this, addAndGetIndex(this, defs));
+ return SimpleSelector.newFactory(this, addAndGetIndex(this, defs), false);
}
public AbstractType<?> getExactTypeIfKnown(String keyspace)
diff --git a/src/java/org/apache/cassandra/schema/CompactionParams.java b/src/java/org/apache/cassandra/schema/CompactionParams.java
index 4859468..eff634f 100644
--- a/src/java/org/apache/cassandra/schema/CompactionParams.java
+++ b/src/java/org/apache/cassandra/schema/CompactionParams.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.utils.FBUtilities;
import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES;
public final class CompactionParams
{
@@ -76,15 +77,15 @@
public static final int DEFAULT_MAX_THRESHOLD = 32;
public static final boolean DEFAULT_ENABLED = true;
- public static final TombstoneOption DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES =
- TombstoneOption.valueOf(System.getProperty("default.provide.overlapping.tombstones", TombstoneOption.NONE.toString()).toUpperCase());
+ public static final TombstoneOption DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE =
+ DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES.getEnum(TombstoneOption.NONE);
public static final Map<String, String> DEFAULT_THRESHOLDS =
ImmutableMap.of(Option.MIN_THRESHOLD.toString(), Integer.toString(DEFAULT_MIN_THRESHOLD),
Option.MAX_THRESHOLD.toString(), Integer.toString(DEFAULT_MAX_THRESHOLD));
public static final CompactionParams DEFAULT =
- new CompactionParams(SizeTieredCompactionStrategy.class, DEFAULT_THRESHOLDS, DEFAULT_ENABLED, DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES);
+ new CompactionParams(SizeTieredCompactionStrategy.class, DEFAULT_THRESHOLDS, DEFAULT_ENABLED, DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE);
private final Class<? extends AbstractCompactionStrategy> klass;
private final ImmutableMap<String, String> options;
@@ -105,7 +106,7 @@
? Boolean.parseBoolean(options.get(Option.ENABLED.toString()))
: DEFAULT_ENABLED;
String overlappingTombstoneParm = options.getOrDefault(Option.PROVIDE_OVERLAPPING_TOMBSTONES.toString(),
- DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES.toString()).toUpperCase();
+ DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE.toString()).toUpperCase();
Optional<TombstoneOption> tombstoneOptional = TombstoneOption.forName(overlappingTombstoneParm);
if (!tombstoneOptional.isPresent())
{
diff --git a/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java b/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java
index 0f0c3e9..0bea577 100644
--- a/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java
+++ b/src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java
@@ -129,7 +129,7 @@
return true;
logger.warn("There are nodes in the cluster with a different schema version than us, from which we did not merge schemas: " +
- "our version: ({}), outstanding versions -> endpoints: {}. Use -D{}}=true to ignore this, " +
+ "our version: ({}), outstanding versions -> endpoints: {}. Use -D{}=true to ignore this, " +
"-D{}=<ep1[,epN]> to skip specific endpoints, or -D{}=<ver1[,verN]> to skip specific schema versions",
Schema.instance.getVersion(),
migrationCoordinator.outstandingVersions(),
diff --git a/src/java/org/apache/cassandra/schema/DistributedSchema.java b/src/java/org/apache/cassandra/schema/DistributedSchema.java
index 4fed9bb..d2f30aa 100644
--- a/src/java/org/apache/cassandra/schema/DistributedSchema.java
+++ b/src/java/org/apache/cassandra/schema/DistributedSchema.java
@@ -91,7 +91,7 @@
ksm.tables.forEach(tm -> Preconditions.checkArgument(tm.keyspace.equals(ksm.name), "Table %s metadata points to keyspace %s while defined in keyspace %s", tm.name, tm.keyspace, ksm.name));
ksm.views.forEach(vm -> Preconditions.checkArgument(vm.keyspace().equals(ksm.name), "View %s metadata points to keyspace %s while defined in keyspace %s", vm.name(), vm.keyspace(), ksm.name));
ksm.types.forEach(ut -> Preconditions.checkArgument(ut.keyspace.equals(ksm.name), "Type %s points to keyspace %s while defined in keyspace %s", ut.name, ut.keyspace, ksm.name));
- ksm.functions.forEach(f -> Preconditions.checkArgument(f.name().keyspace.equals(ksm.name), "Function %s points to keyspace %s while defined in keyspace %s", f.name().name, f.name().keyspace, ksm.name));
+ ksm.userFunctions.forEach(f -> Preconditions.checkArgument(f.name().keyspace.equals(ksm.name), "Function %s points to keyspace %s while defined in keyspace %s", f.name().name, f.name().keyspace, ksm.name));
});
}
}
diff --git a/src/java/org/apache/cassandra/schema/Functions.java b/src/java/org/apache/cassandra/schema/Functions.java
deleted file mode 100644
index c5de3b8..0000000
--- a/src/java/org/apache/cassandra/schema/Functions.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.schema;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.google.common.collect.*;
-
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.UserType;
-
-import static java.util.stream.Collectors.toList;
-
-import static com.google.common.collect.Iterables.any;
-
-/**
- * An immutable container for a keyspace's UDAs and UDFs (and, in case of {@link org.apache.cassandra.db.SystemKeyspace},
- * native functions and aggregates).
- */
-public final class Functions implements Iterable<Function>
-{
- public enum Filter implements Predicate<Function>
- {
- ALL, UDF, UDA;
-
- public boolean test(Function function)
- {
- switch (this)
- {
- case UDF: return function instanceof UDFunction;
- case UDA: return function instanceof UDAggregate;
- default: return true;
- }
- }
- }
-
- private final ImmutableMultimap<FunctionName, Function> functions;
-
- private Functions(Builder builder)
- {
- functions = builder.functions.build();
- }
-
- public static Builder builder()
- {
- return new Builder();
- }
-
- public static Functions none()
- {
- return builder().build();
- }
-
- public static Functions of(Function... funs)
- {
- return builder().add(funs).build();
- }
-
- public Iterator<Function> iterator()
- {
- return functions.values().iterator();
- }
-
- public Stream<Function> stream()
- {
- return functions.values().stream();
- }
-
- public int size()
- {
- return functions.size();
- }
-
- /**
- * @return a stream of keyspace's UDFs
- */
- public Stream<UDFunction> udfs()
- {
- return stream().filter(Filter.UDF).map(f -> (UDFunction) f);
- }
-
- /**
- * @return a stream of keyspace's UDAs
- */
- public Stream<UDAggregate> udas()
- {
- return stream().filter(Filter.UDA).map(f -> (UDAggregate) f);
- }
-
- public Iterable<Function> referencingUserType(ByteBuffer name)
- {
- return Iterables.filter(this, f -> f.referencesUserType(name));
- }
-
- public Functions withUpdatedUserType(UserType udt)
- {
- if (!any(this, f -> f.referencesUserType(udt.name)))
- return this;
-
- Collection<UDFunction> udfs = udfs().map(f -> f.withUpdatedUserType(udt)).collect(toList());
- Collection<UDAggregate> udas = udas().map(f -> f.withUpdatedUserType(udfs, udt)).collect(toList());
-
- return builder().add(udfs).add(udas).build();
- }
-
- /**
- * @return a stream of aggregates that use the provided function as either a state or a final function
- * @param function the referree function
- */
- public Stream<UDAggregate> aggregatesUsingFunction(Function function)
- {
- return udas().filter(uda -> uda.hasReferenceTo(function));
- }
-
- /**
- * Get all function overloads with the specified name
- *
- * @param name fully qualified function name
- * @return an empty list if the function name is not found; a non-empty collection of {@link Function} otherwise
- */
- public Collection<Function> get(FunctionName name)
- {
- return functions.get(name);
- }
-
- /**
- * Get all UDFs overloads with the specified name
- *
- * @param name fully qualified function name
- * @return an empty list if the function name is not found; a non-empty collection of {@link UDFunction} otherwise
- */
- public Collection<UDFunction> getUdfs(FunctionName name)
- {
- return functions.get(name)
- .stream()
- .filter(Filter.UDF)
- .map(f -> (UDFunction) f)
- .collect(Collectors.toList());
- }
-
- /**
- * Get all UDAs overloads with the specified name
- *
- * @param name fully qualified function name
- * @return an empty list if the function name is not found; a non-empty collection of {@link UDAggregate} otherwise
- */
- public Collection<UDAggregate> getUdas(FunctionName name)
- {
- return functions.get(name)
- .stream()
- .filter(Filter.UDA)
- .map(f -> (UDAggregate) f)
- .collect(Collectors.toList());
- }
-
- public Optional<Function> find(FunctionName name, List<AbstractType<?>> argTypes)
- {
- return find(name, argTypes, Filter.ALL);
- }
-
- /**
- * Find the function with the specified name
- *
- * @param name fully qualified function name
- * @param argTypes function argument types
- * @return an empty {@link Optional} if the function name is not found; a non-empty optional of {@link Function} otherwise
- */
- public Optional<Function> find(FunctionName name, List<AbstractType<?>> argTypes, Filter filter)
- {
- return get(name).stream()
- .filter(filter.and(fun -> typesMatch(fun.argTypes(), argTypes)))
- .findAny();
- }
-
- public boolean isEmpty()
- {
- return functions.isEmpty();
- }
-
- /*
- * We need to compare the CQL3 representation of the type because comparing
- * the AbstractType will fail for example if a UDT has been changed.
- * Reason is that UserType.equals() takes the field names and types into account.
- * Example CQL sequence that would fail when comparing AbstractType:
- * CREATE TYPE foo ...
- * CREATE FUNCTION bar ( par foo ) RETURNS foo ...
- * ALTER TYPE foo ADD ...
- * or
- * ALTER TYPE foo ALTER ...
- * or
- * ALTER TYPE foo RENAME ...
- */
- private static boolean typesMatch(AbstractType<?> t1, AbstractType<?> t2)
- {
- return t1.freeze().asCQL3Type().toString().equals(t2.freeze().asCQL3Type().toString());
- }
-
- public static boolean typesMatch(List<AbstractType<?>> t1, List<AbstractType<?>> t2)
- {
- if (t1.size() != t2.size())
- return false;
-
- for (int i = 0; i < t1.size(); i++)
- if (!typesMatch(t1.get(i), t2.get(i)))
- return false;
-
- return true;
- }
-
- public static int typeHashCode(AbstractType<?> t)
- {
- return t.asCQL3Type().toString().hashCode();
- }
-
- public static int typeHashCode(List<AbstractType<?>> types)
- {
- int h = 0;
- for (AbstractType<?> type : types)
- h = h * 31 + typeHashCode(type);
- return h;
- }
-
- public Functions filter(Predicate<Function> predicate)
- {
- Builder builder = builder();
- stream().filter(predicate).forEach(builder::add);
- return builder.build();
- }
-
- /**
- * Create a Functions instance with the provided function added
- */
- public Functions with(Function fun)
- {
- if (find(fun.name(), fun.argTypes()).isPresent())
- throw new IllegalStateException(String.format("Function %s already exists", fun.name()));
-
- return builder().add(this).add(fun).build();
- }
-
- /**
- * Creates a Functions instance with the function with the provided name and argument types removed
- */
- public Functions without(FunctionName name, List<AbstractType<?>> argTypes)
- {
- Function fun =
- find(name, argTypes).orElseThrow(() -> new IllegalStateException(String.format("Function %s doesn't exists", name)));
-
- return without(fun);
- }
-
- public Functions without(Function function)
- {
- return builder().add(Iterables.filter(this, f -> f != function)).build();
- }
-
- public Functions withAddedOrUpdated(Function function)
- {
- return builder().add(Iterables.filter(this, f -> !(f.name().equals(function.name()) && Functions.typesMatch(f.argTypes(), function.argTypes()))))
- .add(function)
- .build();
- }
-
- @Override
- public boolean equals(Object o)
- {
- return this == o || (o instanceof Functions && functions.equals(((Functions) o).functions));
- }
-
- @Override
- public int hashCode()
- {
- return functions.hashCode();
- }
-
- @Override
- public String toString()
- {
- return functions.values().toString();
- }
-
- public static final class Builder
- {
- final ImmutableMultimap.Builder<FunctionName, Function> functions = new ImmutableMultimap.Builder<>();
-
- private Builder()
- {
- // we need deterministic iteration order; otherwise Functions.equals() breaks down
- functions.orderValuesBy(Comparator.comparingInt(Object::hashCode));
- }
-
- public Functions build()
- {
- return new Functions(this);
- }
-
- public Builder add(Function fun)
- {
- functions.put(fun.name(), fun);
- return this;
- }
-
- public Builder add(Function... funs)
- {
- for (Function fun : funs)
- add(fun);
- return this;
- }
-
- public Builder add(Iterable<? extends Function> funs)
- {
- funs.forEach(this::add);
- return this;
- }
- }
-
- @SuppressWarnings("unchecked")
- static FunctionsDiff<UDFunction> udfsDiff(Functions before, Functions after)
- {
- return (FunctionsDiff<UDFunction>) FunctionsDiff.diff(before, after, Filter.UDF);
- }
-
- @SuppressWarnings("unchecked")
- static FunctionsDiff<UDAggregate> udasDiff(Functions before, Functions after)
- {
- return (FunctionsDiff<UDAggregate>) FunctionsDiff.diff(before, after, Filter.UDA);
- }
-
- public static final class FunctionsDiff<T extends Function> extends Diff<Functions, T>
- {
- static final FunctionsDiff NONE = new FunctionsDiff<>(Functions.none(), Functions.none(), ImmutableList.of());
-
- private FunctionsDiff(Functions created, Functions dropped, ImmutableCollection<Altered<T>> altered)
- {
- super(created, dropped, altered);
- }
-
- private static FunctionsDiff diff(Functions before, Functions after, Filter filter)
- {
- if (before == after)
- return NONE;
-
- Functions created = after.filter(filter.and(k -> !before.find(k.name(), k.argTypes(), filter).isPresent()));
- Functions dropped = before.filter(filter.and(k -> !after.find(k.name(), k.argTypes(), filter).isPresent()));
-
- ImmutableList.Builder<Altered<Function>> altered = ImmutableList.builder();
- before.stream().filter(filter).forEach(functionBefore ->
- {
- after.find(functionBefore.name(), functionBefore.argTypes(), filter).ifPresent(functionAfter ->
- {
- functionBefore.compare(functionAfter).ifPresent(kind -> altered.add(new Altered<>(functionBefore, functionAfter, kind)));
- });
- });
-
- return new FunctionsDiff<>(created, dropped, altered.build());
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
index 6d85391..fb2efc8 100644
--- a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
+++ b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
@@ -20,6 +20,7 @@
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Stream;
import javax.annotation.Nullable;
@@ -30,12 +31,13 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CqlBuilder;
import org.apache.cassandra.cql3.SchemaElement;
+import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.UDAggregate;
import org.apache.cassandra.cql3.functions.UDFunction;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
-import org.apache.cassandra.schema.Functions.FunctionsDiff;
+import org.apache.cassandra.schema.UserFunctions.FunctionsDiff;
import org.apache.cassandra.schema.Tables.TablesDiff;
import org.apache.cassandra.schema.Types.TypesDiff;
import org.apache.cassandra.schema.Views.ViewsDiff;
@@ -61,9 +63,9 @@
public final Tables tables;
public final Views views;
public final Types types;
- public final Functions functions;
+ public final UserFunctions userFunctions;
- private KeyspaceMetadata(String name, Kind kind, KeyspaceParams params, Tables tables, Views views, Types types, Functions functions)
+ private KeyspaceMetadata(String name, Kind kind, KeyspaceParams params, Tables tables, Views views, Types types, UserFunctions functions)
{
this.name = name;
this.kind = kind;
@@ -71,57 +73,57 @@
this.tables = tables;
this.views = views;
this.types = types;
- this.functions = functions;
+ this.userFunctions = functions;
}
public static KeyspaceMetadata create(String name, KeyspaceParams params)
{
- return new KeyspaceMetadata(name, Kind.REGULAR, params, Tables.none(), Views.none(), Types.none(), Functions.none());
+ return new KeyspaceMetadata(name, Kind.REGULAR, params, Tables.none(), Views.none(), Types.none(), UserFunctions.none());
}
public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables)
{
- return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, Views.none(), Types.none(), Functions.none());
+ return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, Views.none(), Types.none(), UserFunctions.none());
}
- public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables, Views views, Types types, Functions functions)
+ public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables, Views views, Types types, UserFunctions functions)
{
return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, views, types, functions);
}
public static KeyspaceMetadata virtual(String name, Tables tables)
{
- return new KeyspaceMetadata(name, Kind.VIRTUAL, KeyspaceParams.local(), tables, Views.none(), Types.none(), Functions.none());
+ return new KeyspaceMetadata(name, Kind.VIRTUAL, KeyspaceParams.local(), tables, Views.none(), Types.none(), UserFunctions.none());
}
public KeyspaceMetadata withSwapped(KeyspaceParams params)
{
- return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, userFunctions);
}
public KeyspaceMetadata withSwapped(Tables regular)
{
- return new KeyspaceMetadata(name, kind, params, regular, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, regular, views, types, userFunctions);
}
public KeyspaceMetadata withSwapped(Views views)
{
- return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, userFunctions);
}
public KeyspaceMetadata withSwapped(Types types)
{
- return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
+ return new KeyspaceMetadata(name, kind, params, tables, views, types, userFunctions);
}
- public KeyspaceMetadata withSwapped(Functions functions)
+ public KeyspaceMetadata withSwapped(UserFunctions functions)
{
return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
}
public KeyspaceMetadata empty()
{
- return new KeyspaceMetadata(this.name, this.kind, this.params, Tables.none(), Views.none(), Types.none(), Functions.none());
+ return new KeyspaceMetadata(this.name, this.kind, this.params, Tables.none(), Views.none(), Types.none(), UserFunctions.none());
}
public boolean isVirtual()
@@ -141,7 +143,7 @@
tables.withUpdatedUserType(udt),
views.withUpdatedUserTypes(udt),
types.withUpdatedUserType(udt),
- functions.withUpdatedUserType(udt));
+ userFunctions.withUpdatedUserType(udt));
}
public Iterable<TableMetadata> tablesAndViews()
@@ -173,6 +175,15 @@
return any(tables, t -> t.indexes.has(indexName));
}
+ /**
+ * @param function a user function
+ * @return a stream of tables within this keyspace that have column masks using the specified user function
+ */
+ public Stream<TableMetadata> tablesUsingFunction(Function function)
+ {
+ return tables.stream().filter(table -> table.dependsOn(function));
+ }
+
public String findAvailableIndexName(String baseName)
{
if (!hasIndex(baseName))
@@ -200,7 +211,7 @@
@Override
public int hashCode()
{
- return Objects.hashCode(name, kind, params, tables, views, functions, types);
+ return Objects.hashCode(name, kind, params, tables, views, userFunctions, types);
}
@Override
@@ -215,12 +226,12 @@
KeyspaceMetadata other = (KeyspaceMetadata) o;
return name.equals(other.name)
- && kind == other.kind
- && params.equals(other.params)
- && tables.equals(other.tables)
- && views.equals(other.views)
- && functions.equals(other.functions)
- && types.equals(other.types);
+ && kind == other.kind
+ && params.equals(other.params)
+ && tables.equals(other.tables)
+ && views.equals(other.views)
+ && userFunctions.equals(other.userFunctions)
+ && types.equals(other.types);
}
@Override
@@ -232,7 +243,7 @@
.add("params", params)
.add("tables", tables)
.add("views", views)
- .add("functions", functions)
+ .add("functions", userFunctions)
.add("types", types)
.toString();
}
@@ -385,10 +396,10 @@
@SuppressWarnings("unchecked") FunctionsDiff<UDFunction> udfs = FunctionsDiff.NONE;
@SuppressWarnings("unchecked") FunctionsDiff<UDAggregate> udas = FunctionsDiff.NONE;
- if (before.functions != after.functions)
+ if (before.userFunctions != after.userFunctions)
{
- udfs = Functions.udfsDiff(before.functions, after.functions);
- udas = Functions.udasDiff(before.functions, after.functions);
+ udfs = UserFunctions.udfsDiff(before.userFunctions, after.userFunctions);
+ udas = UserFunctions.udasDiff(before.userFunctions, after.userFunctions);
}
if (before.params.equals(after.params) && tables.isEmpty() && views.isEmpty() && types.isEmpty() && udfs.isEmpty() && udas.isEmpty())
diff --git a/src/java/org/apache/cassandra/schema/MigrationCoordinator.java b/src/java/org/apache/cassandra/schema/MigrationCoordinator.java
index 61ef4c8..5624899 100644
--- a/src/java/org/apache/cassandra/schema/MigrationCoordinator.java
+++ b/src/java/org/apache/cassandra/schema/MigrationCoordinator.java
@@ -46,7 +46,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
-import org.apache.cassandra.utils.NoSpamLogger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,6 +67,7 @@
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Simulate;
import org.apache.cassandra.utils.concurrent.Future;
@@ -254,10 +254,11 @@
void start()
{
- logger.info("Starting migration coordinator and scheduling pulling schema versions every {}", Duration.ofMillis(SCHEMA_PULL_INTERVAL_MS.getLong()));
+ long interval = SCHEMA_PULL_INTERVAL_MS.getLong();
+ logger.info("Starting migration coordinator and scheduling pulling schema versions every {}", Duration.ofMillis(interval));
announce(schemaVersion.get());
periodicPullTask.updateAndGet(curTask -> curTask == null
- ? periodicCheckExecutor.scheduleWithFixedDelay(this::pullUnreceivedSchemaVersions, SCHEMA_PULL_INTERVAL_MS.getLong(), SCHEMA_PULL_INTERVAL_MS.getLong(), TimeUnit.MILLISECONDS)
+ ? periodicCheckExecutor.scheduleWithFixedDelay(this::pullUnreceivedSchemaVersions, interval, interval, TimeUnit.MILLISECONDS)
: curTask);
}
diff --git a/src/java/org/apache/cassandra/schema/Schema.java b/src/java/org/apache/cassandra/schema/Schema.java
index 0dba167..3cf1814 100644
--- a/src/java/org/apache/cassandra/schema/Schema.java
+++ b/src/java/org/apache/cassandra/schema/Schema.java
@@ -18,7 +18,14 @@
package org.apache.cassandra.schema;
import java.time.Duration;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Supplier;
@@ -28,10 +35,17 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.MapDifference;
import org.apache.commons.lang3.ObjectUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.cql3.functions.*;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UserFunction;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.KeyspaceNotDefinedException;
+import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -49,9 +63,6 @@
import org.apache.cassandra.utils.concurrent.Awaitable;
import org.apache.cassandra.utils.concurrent.LoadingMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import static com.google.common.collect.Iterables.size;
import static java.lang.String.format;
import static org.apache.cassandra.config.DatabaseDescriptor.isDaemonInitialized;
@@ -213,6 +224,12 @@
return keyspaceInstances.getIfReady(keyspaceName);
}
+ /**
+ * Returns {@link ColumnFamilyStore} by the table identifier. Note that though, if called for {@link TableMetadata#id},
+ * when metadata points to a secondary index table, the {@link TableMetadata#id} denotes the identifier of the main
+ * table, not the index table. Thus, this method will return CFS of the main table rather than, probably expected,
+ * CFS for the index backing table.
+ */
public ColumnFamilyStore getColumnFamilyStoreInstance(TableId id)
{
TableMetadata metadata = getTableMetadata(id);
@@ -457,13 +474,13 @@
/* Function helpers */
/**
- * Get all function overloads with the specified name
+ * Get all user-defined function overloads with the specified name.
*
* @param name fully qualified function name
* @return an empty list if the keyspace or the function name are not found;
- * a non-empty collection of {@link Function} otherwise
+ * a non-empty collection of {@link UserFunction} otherwise
*/
- public Collection<Function> getFunctions(FunctionName name)
+ public Collection<UserFunction> getUserFunctions(FunctionName name)
{
if (!name.hasKeyspace())
throw new IllegalArgumentException(String.format("Function name must be fully qualified: got %s", name));
@@ -471,26 +488,24 @@
KeyspaceMetadata ksm = getKeyspaceMetadata(name.keyspace);
return ksm == null
? Collections.emptyList()
- : ksm.functions.get(name);
+ : ksm.userFunctions.get(name);
}
/**
- * Find the function with the specified name
+ * Find the function with the specified name and arguments.
*
* @param name fully qualified function name
* @param argTypes function argument types
* @return an empty {@link Optional} if the keyspace or the function name are not found;
* a non-empty optional of {@link Function} otherwise
*/
- public Optional<Function> findFunction(FunctionName name, List<AbstractType<?>> argTypes)
+ public Optional<UserFunction> findUserFunction(FunctionName name, List<AbstractType<?>> argTypes)
{
if (!name.hasKeyspace())
throw new IllegalArgumentException(String.format("Function name must be fully quallified: got %s", name));
- KeyspaceMetadata ksm = getKeyspaceMetadata(name.keyspace);
- return ksm == null
- ? Optional.empty()
- : ksm.functions.find(name, argTypes);
+ return Optional.ofNullable(getKeyspaceMetadata(name.keyspace))
+ .flatMap(ksm -> ksm.userFunctions.find(name, argTypes));
}
/* Version control */
@@ -777,4 +792,4 @@
: Collections.emptyMap();
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/schema/SchemaChangeNotifier.java b/src/java/org/apache/cassandra/schema/SchemaChangeNotifier.java
index c4537e1..449e99b 100644
--- a/src/java/org/apache/cassandra/schema/SchemaChangeNotifier.java
+++ b/src/java/org/apache/cassandra/schema/SchemaChangeNotifier.java
@@ -52,8 +52,8 @@
keyspace.types.forEach(this::notifyCreateType);
keyspace.tables.forEach(this::notifyCreateTable);
keyspace.views.forEach(this::notifyCreateView);
- keyspace.functions.udfs().forEach(this::notifyCreateFunction);
- keyspace.functions.udas().forEach(this::notifyCreateAggregate);
+ keyspace.userFunctions.udfs().forEach(this::notifyCreateFunction);
+ keyspace.userFunctions.udas().forEach(this::notifyCreateAggregate);
}
public void notifyKeyspaceAltered(KeyspaceMetadata.KeyspaceDiff delta, boolean dropData)
@@ -84,8 +84,8 @@
public void notifyKeyspaceDropped(KeyspaceMetadata keyspace, boolean dropData)
{
- keyspace.functions.udas().forEach(this::notifyDropAggregate);
- keyspace.functions.udfs().forEach(this::notifyDropFunction);
+ keyspace.userFunctions.udas().forEach(this::notifyDropAggregate);
+ keyspace.userFunctions.udfs().forEach(this::notifyDropFunction);
keyspace.views.forEach(view -> notifyDropView(view, dropData));
keyspace.tables.forEach(metadata -> notifyDropTable(metadata, dropData));
keyspace.types.forEach(this::notifyDropType);
diff --git a/src/java/org/apache/cassandra/schema/SchemaEvent.java b/src/java/org/apache/cassandra/schema/SchemaEvent.java
index 5703fe2..8cdd0c8 100644
--- a/src/java/org/apache/cassandra/schema/SchemaEvent.java
+++ b/src/java/org/apache/cassandra/schema/SchemaEvent.java
@@ -172,7 +172,7 @@
if (ksm.params != null) ret.put("params", ksm.params.toString());
if (ksm.tables != null) ret.put("tables", ksm.tables.toString());
if (ksm.views != null) ret.put("views", ksm.views.toString());
- if (ksm.functions != null) ret.put("functions", ksm.functions.toString());
+ if (ksm.userFunctions != null) ret.put("functions", ksm.userFunctions.toString());
if (ksm.types != null) ret.put("types", ksm.types.toString());
return ret;
}
diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
index 3f223dd..d11f870 100644
--- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
@@ -35,6 +35,7 @@
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
@@ -56,6 +57,8 @@
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
+import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_CORRUPTED_SCHEMA_TABLES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_FLUSH_LOCAL_SCHEMA_CHANGES;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
import static org.apache.cassandra.schema.SchemaKeyspaceTables.*;
@@ -63,7 +66,7 @@
/**
* system_schema.* tables and methods for manipulating them.
- *
+ *
* Please notice this class is _not_ thread safe and all methods which reads or updates the data in schema keyspace
* should be accessed only from the implementation of {@link SchemaUpdateHandler} in synchronized blocks.
*/
@@ -76,8 +79,8 @@
private static final Logger logger = LoggerFactory.getLogger(SchemaKeyspace.class);
- private static final boolean FLUSH_SCHEMA_TABLES = CassandraRelevantProperties.FLUSH_LOCAL_SCHEMA_CHANGES.getBoolean();
- private static final boolean IGNORE_CORRUPTED_SCHEMA_TABLES = Boolean.parseBoolean(System.getProperty("cassandra.ignore_corrupted_schema_tables", "false"));
+ private static final boolean FLUSH_SCHEMA_TABLES = TEST_FLUSH_LOCAL_SCHEMA_CHANGES.getBoolean();
+ private static final boolean IGNORE_CORRUPTED_SCHEMA_TABLES_PROPERTY_VALUE = IGNORE_CORRUPTED_SCHEMA_TABLES.getBoolean();
/**
* The tables to which we added the cdc column. This is used in {@link #makeUpdateForSchema} below to make sure we skip that
@@ -100,6 +103,7 @@
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "table_name text,"
+ + "allow_auto_snapshot boolean,"
+ "bloom_filter_fp_chance double,"
+ "caching frozen<map<text, text>>,"
+ "comment text,"
@@ -112,6 +116,7 @@
+ "extensions frozen<map<text, blob>>,"
+ "flags frozen<set<text>>," // SUPER, COUNTER, DENSE, COMPOUND
+ "gc_grace_seconds int,"
+ + "incremental_backups boolean,"
+ "id uuid,"
+ "max_index_interval int,"
+ "memtable_flush_period_in_ms int,"
@@ -137,6 +142,20 @@
+ "type text,"
+ "PRIMARY KEY ((keyspace_name), table_name, column_name))");
+ private static final TableMetadata ColumnMasks =
+ parse(COLUMN_MASKS,
+ "column dynamic data masks",
+ "CREATE TABLE %s ("
+ + "keyspace_name text,"
+ + "table_name text,"
+ + "column_name text,"
+ + "function_keyspace text,"
+ + "function_name text,"
+ + "function_argument_types frozen<list<text>>,"
+ + "function_argument_values frozen<list<text>>,"
+ + "function_argument_nulls frozen<list<boolean>>," // arguments that are null
+ + "PRIMARY KEY ((keyspace_name), table_name, column_name))");
+
private static final TableMetadata DroppedColumns =
parse(DROPPED_COLUMNS,
"dropped column registry",
@@ -168,6 +187,7 @@
+ "base_table_id uuid,"
+ "base_table_name text,"
+ "where_clause text,"
+ + "allow_auto_snapshot boolean,"
+ "bloom_filter_fp_chance double,"
+ "caching frozen<map<text, text>>,"
+ "comment text,"
@@ -179,6 +199,7 @@
+ "default_time_to_live int,"
+ "extensions frozen<map<text, blob>>,"
+ "gc_grace_seconds int,"
+ + "incremental_backups boolean,"
+ "id uuid,"
+ "include_all_columns boolean,"
+ "max_index_interval int,"
@@ -240,8 +261,17 @@
+ "state_type text,"
+ "PRIMARY KEY ((keyspace_name), aggregate_name, argument_types))");
- private static final List<TableMetadata> ALL_TABLE_METADATA =
- ImmutableList.of(Keyspaces, Tables, Columns, Triggers, DroppedColumns, Views, Types, Functions, Aggregates, Indexes);
+ private static final List<TableMetadata> ALL_TABLE_METADATA = ImmutableList.of(Keyspaces,
+ Tables,
+ Columns,
+ ColumnMasks,
+ Triggers,
+ DroppedColumns,
+ Views,
+ Types,
+ Functions,
+ Aggregates,
+ Indexes);
private static TableMetadata parse(String name, String description, String cql)
{
@@ -469,8 +499,8 @@
keyspace.tables.forEach(table -> addTableToSchemaMutation(table, true, builder));
keyspace.views.forEach(view -> addViewToSchemaMutation(view, true, builder));
keyspace.types.forEach(type -> addTypeToSchemaMutation(type, builder));
- keyspace.functions.udfs().forEach(udf -> addFunctionToSchemaMutation(udf, builder));
- keyspace.functions.udas().forEach(uda -> addAggregateToSchemaMutation(uda, builder));
+ keyspace.userFunctions.udfs().forEach(udf -> addFunctionToSchemaMutation(udf, builder));
+ keyspace.userFunctions.udas().forEach(uda -> addAggregateToSchemaMutation(uda, builder));
return builder;
}
@@ -563,6 +593,16 @@
// in mixed operation with pre-4.1 versioned node during upgrades.
if (params.memtable != MemtableParams.DEFAULT)
builder.add("memtable", params.memtable.configurationKey());
+
+ // As above, only add the allow_auto_snapshot column if the value is not default (true) and
+ // auto-snapshotting is enabled, to avoid RTE in pre-4.2 versioned node during upgrades
+ if (!params.allowAutoSnapshot)
+ builder.add("allow_auto_snapshot", false);
+
+ // As above, only add the incremental_backups column if the value is not default (true) and
+ // incremental_backups is enabled, to avoid RTE in pre-4.2 versioned node during upgrades
+ if (!params.incrementalBackups)
+ builder.add("incremental_backups", false);
}
private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder)
@@ -674,7 +714,7 @@
{
AbstractType<?> type = column.type;
if (type instanceof ReversedType)
- type = ((ReversedType) type).baseType;
+ type = ((ReversedType<?>) type).baseType;
builder.update(Columns)
.row(table.name, column.name.toString())
@@ -683,6 +723,52 @@
.add("position", column.position())
.add("clustering_order", column.clusteringOrder().toString().toLowerCase())
.add("type", type.asCQL3Type().toString());
+
+ ColumnMask mask = column.getMask();
+ if (SchemaConstants.isReplicatedSystemKeyspace(table.keyspace))
+ {
+ // The propagation of system distributed keyspaces at startup can be problematic for old nodes without DDM,
+ // since those won't know what to do with the mask mutations. Thus, we don't support DDM on those keyspaces.
+ assert mask == null : "Dynamic data masking shouldn't be used on system distributed keyspaces";
+ }
+ else
+ {
+ Row.SimpleBuilder maskBuilder = builder.update(ColumnMasks).row(table.name, column.name.toString());
+
+ if (mask == null)
+ {
+ maskBuilder.delete();
+ }
+ else
+ {
+ FunctionName maskFunctionName = mask.function.name();
+
+ // Some arguments of the masking function can be null, but the CQL's list type that stores them doesn't
+ // accept nulls, so we use a parallel list of booleans to store what arguments are null.
+ List<AbstractType<?>> partialTypes = mask.partialArgumentTypes();
+ List<ByteBuffer> partialValues = mask.partialArgumentValues();
+ int numArgs = partialTypes.size();
+ List<String> types = new ArrayList<>(numArgs);
+ List<String> values = new ArrayList<>(numArgs);
+ List<Boolean> nulls = new ArrayList<>(numArgs);
+ for (int i = 0; i < numArgs; i++)
+ {
+ AbstractType<?> argType = partialTypes.get(i);
+ types.add(argType.asCQL3Type().toString());
+
+ ByteBuffer argValue = partialValues.get(i);
+ boolean isNull = argValue == null;
+ nulls.add(isNull);
+ values.add(isNull ? "" : argType.getString(argValue));
+ }
+
+ maskBuilder.add("function_keyspace", maskFunctionName.keyspace)
+ .add("function_name", maskFunctionName.name)
+ .add("function_argument_types", types)
+ .add("function_argument_values", values)
+ .add("function_argument_nulls", nulls);
+ }
+ }
}
private static void dropColumnFromSchemaMutation(TableMetadata table, ColumnMetadata column, Mutation.SimpleBuilder builder)
@@ -862,9 +948,9 @@
{
KeyspaceParams params = fetchKeyspaceParams(keyspaceName);
Types types = fetchTypes(keyspaceName);
- Tables tables = fetchTables(keyspaceName, types);
- Views views = fetchViews(keyspaceName, types);
- Functions functions = fetchFunctions(keyspaceName, types);
+ UserFunctions functions = fetchFunctions(keyspaceName, types);
+ Tables tables = fetchTables(keyspaceName, types, functions);
+ Views views = fetchViews(keyspaceName, types, functions);
return KeyspaceMetadata.create(keyspaceName, params, tables, views, types, functions);
}
@@ -893,7 +979,7 @@
return types.build();
}
- private static Tables fetchTables(String keyspaceName, Types types)
+ private static Tables fetchTables(String keyspaceName, Types types, UserFunctions functions)
{
String query = format("SELECT table_name FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, TABLES);
@@ -903,7 +989,7 @@
String tableName = row.getString("table_name");
try
{
- tables.add(fetchTable(keyspaceName, tableName, types));
+ tables.add(fetchTable(keyspaceName, tableName, types, functions));
}
catch (MissingColumns exc)
{
@@ -918,13 +1004,13 @@
SchemaConstants.SCHEMA_KEYSPACE_NAME, COLUMNS, keyspaceName, tableName,
SchemaConstants.SCHEMA_KEYSPACE_NAME, COLUMNS);
- if (IGNORE_CORRUPTED_SCHEMA_TABLES)
+ if (IGNORE_CORRUPTED_SCHEMA_TABLES_PROPERTY_VALUE)
{
logger.error(errorMsg, "", exc);
}
else
{
- logger.error(errorMsg, "restart cassandra with -Dcassandra.ignore_corrupted_schema_tables=true and ");
+ logger.error(errorMsg, "restart cassandra with -D{}=true and ", IGNORE_CORRUPTED_SCHEMA_TABLES.getKey());
throw exc;
}
}
@@ -932,7 +1018,7 @@
return tables.build();
}
- private static TableMetadata fetchTable(String keyspaceName, String tableName, Types types)
+ private static TableMetadata fetchTable(String keyspaceName, String tableName, Types types, UserFunctions functions)
{
String query = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, TABLES);
UntypedResultSet rows = query(query, keyspaceName, tableName);
@@ -944,7 +1030,7 @@
return TableMetadata.builder(keyspaceName, tableName, TableId.fromUUID(row.getUUID("id")))
.flags(flags)
.params(createTableParamsFromRow(row))
- .addColumns(fetchColumns(keyspaceName, tableName, types))
+ .addColumns(fetchColumns(keyspaceName, tableName, types, functions))
.droppedColumns(fetchDroppedColumns(keyspaceName, tableName))
.indexes(fetchIndexes(keyspaceName, tableName))
.triggers(fetchTriggers(keyspaceName, tableName))
@@ -954,32 +1040,41 @@
@VisibleForTesting
static TableParams createTableParamsFromRow(UntypedResultSet.Row row)
{
- return TableParams.builder()
- .bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance"))
- .caching(CachingParams.fromMap(row.getFrozenTextMap("caching")))
- .comment(row.getString("comment"))
- .compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction")))
- .compression(CompressionParams.fromMap(row.getFrozenTextMap("compression")))
- .memtable(MemtableParams.getWithFallback(row.has("memtable")
- ? row.getString("memtable")
- : null)) // memtable column was introduced in 4.1
- .defaultTimeToLive(row.getInt("default_time_to_live"))
- .extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance))
- .gcGraceSeconds(row.getInt("gc_grace_seconds"))
- .maxIndexInterval(row.getInt("max_index_interval"))
- .memtableFlushPeriodInMs(row.getInt("memtable_flush_period_in_ms"))
- .minIndexInterval(row.getInt("min_index_interval"))
- .crcCheckChance(row.getDouble("crc_check_chance"))
- .speculativeRetry(SpeculativeRetryPolicy.fromString(row.getString("speculative_retry")))
- .additionalWritePolicy(row.has("additional_write_policy") ?
- SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) :
- SpeculativeRetryPolicy.fromString("99PERCENTILE"))
- .cdc(row.has("cdc") && row.getBoolean("cdc"))
- .readRepair(getReadRepairStrategy(row))
- .build();
+ TableParams.Builder builder = TableParams.builder()
+ .bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance"))
+ .caching(CachingParams.fromMap(row.getFrozenTextMap("caching")))
+ .comment(row.getString("comment"))
+ .compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction")))
+ .compression(CompressionParams.fromMap(row.getFrozenTextMap("compression")))
+ .memtable(MemtableParams.getWithFallback(row.has("memtable")
+ ? row.getString("memtable")
+ : null)) // memtable column was introduced in 4.1
+ .defaultTimeToLive(row.getInt("default_time_to_live"))
+ .extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance))
+ .gcGraceSeconds(row.getInt("gc_grace_seconds"))
+ .maxIndexInterval(row.getInt("max_index_interval"))
+ .memtableFlushPeriodInMs(row.getInt("memtable_flush_period_in_ms"))
+ .minIndexInterval(row.getInt("min_index_interval"))
+ .crcCheckChance(row.getDouble("crc_check_chance"))
+ .speculativeRetry(SpeculativeRetryPolicy.fromString(row.getString("speculative_retry")))
+ .additionalWritePolicy(row.has("additional_write_policy") ?
+ SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) :
+ SpeculativeRetryPolicy.fromString("99PERCENTILE"))
+ .cdc(row.has("cdc") && row.getBoolean("cdc"))
+ .readRepair(getReadRepairStrategy(row));
+
+ // allow_auto_snapshot column was introduced in 4.2
+ if (row.has("allow_auto_snapshot"))
+ builder.allowAutoSnapshot(row.getBoolean("allow_auto_snapshot"));
+
+ // incremental_backups column was introduced in 4.2
+ if (row.has("incremental_backups"))
+ builder.incrementalBackups(row.getBoolean("incremental_backups"));
+
+ return builder.build();
}
- private static List<ColumnMetadata> fetchColumns(String keyspace, String table, Types types)
+ private static List<ColumnMetadata> fetchColumns(String keyspace, String table, Types types, UserFunctions functions)
{
String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, COLUMNS);
UntypedResultSet columnRows = query(query, keyspace, table);
@@ -987,7 +1082,7 @@
throw new MissingColumns("Columns not found in schema table for " + keyspace + '.' + table);
List<ColumnMetadata> columns = new ArrayList<>();
- columnRows.forEach(row -> columns.add(createColumnFromRow(row, types)));
+ columnRows.forEach(row -> columns.add(createColumnFromRow(row, types, functions)));
if (columns.stream().noneMatch(ColumnMetadata::isPartitionKey))
throw new MissingColumns("No partition key columns found in schema table for " + keyspace + "." + table);
@@ -996,7 +1091,7 @@
}
@VisibleForTesting
- static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types types)
+ public static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types types, UserFunctions functions)
{
String keyspace = row.getString("keyspace_name");
String table = row.getString("table_name");
@@ -1012,7 +1107,53 @@
ColumnIdentifier name = new ColumnIdentifier(row.getBytes("column_name_bytes"), row.getString("column_name"));
- return new ColumnMetadata(keyspace, table, name, type, position, kind);
+ ColumnMask mask = null;
+ String query = format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ? AND column_name = ?",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME, COLUMN_MASKS);
+ UntypedResultSet columnMasks = query(query, keyspace, table, name.toString());
+ if (!columnMasks.isEmpty())
+ {
+ UntypedResultSet.Row maskRow = columnMasks.one();
+ FunctionName functionName = new FunctionName(maskRow.getString("function_keyspace"), maskRow.getString("function_name"));
+
+ List<String> partialArgumentTypes = maskRow.getFrozenList("function_argument_types", UTF8Type.instance);
+ List<AbstractType<?>> argumentTypes = new ArrayList<>(1 + partialArgumentTypes.size());
+ argumentTypes.add(type);
+ for (String argumentType : partialArgumentTypes)
+ {
+ argumentTypes.add(CQLTypeParser.parse(keyspace, argumentType, types));
+ }
+
+ Function function = FunctionResolver.get(keyspace, functionName, argumentTypes, null, null, null, functions);
+ if (function == null)
+ {
+ throw new AssertionError(format("Unable to find masking function %s(%s) for column %s.%s.%s",
+ functionName, argumentTypes, keyspace, table, name));
+ }
+ else if (!(function instanceof ScalarFunction))
+ {
+ throw new AssertionError(format("Column %s.%s.%s is unexpectedly masked with function %s " +
+ "which is not a scalar masking function",
+ keyspace, table, name, function));
+ }
+
+ // Some arguments of the masking function can be null, but the CQL's list type that stores them doesn't
+ // accept nulls, so we use a parallel list of booleans to store what arguments are null.
+ List<Boolean> nulls = maskRow.getFrozenList("function_argument_nulls", BooleanType.instance);
+ List<String> valuesAsCQL = maskRow.getFrozenList("function_argument_values", UTF8Type.instance);
+ ByteBuffer[] values = new ByteBuffer[valuesAsCQL.size()];
+ for (int i = 0; i < valuesAsCQL.size(); i++)
+ {
+ if (nulls.get(i))
+ values[i] = null;
+ else
+ values[i] = argumentTypes.get(i + 1).fromString(valuesAsCQL.get(i));
+ }
+
+ mask = ColumnMask.build((ScalarFunction) function, values);
+ }
+
+ return new ColumnMetadata(keyspace, table, name, type, position, kind, mask);
}
private static Map<ByteBuffer, DroppedColumn> fetchDroppedColumns(String keyspace, String table)
@@ -1044,7 +1185,7 @@
assert kind == ColumnMetadata.Kind.REGULAR || kind == ColumnMetadata.Kind.STATIC
: "Unexpected dropped column kind: " + kind;
- ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind);
+ ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind, null);
long droppedTime = TimeUnit.MILLISECONDS.toMicros(row.getLong("dropped_time"));
return new DroppedColumn(column, droppedTime);
}
@@ -1080,17 +1221,17 @@
return new TriggerMetadata(name, classOption);
}
- private static Views fetchViews(String keyspaceName, Types types)
+ private static Views fetchViews(String keyspaceName, Types types, UserFunctions functions)
{
String query = format("SELECT view_name FROM %s.%s WHERE keyspace_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, VIEWS);
Views.Builder views = org.apache.cassandra.schema.Views.builder();
for (UntypedResultSet.Row row : query(query, keyspaceName))
- views.put(fetchView(keyspaceName, row.getString("view_name"), types));
+ views.put(fetchView(keyspaceName, row.getString("view_name"), types, functions));
return views.build();
}
- private static ViewMetadata fetchView(String keyspaceName, String viewName, Types types)
+ private static ViewMetadata fetchView(String keyspaceName, String viewName, Types types, UserFunctions functions)
{
String query = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND view_name = ?", SchemaConstants.SCHEMA_KEYSPACE_NAME, VIEWS);
UntypedResultSet rows = query(query, keyspaceName, viewName);
@@ -1103,7 +1244,7 @@
boolean includeAll = row.getBoolean("include_all_columns");
String whereClauseString = row.getString("where_clause");
- List<ColumnMetadata> columns = fetchColumns(keyspaceName, viewName, types);
+ List<ColumnMetadata> columns = fetchColumns(keyspaceName, viewName, types, functions);
TableMetadata metadata =
TableMetadata.builder(keyspaceName, viewName, TableId.fromUUID(row.getUUID("id")))
@@ -1127,12 +1268,12 @@
return new ViewMetadata(baseTableId, baseTableName, includeAll, whereClause, metadata);
}
- private static Functions fetchFunctions(String keyspaceName, Types types)
+ private static UserFunctions fetchFunctions(String keyspaceName, Types types)
{
Collection<UDFunction> udfs = fetchUDFs(keyspaceName, types);
Collection<UDAggregate> udas = fetchUDAs(keyspaceName, udfs, types);
- return org.apache.cassandra.schema.Functions.builder().add(udfs).add(udas).build();
+ return UserFunctions.builder().add(udfs).add(udas).build();
}
private static Collection<UDFunction> fetchUDFs(String keyspaceName, Types types)
@@ -1169,7 +1310,7 @@
* TODO: find a way to get rid of Schema.instance dependency; evaluate if the opimisation below makes a difference
* in the first place. Remove if it isn't.
*/
- org.apache.cassandra.cql3.functions.Function existing = Schema.instance.findFunction(name, argTypes).orElse(null);
+ UserFunction existing = Schema.instance.findUserFunction(name, argTypes).orElse(null);
if (existing instanceof UDFunction)
{
// This check prevents duplicate compilation of effectively the same UDF.
diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspaceTables.java b/src/java/org/apache/cassandra/schema/SchemaKeyspaceTables.java
index c00a4f7..3fc9c8b 100644
--- a/src/java/org/apache/cassandra/schema/SchemaKeyspaceTables.java
+++ b/src/java/org/apache/cassandra/schema/SchemaKeyspaceTables.java
@@ -24,6 +24,7 @@
public static final String KEYSPACES = "keyspaces";
public static final String TABLES = "tables";
public static final String COLUMNS = "columns";
+ public static final String COLUMN_MASKS = "column_masks";
public static final String DROPPED_COLUMNS = "dropped_columns";
public static final String TRIGGERS = "triggers";
public static final String VIEWS = "views";
@@ -45,7 +46,8 @@
*
* See CASSANDRA-12213 for more details.
*/
- public static final ImmutableList<String> ALL = ImmutableList.of(COLUMNS,
+ public static final ImmutableList<String> ALL = ImmutableList.of(COLUMN_MASKS,
+ COLUMNS,
DROPPED_COLUMNS,
TRIGGERS,
TYPES,
diff --git a/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactoryProvider.java b/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactoryProvider.java
index 9411a92..63d170a 100644
--- a/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactoryProvider.java
+++ b/src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactoryProvider.java
@@ -25,12 +25,17 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SCHEMA_UPDATE_HANDLER_FACTORY_CLASS;
+
/**
- * Provides the instance of SchemaUpdateHandler factory pointed by {@link #SUH_FACTORY_CLASS_PROPERTY} system property.
+ * Provides the instance of SchemaUpdateHandler factory pointed by
+ * {@link org.apache.cassandra.config.CassandraRelevantProperties#SCHEMA_UPDATE_HANDLER_FACTORY_CLASS} system property.
* If the property is not defined, the default factory {@link DefaultSchemaUpdateHandler} instance is returned.
*/
public class SchemaUpdateHandlerFactoryProvider implements Provider<SchemaUpdateHandlerFactory>
{
+ /** @deprecated Use CassandraRelevantProperties.SCHEMA_UPDATE_HANDLER_FACTORY_CLASS instead. */
+ @Deprecated
public static final String SUH_FACTORY_CLASS_PROPERTY = "cassandra.schema.update_handler_factory.class";
public final static SchemaUpdateHandlerFactoryProvider instance = new SchemaUpdateHandlerFactoryProvider();
@@ -38,7 +43,7 @@
@Override
public SchemaUpdateHandlerFactory get()
{
- String suhFactoryClassName = StringUtils.trimToNull(System.getProperty(SUH_FACTORY_CLASS_PROPERTY));
+ String suhFactoryClassName = StringUtils.trimToNull(SCHEMA_UPDATE_HANDLER_FACTORY_CLASS.getString());
if (suhFactoryClassName == null)
{
return DefaultSchemaUpdateHandlerFactory.instance;
@@ -53,7 +58,7 @@
catch (InstantiationException | IllegalAccessException ex)
{
throw new ConfigurationException(String.format("Failed to initialize schema update handler factory class %s defined in %s system property.",
- suhFactoryClassName, SUH_FACTORY_CLASS_PROPERTY), ex);
+ suhFactoryClassName, SCHEMA_UPDATE_HANDLER_FACTORY_CLASS.getKey()), ex);
}
}
}
diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
index dc40093..c11b2c4 100644
--- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
@@ -174,7 +174,7 @@
{
Collection<Range<Token>> ranges = options.getRanges();
String query = "INSERT INTO %s.%s (parent_id, keyspace_name, columnfamily_names, requested_ranges, started_at, options)"+
- " VALUES (%s, '%s', { '%s' }, { '%s' }, toTimestamp(now()), { %s })";
+ " VALUES (%s, '%s', { '%s' }, { '%s' }, to_timestamp(now()), { %s })";
String fmtQry = format(query,
SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
PARENT_REPAIR_HISTORY,
@@ -206,7 +206,7 @@
public static void failParentRepair(TimeUUID parent_id, Throwable t)
{
- String query = "UPDATE %s.%s SET finished_at = toTimestamp(now()), exception_message=?, exception_stacktrace=? WHERE parent_id=%s";
+ String query = "UPDATE %s.%s SET finished_at = to_timestamp(now()), exception_message=?, exception_stacktrace=? WHERE parent_id=%s";
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
@@ -218,7 +218,7 @@
public static void successfulParentRepair(TimeUUID parent_id, Collection<Range<Token>> successfulRanges)
{
- String query = "UPDATE %s.%s SET finished_at = toTimestamp(now()), successful_ranges = {'%s'} WHERE parent_id=%s";
+ String query = "UPDATE %s.%s SET finished_at = to_timestamp(now()), successful_ranges = {'%s'} WHERE parent_id=%s";
String fmtQuery = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, PARENT_REPAIR_HISTORY, Joiner.on("','").join(successfulRanges), parent_id.toString());
processSilent(fmtQuery);
}
@@ -241,10 +241,10 @@
String query =
"INSERT INTO %s.%s (keyspace_name, columnfamily_name, id, parent_id, range_begin, range_end, coordinator, coordinator_port, participants, participants_v2, status, started_at) " +
- "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', %d, { '%s' }, { '%s' }, '%s', toTimestamp(now()))";
+ "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', %d, { '%s' }, { '%s' }, '%s', to_timestamp(now()))";
String queryWithoutNewColumns =
"INSERT INTO %s.%s (keyspace_name, columnfamily_name, id, parent_id, range_begin, range_end, coordinator, participants, status, started_at) " +
- "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', { '%s' }, '%s', toTimestamp(now()))";
+ "VALUES ( '%s', '%s', %s, %s, '%s', '%s', '%s', { '%s' }, '%s', to_timestamp(now()))";
for (String cfname : cfnames)
{
@@ -292,7 +292,7 @@
public static void successfulRepairJob(TimeUUID id, String keyspaceName, String cfname)
{
- String query = "UPDATE %s.%s SET status = '%s', finished_at = toTimestamp(now()) WHERE keyspace_name = '%s' AND columnfamily_name = '%s' AND id = %s";
+ String query = "UPDATE %s.%s SET status = '%s', finished_at = to_timestamp(now()) WHERE keyspace_name = '%s' AND columnfamily_name = '%s' AND id = %s";
String fmtQuery = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY,
RepairState.SUCCESS.toString(),
keyspaceName,
@@ -303,7 +303,7 @@
public static void failedRepairJob(TimeUUID id, String keyspaceName, String cfname, Throwable t)
{
- String query = "UPDATE %s.%s SET status = '%s', finished_at = toTimestamp(now()), exception_message=?, exception_stacktrace=? WHERE keyspace_name = '%s' AND columnfamily_name = '%s' AND id = %s";
+ String query = "UPDATE %s.%s SET status = '%s', finished_at = to_timestamp(now()), exception_message=?, exception_stacktrace=? WHERE keyspace_name = '%s' AND columnfamily_name = '%s' AND id = %s";
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
t.printStackTrace(pw);
diff --git a/src/java/org/apache/cassandra/schema/TableId.java b/src/java/org/apache/cassandra/schema/TableId.java
index fd47a47..ceeeec3 100644
--- a/src/java/org/apache/cassandra/schema/TableId.java
+++ b/src/java/org/apache/cassandra/schema/TableId.java
@@ -20,11 +20,15 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.UUID;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang3.ArrayUtils;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -62,6 +66,27 @@
return new TableId(UUID.fromString(idString));
}
+ @Nullable
+ public static Pair<String, TableId> tableNameAndIdFromFilename(String filename)
+ {
+ int dash = filename.lastIndexOf('-');
+ if (dash <= 0 || dash != filename.length() - 32 - 1)
+ return null;
+
+ TableId id = fromHexString(filename.substring(dash + 1));
+ String tableName = filename.substring(0, dash);
+
+ return Pair.create(tableName, id);
+ }
+
+ private static TableId fromHexString(String nonDashUUID)
+ {
+ ByteBuffer bytes = ByteBufferUtil.hexToBytes(nonDashUUID);
+ long msb = bytes.getLong(0);
+ long lsb = bytes.getLong(8);
+ return fromUUID(new UUID(msb, lsb));
+ }
+
/**
* Creates the UUID of a system table.
*
diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java
index 2e9d507..53a22ab 100644
--- a/src/java/org/apache/cassandra/schema/TableMetadata.java
+++ b/src/java/org/apache/cassandra/schema/TableMetadata.java
@@ -18,14 +18,27 @@
package org.apache.cassandra.schema;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
-
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
import javax.annotation.Nullable;
import com.google.common.base.MoreObjects;
-import com.google.common.collect.*;
-
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,8 +47,20 @@
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.CqlBuilder;
import org.apache.cassandra.cql3.SchemaElement;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.EmptyType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -263,6 +288,11 @@
{
return false;
}
+
+ public boolean isIncrementalBackupsEnabled()
+ {
+ return params.incrementalBackups;
+ }
public boolean isStaticCompactTable()
{
@@ -414,6 +444,35 @@
return !staticColumns().isEmpty();
}
+ /**
+ * @return {@code true} if the table has any masked column, {@code false} otherwise.
+ */
+ public boolean hasMaskedColumns()
+ {
+ for (ColumnMetadata column : columns.values())
+ {
+ if (column.isMasked())
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * @param function a user function
+ * @return {@code true} if the table has any masked column depending on the specified user function,
+ * {@code false} otherwise.
+ */
+ public boolean dependsOn(Function function)
+ {
+ for (ColumnMetadata column : columns.values())
+ {
+ ColumnMask mask = column.getMask();
+ if (mask != null && mask.function.name().equals(function.name()))
+ return true;
+ }
+ return false;
+ }
+
public void validate()
{
if (!isNameValid(keyspace))
@@ -469,7 +528,7 @@
return !columnName.bytes.hasRemaining();
}
- void validateCompatibility(TableMetadata previous)
+ public void validateCompatibility(TableMetadata previous)
{
if (isIndex())
return;
@@ -777,6 +836,12 @@
return this;
}
+ public Builder allowAutoSnapshot(boolean val)
+ {
+ params.allowAutoSnapshot(val);
+ return this;
+ }
+
public Builder bloomFilterFpChance(double val)
{
params.bloomFilterFpChance(val);
@@ -897,44 +962,84 @@
return this;
}
- public Builder addPartitionKeyColumn(String name, AbstractType type)
+ public Builder addPartitionKeyColumn(String name, AbstractType<?> type)
{
- return addPartitionKeyColumn(ColumnIdentifier.getInterned(name, false), type);
+ return addPartitionKeyColumn(name, type, null);
}
- public Builder addPartitionKeyColumn(ColumnIdentifier name, AbstractType type)
+ public Builder addPartitionKeyColumn(String name, AbstractType<?> type, @Nullable ColumnMask mask)
{
- return addColumn(new ColumnMetadata(keyspace, this.name, name, type, partitionKeyColumns.size(), ColumnMetadata.Kind.PARTITION_KEY));
+ return addPartitionKeyColumn(ColumnIdentifier.getInterned(name, false), type, mask);
}
- public Builder addClusteringColumn(String name, AbstractType type)
+ public Builder addPartitionKeyColumn(ColumnIdentifier name, AbstractType<?> type)
{
- return addClusteringColumn(ColumnIdentifier.getInterned(name, false), type);
+ return addPartitionKeyColumn(name, type, null);
}
- public Builder addClusteringColumn(ColumnIdentifier name, AbstractType type)
+ public Builder addPartitionKeyColumn(ColumnIdentifier name, AbstractType<?> type, @Nullable ColumnMask mask)
{
- return addColumn(new ColumnMetadata(keyspace, this.name, name, type, clusteringColumns.size(), ColumnMetadata.Kind.CLUSTERING));
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, partitionKeyColumns.size(), ColumnMetadata.Kind.PARTITION_KEY, mask));
}
- public Builder addRegularColumn(String name, AbstractType type)
+ public Builder addClusteringColumn(String name, AbstractType<?> type)
{
- return addRegularColumn(ColumnIdentifier.getInterned(name, false), type);
+ return addClusteringColumn(name, type, null);
}
- public Builder addRegularColumn(ColumnIdentifier name, AbstractType type)
+ public Builder addClusteringColumn(String name, AbstractType<?> type, @Nullable ColumnMask mask)
{
- return addColumn(new ColumnMetadata(keyspace, this.name, name, type, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR));
+ return addClusteringColumn(ColumnIdentifier.getInterned(name, false), type, mask);
}
- public Builder addStaticColumn(String name, AbstractType type)
+ public Builder addClusteringColumn(ColumnIdentifier name, AbstractType<?> type)
{
- return addStaticColumn(ColumnIdentifier.getInterned(name, false), type);
+ return addClusteringColumn(name, type, null);
}
- public Builder addStaticColumn(ColumnIdentifier name, AbstractType type)
+ public Builder addClusteringColumn(ColumnIdentifier name, AbstractType<?> type, @Nullable ColumnMask mask)
{
- return addColumn(new ColumnMetadata(keyspace, this.name, name, type, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.STATIC));
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, clusteringColumns.size(), ColumnMetadata.Kind.CLUSTERING, mask));
+ }
+
+ public Builder addRegularColumn(String name, AbstractType<?> type)
+ {
+ return addRegularColumn(name, type, null);
+ }
+
+ public Builder addRegularColumn(String name, AbstractType<?> type, @Nullable ColumnMask mask)
+ {
+ return addRegularColumn(ColumnIdentifier.getInterned(name, false), type, mask);
+ }
+
+ public Builder addRegularColumn(ColumnIdentifier name, AbstractType<?> type)
+ {
+ return addRegularColumn(name, type, null);
+ }
+
+ public Builder addRegularColumn(ColumnIdentifier name, AbstractType<?> type, @Nullable ColumnMask mask)
+ {
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR, mask));
+ }
+
+ public Builder addStaticColumn(String name, AbstractType<?> type)
+ {
+ return addStaticColumn(name, type, null);
+ }
+
+ public Builder addStaticColumn(String name, AbstractType<?> type, @Nullable ColumnMask mask)
+ {
+ return addStaticColumn(ColumnIdentifier.getInterned(name, false), type, mask);
+ }
+
+ public Builder addStaticColumn(ColumnIdentifier name, AbstractType<?> type)
+ {
+ return addStaticColumn(name, type, null);
+ }
+
+ public Builder addStaticColumn(ColumnIdentifier name, AbstractType<?> type, @Nullable ColumnMask mask)
+ {
+ return addColumn(new ColumnMetadata(keyspace, this.name, name, type, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.STATIC, mask));
}
public Builder addColumn(ColumnMetadata column)
@@ -1059,6 +1164,19 @@
return this;
}
+ public Builder alterColumnMask(ColumnIdentifier name, @Nullable ColumnMask mask)
+ {
+ ColumnMetadata column = columns.get(name.bytes);
+ if (column == null)
+ throw new IllegalArgumentException();
+
+ ColumnMetadata newColumn = column.withNewMask(mask);
+
+ updateColumn(column, newColumn);
+
+ return this;
+ }
+
Builder alterColumnType(ColumnIdentifier name, AbstractType<?> type)
{
ColumnMetadata column = columns.get(name.bytes);
@@ -1067,6 +1185,13 @@
ColumnMetadata newColumn = column.withNewType(type);
+ updateColumn(column, newColumn);
+
+ return this;
+ }
+
+ private void updateColumn(ColumnMetadata column, ColumnMetadata newColumn)
+ {
switch (column.kind)
{
case PARTITION_KEY:
@@ -1083,8 +1208,6 @@
}
columns.put(column.name.bytes, newColumn);
-
- return this;
}
}
@@ -1509,7 +1632,7 @@
for (ColumnMetadata c : regularAndStaticColumns)
{
if (c.isStatic())
- columns.add(new ColumnMetadata(c.ksName, c.cfName, c.name, c.type, -1, ColumnMetadata.Kind.REGULAR));
+ columns.add(new ColumnMetadata(c.ksName, c.cfName, c.name, c.type, -1, ColumnMetadata.Kind.REGULAR, c.getMask()));
}
otherColumns = columns.iterator();
}
@@ -1596,4 +1719,4 @@
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java
index 440729c..8f883f8 100644
--- a/src/java/org/apache/cassandra/schema/TableParams.java
+++ b/src/java/org/apache/cassandra/schema/TableParams.java
@@ -36,11 +36,13 @@
import static java.lang.String.format;
import static java.util.stream.Collectors.toMap;
+import static org.apache.cassandra.schema.TableParams.Option.*;
public final class TableParams
{
public enum Option
{
+ ALLOW_AUTO_SNAPSHOT,
BLOOM_FILTER_FP_CHANCE,
CACHING,
COMMENT,
@@ -50,6 +52,7 @@
DEFAULT_TIME_TO_LIVE,
EXTENSIONS,
GC_GRACE_SECONDS,
+ INCREMENTAL_BACKUPS,
MAX_INDEX_INTERVAL,
MEMTABLE_FLUSH_PERIOD_IN_MS,
MIN_INDEX_INTERVAL,
@@ -67,9 +70,11 @@
}
public final String comment;
+ public final boolean allowAutoSnapshot;
public final double bloomFilterFpChance;
public final double crcCheckChance;
public final int gcGraceSeconds;
+ public final boolean incrementalBackups;
public final int defaultTimeToLive;
public final int memtableFlushPeriodInMs;
public final int minIndexInterval;
@@ -87,11 +92,13 @@
private TableParams(Builder builder)
{
comment = builder.comment;
- bloomFilterFpChance = builder.bloomFilterFpChance == null
+ allowAutoSnapshot = builder.allowAutoSnapshot;
+ bloomFilterFpChance = builder.bloomFilterFpChance == -1
? builder.compaction.defaultBloomFilterFbChance()
: builder.bloomFilterFpChance;
crcCheckChance = builder.crcCheckChance;
gcGraceSeconds = builder.gcGraceSeconds;
+ incrementalBackups = builder.incrementalBackups;
defaultTimeToLive = builder.defaultTimeToLive;
memtableFlushPeriodInMs = builder.memtableFlushPeriodInMs;
minIndexInterval = builder.minIndexInterval;
@@ -114,7 +121,8 @@
public static Builder builder(TableParams params)
{
- return new Builder().bloomFilterFpChance(params.bloomFilterFpChance)
+ return new Builder().allowAutoSnapshot(params.allowAutoSnapshot)
+ .bloomFilterFpChance(params.bloomFilterFpChance)
.caching(params.caching)
.comment(params.comment)
.compaction(params.compaction)
@@ -123,6 +131,7 @@
.crcCheckChance(params.crcCheckChance)
.defaultTimeToLive(params.defaultTimeToLive)
.gcGraceSeconds(params.gcGraceSeconds)
+ .incrementalBackups(params.incrementalBackups)
.maxIndexInterval(params.maxIndexInterval)
.memtableFlushPeriodInMs(params.memtableFlushPeriodInMs)
.minIndexInterval(params.minIndexInterval)
@@ -147,7 +156,7 @@
if (bloomFilterFpChance <= minBloomFilterFpChanceValue || bloomFilterFpChance > 1)
{
fail("%s must be larger than %s and less than or equal to 1.0 (got %s)",
- Option.BLOOM_FILTER_FP_CHANCE,
+ BLOOM_FILTER_FP_CHANCE,
minBloomFilterFpChanceValue,
bloomFilterFpChance);
}
@@ -155,33 +164,33 @@
if (crcCheckChance < 0 || crcCheckChance > 1.0)
{
fail("%s must be larger than or equal to 0 and smaller than or equal to 1.0 (got %s)",
- Option.CRC_CHECK_CHANCE,
+ CRC_CHECK_CHANCE,
crcCheckChance);
}
if (defaultTimeToLive < 0)
- fail("%s must be greater than or equal to 0 (got %s)", Option.DEFAULT_TIME_TO_LIVE, defaultTimeToLive);
+ fail("%s must be greater than or equal to 0 (got %s)", DEFAULT_TIME_TO_LIVE, defaultTimeToLive);
if (defaultTimeToLive > Attributes.MAX_TTL)
- fail("%s must be less than or equal to %d (got %s)", Option.DEFAULT_TIME_TO_LIVE, Attributes.MAX_TTL, defaultTimeToLive);
+ fail("%s must be less than or equal to %d (got %s)", DEFAULT_TIME_TO_LIVE, Attributes.MAX_TTL, defaultTimeToLive);
if (gcGraceSeconds < 0)
- fail("%s must be greater than or equal to 0 (got %s)", Option.GC_GRACE_SECONDS, gcGraceSeconds);
+ fail("%s must be greater than or equal to 0 (got %s)", GC_GRACE_SECONDS, gcGraceSeconds);
if (minIndexInterval < 1)
- fail("%s must be greater than or equal to 1 (got %s)", Option.MIN_INDEX_INTERVAL, minIndexInterval);
+ fail("%s must be greater than or equal to 1 (got %s)", MIN_INDEX_INTERVAL, minIndexInterval);
if (maxIndexInterval < minIndexInterval)
{
fail("%s must be greater than or equal to %s (%s) (got %s)",
- Option.MAX_INDEX_INTERVAL,
- Option.MIN_INDEX_INTERVAL,
+ MAX_INDEX_INTERVAL,
+ MIN_INDEX_INTERVAL,
minIndexInterval,
maxIndexInterval);
}
if (memtableFlushPeriodInMs < 0)
- fail("%s must be greater than or equal to 0 (got %s)", Option.MEMTABLE_FLUSH_PERIOD_IN_MS, memtableFlushPeriodInMs);
+ fail("%s must be greater than or equal to 0 (got %s)", MEMTABLE_FLUSH_PERIOD_IN_MS, memtableFlushPeriodInMs);
if (cdc && memtable.factory().writesShouldSkipCommitLog())
fail("CDC cannot work if writes skip the commit log. Check your memtable configuration.");
@@ -204,9 +213,12 @@
TableParams p = (TableParams) o;
return comment.equals(p.comment)
+ && additionalWritePolicy.equals(p.additionalWritePolicy)
+ && allowAutoSnapshot == p.allowAutoSnapshot
&& bloomFilterFpChance == p.bloomFilterFpChance
&& crcCheckChance == p.crcCheckChance
- && gcGraceSeconds == p.gcGraceSeconds
+ && gcGraceSeconds == p.gcGraceSeconds
+ && incrementalBackups == p.incrementalBackups
&& defaultTimeToLive == p.defaultTimeToLive
&& memtableFlushPeriodInMs == p.memtableFlushPeriodInMs
&& minIndexInterval == p.minIndexInterval
@@ -225,9 +237,12 @@
public int hashCode()
{
return Objects.hashCode(comment,
+ additionalWritePolicy,
+ allowAutoSnapshot,
bloomFilterFpChance,
crcCheckChance,
gcGraceSeconds,
+ incrementalBackups,
defaultTimeToLive,
memtableFlushPeriodInMs,
minIndexInterval,
@@ -246,22 +261,25 @@
public String toString()
{
return MoreObjects.toStringHelper(this)
- .add(Option.COMMENT.toString(), comment)
- .add(Option.BLOOM_FILTER_FP_CHANCE.toString(), bloomFilterFpChance)
- .add(Option.CRC_CHECK_CHANCE.toString(), crcCheckChance)
- .add(Option.GC_GRACE_SECONDS.toString(), gcGraceSeconds)
- .add(Option.DEFAULT_TIME_TO_LIVE.toString(), defaultTimeToLive)
- .add(Option.MEMTABLE_FLUSH_PERIOD_IN_MS.toString(), memtableFlushPeriodInMs)
- .add(Option.MIN_INDEX_INTERVAL.toString(), minIndexInterval)
- .add(Option.MAX_INDEX_INTERVAL.toString(), maxIndexInterval)
- .add(Option.SPECULATIVE_RETRY.toString(), speculativeRetry)
- .add(Option.CACHING.toString(), caching)
- .add(Option.COMPACTION.toString(), compaction)
- .add(Option.COMPRESSION.toString(), compression)
- .add(Option.MEMTABLE.toString(), memtable)
- .add(Option.EXTENSIONS.toString(), extensions)
- .add(Option.CDC.toString(), cdc)
- .add(Option.READ_REPAIR.toString(), readRepair)
+ .add(COMMENT.toString(), comment)
+ .add(ADDITIONAL_WRITE_POLICY.toString(), additionalWritePolicy)
+ .add(ALLOW_AUTO_SNAPSHOT.toString(), allowAutoSnapshot)
+ .add(BLOOM_FILTER_FP_CHANCE.toString(), bloomFilterFpChance)
+ .add(CRC_CHECK_CHANCE.toString(), crcCheckChance)
+ .add(GC_GRACE_SECONDS.toString(), gcGraceSeconds)
+ .add(DEFAULT_TIME_TO_LIVE.toString(), defaultTimeToLive)
+ .add(INCREMENTAL_BACKUPS.toString(), incrementalBackups)
+ .add(MEMTABLE_FLUSH_PERIOD_IN_MS.toString(), memtableFlushPeriodInMs)
+ .add(MIN_INDEX_INTERVAL.toString(), minIndexInterval)
+ .add(MAX_INDEX_INTERVAL.toString(), maxIndexInterval)
+ .add(SPECULATIVE_RETRY.toString(), speculativeRetry)
+ .add(CACHING.toString(), caching)
+ .add(COMPACTION.toString(), compaction)
+ .add(COMPRESSION.toString(), compression)
+ .add(MEMTABLE.toString(), memtable)
+ .add(EXTENSIONS.toString(), extensions)
+ .add(CDC.toString(), cdc)
+ .add(READ_REPAIR.toString(), readRepair)
.toString();
}
@@ -270,6 +288,8 @@
// option names should be in alphabetical order
builder.append("additional_write_policy = ").appendWithSingleQuotes(additionalWritePolicy.toString())
.newLine()
+ .append("AND allow_auto_snapshot = ").append(allowAutoSnapshot)
+ .newLine()
.append("AND bloom_filter_fp_chance = ").append(bloomFilterFpChance)
.newLine()
.append("AND caching = ").append(caching.asMap())
@@ -301,6 +321,8 @@
.newLine()
.append("AND gc_grace_seconds = ").append(gcGraceSeconds)
.newLine()
+ .append("AND incremental_backups = ").append(incrementalBackups)
+ .newLine()
.append("AND max_index_interval = ").append(maxIndexInterval)
.newLine()
.append("AND memtable_flush_period_in_ms = ").append(memtableFlushPeriodInMs)
@@ -315,9 +337,11 @@
public static final class Builder
{
private String comment = "";
- private Double bloomFilterFpChance;
+ private boolean allowAutoSnapshot = true;
+ private double bloomFilterFpChance = -1;
private double crcCheckChance = 1.0;
private int gcGraceSeconds = 864000; // 10 days
+ private boolean incrementalBackups = true;
private int defaultTimeToLive = 0;
private int memtableFlushPeriodInMs = 0;
private int minIndexInterval = 128;
@@ -347,6 +371,12 @@
return this;
}
+ public Builder allowAutoSnapshot(boolean val)
+ {
+ allowAutoSnapshot = val;
+ return this;
+ }
+
public Builder bloomFilterFpChance(double val)
{
bloomFilterFpChance = val;
@@ -365,6 +395,12 @@
return this;
}
+ public Builder incrementalBackups(boolean val)
+ {
+ incrementalBackups = val;
+ return this;
+ }
+
public Builder defaultTimeToLive(int val)
{
defaultTimeToLive = val;
diff --git a/src/java/org/apache/cassandra/schema/UserFunctions.java b/src/java/org/apache/cassandra/schema/UserFunctions.java
new file mode 100644
index 0000000..b40c704
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/UserFunctions.java
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.*;
+
+import org.apache.cassandra.cql3.functions.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
+
+import static java.util.stream.Collectors.toList;
+
+import static com.google.common.collect.Iterables.any;
+
+/**
+ * An immutable container for a keyspace's UDAs and UDFs.
+ */
+public final class UserFunctions implements Iterable<UserFunction>
+{
+ public enum Filter implements Predicate<UserFunction>
+ {
+ ALL, UDF, UDA;
+
+ public boolean test(UserFunction function)
+ {
+ switch (this)
+ {
+ case UDF: return function instanceof UDFunction;
+ case UDA: return function instanceof UDAggregate;
+ default: return true;
+ }
+ }
+ }
+
+ private final ImmutableMultimap<FunctionName, UserFunction> functions;
+
+ private UserFunctions(Builder builder)
+ {
+ functions = builder.functions.build();
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ public static UserFunctions none()
+ {
+ return builder().build();
+ }
+
+ public Iterator<UserFunction> iterator()
+ {
+ return functions.values().iterator();
+ }
+
+ public Stream<UserFunction> stream()
+ {
+ return functions.values().stream();
+ }
+
+ public int size()
+ {
+ return functions.size();
+ }
+
+ /**
+ * @return a stream of keyspace's UDFs
+ */
+ public Stream<UDFunction> udfs()
+ {
+ return stream().filter(Filter.UDF).map(f -> (UDFunction) f);
+ }
+
+ /**
+ * @return a stream of keyspace's UDAs
+ */
+ public Stream<UDAggregate> udas()
+ {
+ return stream().filter(Filter.UDA).map(f -> (UDAggregate) f);
+ }
+
+ public Iterable<UserFunction> referencingUserType(ByteBuffer name)
+ {
+ return Iterables.filter(this, f -> f.referencesUserType(name));
+ }
+
+ public UserFunctions withUpdatedUserType(UserType udt)
+ {
+ if (!any(this, f -> f.referencesUserType(udt.name)))
+ return this;
+
+ Collection<UDFunction> udfs = udfs().map(f -> f.withUpdatedUserType(udt)).collect(toList());
+ Collection<UDAggregate> udas = udas().map(f -> f.withUpdatedUserType(udfs, udt)).collect(toList());
+
+ return builder().add(udfs).add(udas).build();
+ }
+
+ /**
+ * @return a stream of aggregates that use the provided function as either a state or a final function
+ * @param function the referree function
+ */
+ public Stream<UDAggregate> aggregatesUsingFunction(Function function)
+ {
+ return udas().filter(uda -> uda.hasReferenceTo(function));
+ }
+
+ /**
+ * Get all function overloads with the specified name
+ *
+ * @param name fully qualified function name
+ * @return an empty list if the function name is not found; a non-empty collection of {@link Function} otherwise
+ */
+ public Collection<UserFunction> get(FunctionName name)
+ {
+ return functions.get(name);
+ }
+
+ /**
+ * Get all UDFs overloads with the specified name
+ *
+ * @param name fully qualified function name
+ * @return an empty list if the function name is not found; a non-empty collection of {@link UDFunction} otherwise
+ */
+ public Collection<UDFunction> getUdfs(FunctionName name)
+ {
+ return functions.get(name)
+ .stream()
+ .filter(Filter.UDF)
+ .map(f -> (UDFunction) f)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Get all UDAs overloads with the specified name
+ *
+ * @param name fully qualified function name
+ * @return an empty list if the function name is not found; a non-empty collection of {@link UDAggregate} otherwise
+ */
+ public Collection<UDAggregate> getUdas(FunctionName name)
+ {
+ return functions.get(name)
+ .stream()
+ .filter(Filter.UDA)
+ .map(f -> (UDAggregate) f)
+ .collect(Collectors.toList());
+ }
+
+ public Optional<UserFunction> find(FunctionName name, List<AbstractType<?>> argTypes)
+ {
+ return find(name, argTypes, Filter.ALL);
+ }
+
+ /**
+ * Find the function with the specified name
+ *
+ * @param name fully qualified function name
+ * @param argTypes function argument types
+ * @return an empty {@link Optional} if the function name is not found; a non-empty optional of {@link Function} otherwise
+ */
+ public Optional<UserFunction> find(FunctionName name, List<AbstractType<?>> argTypes, Filter filter)
+ {
+ return get(name).stream()
+ .filter(filter.and(fun -> fun.typesMatch(argTypes)))
+ .findAny();
+ }
+
+ public boolean isEmpty()
+ {
+ return functions.isEmpty();
+ }
+
+ public static int typeHashCode(AbstractType<?> t)
+ {
+ return t.asCQL3Type().toString().hashCode();
+ }
+
+ public static int typeHashCode(List<AbstractType<?>> types)
+ {
+ int h = 0;
+ for (AbstractType<?> type : types)
+ h = h * 31 + typeHashCode(type);
+ return h;
+ }
+
+ public UserFunctions filter(Predicate<UserFunction> predicate)
+ {
+ Builder builder = builder();
+ stream().filter(predicate).forEach(builder::add);
+ return builder.build();
+ }
+
+ /**
+ * Create a Functions instance with the provided function added
+ */
+ public UserFunctions with(UserFunction fun)
+ {
+ if (find(fun.name(), fun.argTypes()).isPresent())
+ throw new IllegalStateException(String.format("Function %s already exists", fun.name()));
+
+ return builder().add(this).add(fun).build();
+ }
+
+ /**
+ * Creates a Functions instance with the function with the provided name and argument types removed
+ */
+ public UserFunctions without(FunctionName name, List<AbstractType<?>> argTypes)
+ {
+ Function fun =
+ find(name, argTypes).orElseThrow(() -> new IllegalStateException(String.format("Function %s doesn't exists", name)));
+
+ return without(fun);
+ }
+
+ public UserFunctions without(Function function)
+ {
+ return builder().add(Iterables.filter(this, f -> f != function)).build();
+ }
+
+ public UserFunctions withAddedOrUpdated(UserFunction function)
+ {
+ return builder().add(Iterables.filter(this, f -> !(f.name().equals(function.name()) && f.typesMatch(function.argTypes()))))
+ .add(function)
+ .build();
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ return this == o || (o instanceof UserFunctions && functions.equals(((UserFunctions) o).functions));
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return functions.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return functions.values().toString();
+ }
+
+ public static final class Builder
+ {
+ final ImmutableMultimap.Builder<FunctionName, UserFunction> functions = new ImmutableMultimap.Builder<>();
+
+ private Builder()
+ {
+ // we need deterministic iteration order; otherwise Functions.equals() breaks down
+ functions.orderValuesBy(Comparator.comparingInt(Object::hashCode));
+ }
+
+ public UserFunctions build()
+ {
+ return new UserFunctions(this);
+ }
+
+ public Builder add(UserFunction fun)
+ {
+ functions.put(fun.name(), fun);
+ return this;
+ }
+
+ public Builder add(UserFunction... funs)
+ {
+ for (UserFunction fun : funs)
+ add(fun);
+ return this;
+ }
+
+ public Builder add(Iterable<? extends UserFunction> funs)
+ {
+ funs.forEach(this::add);
+ return this;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ static FunctionsDiff<UDFunction> udfsDiff(UserFunctions before, UserFunctions after)
+ {
+ return (FunctionsDiff<UDFunction>) FunctionsDiff.diff(before, after, Filter.UDF);
+ }
+
+ @SuppressWarnings("unchecked")
+ static FunctionsDiff<UDAggregate> udasDiff(UserFunctions before, UserFunctions after)
+ {
+ return (FunctionsDiff<UDAggregate>) FunctionsDiff.diff(before, after, Filter.UDA);
+ }
+
+ public static final class FunctionsDiff<T extends Function> extends Diff<UserFunctions, T>
+ {
+ static final FunctionsDiff NONE = new FunctionsDiff<>(UserFunctions.none(), UserFunctions.none(), ImmutableList.of());
+
+ private FunctionsDiff(UserFunctions created, UserFunctions dropped, ImmutableCollection<Altered<T>> altered)
+ {
+ super(created, dropped, altered);
+ }
+
+ private static FunctionsDiff diff(UserFunctions before, UserFunctions after, Filter filter)
+ {
+ if (before == after)
+ return NONE;
+
+ UserFunctions created = after.filter(filter.and(k -> !before.find(k.name(), k.argTypes(), filter).isPresent()));
+ UserFunctions dropped = before.filter(filter.and(k -> !after.find(k.name(), k.argTypes(), filter).isPresent()));
+
+ ImmutableList.Builder<Altered<UserFunction>> altered = ImmutableList.builder();
+ before.stream().filter(filter).forEach(functionBefore ->
+ {
+ after.find(functionBefore.name(), functionBefore.argTypes(), filter).ifPresent(functionAfter ->
+ {
+ functionBefore.compare(functionAfter).ifPresent(kind -> altered.add(new Altered<>(functionBefore, functionAfter, kind)));
+ });
+ });
+
+ return new FunctionsDiff<>(created, dropped, altered.build());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/ViewMetadata.java b/src/java/org/apache/cassandra/schema/ViewMetadata.java
index 0053249..df26a13 100644
--- a/src/java/org/apache/cassandra/schema/ViewMetadata.java
+++ b/src/java/org/apache/cassandra/schema/ViewMetadata.java
@@ -20,10 +20,13 @@
import java.nio.ByteBuffer;
import java.util.Optional;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.functions.masking.ColumnMask;
import org.apache.cassandra.db.marshal.UserType;
public final class ViewMetadata implements SchemaElement
@@ -158,6 +161,15 @@
metadata.unbuild().addColumn(column).build());
}
+ public ViewMetadata withNewColumnMask(ColumnIdentifier name, @Nullable ColumnMask mask)
+ {
+ return new ViewMetadata(baseTableId,
+ baseTableName,
+ includeAllColumns,
+ whereClause,
+ metadata.unbuild().alterColumnMask(name, mask).build());
+ }
+
public void appendCqlTo(CqlBuilder builder,
boolean internals,
boolean ifNotExists)
diff --git a/src/java/org/apache/cassandra/security/AbstractSslContextFactory.java b/src/java/org/apache/cassandra/security/AbstractSslContextFactory.java
index c2ef851..406c472 100644
--- a/src/java/org/apache/cassandra/security/AbstractSslContextFactory.java
+++ b/src/java/org/apache/cassandra/security/AbstractSslContextFactory.java
@@ -35,7 +35,8 @@
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
-import org.apache.cassandra.config.Config;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
/**
* Abstract class implementing {@code ISslContextFacotry} to provide most of the functionality that any
@@ -109,12 +110,13 @@
/**
* Dervies if {@code OpenSSL} is available. It allows in-jvm dtests to disable tcnative openssl support by
- * setting {@code cassandra.disable_tcactive_openssl} system property as {@code true}. Otherwise, it creates a
- * circular reference that prevents the instance class loader from being garbage collected.
+ * setting {@link org.apache.cassandra.config.CassandraRelevantProperties#DISABLE_TCACTIVE_OPENSSL}
+ * system property as {@code true}. Otherwise, it creates a circular reference that prevents the instance
+ * class loader from being garbage collected.
*/
protected void deriveIfOpenSslAvailable()
{
- if (Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_tcactive_openssl"))
+ if (DISABLE_TCACTIVE_OPENSSL.getBoolean())
openSslIsAvailable = false;
else
openSslIsAvailable = OpenSsl.isAvailable();
@@ -178,15 +180,16 @@
key file in PEM format (see {@link SslContextBuilder#forServer(File, File, String)}). However, we are
not supporting that now to keep the config/yaml API simple.
*/
- KeyManagerFactory kmf = buildKeyManagerFactory();
SslContextBuilder builder;
if (socketType == SocketType.SERVER)
{
+ KeyManagerFactory kmf = buildKeyManagerFactory();
builder = SslContextBuilder.forServer(kmf).clientAuth(this.require_client_auth ? ClientAuth.REQUIRE :
ClientAuth.NONE);
}
else
{
+ KeyManagerFactory kmf = buildOutboundKeyManagerFactory();
builder = SslContextBuilder.forClient().keyManager(kmf);
}
@@ -263,4 +266,12 @@
abstract protected KeyManagerFactory buildKeyManagerFactory() throws SSLException;
abstract protected TrustManagerFactory buildTrustManagerFactory() throws SSLException;
+
+ /**
+ * Create a {@code KeyManagerFactory} for outbound connections.
+ * It provides a seperate keystore for internode mTLS outbound connections.
+ * @return {@code KeyManagerFactory}
+ * @throws SSLException
+ */
+ abstract protected KeyManagerFactory buildOutboundKeyManagerFactory() throws SSLException;
}
diff --git a/src/java/org/apache/cassandra/security/DisableSslContextFactory.java b/src/java/org/apache/cassandra/security/DisableSslContextFactory.java
index 9dab062..8058d0a 100644
--- a/src/java/org/apache/cassandra/security/DisableSslContextFactory.java
+++ b/src/java/org/apache/cassandra/security/DisableSslContextFactory.java
@@ -37,12 +37,24 @@
}
@Override
+ protected KeyManagerFactory buildOutboundKeyManagerFactory() throws SSLException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public boolean hasKeystore()
{
return false;
}
@Override
+ public boolean hasOutboundKeystore()
+ {
+ return false;
+ }
+
+ @Override
public void initHotReloading() throws SSLException
{
}
diff --git a/src/java/org/apache/cassandra/security/FileBasedSslContextFactory.java b/src/java/org/apache/cassandra/security/FileBasedSslContextFactory.java
index 9876eb4..fdf6696 100644
--- a/src/java/org/apache/cassandra/security/FileBasedSslContextFactory.java
+++ b/src/java/org/apache/cassandra/security/FileBasedSslContextFactory.java
@@ -32,7 +32,6 @@
import javax.net.ssl.SSLException;
import javax.net.ssl.TrustManagerFactory;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,38 +46,32 @@
* {@code CAUTION:} While this is a useful abstraction, please be careful if you need to modify this class
* given possible custom implementations out there!
*/
-abstract public class FileBasedSslContextFactory extends AbstractSslContextFactory
+public abstract class FileBasedSslContextFactory extends AbstractSslContextFactory
{
private static final Logger logger = LoggerFactory.getLogger(FileBasedSslContextFactory.class);
-
- @VisibleForTesting
- protected volatile boolean checkedExpiry = false;
+ protected FileBasedStoreContext keystoreContext;
+ protected FileBasedStoreContext outboundKeystoreContext;
+ protected FileBasedStoreContext trustStoreContext;
/**
* List of files that trigger hot reloading of SSL certificates
*/
protected volatile List<HotReloadableFile> hotReloadableFiles = new ArrayList<>();
- protected String keystore;
- protected String keystore_password;
- protected String truststore;
- protected String truststore_password;
-
public FileBasedSslContextFactory()
{
- keystore = "conf/.keystore";
- keystore_password = "cassandra";
- truststore = "conf/.truststore";
- truststore_password = "cassandra";
+ keystoreContext = new FileBasedStoreContext("conf/.keystore", "cassandra");
+ outboundKeystoreContext = new FileBasedStoreContext("conf/.keystore", "cassandra");
+ trustStoreContext = new FileBasedStoreContext("conf/.truststore", "cassandra");
}
public FileBasedSslContextFactory(Map<String, Object> parameters)
{
super(parameters);
- keystore = getString("keystore");
- keystore_password = getString("keystore_password");
- truststore = getString("truststore");
- truststore_password = getString("truststore_password");
+ keystoreContext = new FileBasedStoreContext(getString("keystore"), getString("keystore_password"));
+ outboundKeystoreContext = new FileBasedStoreContext(StringUtils.defaultString(getString("outbound_keystore"), keystoreContext.filePath),
+ StringUtils.defaultString(getString("outbound_keystore_password"), keystoreContext.password));
+ trustStoreContext = new FileBasedStoreContext(getString("truststore"), getString("truststore_password"));
}
@Override
@@ -90,30 +83,41 @@
@Override
public boolean hasKeystore()
{
- return keystore != null && new File(keystore).exists();
+ return keystoreContext.hasKeystore();
+ }
+
+ @Override
+ public boolean hasOutboundKeystore()
+ {
+ return outboundKeystoreContext.hasKeystore();
}
private boolean hasTruststore()
{
- return truststore != null && new File(truststore).exists();
+ return trustStoreContext.filePath != null && new File(trustStoreContext.filePath).exists();
}
@Override
public synchronized void initHotReloading()
{
boolean hasKeystore = hasKeystore();
+ boolean hasOutboundKeystore = hasOutboundKeystore();
boolean hasTruststore = hasTruststore();
- if (hasKeystore || hasTruststore)
+ if (hasKeystore || hasOutboundKeystore || hasTruststore)
{
List<HotReloadableFile> fileList = new ArrayList<>();
if (hasKeystore)
{
- fileList.add(new HotReloadableFile(keystore));
+ fileList.add(new HotReloadableFile(keystoreContext.filePath));
+ }
+ if (hasOutboundKeystore)
+ {
+ fileList.add(new HotReloadableFile(outboundKeystoreContext.filePath));
}
if (hasTruststore)
{
- fileList.add(new HotReloadableFile(truststore));
+ fileList.add(new HotReloadableFile(trustStoreContext.filePath));
}
hotReloadableFiles = fileList;
}
@@ -122,15 +126,18 @@
/**
* Validates the given keystore password.
*
+ * @param isOutboundKeystore {@code true} for the {@code outbound_keystore_password};{@code false} otherwise
* @param password value
* @throws IllegalArgumentException if the {@code password} is empty as per the definition of {@link StringUtils#isEmpty(CharSequence)}
*/
- protected void validatePassword(String password)
+ protected void validatePassword(boolean isOutboundKeystore, String password)
{
boolean keystorePasswordEmpty = StringUtils.isEmpty(password);
if (keystorePasswordEmpty)
{
- throw new IllegalArgumentException("'keystore_password' must be specified");
+ String keyName = isOutboundKeystore ? "outbound_" : "";
+ final String msg = String.format("'%skeystore_password' must be specified", keyName);
+ throw new IllegalArgumentException(msg);
}
}
@@ -140,6 +147,8 @@
*
* @return KeyManagerFactory built from the file based keystore.
* @throws SSLException if any issues encountered during the build process
+ * @throws IllegalArgumentException if the validation for the {@code keystore_password} fails
+ * @see #validatePassword(boolean, String)
*/
@Override
protected KeyManagerFactory buildKeyManagerFactory() throws SSLException
@@ -148,26 +157,19 @@
* Validation of the password is delayed until this point to allow nullable keystore passwords
* for other use-cases (CASSANDRA-18124).
*/
- validatePassword(keystore_password);
+ validatePassword(false, keystoreContext.password);
+ return getKeyManagerFactory(keystoreContext);
+ }
- try (InputStream ksf = Files.newInputStream(File.getPath(keystore)))
- {
- final String algorithm = this.algorithm == null ? KeyManagerFactory.getDefaultAlgorithm() : this.algorithm;
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm);
- KeyStore ks = KeyStore.getInstance(store_type);
- ks.load(ksf, keystore_password.toCharArray());
- if (!checkedExpiry)
- {
- checkExpiredCerts(ks);
- checkedExpiry = true;
- }
- kmf.init(ks, keystore_password.toCharArray());
- return kmf;
- }
- catch (Exception e)
- {
- throw new SSLException("failed to build key manager store for secure connections", e);
- }
+ @Override
+ protected KeyManagerFactory buildOutboundKeyManagerFactory() throws SSLException
+ {
+ /*
+ * Validation of the password is delayed until this point to allow nullable keystore passwords
+ * for other use-cases (CASSANDRA-18124).
+ */
+ validatePassword(true, outboundKeystoreContext.password);
+ return getKeyManagerFactory(outboundKeystoreContext);
}
/**
@@ -179,13 +181,13 @@
@Override
protected TrustManagerFactory buildTrustManagerFactory() throws SSLException
{
- try (InputStream tsf = Files.newInputStream(File.getPath(truststore)))
+ try (InputStream tsf = Files.newInputStream(File.getPath(trustStoreContext.filePath)))
{
final String algorithm = this.algorithm == null ? TrustManagerFactory.getDefaultAlgorithm() : this.algorithm;
TrustManagerFactory tmf = TrustManagerFactory.getInstance(algorithm);
KeyStore ts = KeyStore.getInstance(store_type);
- final char[] truststorePassword = StringUtils.isEmpty(truststore_password) ? null : truststore_password.toCharArray();
+ final char[] truststorePassword = StringUtils.isEmpty(trustStoreContext.password) ? null : trustStoreContext.password.toCharArray();
ts.load(tsf, truststorePassword);
tmf.init(ts);
return tmf;
@@ -196,6 +198,29 @@
}
}
+ private KeyManagerFactory getKeyManagerFactory(final FileBasedStoreContext context) throws SSLException
+ {
+ try (InputStream ksf = Files.newInputStream(File.getPath(context.filePath)))
+ {
+ final String algorithm = this.algorithm == null ? KeyManagerFactory.getDefaultAlgorithm() : this.algorithm;
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm);
+ KeyStore ks = KeyStore.getInstance(store_type);
+ ks.load(ksf, context.password.toCharArray());
+
+ if (!context.checkedExpiry)
+ {
+ checkExpiredCerts(ks);
+ context.checkedExpiry = true;
+ }
+ kmf.init(ks, context.password.toCharArray());
+ return kmf;
+ }
+ catch (Exception e)
+ {
+ throw new SSLException("failed to build key manager store for secure connections", e);
+ }
+ }
+
protected boolean checkExpiredCerts(KeyStore ks) throws KeyStoreException
{
boolean hasExpiredCerts = false;
@@ -247,4 +272,27 @@
'}';
}
}
+
+ protected static class FileBasedStoreContext
+ {
+ public volatile boolean checkedExpiry = false;
+ public String filePath;
+ public String password;
+
+ public FileBasedStoreContext(String keystore, String keystorePassword)
+ {
+ this.filePath = keystore;
+ this.password = keystorePassword;
+ }
+
+ protected boolean hasKeystore()
+ {
+ return filePath != null && new File(filePath).exists();
+ }
+
+ protected boolean passwordMatchesIfPresent(String keyPassword)
+ {
+ return StringUtils.isEmpty(password) || keyPassword.equals(password);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/security/ISslContextFactory.java b/src/java/org/apache/cassandra/security/ISslContextFactory.java
index 579c95e..11c4717 100644
--- a/src/java/org/apache/cassandra/security/ISslContextFactory.java
+++ b/src/java/org/apache/cassandra/security/ISslContextFactory.java
@@ -100,6 +100,16 @@
}
/**
+ * Returns if this factory uses outbound keystore.
+ *
+ * @return {@code true} by default unless the implementation overrides this
+ */
+ default boolean hasOutboundKeystore()
+ {
+ return false;
+ }
+
+ /**
* Returns the prepared list of accepted protocols.
*
* @return array of protocol names suitable for passing to Netty's SslContextBuilder.protocols, or null if the
diff --git a/src/java/org/apache/cassandra/security/PEMBasedSslContextFactory.java b/src/java/org/apache/cassandra/security/PEMBasedSslContextFactory.java
index d62aef5..62e2d4c 100644
--- a/src/java/org/apache/cassandra/security/PEMBasedSslContextFactory.java
+++ b/src/java/org/apache/cassandra/security/PEMBasedSslContextFactory.java
@@ -89,42 +89,50 @@
{
public static final String DEFAULT_TARGET_STORETYPE = "PKCS12";
private static final Logger logger = LoggerFactory.getLogger(PEMBasedSslContextFactory.class);
- private String pemEncodedKey;
- private String keyPassword;
- private String pemEncodedCertificates;
- private boolean maybeFileBasedPrivateKey;
- private boolean maybeFileBasedTrustedCertificates;
+ private PEMBasedKeyStoreContext pemEncodedTrustCertificates;
+ private PEMBasedKeyStoreContext pemEncodedKeyContext;
+ private PEMBasedKeyStoreContext pemEncodedOutboundKeyContext;
public PEMBasedSslContextFactory()
{
}
+ private void validatePasswords()
+ {
+ boolean shouldThrow = !keystoreContext.passwordMatchesIfPresent(pemEncodedKeyContext.password)
+ || !outboundKeystoreContext.passwordMatchesIfPresent(pemEncodedOutboundKeyContext.password);
+ boolean outboundPasswordMismatch = !outboundKeystoreContext.passwordMatchesIfPresent(pemEncodedOutboundKeyContext.password);
+ String keyName = outboundPasswordMismatch ? "outbound_" : "";
+
+ if (shouldThrow)
+ {
+ final String msg = String.format("'%skeystore_password' and '%skey_password' both configurations are given and the values do not match", keyName, keyName);
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
public PEMBasedSslContextFactory(Map<String, Object> parameters)
{
super(parameters);
- pemEncodedKey = getString(ConfigKey.ENCODED_KEY.getKeyName());
- keyPassword = getString(ConfigKey.KEY_PASSWORD.getKeyName());
- if (StringUtils.isEmpty(keyPassword))
- {
- keyPassword = keystore_password;
- }
- else if (!StringUtils.isEmpty(keystore_password) && !keyPassword.equals(keystore_password))
- {
- throw new IllegalArgumentException("'keystore_password' and 'key_password' both configurations are given and the " +
- "values do not match");
- }
+ final String pemEncodedKey = getString(ConfigKey.ENCODED_KEY.getKeyName());
+ final String pemEncodedKeyPassword = StringUtils.defaultString(getString(ConfigKey.KEY_PASSWORD.getKeyName()), keystoreContext.password);
+ pemEncodedKeyContext = new PEMBasedKeyStoreContext(pemEncodedKey, pemEncodedKeyPassword, StringUtils.isEmpty(pemEncodedKey), keystoreContext);
- if (!StringUtils.isEmpty(truststore_password))
+ final String pemEncodedOutboundKey = StringUtils.defaultString(getString(ConfigKey.OUTBOUND_ENCODED_KEY.getKeyName()), pemEncodedKey);
+ final String outboundKeyPassword = StringUtils.defaultString(StringUtils.defaultString(getString(ConfigKey.OUTBOUND_ENCODED_KEY_PASSWORD.getKeyName()),
+ outboundKeystoreContext.password), pemEncodedKeyPassword);
+ pemEncodedOutboundKeyContext = new PEMBasedKeyStoreContext(pemEncodedKey, outboundKeyPassword, StringUtils.isEmpty(pemEncodedOutboundKey), outboundKeystoreContext);
+
+ validatePasswords();
+
+ if (!StringUtils.isEmpty(trustStoreContext.password))
{
logger.warn("PEM based truststore should not be using password. Ignoring the given value in " +
"'truststore_password' configuration.");
}
- pemEncodedCertificates = getString(ConfigKey.ENCODED_CERTIFICATES.getKeyName());
-
- maybeFileBasedPrivateKey = StringUtils.isEmpty(pemEncodedKey);
- maybeFileBasedTrustedCertificates = StringUtils.isEmpty(pemEncodedCertificates);
-
+ final String pemEncodedCerts = getString(ConfigKey.ENCODED_CERTIFICATES.getKeyName());
+ pemEncodedTrustCertificates = new PEMBasedKeyStoreContext(pemEncodedCerts, null, StringUtils.isEmpty(pemEncodedCerts), trustStoreContext);
enforceSinglePrivateKeySource();
enforceSingleTurstedCertificatesSource();
}
@@ -137,18 +145,22 @@
@Override
public boolean hasKeystore()
{
- return maybeFileBasedPrivateKey ? keystoreFileExists() :
- !StringUtils.isEmpty(pemEncodedKey);
+ return pemEncodedKeyContext.maybeFilebasedKey
+ ? keystoreContext.hasKeystore()
+ : !StringUtils.isEmpty(pemEncodedKeyContext.key);
}
/**
- * Checks if the keystore file exists.
+ * Decides if this factory has an outbound keystore defined - key material specified in files or inline to the configuration.
*
- * @return {@code true} if keystore file exists; {@code false} otherwise
+ * @return {@code true} if there is an outbound keystore defined; {@code false} otherwise
*/
- private boolean keystoreFileExists()
+ @Override
+ public boolean hasOutboundKeystore()
{
- return keystore != null && new File(keystore).exists();
+ return pemEncodedOutboundKeyContext.maybeFilebasedKey
+ ? outboundKeystoreContext.hasKeystore()
+ : !StringUtils.isEmpty(pemEncodedOutboundKeyContext.key);
}
/**
@@ -159,8 +171,8 @@
*/
private boolean hasTruststore()
{
- return maybeFileBasedTrustedCertificates ? truststoreFileExists() :
- !StringUtils.isEmpty(pemEncodedCertificates);
+ return pemEncodedTrustCertificates.maybeFilebasedKey ? truststoreFileExists() :
+ !StringUtils.isEmpty(pemEncodedTrustCertificates.key);
}
/**
@@ -170,7 +182,7 @@
*/
private boolean truststoreFileExists()
{
- return truststore != null && new File(truststore).exists();
+ return trustStoreContext.filePath != null && new File(trustStoreContext.filePath).exists();
}
/**
@@ -180,13 +192,17 @@
public synchronized void initHotReloading()
{
List<HotReloadableFile> fileList = new ArrayList<>();
- if (maybeFileBasedPrivateKey && hasKeystore())
+ if (pemEncodedKeyContext.maybeFilebasedKey && hasKeystore())
{
- fileList.add(new HotReloadableFile(keystore));
+ fileList.add(new HotReloadableFile(keystoreContext.filePath));
}
- if (maybeFileBasedTrustedCertificates && hasTruststore())
+ if (pemEncodedOutboundKeyContext.maybeFilebasedKey && hasOutboundKeystore())
{
- fileList.add(new HotReloadableFile(truststore));
+ fileList.add(new HotReloadableFile(outboundKeystoreContext.filePath));
+ }
+ if (pemEncodedTrustCertificates.maybeFilebasedKey && hasTruststore())
+ {
+ fileList.add(new HotReloadableFile(trustStoreContext.filePath));
}
if (!fileList.isEmpty())
{
@@ -204,29 +220,40 @@
@Override
protected KeyManagerFactory buildKeyManagerFactory() throws SSLException
{
+ return buildKeyManagerFactory(pemEncodedKeyContext, keystoreContext);
+ }
+
+ @Override
+ protected KeyManagerFactory buildOutboundKeyManagerFactory() throws SSLException
+ {
+ return buildKeyManagerFactory(pemEncodedOutboundKeyContext, outboundKeystoreContext);
+ }
+
+ private KeyManagerFactory buildKeyManagerFactory(PEMBasedKeyStoreContext pemBasedKeyStoreContext, FileBasedStoreContext keyStoreContext) throws SSLException
+ {
try
{
- if (hasKeystore())
+ if (pemBasedKeyStoreContext.hasKey())
{
- if (maybeFileBasedPrivateKey)
+ if (pemBasedKeyStoreContext.maybeFilebasedKey)
{
- pemEncodedKey = readPEMFile(keystore); // read PEM from the file
+ pemBasedKeyStoreContext.key = readPEMFile(keyStoreContext.filePath); // read PEM from the file
}
KeyManagerFactory kmf = KeyManagerFactory.getInstance(
algorithm == null ? KeyManagerFactory.getDefaultAlgorithm() : algorithm);
- KeyStore ks = buildKeyStore();
- if (!checkedExpiry)
+ KeyStore ks = buildKeyStore(pemBasedKeyStoreContext.key, pemBasedKeyStoreContext.password);
+ if (!keyStoreContext.checkedExpiry)
{
checkExpiredCerts(ks);
- checkedExpiry = true;
+ keyStoreContext.checkedExpiry = true;
}
- kmf.init(ks, keyPassword != null ? keyPassword.toCharArray() : null);
+ kmf.init(ks, pemBasedKeyStoreContext.password != null ? pemBasedKeyStoreContext.password.toCharArray() : null);
return kmf;
}
else
{
- throw new SSLException("Must provide keystore or private_key in configuration for PEMBasedSSlContextFactory");
+ throw new SSLException("Must provide outbound_keystore or outbound_private_key in configuration for PEMBasedSSlContextFactory");
}
}
catch (Exception e)
@@ -248,9 +275,9 @@
{
if (hasTruststore())
{
- if (maybeFileBasedTrustedCertificates)
+ if (pemEncodedTrustCertificates.maybeFilebasedKey)
{
- pemEncodedCertificates = readPEMFile(truststore); // read PEM from the file
+ pemEncodedTrustCertificates.key = readPEMFile(trustStoreContext.filePath); // read PEM from the file
}
TrustManagerFactory tmf = TrustManagerFactory.getInstance(
@@ -280,7 +307,7 @@
* Builds KeyStore object given the {@link #DEFAULT_TARGET_STORETYPE} out of the PEM formatted private key material.
* It uses {@code cassandra-ssl-keystore} as the alias for the created key-entry.
*/
- private KeyStore buildKeyStore() throws GeneralSecurityException, IOException
+ private static KeyStore buildKeyStore(final String pemEncodedKey, final String keyPassword) throws GeneralSecurityException, IOException
{
char[] keyPasswordArray = keyPassword != null ? keyPassword.toCharArray() : null;
PrivateKey privateKey = PEMReader.extractPrivateKey(pemEncodedKey, keyPassword);
@@ -304,7 +331,7 @@
*/
private KeyStore buildTrustStore() throws GeneralSecurityException, IOException
{
- Certificate[] certChainArray = PEMReader.extractCertificates(pemEncodedCertificates);
+ Certificate[] certChainArray = PEMReader.extractCertificates(pemEncodedTrustCertificates.key);
if (certChainArray == null || certChainArray.length == 0)
{
throw new SSLException("Could not read any certificates from the given PEM");
@@ -325,11 +352,16 @@
*/
private void enforceSinglePrivateKeySource()
{
- if (keystoreFileExists() && !StringUtils.isEmpty(pemEncodedKey))
+ if (keystoreContext.hasKeystore() && !StringUtils.isEmpty(pemEncodedKeyContext.key))
{
throw new IllegalArgumentException("Configuration must specify value for either keystore or private_key, " +
"not both for PEMBasedSSlContextFactory");
}
+ if (outboundKeystoreContext.hasKeystore() && !StringUtils.isEmpty(pemEncodedOutboundKeyContext.key))
+ {
+ throw new IllegalArgumentException("Configuration must specify value for either outbound_keystore or outbound_private_key, " +
+ "not both for PEMBasedSSlContextFactory");
+ }
}
/**
@@ -338,17 +370,43 @@
*/
private void enforceSingleTurstedCertificatesSource()
{
- if (truststoreFileExists() && !StringUtils.isEmpty(pemEncodedCertificates))
+ if (truststoreFileExists() && !StringUtils.isEmpty(pemEncodedTrustCertificates.key))
{
throw new IllegalArgumentException("Configuration must specify value for either truststore or " +
"trusted_certificates, not both for PEMBasedSSlContextFactory");
}
}
+ public static class PEMBasedKeyStoreContext
+ {
+ public String key;
+ public final String password;
+ public final boolean maybeFilebasedKey;
+ public final FileBasedStoreContext filebasedKeystoreContext;
+
+ public PEMBasedKeyStoreContext(final String encodedKey, final String getEncodedKeyPassword,
+ final boolean maybeFilebasedKey, final FileBasedStoreContext filebasedKeystoreContext)
+ {
+ this.key = encodedKey;
+ this.password = getEncodedKeyPassword;
+ this.maybeFilebasedKey = maybeFilebasedKey;
+ this.filebasedKeystoreContext = filebasedKeystoreContext;
+ }
+
+ public boolean hasKey()
+ {
+ return maybeFilebasedKey
+ ? filebasedKeystoreContext.hasKeystore()
+ : !StringUtils.isEmpty(key);
+ }
+ }
+
public enum ConfigKey
{
ENCODED_KEY("private_key"),
KEY_PASSWORD("private_key_password"),
+ OUTBOUND_ENCODED_KEY("outbound_private_key"),
+ OUTBOUND_ENCODED_KEY_PASSWORD("outbound_private_key_password"),
ENCODED_CERTIFICATES("trusted_certificates");
final String keyName;
diff --git a/src/java/org/apache/cassandra/security/SSLFactory.java b/src/java/org/apache/cassandra/security/SSLFactory.java
index e06da1f..e6da577 100644
--- a/src/java/org/apache/cassandra/security/SSLFactory.java
+++ b/src/java/org/apache/cassandra/security/SSLFactory.java
@@ -41,11 +41,12 @@
import io.netty.handler.ssl.SslContext;
import io.netty.util.ReferenceCountUtil;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.security.ISslContextFactory.SocketType;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
+
/**
* A Factory for providing and setting up client {@link SSLSocket}s. Also provides
* methods for creating both JSSE {@link SSLContext} instances as well as netty {@link SslContext} instances.
@@ -64,7 +65,7 @@
static private final boolean openSslIsAvailable;
static
{
- if (Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_tcactive_openssl"))
+ if (DISABLE_TCACTIVE_OPENSSL.getBoolean())
{
openSslIsAvailable = false;
}
diff --git a/src/java/org/apache/cassandra/serializers/AbstractMapSerializer.java b/src/java/org/apache/cassandra/serializers/AbstractMapSerializer.java
new file mode 100644
index 0000000..27bae6e
--- /dev/null
+++ b/src/java/org/apache/cassandra/serializers/AbstractMapSerializer.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+
+import com.google.common.collect.Range;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Common superclass for {@link SetSerializer} and {@link MapSerializer}, considering a set as a map without values.
+ */
+abstract class AbstractMapSerializer<T> extends CollectionSerializer<T>
+{
+ private final boolean hasValues;
+ private final String name;
+
+ protected AbstractMapSerializer(boolean hasValues)
+ {
+ this.hasValues = hasValues;
+ name = hasValues ? "map" : "set";
+ }
+
+ @Override
+ public ByteBuffer getSliceFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator,
+ boolean frozen)
+ {
+ if (from == ByteBufferUtil.UNSET_BYTE_BUFFER && to == ByteBufferUtil.UNSET_BYTE_BUFFER)
+ return collection;
+
+ try
+ {
+ ByteBuffer input = collection.duplicate();
+ int n = readCollectionSize(input, ByteBufferAccessor.instance);
+ input.position(input.position() + sizeOfCollectionSize());
+ int startPos = input.position();
+ int count = 0;
+ boolean inSlice = from == ByteBufferUtil.UNSET_BYTE_BUFFER;
+
+ for (int i = 0; i < n; i++)
+ {
+ int pos = input.position();
+ ByteBuffer key = readValue(input, ByteBufferAccessor.instance, 0);
+ input.position(input.position() + sizeOfValue(key, ByteBufferAccessor.instance));
+
+ // If we haven't passed the start already, check if we have now
+ if (!inSlice)
+ {
+ int comparison = comparator.compareForCQL(from, key);
+ if (comparison <= 0)
+ {
+ // We're now within the slice
+ inSlice = true;
+ startPos = pos;
+ }
+ else
+ {
+ // We're before the slice, so we know we don't care about this element
+ skipMapValue(input);
+ continue;
+ }
+ }
+
+ // Now check if we're done
+ int comparison = to == ByteBufferUtil.UNSET_BYTE_BUFFER ? -1 : comparator.compareForCQL(key, to);
+ if (comparison > 0)
+ {
+ // We're done and shouldn't include the key we just read
+ input.position(pos);
+ break;
+ }
+
+ // Otherwise, we'll include that element
+ skipMapValue(input); // value
+ ++count;
+
+ // But if we know it was the last of the slice, we break early
+ if (comparison == 0)
+ break;
+ }
+
+ if (count == 0 && !frozen)
+ return null;
+
+ return copyAsNewCollection(collection, count, startPos, input.position());
+ }
+ catch (BufferUnderflowException | IndexOutOfBoundsException e)
+ {
+ throw new MarshalException("Not enough bytes to read a " + name);
+ }
+ }
+
+ @Override
+ public int getIndexFromSerialized(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator)
+ {
+ try
+ {
+ ByteBuffer input = collection.duplicate();
+ int n = readCollectionSize(input, ByteBufferAccessor.instance);
+ int offset = sizeOfCollectionSize();
+ for (int i = 0; i < n; i++)
+ {
+ ByteBuffer kbb = readValue(input, ByteBufferAccessor.instance, offset);
+ offset += sizeOfValue(kbb, ByteBufferAccessor.instance);
+ int comparison = comparator.compareForCQL(kbb, key);
+
+ if (comparison == 0)
+ return i;
+
+ if (comparison > 0)
+ // since the set is in sorted order, we know we've gone too far and the element doesn't exist
+ return -1;
+
+ // comparison < 0
+ if (hasValues)
+ offset += skipValue(input, ByteBufferAccessor.instance, offset);
+ }
+ return -1;
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new MarshalException("Not enough bytes to read a " + name);
+ }
+ }
+
+ @Override
+ public Range<Integer> getIndexesRangeFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator)
+ {
+ if (from == ByteBufferUtil.UNSET_BYTE_BUFFER && to == ByteBufferUtil.UNSET_BYTE_BUFFER)
+ return Range.closed(0, Integer.MAX_VALUE);
+
+ try
+ {
+ ByteBuffer input = collection.duplicate();
+ int n = readCollectionSize(input, ByteBufferAccessor.instance);
+ input.position(input.position() + sizeOfCollectionSize());
+ int start = from == ByteBufferUtil.UNSET_BYTE_BUFFER ? 0 : -1;
+ int end = to == ByteBufferUtil.UNSET_BYTE_BUFFER ? n : -1;
+
+ for (int i = 0; i < n; i++)
+ {
+ if (start >= 0 && end >= 0)
+ break;
+ else if (i > 0)
+ skipMapValue(input);
+
+ ByteBuffer key = readValue(input, ByteBufferAccessor.instance, 0);
+ input.position(input.position() + sizeOfValue(key, ByteBufferAccessor.instance));
+
+ if (start < 0)
+ {
+ int comparison = comparator.compareForCQL(from, key);
+ if (comparison <= 0)
+ start = i;
+ else
+ continue;
+ }
+
+ if (end < 0)
+ {
+ int comparison = comparator.compareForCQL(key, to);
+ if (comparison > 0)
+ end = i;
+ }
+ }
+
+ if (start < 0)
+ return Range.closedOpen(0, 0);
+
+ if (end < 0)
+ return Range.closedOpen(start, n);
+
+ return Range.closedOpen(start, end);
+ }
+ catch (BufferUnderflowException e)
+ {
+ throw new MarshalException("Not enough bytes to read a " + name);
+ }
+ }
+
+ private void skipMapValue(ByteBuffer input)
+ {
+ if (hasValues)
+ skipValue(input);
+ }
+}
diff --git a/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java b/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java
new file mode 100644
index 0000000..1be4d61
--- /dev/null
+++ b/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.TypeParser;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class AbstractTypeSerializer
+{
+ public void serialize(AbstractType<?> type, DataOutputPlus out) throws IOException
+ {
+ ByteBufferUtil.writeWithVIntLength(UTF8Type.instance.decompose(type.toString()), out);
+ }
+
+ public void serializeList(List<AbstractType<?>> types, DataOutputPlus out) throws IOException
+ {
+ out.writeUnsignedVInt32(types.size());
+ for (AbstractType<?> type : types)
+ serialize(type, out);
+ }
+
+ public AbstractType<?> deserialize(DataInputPlus in) throws IOException
+ {
+ ByteBuffer raw = ByteBufferUtil.readWithVIntLength(in);
+ return TypeParser.parse(UTF8Type.instance.compose(raw));
+ }
+
+ public List<AbstractType<?>> deserializeList(DataInputPlus in) throws IOException
+ {
+ int size = (int) in.readUnsignedVInt();
+ List<AbstractType<?>> types = new ArrayList<>(size);
+ for (int i = 0; i < size; i++)
+ types.add(deserialize(in));
+ return types;
+ }
+
+ public long serializedSize(AbstractType<?> type)
+ {
+ return ByteBufferUtil.serializedSizeWithVIntLength(UTF8Type.instance.decompose(type.toString()));
+ }
+
+ public long serializedListSize(List<AbstractType<?>> types)
+ {
+ long size = TypeSizes.sizeofUnsignedVInt(types.size());
+ for (AbstractType<?> type : types)
+ size += serializedSize(type);
+ return size;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/serializers/BooleanSerializer.java b/src/java/org/apache/cassandra/serializers/BooleanSerializer.java
index d372a2a..403e6b7 100644
--- a/src/java/org/apache/cassandra/serializers/BooleanSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/BooleanSerializer.java
@@ -24,8 +24,8 @@
public class BooleanSerializer extends TypeSerializer<Boolean>
{
- private static final ByteBuffer TRUE = ByteBuffer.wrap(new byte[] {1});
- private static final ByteBuffer FALSE = ByteBuffer.wrap(new byte[] {0});
+ public static final ByteBuffer TRUE = ByteBuffer.wrap(new byte[] {1});
+ public static final ByteBuffer FALSE = ByteBuffer.wrap(new byte[] {0});
public static final BooleanSerializer instance = new BooleanSerializer();
diff --git a/src/java/org/apache/cassandra/serializers/CollectionSerializer.java b/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
index 1722c3d..b514d24 100644
--- a/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/CollectionSerializer.java
@@ -18,14 +18,17 @@
package org.apache.cassandra.serializers;
+import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
+import java.util.function.Consumer;
+
+import com.google.common.collect.Range;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.ValueAccessor;
-import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -34,79 +37,49 @@
protected abstract List<ByteBuffer> serializeValues(T value);
protected abstract int getElementCount(T value);
- public abstract <V> T deserializeForNativeProtocol(V value, ValueAccessor<V> accessor, ProtocolVersion version);
-
- public T deserializeForNativeProtocol(ByteBuffer value, ProtocolVersion version)
- {
- return deserializeForNativeProtocol(value, ByteBufferAccessor.instance, version);
- }
-
- public abstract <V> void validateForNativeProtocol(V value, ValueAccessor<V> accessor, ProtocolVersion version);
-
+ @Override
public ByteBuffer serialize(T input)
{
List<ByteBuffer> values = serializeValues(input);
- // See deserialize() for why using the protocol v3 variant is the right thing to do.
- return pack(values, ByteBufferAccessor.instance, getElementCount(input), ProtocolVersion.V3);
+ return pack(values, ByteBufferAccessor.instance, getElementCount(input));
}
- public <V> T deserialize(V value, ValueAccessor<V> accessor)
+ public static ByteBuffer pack(Collection<ByteBuffer> values, int elements)
{
- // The only cases we serialize/deserialize collections internally (i.e. not for the protocol sake),
- // is:
- // 1) when collections are frozen
- // 2) for internal calls.
- // In both case, using the protocol 3 version variant is the right thing to do.
- return deserializeForNativeProtocol(value, accessor, ProtocolVersion.V3);
+ return pack(values, ByteBufferAccessor.instance, elements);
}
- public <T1> void validate(T1 value, ValueAccessor<T1> accessor) throws MarshalException
- {
- // Same thing as above
- validateForNativeProtocol(value, accessor, ProtocolVersion.V3);
- }
-
- public static ByteBuffer pack(Collection<ByteBuffer> values, int elements, ProtocolVersion version)
- {
- return pack(values, ByteBufferAccessor.instance, elements, version);
- }
-
- public static <V> V pack(Collection<V> values, ValueAccessor<V> accessor, int elements, ProtocolVersion version)
+ public static <V> V pack(Collection<V> values, ValueAccessor<V> accessor, int elements)
{
int size = 0;
for (V value : values)
- size += sizeOfValue(value, accessor, version);
+ size += sizeOfValue(value, accessor);
- ByteBuffer result = ByteBuffer.allocate(sizeOfCollectionSize(elements, version) + size);
- writeCollectionSize(result, elements, version);
+ ByteBuffer result = ByteBuffer.allocate(sizeOfCollectionSize() + size);
+ writeCollectionSize(result, elements);
for (V value : values)
{
- writeValue(result, value, accessor, version);
+ writeValue(result, value, accessor);
}
return accessor.valueOf((ByteBuffer) result.flip());
}
- protected static void writeCollectionSize(ByteBuffer output, int elements, ProtocolVersion version)
+ protected static void writeCollectionSize(ByteBuffer output, int elements)
{
output.putInt(elements);
}
- public static int readCollectionSize(ByteBuffer input, ProtocolVersion version)
- {
- return readCollectionSize(input, ByteBufferAccessor.instance, version);
- }
-
- public static <V> int readCollectionSize(V value, ValueAccessor<V> accessor, ProtocolVersion version)
+ public static <V> int readCollectionSize(V value, ValueAccessor<V> accessor)
{
return accessor.toInt(value);
}
- public static int sizeOfCollectionSize(int elements, ProtocolVersion version)
+ public static int sizeOfCollectionSize()
{
return TypeSizes.INT_SIZE;
}
- public static <V> void writeValue(ByteBuffer output, V value, ValueAccessor<V> accessor, ProtocolVersion version)
+ public static <V> void writeValue(ByteBuffer output, V value, ValueAccessor<V> accessor)
{
if (value == null)
{
@@ -118,7 +91,7 @@
accessor.write(value, output);
}
- public static <V> V readValue(V input, ValueAccessor<V> accessor, int offset, ProtocolVersion version)
+ public static <V> V readValue(V input, ValueAccessor<V> accessor, int offset)
{
int size = accessor.getInt(input, offset);
if (size < 0)
@@ -127,27 +100,27 @@
return accessor.slice(input, offset + TypeSizes.INT_SIZE, size);
}
- public static <V> V readNonNullValue(V input, ValueAccessor<V> accessor, int offset, ProtocolVersion version)
+ public static <V> V readNonNullValue(V input, ValueAccessor<V> accessor, int offset)
{
- V value = readValue(input, accessor, offset, version);
+ V value = readValue(input, accessor, offset);
if (value == null)
throw new MarshalException("Null value read when not allowed");
return value;
}
- protected static void skipValue(ByteBuffer input, ProtocolVersion version)
+ protected static void skipValue(ByteBuffer input)
{
int size = input.getInt();
input.position(input.position() + size);
}
- public static <V> int skipValue(V input, ValueAccessor<V> accessor, int offset, ProtocolVersion version)
+ public static <V> int skipValue(V input, ValueAccessor<V> accessor, int offset)
{
int size = accessor.getInt(input, offset);
return TypeSizes.sizeof(size) + size;
}
- public static <V> int sizeOfValue(V value, ValueAccessor<V> accessor, ProtocolVersion version)
+ public static <V> int sizeOfValue(V value, ValueAccessor<V> accessor)
{
return value == null ? 4 : 4 + accessor.size(value);
}
@@ -188,20 +161,72 @@
boolean frozen);
/**
+ * Returns the index of an element in a serialized collection.
+ * <p>
+ * Note that this is only supported by sets and maps, but not by lists.
+ *
+ * @param collection The serialized collection. This cannot be {@code null}.
+ * @param key The key for which the index must be found. This cannot be {@code null} nor
+ * {@link ByteBufferUtil#UNSET_BYTE_BUFFER}).
+ * @param comparator The type to use to compare the {@code key} value to those in the collection.
+ * @return The index of the element associated with {@code key} if one exists, {@code -1} otherwise.
+ */
+ public abstract int getIndexFromSerialized(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator);
+
+ /**
+ * Returns the range of indexes corresponding to the specified range of elements in the serialized collection.
+ * <p>
+ * Note that this is only supported by sets and maps, but not by lists.
+ *
+ * @param collection The serialized collection. This cannot be {@code null}.
+ * @param from The left bound of the slice to extract. This cannot be {@code null} but if this is
+ * {@link ByteBufferUtil#UNSET_BYTE_BUFFER}, then the returned slice starts at the beginning of the collection.
+ * @param to The left bound of the slice to extract. This cannot be {@code null} but if this is
+ * {@link ByteBufferUtil#UNSET_BYTE_BUFFER}, then the returned slice ends at the end of the collection.
+ * @param comparator The type to use to compare the {@code from} and {@code to} values to those in the collection.
+ * @return The range of indexes corresponding to specified range of elements.
+ */
+ public abstract Range<Integer> getIndexesRangeFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator);
+
+ /**
* Creates a new serialized map composed from the data from {@code input} between {@code startPos}
* (inclusive) and {@code endPos} (exclusive), assuming that data holds {@code count} elements.
*/
- protected ByteBuffer copyAsNewCollection(ByteBuffer input, int count, int startPos, int endPos, ProtocolVersion version)
+ protected ByteBuffer copyAsNewCollection(ByteBuffer input, int count, int startPos, int endPos)
{
- int sizeLen = sizeOfCollectionSize(count, version);
+ int sizeLen = sizeOfCollectionSize();
if (count == 0)
return ByteBuffer.allocate(sizeLen);
int bodyLen = endPos - startPos;
ByteBuffer output = ByteBuffer.allocate(sizeLen + bodyLen);
- writeCollectionSize(output, count, version);
+ writeCollectionSize(output, count);
output.position(0);
ByteBufferUtil.copyBytes(input, startPos, output, sizeLen, bodyLen);
return output;
}
+
+ public void forEach(ByteBuffer input, Consumer<ByteBuffer> action)
+ {
+ try
+ {
+ int collectionSize = readCollectionSize(input, ByteBufferAccessor.instance);
+ int offset = sizeOfCollectionSize();
+
+ for (int i = 0; i < collectionSize; i++)
+ {
+ ByteBuffer value = readValue(input, ByteBufferAccessor.instance, offset);
+ offset += sizeOfValue(value, ByteBufferAccessor.instance);
+
+ action.accept(value);
+ }
+ }
+ catch (BufferUnderflowException | IndexOutOfBoundsException e)
+ {
+ throw new MarshalException("Not enough bytes to read a set");
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/serializers/DurationSerializer.java b/src/java/org/apache/cassandra/serializers/DurationSerializer.java
index 254b2b0..78326e1 100644
--- a/src/java/org/apache/cassandra/serializers/DurationSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/DurationSerializer.java
@@ -17,9 +17,6 @@
*/
package org.apache.cassandra.serializers;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.io.util.DataInputBuffer;
@@ -27,6 +24,9 @@
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.vint.VIntCoding;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
public final class DurationSerializer extends TypeSerializer<Duration>
{
public static final DurationSerializer instance = new DurationSerializer();
@@ -65,8 +65,8 @@
try (DataInputBuffer in = new DataInputBuffer(accessor.toBuffer(value), true)) // TODO: make a value input buffer
{
- int months = (int) in.readVInt();
- int days = (int) in.readVInt();
+ int months = in.readVInt32();
+ int days = in.readVInt32();
long nanoseconds = in.readVInt();
return Duration.newInstance(months, days, nanoseconds);
}
diff --git a/src/java/org/apache/cassandra/serializers/ListSerializer.java b/src/java/org/apache/cassandra/serializers/ListSerializer.java
index 7a2f634..f10e8f7 100644
--- a/src/java/org/apache/cassandra/serializers/ListSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/ListSerializer.java
@@ -25,24 +25,27 @@
import java.util.concurrent.ConcurrentMap;
import java.util.function.Predicate;
+import com.google.common.collect.Range;
+
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.ValueAccessor;
-import org.apache.cassandra.transport.ProtocolVersion;
public class ListSerializer<T> extends CollectionSerializer<List<T>>
{
// interning instances
- private static final ConcurrentMap<TypeSerializer<?>, ListSerializer> instances = new ConcurrentHashMap<TypeSerializer<?>, ListSerializer>();
+ @SuppressWarnings("rawtypes")
+ private static final ConcurrentMap<TypeSerializer<?>, ListSerializer> instances = new ConcurrentHashMap<>();
public final TypeSerializer<T> elements;
+ @SuppressWarnings("unchecked")
public static <T> ListSerializer<T> getInstance(TypeSerializer<T> elements)
{
ListSerializer<T> t = instances.get(elements);
if (t == null)
- t = instances.computeIfAbsent(elements, k -> new ListSerializer<>(k) );
+ t = instances.computeIfAbsent(elements, ListSerializer::new);
return t;
}
@@ -51,6 +54,7 @@
this.elements = elements;
}
+ @Override
protected List<ByteBuffer> serializeValues(List<T> values)
{
List<ByteBuffer> output = new ArrayList<>(values.size());
@@ -59,21 +63,23 @@
return output;
}
+ @Override
public int getElementCount(List<T> value)
{
return value.size();
}
- public <V> void validateForNativeProtocol(V input, ValueAccessor<V> accessor, ProtocolVersion version)
+ @Override
+ public <V> void validate(V input, ValueAccessor<V> accessor)
{
try
{
- int n = readCollectionSize(input, accessor, version);
- int offset = sizeOfCollectionSize(n, version);
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
for (int i = 0; i < n; i++)
{
- V value = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(value, accessor, version);
+ V value = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(value, accessor);
elements.validate(value, accessor);
}
@@ -86,12 +92,13 @@
}
}
- public <V> List<T> deserializeForNativeProtocol(V input, ValueAccessor<V> accessor, ProtocolVersion version)
+ @Override
+ public <V> List<T> deserialize(V input, ValueAccessor<V> accessor)
{
try
{
- int n = readCollectionSize(input, accessor, version);
- int offset = sizeOfCollectionSize(n, version);
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
if (n < 0)
throw new MarshalException("The data cannot be deserialized as a list");
@@ -100,7 +107,7 @@
// In such a case we do not want to initialize the list with that size as it can result
// in an OOM (see CASSANDRA-12618). On the other hand we do not want to have to resize the list
// if we can avoid it, so we put a reasonable limit on the initialCapacity.
- List<T> l = new ArrayList<T>(Math.min(n, 256));
+ List<T> l = new ArrayList<>(Math.min(n, 256));
for (int i = 0; i < n; i++)
{
// CASSANDRA-6839: "We can have nulls in lists that are used for IN values"
@@ -109,8 +116,8 @@
// for it, but should likely be changed to readNonNull. Validate has been
// changed to throw on null elements as otherwise it would NPE, and it's unclear
// if callers could handle null elements.
- V databb = readValue(input, accessor, offset, version);
- offset += sizeOfValue(databb, accessor, version);
+ V databb = readValue(input, accessor, offset);
+ offset += sizeOfValue(databb, accessor);
if (databb != null)
{
elements.validate(databb, accessor);
@@ -142,8 +149,8 @@
{
try
{
- int s = readCollectionSize(input, accessor, ProtocolVersion.V3);
- int offset = sizeOfCollectionSize(s, ProtocolVersion.V3);
+ int s = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
for (int i = 0; i < s; i++)
{
@@ -178,8 +185,8 @@
{
try
{
- int n = readCollectionSize(input, accessor, ProtocolVersion.V3);
- int offset = sizeOfCollectionSize(n, ProtocolVersion.V3);
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
if (n <= index)
return null;
@@ -188,7 +195,7 @@
int length = accessor.getInt(input, offset);
offset += TypeSizes.INT_SIZE + length;
}
- return readValue(input, accessor, offset, ProtocolVersion.V3);
+ return readValue(input, accessor, offset);
}
catch (BufferUnderflowException | IndexOutOfBoundsException e)
{
@@ -201,6 +208,7 @@
return getElement(input, ByteBufferAccessor.instance, index);
}
+ @Override
public String toString(List<T> value)
{
StringBuilder sb = new StringBuilder();
@@ -218,6 +226,8 @@
return sb.toString();
}
+ @Override
+ @SuppressWarnings({ "rawtypes", "unchecked" })
public Class<List<T>> getType()
{
return (Class) List.class;
@@ -226,7 +236,7 @@
@Override
public ByteBuffer getSerializedValue(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator)
{
- // We don't allow selecting an element of a list so we don't need this.
+ // We don't allow selecting an element of a list, so we don't need this.
throw new UnsupportedOperationException();
}
@@ -237,7 +247,22 @@
AbstractType<?> comparator,
boolean frozen)
{
- // We don't allow slicing of list so we don't need this.
+ // We don't allow slicing of lists, so we don't need this.
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getIndexFromSerialized(ByteBuffer collection, ByteBuffer key, AbstractType<?> comparator)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Range<Integer> getIndexesRangeFromSerialized(ByteBuffer collection,
+ ByteBuffer from,
+ ByteBuffer to,
+ AbstractType<?> comparator)
+ {
throw new UnsupportedOperationException();
}
}
diff --git a/src/java/org/apache/cassandra/serializers/MapSerializer.java b/src/java/org/apache/cassandra/serializers/MapSerializer.java
index 2e363b4..9eb0642 100644
--- a/src/java/org/apache/cassandra/serializers/MapSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/MapSerializer.java
@@ -20,49 +20,55 @@
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.function.Consumer;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.marshal.ValueComparators;
import org.apache.cassandra.db.marshal.ValueAccessor;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.db.marshal.ValueComparators;
import org.apache.cassandra.utils.Pair;
-public class MapSerializer<K, V> extends CollectionSerializer<Map<K, V>>
+public class MapSerializer<K, V> extends AbstractMapSerializer<Map<K, V>>
{
// interning instances
+ @SuppressWarnings("rawtypes")
private static final ConcurrentMap<Pair<TypeSerializer<?>, TypeSerializer<?>>, MapSerializer> instances = new ConcurrentHashMap<>();
public final TypeSerializer<K> keys;
public final TypeSerializer<V> values;
private final ValueComparators comparators;
+ @SuppressWarnings("unchecked")
public static <K, V> MapSerializer<K, V> getInstance(TypeSerializer<K> keys, TypeSerializer<V> values, ValueComparators comparators)
{
Pair<TypeSerializer<?>, TypeSerializer<?>> p = Pair.create(keys, values);
MapSerializer<K, V> t = instances.get(p);
if (t == null)
- t = instances.computeIfAbsent(p, k -> new MapSerializer<>(k.left, k.right, comparators) );
+ t = instances.computeIfAbsent(p, k -> new MapSerializer<>(k.left, k.right, comparators));
return t;
}
private MapSerializer(TypeSerializer<K> keys, TypeSerializer<V> values, ValueComparators comparators)
{
+ super(true);
this.keys = keys;
this.values = values;
this.comparators = comparators;
}
+ @Override
public List<ByteBuffer> serializeValues(Map<K, V> map)
{
List<Pair<ByteBuffer, ByteBuffer>> pairs = new ArrayList<>(map.size());
for (Map.Entry<K, V> entry : map.entrySet())
pairs.add(Pair.create(keys.serialize(entry.getKey()), values.serialize(entry.getValue())));
- Collections.sort(pairs, (l, r) -> comparators.buffer.compare(l.left, r.left));
+ pairs.sort((l, r) -> comparators.buffer.compare(l.left, r.left));
List<ByteBuffer> buffers = new ArrayList<>(pairs.size() * 2);
for (Pair<ByteBuffer, ByteBuffer> p : pairs)
{
@@ -72,28 +78,30 @@
return buffers;
}
+ @Override
public int getElementCount(Map<K, V> value)
{
return value.size();
}
- public <T> void validateForNativeProtocol(T input, ValueAccessor<T> accessor, ProtocolVersion version)
+ @Override
+ public <T> void validate(T input, ValueAccessor<T> accessor)
{
try
{
// Empty values are still valid.
if (accessor.isEmpty(input)) return;
-
- int n = readCollectionSize(input, accessor, version);
- int offset = sizeOfCollectionSize(n, version);
+
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
for (int i = 0; i < n; i++)
{
- T key = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(key, accessor, version);
+ T key = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(key, accessor);
keys.validate(key, accessor);
- T value = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(value, accessor, version);
+ T value = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(value, accessor);
values.validate(value, accessor);
}
if (!accessor.isEmptyFromOffset(input, offset))
@@ -105,12 +113,13 @@
}
}
- public <I> Map<K, V> deserializeForNativeProtocol(I input, ValueAccessor<I> accessor, ProtocolVersion version)
+ @Override
+ public <I> Map<K, V> deserialize(I input, ValueAccessor<I> accessor)
{
try
{
- int n = readCollectionSize(input, accessor, version);
- int offset = sizeOfCollectionSize(n, version);
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
if (n < 0)
throw new MarshalException("The data cannot be deserialized as a map");
@@ -119,15 +128,15 @@
// In such a case we do not want to initialize the map with that initialCapacity as it can result
// in an OOM when put is called (see CASSANDRA-12618). On the other hand we do not want to have to resize
// the map if we can avoid it, so we put a reasonable limit on the initialCapacity.
- Map<K, V> m = new LinkedHashMap<K, V>(Math.min(n, 256));
+ Map<K, V> m = new LinkedHashMap<>(Math.min(n, 256));
for (int i = 0; i < n; i++)
{
- I key = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(key, accessor, version);
+ I key = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(key, accessor);
keys.validate(key, accessor);
- I value = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(value, accessor, version);
+ I value = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(value, accessor);
values.validate(value, accessor);
m.put(keys.deserialize(key, accessor), values.deserialize(value, accessor));
@@ -148,20 +157,20 @@
try
{
ByteBuffer input = collection.duplicate();
- int n = readCollectionSize(input, ProtocolVersion.V3);
- int offset = sizeOfCollectionSize(n, ProtocolVersion.V3);
+ int n = readCollectionSize(input, ByteBufferAccessor.instance);
+ int offset = sizeOfCollectionSize();
for (int i = 0; i < n; i++)
{
- ByteBuffer kbb = readValue(input, ByteBufferAccessor.instance, offset, ProtocolVersion.V3);
- offset += sizeOfValue(kbb, ByteBufferAccessor.instance, ProtocolVersion.V3);
+ ByteBuffer kbb = readValue(input, ByteBufferAccessor.instance, offset);
+ offset += sizeOfValue(kbb, ByteBufferAccessor.instance);
int comparison = comparator.compareForCQL(kbb, key);
if (comparison == 0)
- return readValue(input, ByteBufferAccessor.instance, offset, ProtocolVersion.V3);
+ return readValue(input, ByteBufferAccessor.instance, offset);
else if (comparison > 0)
// since the map is in sorted order, we know we've gone too far and the element doesn't exist
return null;
else // comparison < 0
- offset += skipValue(input, ByteBufferAccessor.instance, offset, ProtocolVersion.V3);
+ offset += skipValue(input, ByteBufferAccessor.instance, offset);
}
return null;
}
@@ -172,77 +181,6 @@
}
@Override
- public ByteBuffer getSliceFromSerialized(ByteBuffer collection,
- ByteBuffer from,
- ByteBuffer to,
- AbstractType<?> comparator,
- boolean frozen)
- {
- if (from == ByteBufferUtil.UNSET_BYTE_BUFFER && to == ByteBufferUtil.UNSET_BYTE_BUFFER)
- return collection;
-
- try
- {
- ByteBuffer input = collection.duplicate();
- int n = readCollectionSize(input, ProtocolVersion.V3);
- input.position(input.position() + sizeOfCollectionSize(n, ProtocolVersion.V3));
- int startPos = input.position();
- int count = 0;
- boolean inSlice = from == ByteBufferUtil.UNSET_BYTE_BUFFER;
-
- for (int i = 0; i < n; i++)
- {
- int pos = input.position();
- ByteBuffer kbb = readValue(input, ByteBufferAccessor.instance, 0, ProtocolVersion.V3); // key
- input.position(input.position() + sizeOfValue(kbb, ByteBufferAccessor.instance, ProtocolVersion.V3));
-
- // If we haven't passed the start already, check if we have now
- if (!inSlice)
- {
- int comparison = comparator.compareForCQL(from, kbb);
- if (comparison <= 0)
- {
- // We're now within the slice
- inSlice = true;
- startPos = pos;
- }
- else
- {
- // We're before the slice so we know we don't care about this element
- skipValue(input, ProtocolVersion.V3); // value
- continue;
- }
- }
-
- // Now check if we're done
- int comparison = to == ByteBufferUtil.UNSET_BYTE_BUFFER ? -1 : comparator.compareForCQL(kbb, to);
- if (comparison > 0)
- {
- // We're done and shouldn't include the key we just read
- input.position(pos);
- break;
- }
-
- // Otherwise, we'll include that element
- skipValue(input, ProtocolVersion.V3); // value
- ++count;
-
- // But if we know if was the last of the slice, we break early
- if (comparison == 0)
- break;
- }
-
- if (count == 0 && !frozen)
- return null;
-
- return copyAsNewCollection(collection, count, startPos, input.position(), ProtocolVersion.V3);
- }
- catch (BufferUnderflowException | IndexOutOfBoundsException e)
- {
- throw new MarshalException("Not enough bytes to read a map");
- }
- }
-
public String toString(Map<K, V> value)
{
StringBuilder sb = new StringBuilder();
@@ -262,8 +200,16 @@
return sb.toString();
}
+ @Override
+ @SuppressWarnings({ "rawtypes", "unchecked" })
public Class<Map<K, V>> getType()
{
- return (Class)Map.class;
+ return (Class) Map.class;
+ }
+
+ @Override
+ public void forEach(ByteBuffer input, Consumer<ByteBuffer> action)
+ {
+ throw new UnsupportedOperationException();
}
}
diff --git a/src/java/org/apache/cassandra/serializers/SetSerializer.java b/src/java/org/apache/cassandra/serializers/SetSerializer.java
index bf565be..599c2a8 100644
--- a/src/java/org/apache/cassandra/serializers/SetSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/SetSerializer.java
@@ -20,26 +20,28 @@
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.marshal.ValueComparators;
-import org.apache.cassandra.db.marshal.ValueAccessor;
-import org.apache.cassandra.transport.ProtocolVersion;
-
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.db.marshal.ValueComparators;
-public class SetSerializer<T> extends CollectionSerializer<Set<T>>
+public class SetSerializer<T> extends AbstractMapSerializer<Set<T>>
{
// interning instances
- private static final ConcurrentMap<TypeSerializer<?>, SetSerializer> instances = new ConcurrentHashMap<TypeSerializer<?>, SetSerializer>();
+ @SuppressWarnings("rawtypes")
+ private static final ConcurrentMap<TypeSerializer<?>, SetSerializer> instances = new ConcurrentHashMap<>();
public final TypeSerializer<T> elements;
private final ValueComparators comparators;
+ @SuppressWarnings("unchecked")
public static <T> SetSerializer<T> getInstance(TypeSerializer<T> elements, ValueComparators comparators)
{
SetSerializer<T> t = instances.get(elements);
@@ -50,37 +52,41 @@
public SetSerializer(TypeSerializer<T> elements, ValueComparators comparators)
{
+ super(false);
this.elements = elements;
this.comparators = comparators;
}
+ @Override
public List<ByteBuffer> serializeValues(Set<T> values)
{
List<ByteBuffer> buffers = new ArrayList<>(values.size());
for (T value : values)
buffers.add(elements.serialize(value));
- Collections.sort(buffers, comparators.buffer);
+ buffers.sort(comparators.buffer);
return buffers;
}
+ @Override
public int getElementCount(Set<T> value)
{
return value.size();
}
- public <V> void validateForNativeProtocol(V input, ValueAccessor<V> accessor, ProtocolVersion version)
+ @Override
+ public <V> void validate(V input, ValueAccessor<V> accessor)
{
try
{
// Empty values are still valid.
if (accessor.isEmpty(input)) return;
- int n = readCollectionSize(input, accessor, version);
- int offset = sizeOfCollectionSize(n, version);
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
for (int i = 0; i < n; i++)
{
- V value = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(value, accessor, version);
+ V value = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(value, accessor);
elements.validate(value, accessor);
}
if (!accessor.isEmptyFromOffset(input, offset))
@@ -92,12 +98,13 @@
}
}
- public <V> Set<T> deserializeForNativeProtocol(V input, ValueAccessor<V> accessor, ProtocolVersion version)
+ @Override
+ public <V> Set<T> deserialize(V input, ValueAccessor<V> accessor)
{
try
{
- int n = readCollectionSize(input, accessor, version);
- int offset = sizeOfCollectionSize(n, version);
+ int n = readCollectionSize(input, accessor);
+ int offset = sizeOfCollectionSize();
if (n < 0)
throw new MarshalException("The data cannot be deserialized as a set");
@@ -106,12 +113,12 @@
// In such a case we do not want to initialize the set with that initialCapacity as it can result
// in an OOM when add is called (see CASSANDRA-12618). On the other hand we do not want to have to resize
// the set if we can avoid it, so we put a reasonable limit on the initialCapacity.
- Set<T> l = new LinkedHashSet<T>(Math.min(n, 256));
+ Set<T> l = new LinkedHashSet<>(Math.min(n, 256));
for (int i = 0; i < n; i++)
{
- V value = readNonNullValue(input, accessor, offset, version);
- offset += sizeOfValue(value, accessor, version);
+ V value = readNonNullValue(input, accessor, offset);
+ offset += sizeOfValue(value, accessor);
elements.validate(value, accessor);
l.add(elements.deserialize(value, accessor));
}
@@ -125,6 +132,7 @@
}
}
+ @Override
public String toString(Set<T> value)
{
StringBuilder sb = new StringBuilder();
@@ -146,6 +154,8 @@
return sb.toString();
}
+ @Override
+ @SuppressWarnings({ "rawtypes", "unchecked" })
public Class<Set<T>> getType()
{
return (Class) Set.class;
@@ -156,13 +166,13 @@
{
try
{
- int n = readCollectionSize(input, ProtocolVersion.V3);
- int offset = sizeOfCollectionSize(n, ProtocolVersion.V3);
+ int n = readCollectionSize(input, ByteBufferAccessor.instance);
+ int offset = sizeOfCollectionSize();
for (int i = 0; i < n; i++)
{
- ByteBuffer value = readValue(input, ByteBufferAccessor.instance, offset, ProtocolVersion.V3);
- offset += sizeOfValue(value, ByteBufferAccessor.instance, ProtocolVersion.V3);
+ ByteBuffer value = readValue(input, ByteBufferAccessor.instance, offset);
+ offset += sizeOfValue(value, ByteBufferAccessor.instance);
int comparison = comparator.compareForCQL(value, key);
if (comparison == 0)
return value;
@@ -178,74 +188,4 @@
throw new MarshalException("Not enough bytes to read a set");
}
}
-
- @Override
- public ByteBuffer getSliceFromSerialized(ByteBuffer collection,
- ByteBuffer from,
- ByteBuffer to,
- AbstractType<?> comparator,
- boolean frozen)
- {
- if (from == ByteBufferUtil.UNSET_BYTE_BUFFER && to == ByteBufferUtil.UNSET_BYTE_BUFFER)
- return collection;
-
- try
- {
- ByteBuffer input = collection.duplicate();
- int n = readCollectionSize(input, ProtocolVersion.V3);
- input.position(input.position() + sizeOfCollectionSize(n, ProtocolVersion.V3));
- int startPos = input.position();
- int count = 0;
- boolean inSlice = from == ByteBufferUtil.UNSET_BYTE_BUFFER;
-
- for (int i = 0; i < n; i++)
- {
- int pos = input.position();
- ByteBuffer value = readValue(input, ByteBufferAccessor.instance, 0, ProtocolVersion.V3);
- input.position(input.position() + sizeOfValue(value, ByteBufferAccessor.instance, ProtocolVersion.V3));
-
- // If we haven't passed the start already, check if we have now
- if (!inSlice)
- {
- int comparison = comparator.compareForCQL(from, value);
- if (comparison <= 0)
- {
- // We're now within the slice
- inSlice = true;
- startPos = pos;
- }
- else
- {
- // We're before the slice so we know we don't care about this value
- continue;
- }
- }
-
- // Now check if we're done
- int comparison = to == ByteBufferUtil.UNSET_BYTE_BUFFER ? -1 : comparator.compareForCQL(value, to);
- if (comparison > 0)
- {
- // We're done and shouldn't include the value we just read
- input.position(pos);
- break;
- }
-
- // Otherwise, we'll include that value
- ++count;
-
- // But if we know if was the last of the slice, we break early
- if (comparison == 0)
- break;
- }
-
- if (count == 0 && !frozen)
- return null;
-
- return copyAsNewCollection(collection, count, startPos, input.position(), ProtocolVersion.V3);
- }
- catch (BufferUnderflowException | IndexOutOfBoundsException e)
- {
- throw new MarshalException("Not enough bytes to read a set");
- }
- }
}
diff --git a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
index ce28266..d2c54f9 100644
--- a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
@@ -22,7 +22,9 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.function.Function;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
@@ -114,34 +116,42 @@
{
long timeoutNanos = currentTimeoutNanos();
- boolean success;
+ boolean signaled;
try
{
- success = condition.await(timeoutNanos, NANOSECONDS);
+ signaled = condition.await(timeoutNanos, NANOSECONDS);
}
catch (InterruptedException e)
{
throw new UncheckedInterruptedException(e);
}
- if (!success)
- {
- int blockedFor = blockFor();
- int acks = ackCount();
- // It's pretty unlikely, but we can race between exiting await above and here, so
- // that we could now have enough acks. In that case, we "lie" on the acks count to
- // avoid sending confusing info to the user (see CASSANDRA-6491).
- if (acks >= blockedFor)
- acks = blockedFor - 1;
- throw new WriteTimeoutException(writeType, replicaPlan.consistencyLevel(), acks, blockedFor);
- }
+ if (!signaled)
+ throwTimeout();
if (blockFor() + failures > candidateReplicaCount())
{
- throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, failureReasonByEndpoint);
+ if (RequestCallback.isTimeout(this.failureReasonByEndpoint.keySet().stream()
+ .filter(this::waitingFor) // DatacenterWriteResponseHandler filters errors from remote DCs
+ .collect(Collectors.toMap(Function.identity(), this.failureReasonByEndpoint::get))))
+ throwTimeout();
+
+ throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, this.failureReasonByEndpoint);
}
}
+ private void throwTimeout()
+ {
+ int blockedFor = blockFor();
+ int acks = ackCount();
+ // It's pretty unlikely, but we can race between exiting await above and here, so
+ // that we could now have enough acks. In that case, we "lie" on the acks count to
+ // avoid sending confusing info to the user (see CASSANDRA-6491).
+ if (acks >= blockedFor)
+ acks = blockedFor - 1;
+ throw new WriteTimeoutException(writeType, replicaPlan.consistencyLevel(), acks, blockedFor);
+ }
+
public final long currentTimeoutNanos()
{
long requestTimeout = writeType == COUNTER
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index a0716b1..5d69507 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -45,6 +45,7 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.EndpointsByRange;
import org.apache.cassandra.locator.EndpointsForRange;
@@ -118,6 +119,11 @@
import static java.util.Collections.synchronizedSet;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PARENT_REPAIR_STATUS_CACHE_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PARENT_REPAIR_STATUS_EXPIRY_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_ALLOW_MULTIPLE_PENDING_UNSAFE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES;
import static org.apache.cassandra.config.Config.RepairCommandPoolFullStrategy.reject;
import static org.apache.cassandra.config.DatabaseDescriptor.*;
import static org.apache.cassandra.net.Message.out;
@@ -220,13 +226,11 @@
this.failureDetector = failureDetector;
this.gossiper = gossiper;
this.repairStatusByCmd = CacheBuilder.newBuilder()
- .expireAfterWrite(
- Long.getLong("cassandra.parent_repair_status_expiry_seconds",
- TimeUnit.SECONDS.convert(1, TimeUnit.DAYS)), TimeUnit.SECONDS)
+ .expireAfterWrite(PARENT_REPAIR_STATUS_EXPIRY_SECONDS.getLong(), TimeUnit.SECONDS)
// using weight wouldn't work so well, since it doesn't reflect mutation of cached data
// see https://github.com/google/guava/wiki/CachesExplained
// We assume each entry is unlikely to be much more than 100 bytes, so bounding the size should be sufficient.
- .maximumSize(Long.getLong("cassandra.parent_repair_status_cache_size", 100_000))
+ .maximumSize(PARENT_REPAIR_STATUS_CACHE_SIZE.getLong())
.build();
DurationSpec.LongNanosecondsBound duration = getRepairStateExpires();
@@ -294,18 +298,44 @@
return DatabaseDescriptor.getRepairSessionSpaceInMiB();
}
+ @Deprecated
@Override
public void setRepairSessionSpaceInMebibytes(int sizeInMebibytes)
{
DatabaseDescriptor.setRepairSessionSpaceInMiB(sizeInMebibytes);
}
+ @Deprecated
@Override
public int getRepairSessionSpaceInMebibytes()
{
return DatabaseDescriptor.getRepairSessionSpaceInMiB();
}
+ @Override
+ public void setRepairSessionSpaceInMiB(int sizeInMebibytes)
+ {
+ try
+ {
+ DatabaseDescriptor.setRepairSessionSpaceInMiB(sizeInMebibytes);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+
+ /*
+ * In CASSANDRA-17668, JMX setters that did not throw standard exceptions were deprecated in favor of ones that do.
+ * For consistency purposes, the respective getter "getRepairSessionSpaceInMebibytes" was also deprecated and
+ * replaced by this method.
+ */
+ @Override
+ public int getRepairSessionSpaceInMiB()
+ {
+ return DatabaseDescriptor.getRepairSessionSpaceInMiB();
+ }
+
public List<CompositeData> getRepairStats(List<String> schemaArgs, String rangeString)
{
List<CompositeData> stats = new ArrayList<>();
@@ -1028,21 +1058,21 @@
throw new RuntimeException(String.format("Insufficient live nodes to repair paxos for %s in %s for %s.\n" +
"There must be enough live nodes to satisfy EACH_QUORUM, but the following nodes are down: %s\n" +
"This check can be skipped by setting either the yaml property skip_paxos_repair_on_topology_change or " +
- "the system property cassandra.skip_paxos_repair_on_topology_change to false. The jmx property " +
+ "the system property %s to false. The jmx property " +
"StorageService.SkipPaxosRepairOnTopologyChange can also be set to false to temporarily disable without " +
"restarting the node\n" +
"Individual keyspaces can be skipped with the yaml property skip_paxos_repair_on_topology_change_keyspaces, the" +
- "system property cassandra.skip_paxos_repair_on_topology_change_keyspaces, or temporarily with the jmx" +
+ "system property %s, or temporarily with the jmx" +
"property StorageService.SkipPaxosRepairOnTopologyChangeKeyspaces\n" +
"Skipping this check can lead to paxos correctness issues",
- range, ksName, reason, downEndpoints));
+ range, ksName, reason, downEndpoints, SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE.getKey(), SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES.getKey()));
}
EndpointsForToken pending = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(range.right, ksName);
- if (pending.size() > 1 && !Boolean.getBoolean("cassandra.paxos_repair_allow_multiple_pending_unsafe"))
+ if (pending.size() > 1 && !PAXOS_REPAIR_ALLOW_MULTIPLE_PENDING_UNSAFE.getBoolean())
{
throw new RuntimeException(String.format("Cannot begin paxos auto repair for %s in %s.%s, multiple pending endpoints exist for range (%s). " +
- "Set -Dcassandra.paxos_repair_allow_multiple_pending_unsafe=true to skip this check",
- range, table.keyspace, table.name, pending));
+ "Set -D%s=true to skip this check",
+ range, table.keyspace, table.name, pending, PAXOS_REPAIR_ALLOW_MULTIPLE_PENDING_UNSAFE.getKey()));
}
Future<Void> future = PaxosCleanup.cleanup(endpoints, table, Collections.singleton(range), false, repairCommandExecutor());
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
index 009ad56..9a60663 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
@@ -34,9 +34,20 @@
@Deprecated
public int getRepairSessionSpaceInMegabytes();
+ /**
+ * @deprecated use setRepairSessionSpaceInMiB instead as it will not throw non-standard exceptions
+ */
+ @Deprecated
public void setRepairSessionSpaceInMebibytes(int sizeInMebibytes);
+ /**
+ * @deprecated use getRepairSessionSpaceInMiB instead
+ */
+ @Deprecated
public int getRepairSessionSpaceInMebibytes();
+ public void setRepairSessionSpaceInMiB(int sizeInMebibytes);
+ public int getRepairSessionSpaceInMiB();
+
public boolean getUseOffheapMerkleTrees();
public void setUseOffheapMerkleTrees(boolean value);
diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java
index 9c23e71..0dc3110 100644
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@ -19,31 +19,51 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.cache.*;
+import org.apache.cassandra.cache.AutoSavingCache;
import org.apache.cassandra.cache.AutoSavingCache.CacheSerializer;
+import org.apache.cassandra.cache.CacheProvider;
+import org.apache.cassandra.cache.CaffeineCache;
+import org.apache.cassandra.cache.CounterCacheKey;
+import org.apache.cassandra.cache.ICache;
+import org.apache.cassandra.cache.IRowCacheEntry;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.cache.RowCacheKey;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ClockAndCount;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.context.CounterContext;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.partitions.CachedBTreePartition;
import org.apache.cassandra.db.partitions.CachedPartition;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
-import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteArrayUtil;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -80,7 +100,7 @@
public final static CacheService instance = new CacheService();
- public final AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache;
+ public final AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache;
public final AutoSavingCache<RowCacheKey, IRowCacheEntry> rowCache;
public final AutoSavingCache<CounterCacheKey, ClockAndCount> counterCache;
@@ -96,7 +116,7 @@
/**
* @return auto saving cache object
*/
- private AutoSavingCache<KeyCacheKey, RowIndexEntry> initKeyCache()
+ private AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> initKeyCache()
{
logger.info("Initializing key cache with capacity of {} MiBs.", DatabaseDescriptor.getKeyCacheSizeInMiB());
@@ -104,9 +124,9 @@
// as values are constant size we can use singleton weigher
// where 48 = 40 bytes (average size of the key) + 8 bytes (size of value)
- ICache<KeyCacheKey, RowIndexEntry> kc;
+ ICache<KeyCacheKey, AbstractRowIndexEntry> kc;
kc = CaffeineCache.create(keyCacheInMemoryCapacity);
- AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer());
+ AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer());
int keyCacheKeysToSave = DatabaseDescriptor.getKeyCacheKeysToSave();
@@ -340,166 +360,201 @@
logger.debug("cache saves completed");
}
- public static class CounterCacheSerializer implements CacheSerializer<CounterCacheKey, ClockAndCount>
+ public static class CounterCacheSerializer extends CacheSerializer<CounterCacheKey, ClockAndCount>
{
public void serialize(CounterCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
assert(cfs.metadata().isCounter());
- TableMetadata tableMetadata = cfs.metadata();
- tableMetadata.id.serialize(out);
- out.writeUTF(tableMetadata.indexName().orElse(""));
+ writeCFS(out, cfs);
key.write(out);
}
- public Future<Pair<CounterCacheKey, ClockAndCount>> deserialize(DataInputPlus in, final ColumnFamilyStore cfs) throws IOException
+ public Future<Pair<CounterCacheKey, ClockAndCount>> deserialize(DataInputPlus in) throws IOException
{
//Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a
//parameter so they aren't deserialized here, even though they are serialized by this serializer
+ ColumnFamilyStore cfs = readCFS(in);
if (cfs == null)
return null;
final CounterCacheKey cacheKey = CounterCacheKey.read(cfs.metadata(), in);
if (!cfs.metadata().isCounter() || !cfs.isCounterCacheEnabled())
return null;
- return Stage.READ.submit(new Callable<Pair<CounterCacheKey, ClockAndCount>>()
- {
- public Pair<CounterCacheKey, ClockAndCount> call() throws Exception
- {
- ByteBuffer value = cacheKey.readCounterValue(cfs);
- return value == null
- ? null
- : Pair.create(cacheKey, CounterContext.instance().getLocalClockAndCount(value));
- }
+ return Stage.READ.submit(() -> {
+ ByteBuffer value = cacheKey.readCounterValue(cfs);
+ return value == null
+ ? null
+ : Pair.create(cacheKey, CounterContext.instance().getLocalClockAndCount(value));
});
}
}
- public static class RowCacheSerializer implements CacheSerializer<RowCacheKey, IRowCacheEntry>
+ public static class RowCacheSerializer extends CacheSerializer<RowCacheKey, IRowCacheEntry>
{
public void serialize(RowCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
assert(!cfs.isIndex());//Shouldn't have row cache entries for indexes
- TableMetadata tableMetadata = cfs.metadata();
- tableMetadata.id.serialize(out);
- out.writeUTF(tableMetadata.indexName().orElse(""));
+ writeCFS(out, cfs);
ByteArrayUtil.writeWithLength(key.key, out);
}
- public Future<Pair<RowCacheKey, IRowCacheEntry>> deserialize(DataInputPlus in, final ColumnFamilyStore cfs) throws IOException
+ public Future<Pair<RowCacheKey, IRowCacheEntry>> deserialize(DataInputPlus in) throws IOException
{
//Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a
//parameter so they aren't deserialized here, even though they are serialized by this serializer
+ ColumnFamilyStore cfs = readCFS(in);
final ByteBuffer buffer = ByteBufferUtil.readWithLength(in);
if (cfs == null || !cfs.isRowCacheEnabled())
return null;
final int rowsToCache = cfs.metadata().params.caching.rowsPerPartitionToCache();
assert(!cfs.isIndex());//Shouldn't have row cache entries for indexes
- return Stage.READ.submit(new Callable<Pair<RowCacheKey, IRowCacheEntry>>()
- {
- public Pair<RowCacheKey, IRowCacheEntry> call() throws Exception
+ return Stage.READ.submit(() -> {
+ DecoratedKey key = cfs.decorateKey(buffer);
+ int nowInSec = FBUtilities.nowInSeconds();
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata(), nowInSec, key);
+ try (ReadExecutionController controller = cmd.executionController(); UnfilteredRowIterator iter = cmd.queryMemtableAndDisk(cfs, controller))
{
- DecoratedKey key = cfs.decorateKey(buffer);
- int nowInSec = FBUtilities.nowInSeconds();
- SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata(), nowInSec, key);
- try (ReadExecutionController controller = cmd.executionController(); UnfilteredRowIterator iter = cmd.queryMemtableAndDisk(cfs, controller))
- {
- CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec);
- return Pair.create(new RowCacheKey(cfs.metadata(), key), toCache);
- }
+ CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec);
+ return Pair.create(new RowCacheKey(cfs.metadata(), key), toCache);
}
});
}
}
- public static class KeyCacheSerializer implements CacheSerializer<KeyCacheKey, RowIndexEntry>
+ public static class KeyCacheSerializer extends CacheSerializer<KeyCacheKey, AbstractRowIndexEntry>
{
- // For column families with many SSTables the linear nature of getSSTables slowed down KeyCache loading
- // by orders of magnitude. So we cache the sstables once and rely on cleanupAfterDeserialize to cleanup any
- // cached state we may have accumulated during the load.
- Map<Pair<String, String>, Map<SSTableId, SSTableReader>> cachedSSTableReaders = new ConcurrentHashMap<>();
+ private final ArrayList<Pair<KeyCacheSupport<?>, SSTableFormat<?, ?>>> readers = new ArrayList<>();
+ private final LinkedHashMap<Descriptor, Pair<Integer, ColumnFamilyStore>> readerOrdinals = new LinkedHashMap<>();
+
+ @Override
+ public void serializeMetadata(DataOutputPlus out) throws IOException
+ {
+ super.serializeMetadata(out);
+ out.writeUnsignedVInt32(readerOrdinals.size());
+ Descriptor desc;
+ for (Map.Entry<Descriptor, Pair<Integer, ColumnFamilyStore>> table : readerOrdinals.entrySet())
+ {
+ desc = table.getKey();
+ ColumnFamilyStore cfs = table.getValue().right;
+ super.writeCFS(out, cfs);
+ out.writeUTF(desc.version.format.name());
+ out.writeUTF(desc.version.toString());
+ ByteBufferUtil.writeWithShortLength(desc.id.asBytes(), out);
+ }
+ }
+
+ @Override
+ public void deserializeMetadata(DataInputPlus in) throws IOException
+ {
+ super.deserializeMetadata(in);
+ Map<ColumnFamilyStore, Map<ImmutableTriple<SSTableId, String, SSTableFormat<?, ?>>, SSTableReader>> tmpReaders = new HashMap<>();
+ int sstablesNum = in.readUnsignedVInt32();
+ readers.clear();
+ readers.ensureCapacity(sstablesNum);
+ for (int i = 0; i < sstablesNum; i++)
+ {
+ ColumnFamilyStore cfs = readCFS(in);
+ String formatName = in.readUTF();
+ SSTableFormat<?, ?> format = Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(formatName), "Unknown SSTable format: " + formatName);
+ String version = in.readUTF();
+ SSTableId id = SSTableIdFactory.instance.fromBytes(ByteBufferUtil.readWithShortLength(in));
+
+ SSTableReader reader = null;
+ if (cfs != null)
+ {
+ Map<ImmutableTriple<SSTableId, String, SSTableFormat<?, ?>>, SSTableReader> readersMap = tmpReaders.get(cfs);
+ if (readersMap == null)
+ {
+ Set<SSTableReader> liveReaders = cfs.getLiveSSTables();
+ readersMap = new HashMap<>(liveReaders.size());
+ for (SSTableReader r : liveReaders)
+ readersMap.put(ImmutableTriple.of(r.descriptor.id, r.descriptor.version.toString(), r.descriptor.version.format), r);
+ tmpReaders.put(cfs, readersMap);
+ }
+ reader = readersMap.get(ImmutableTriple.of(id, version, format));
+ }
+ if (reader instanceof KeyCacheSupport<?>)
+ readers.add(Pair.create((KeyCacheSupport<?>) reader, format));
+ else
+ readers.add(Pair.create(null, format));
+ }
+ }
public void serialize(KeyCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
- RowIndexEntry entry = CacheService.instance.keyCache.getInternal(key);
+ AbstractRowIndexEntry entry = CacheService.instance.keyCache.getInternal(key);
if (entry == null)
return;
- TableMetadata tableMetadata = cfs.metadata();
- tableMetadata.id.serialize(out);
- out.writeUTF(tableMetadata.indexName().orElse(""));
- ByteArrayUtil.writeWithLength(key.key, out);
- if (key.desc.id instanceof SequenceBasedSSTableId)
- {
- out.writeInt(((SequenceBasedSSTableId) key.desc.id).generation);
- }
- else
- {
- out.writeInt(Integer.MIN_VALUE); // backwards compatibility for "int based generation only"
- ByteBufferUtil.writeWithShortLength(key.desc.id.asBytes(), out);
- }
- out.writeBoolean(true);
-
- SerializationHeader header = new SerializationHeader(false, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
- key.desc.getFormat().getIndexSerializer(cfs.metadata(), key.desc.version, header).serializeForCache(entry, out);
+ writeSSTable(cfs, key.desc, out);
+ out.writeInt(key.key.length);
+ out.write(key.key);
+ entry.serializeForCache(out);
}
- public Future<Pair<KeyCacheKey, RowIndexEntry>> deserialize(DataInputPlus input, ColumnFamilyStore cfs) throws IOException
+ public Future<Pair<KeyCacheKey, AbstractRowIndexEntry>> deserialize(DataInputPlus input) throws IOException
{
- boolean skipEntry = cfs == null || !cfs.isKeyCacheEnabled();
+ Pair<KeyCacheSupport<?>, SSTableFormat<?, ?>> reader = readSSTable(input);
+ boolean skipEntry = reader.left == null || !reader.left.getKeyCache().isEnabled();
- //Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a
- //parameter so they aren't deserialized here, even though they are serialized by this serializer
int keyLength = input.readInt();
if (keyLength > FBUtilities.MAX_UNSIGNED_SHORT)
- {
throw new IOException(String.format("Corrupted key cache. Key length of %d is longer than maximum of %d",
keyLength, FBUtilities.MAX_UNSIGNED_SHORT));
- }
ByteBuffer key = ByteBufferUtil.read(input, keyLength);
- int generation = input.readInt();
- SSTableId generationId = generation == Integer.MIN_VALUE
- ? SSTableIdFactory.instance.fromBytes(ByteBufferUtil.readWithShortLength(input))
- : new SequenceBasedSSTableId(generation); // Backwards compatibility for "int based generation sstables"
- input.readBoolean(); // backwards compatibility for "promoted indexes" boolean
- SSTableReader reader = null;
- if (!skipEntry)
- {
- Pair<String, String> qualifiedName = Pair.create(cfs.metadata.keyspace, cfs.metadata.name);
- Map<SSTableId, SSTableReader> generationToSSTableReader = cachedSSTableReaders.get(qualifiedName);
- if (generationToSSTableReader == null)
- {
- generationToSSTableReader = new HashMap<>(cfs.getLiveSSTables().size());
- for (SSTableReader ssTableReader : cfs.getSSTables(SSTableSet.CANONICAL))
- {
- generationToSSTableReader.put(ssTableReader.descriptor.id, ssTableReader);
- }
- cachedSSTableReaders.putIfAbsent(qualifiedName, generationToSSTableReader);
- }
- reader = generationToSSTableReader.get(generationId);
- }
-
- if (skipEntry || reader == null)
+ if (skipEntry)
{
// The sstable doesn't exist anymore, so we can't be sure of the exact version and assume its the current version. The only case where we'll be
// wrong is during upgrade, in which case we fail at deserialization. This is not a huge deal however since 1) this is unlikely enough that
// this won't affect many users (if any) and only once, 2) this doesn't prevent the node from starting and 3) CASSANDRA-10219 shows that this
// part of the code has been broken for a while without anyone noticing (it is, btw, still broken until CASSANDRA-10219 is fixed).
- RowIndexEntry.Serializer.skipForCache(input);
+ SSTableFormat.KeyCacheValueSerializer<?, ?> serializer = reader.right.getKeyCacheValueSerializer();
+
+ serializer.skip(input);
return null;
}
+ long pos = ((RandomAccessReader) input).getPosition();
+ AbstractRowIndexEntry cacheValue;
+ try
+ {
+ cacheValue = reader.left.deserializeKeyCacheValue(input);
+ } catch (RuntimeException | Error ex)
+ {
+ logger.error("Deserializing key cache entry at {} for {}", pos, reader.left);
+ throw ex;
+ }
+ KeyCacheKey cacheKey = reader.left.getCacheKey(key);
+ return ImmediateFuture.success(Pair.create(cacheKey, cacheValue));
+ }
- RowIndexEntry.IndexSerializer<?> indexSerializer = reader.descriptor.getFormat().getIndexSerializer(reader.metadata(),
- reader.descriptor.version,
- reader.header);
- RowIndexEntry<?> entry = indexSerializer.deserializeForCache(input);
- return ImmediateFuture.success(Pair.create(new KeyCacheKey(cfs.metadata(), reader.descriptor, key), entry));
+ private void writeSSTable(ColumnFamilyStore cfs, Descriptor desc, DataOutputPlus out) throws IOException
+ {
+ getOrCreateCFSOrdinal(cfs);
+ Pair<Integer, ColumnFamilyStore> existing = readerOrdinals.putIfAbsent(desc, Pair.create(readerOrdinals.size(), cfs));
+ int ordinal = existing == null ? readerOrdinals.size() - 1 : existing.left;
+ out.writeUnsignedVInt32(ordinal);
+ }
+
+ private Pair<KeyCacheSupport<?>, SSTableFormat<?, ?>> readSSTable(DataInputPlus input) throws IOException
+ {
+ int ordinal = input.readUnsignedVInt32();
+ if (ordinal >= readers.size())
+ throw new IOException("Corrupted key cache. Failed to deserialize key of key cache - invalid sstable ordinal " + ordinal);
+ return readers.get(ordinal);
}
public void cleanupAfterDeserialize()
{
- cachedSSTableReaders.clear();
+ super.cleanupAfterDeserialize();
+ readers.clear();
+ }
+
+ public void cleanupAfterSerialize()
+ {
+ super.cleanupAfterSerialize();
+ readerOrdinals.clear();
}
}
}
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index f1bca66..84fc5b1 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -87,15 +87,22 @@
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
+import org.apache.cassandra.utils.logging.LoggingSupportFactory;
+import org.apache.cassandra.utils.logging.VirtualTableAppender;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_FOREGROUND;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_REMOTE_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_PID_FILE;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_CLASS_PATH;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_RMI_SERVER_RANDOM_ID;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_VERSION;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_VM_NAME;
+import static org.apache.cassandra.config.CassandraRelevantProperties.METRICS_REPORTER_CONFIG_FILE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SIZE_RECORDER_INTERVAL;
+import static org.apache.cassandra.config.CassandraRelevantProperties.START_NATIVE_TRANSPORT;
/**
* The <code>CassandraDaemon</code> is an abstraction for a Cassandra daemon
@@ -149,7 +156,7 @@
return;
}
- System.setProperty("java.rmi.server.randomIDs", "true");
+ JAVA_RMI_SERVER_RANDOM_ID.setBoolean(true);
// If a remote port has been specified then use that to set up a JMX
// connector server which can be accessed remotely. Otherwise, look
@@ -165,7 +172,7 @@
if (jmxPort == null)
{
localOnly = true;
- jmxPort = System.getProperty("cassandra.jmx.local.port");
+ jmxPort = CASSANDRA_JMX_LOCAL_PORT.getString();
}
if (jmxPort == null)
@@ -247,7 +254,7 @@
ThreadAwareSecurityManager.install();
- logSystemInfo();
+ logSystemInfo(logger);
NativeLibrary.tryMlockall();
@@ -290,24 +297,13 @@
SSTableHeaderFix.fixNonFrozenUDTIfUpgradeFrom30();
- // clean up debris in the rest of the keyspaces
- for (String keyspaceName : Schema.instance.getKeyspaces())
+ try
{
- // Skip system as we've already cleaned it
- if (keyspaceName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
- continue;
-
- for (TableMetadata cfm : Schema.instance.getTablesAndViews(keyspaceName))
- {
- try
- {
- ColumnFamilyStore.scrubDataDirectories(cfm);
- }
- catch (StartupException e)
- {
- exitOrFail(e.returnCode, e.getMessage(), e.getCause());
- }
- }
+ scrubDataDirectories();
+ }
+ catch (StartupException e)
+ {
+ exitOrFail(e.returnCode, e.getMessage(), e.getCause());
}
Keyspace.setInitialized();
@@ -377,7 +373,7 @@
// schedule periodic dumps of table size estimates into SystemKeyspace.SIZE_ESTIMATES_CF
// set cassandra.size_recorder_interval to 0 to disable
- int sizeRecorderInterval = Integer.getInteger("cassandra.size_recorder_interval", 5 * 60);
+ int sizeRecorderInterval = SIZE_RECORDER_INTERVAL.getInt();
if (sizeRecorderInterval > 0)
ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, sizeRecorderInterval, TimeUnit.SECONDS);
@@ -388,7 +384,7 @@
QueryProcessor.instance.preloadPreparedStatements();
// Metrics
- String metricsReporterConfigFile = System.getProperty("cassandra.metricsReporterConfigFile");
+ String metricsReporterConfigFile = METRICS_REPORTER_CONFIG_FILE.getString();
if (metricsReporterConfigFile != null)
{
logger.info("Trying to load metrics-reporter-config from file: {}", metricsReporterConfigFile);
@@ -578,6 +574,28 @@
{
VirtualKeyspaceRegistry.instance.register(VirtualSchemaKeyspace.instance);
VirtualKeyspaceRegistry.instance.register(SystemViewsKeyspace.instance);
+
+ // flush log messages to system_views.system_logs virtual table as there were messages already logged
+ // before that virtual table was instantiated
+ LoggingSupportFactory.getLoggingSupport()
+ .getAppender(VirtualTableAppender.class, VirtualTableAppender.APPENDER_NAME)
+ .ifPresent(appender -> ((VirtualTableAppender) appender).flushBuffer());
+ }
+
+ public void scrubDataDirectories() throws StartupException
+ {
+ // clean up debris in the rest of the keyspaces
+ for (String keyspaceName : Schema.instance.getKeyspaces())
+ {
+ // Skip system as we've already cleaned it
+ if (keyspaceName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
+ continue;
+
+ for (TableMetadata cfm : Schema.instance.getTablesAndViews(keyspaceName))
+ {
+ ColumnFamilyStore.scrubDataDirectories(cfm);
+ }
+ }
}
public synchronized void initializeClientTransports()
@@ -614,7 +632,7 @@
return setupCompleted;
}
- private void logSystemInfo()
+ public static void logSystemInfo(Logger logger)
{
if (logger.isInfoEnabled())
{
@@ -685,8 +703,8 @@
private void startClientTransports()
{
- String nativeFlag = System.getProperty("cassandra.start_native_transport");
- if ((nativeFlag != null && Boolean.parseBoolean(nativeFlag)) || (nativeFlag == null && DatabaseDescriptor.startNativeTransport()))
+ String nativeFlag = START_NATIVE_TRANSPORT.getString();
+ if (START_NATIVE_TRANSPORT.getBoolean() || (nativeFlag == null && DatabaseDescriptor.startNativeTransport()))
{
startNativeTransport();
StorageService.instance.setRpcReady(true);
diff --git a/src/java/org/apache/cassandra/service/ClientState.java b/src/java/org/apache/cassandra/service/ClientState.java
index 9e35c7f..f03a971 100644
--- a/src/java/org/apache/cassandra/service/ClientState.java
+++ b/src/java/org/apache/cassandra/service/ClientState.java
@@ -58,6 +58,7 @@
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MD5Digest;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_QUERY_HANDLER_CLASS;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
@@ -101,13 +102,14 @@
static
{
QueryHandler handler = QueryProcessor.instance;
- String customHandlerClass = System.getProperty("cassandra.custom_query_handler_class");
+ String customHandlerClass = CUSTOM_QUERY_HANDLER_CLASS.getString();
if (customHandlerClass != null)
{
try
{
handler = FBUtilities.construct(customHandlerClass, "QueryHandler");
- logger.info("Using {} as query handler for native protocol queries (as requested with -Dcassandra.custom_query_handler_class)", customHandlerClass);
+ logger.info("Using {} as a query handler for native protocol queries (as requested by the {} system property)",
+ customHandlerClass, CUSTOM_QUERY_HANDLER_CLASS.getKey());
}
catch (Exception e)
{
@@ -414,6 +416,27 @@
ensurePermission(table.keyspace, perm, table.resource);
}
+ public boolean hasTablePermission(TableMetadata table, Permission perm)
+ {
+ if (isInternal)
+ return true;
+
+ validateLogin();
+
+ if (!DatabaseDescriptor.getAuthorizer().requireAuthorization())
+ return true;
+
+ List<? extends IResource> resources = Resources.chain(table.resource);
+ if (DatabaseDescriptor.getAuthFromRoot())
+ resources = Lists.reverse(resources);
+
+ for (IResource r : resources)
+ if (authorize(r).contains(perm))
+ return true;
+
+ return false;
+ }
+
private void ensurePermission(String keyspace, Permission perm, DataResource resource)
{
validateKeyspace(keyspace);
diff --git a/src/java/org/apache/cassandra/service/DataResurrectionCheck.java b/src/java/org/apache/cassandra/service/DataResurrectionCheck.java
index 2c3b035..4cf3278 100644
--- a/src/java/org/apache/cassandra/service/DataResurrectionCheck.java
+++ b/src/java/org/apache/cassandra/service/DataResurrectionCheck.java
@@ -45,7 +45,7 @@
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JsonUtils;
import org.apache.cassandra.utils.Pair;
import static java.lang.String.format;
@@ -86,12 +86,12 @@
public void serializeToJsonFile(File outputFile) throws IOException
{
- FBUtilities.serializeToJsonFile(this, outputFile);
+ JsonUtils.serializeToJsonFile(this, outputFile);
}
public static Heartbeat deserializeFromJsonFile(File file) throws IOException
{
- return FBUtilities.deserializeFromJsonFile(Heartbeat.class, file);
+ return JsonUtils.deserializeFromJsonFile(Heartbeat.class, file);
}
@Override
diff --git a/src/java/org/apache/cassandra/service/LoadBroadcaster.java b/src/java/org/apache/cassandra/service/LoadBroadcaster.java
index ebda3cd..da93ebd 100644
--- a/src/java/org/apache/cassandra/service/LoadBroadcaster.java
+++ b/src/java/org/apache/cassandra/service/LoadBroadcaster.java
@@ -30,9 +30,11 @@
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.gms.*;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BROADCAST_INTERVAL_MS;
+
public class LoadBroadcaster implements IEndpointStateChangeSubscriber
{
- static final int BROADCAST_INTERVAL = Integer.getInteger("cassandra.broadcast_interval_ms", 60 * 1000);
+ static final int BROADCAST_INTERVAL = BROADCAST_INTERVAL_MS.getInt();
public static final LoadBroadcaster instance = new LoadBroadcaster();
diff --git a/src/java/org/apache/cassandra/service/NativeTransportService.java b/src/java/org/apache/cassandra/service/NativeTransportService.java
index f131d74..cc6ee37 100644
--- a/src/java/org/apache/cassandra/service/NativeTransportService.java
+++ b/src/java/org/apache/cassandra/service/NativeTransportService.java
@@ -40,6 +40,8 @@
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.NativeLibrary;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NATIVE_EPOLL_ENABLED;
+
/**
* Handles native transport server lifecycle and associated resources. Lazily initialized.
*/
@@ -160,7 +162,7 @@
*/
public static boolean useEpoll()
{
- final boolean enableEpoll = Boolean.parseBoolean(System.getProperty("cassandra.native.epoll.enabled", "true"));
+ final boolean enableEpoll = NATIVE_EPOLL_ENABLED.getBoolean();
if (enableEpoll && !Epoll.isAvailable() && NativeLibrary.osType == NativeLibrary.OSType.LINUX)
logger.warn("epoll not available", Epoll.unavailabilityCause());
diff --git a/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java b/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java
index de78892..6a7baf3 100644
--- a/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java
+++ b/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java
@@ -36,7 +36,7 @@
import org.apache.cassandra.db.lifecycle.Tracker;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.VersionAndType;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.notifications.INotification;
import org.apache.cassandra.notifications.INotificationConsumer;
import org.apache.cassandra.notifications.InitialSSTableAddedNotification;
@@ -78,23 +78,23 @@
private final Set<Descriptor> allSSTables = ConcurrentHashMap.newKeySet();
- private final VersionAndType currentVersion;
+ private final Version currentVersion;
private int sstablesForCurrentVersion;
- private final Map<VersionAndType, Integer> sstablesForOtherVersions = new HashMap<>();
+ private final Map<Version, Integer> sstablesForOtherVersions = new HashMap<>();
- private volatile ImmutableSet<VersionAndType> versionsInUse = ImmutableSet.of();
+ private volatile ImmutableSet<Version> versionsInUse = ImmutableSet.of();
private final Set<INotificationConsumer> subscribers = new CopyOnWriteArraySet<>();
- public SSTablesGlobalTracker(SSTableFormat.Type currentSSTableFormat)
+ public SSTablesGlobalTracker(SSTableFormat<?, ?> currentSSTableFormat)
{
- this.currentVersion = new VersionAndType(currentSSTableFormat.info.getLatestVersion(), currentSSTableFormat);
+ this.currentVersion = currentSSTableFormat.getLatestVersion();
}
/**
* The set of all sstable versions currently in use on this node.
*/
- public Set<VersionAndType> versionsInUse()
+ public Set<Version> versionsInUse()
{
return versionsInUse;
}
@@ -151,7 +151,7 @@
synchronized block.
*/
int currentDelta = 0;
- Map<VersionAndType, Integer> othersDelta = null;
+ Map<Version, Integer> othersDelta = null;
/*
Note: we deal with removes first as if a notification both removes and adds, it's a compaction and while
it should never remove and add the same descriptor in practice, doing the remove first is more logical.
@@ -161,7 +161,7 @@
if (!allSSTables.remove(desc))
continue;
- VersionAndType version = version(desc);
+ Version version = desc.version;
if (currentVersion.equals(version))
--currentDelta;
else
@@ -172,7 +172,7 @@
if (!allSSTables.add(desc))
continue;
- VersionAndType version = version(desc);
+ Version version = desc.version;
if (currentVersion.equals(version))
++currentDelta;
else
@@ -196,9 +196,9 @@
if (othersDelta != null)
{
- for (Map.Entry<VersionAndType, Integer> entry : othersDelta.entrySet())
+ for (Map.Entry<Version, Integer> entry : othersDelta.entrySet())
{
- VersionAndType version = entry.getKey();
+ Version version = entry.getKey();
int delta = entry.getValue();
/*
Updates the count, removing the version if it reaches 0 (note: we could use Map#compute for this,
@@ -221,16 +221,16 @@
return triggerUpdate;
}
- private static ImmutableSet<VersionAndType> computeVersionsInUse(int sstablesForCurrentVersion, VersionAndType currentVersion, Map<VersionAndType, Integer> sstablesForOtherVersions)
+ private static ImmutableSet<Version> computeVersionsInUse(int sstablesForCurrentVersion, Version currentVersion, Map<Version, Integer> sstablesForOtherVersions)
{
- ImmutableSet.Builder<VersionAndType> builder = ImmutableSet.builder();
+ ImmutableSet.Builder<Version> builder = ImmutableSet.builder();
if (sstablesForCurrentVersion > 0)
builder.add(currentVersion);
builder.addAll(sstablesForOtherVersions.keySet());
return builder.build();
}
- private static int sanitizeSSTablesCount(int sstableCount, VersionAndType version)
+ private static int sanitizeSSTablesCount(int sstableCount, Version version)
{
if (sstableCount >= 0)
return sstableCount;
@@ -241,7 +241,7 @@
*/
noSpamLogger.error("Invalid state while handling sstables change notification: the number of sstables for " +
"version {} was computed to {}. This indicate a bug and please report it, but it should " +
- "not have adverse consequences.", version, sstableCount, new RuntimeException());
+ "not have adverse consequences.", version.toFormatAndVersionString(), sstableCount, new RuntimeException());
return 0;
}
@@ -267,18 +267,13 @@
return Collections.emptyList();
}
- private static Map<VersionAndType, Integer> update(Map<VersionAndType, Integer> counts,
- VersionAndType toUpdate,
+ private static Map<Version, Integer> update(Map<Version, Integer> counts,
+ Version toUpdate,
int delta)
{
- Map<VersionAndType, Integer> m = counts == null ? new HashMap<>() : counts;
+ Map<Version, Integer> m = counts == null ? new HashMap<>() : counts;
m.merge(toUpdate, delta, (a, b) -> (a + b == 0) ? null : (a + b));
return m;
}
- @VisibleForTesting
- static VersionAndType version(Descriptor sstable)
- {
- return new VersionAndType(sstable.version, sstable.formatType);
- }
}
diff --git a/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java b/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java
index 352bec7..08f67a4 100644
--- a/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java
+++ b/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java
@@ -18,9 +18,11 @@
package org.apache.cassandra.service;
+import java.util.stream.Collectors;
+
import com.google.common.collect.ImmutableSet;
-import org.apache.cassandra.io.sstable.format.VersionAndType;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.notifications.INotification;
/**
@@ -35,9 +37,9 @@
/**
* The set of all sstable versions in use on this node at the time of this notification.
*/
- public final ImmutableSet<VersionAndType> versionsInUse;
+ public final ImmutableSet<Version> versionsInUse;
- SSTablesVersionsInUseChangeNotification(ImmutableSet<VersionAndType> versionsInUse)
+ SSTablesVersionsInUseChangeNotification(ImmutableSet<Version> versionsInUse)
{
this.versionsInUse = versionsInUse;
}
@@ -45,6 +47,6 @@
@Override
public String toString()
{
- return String.format("SSTablesInUseChangeNotification(%s)", versionsInUse);
+ return String.format("SSTablesInUseChangeNotification(%s)", versionsInUse.stream().map(Version::toFormatAndVersionString).collect(Collectors.toList()));
}
}
diff --git a/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java b/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
index 850c982..99b5105 100644
--- a/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
+++ b/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
@@ -17,16 +17,23 @@
*/
package org.apache.cassandra.service;
+import java.util.Collections;
+import java.util.List;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.SnapshotCommand;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SnapshotCommand;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
+import static org.apache.cassandra.net.ParamType.SNAPSHOT_RANGES;
+
public class SnapshotVerbHandler implements IVerbHandler<SnapshotCommand>
{
public static final SnapshotVerbHandler instance = new SnapshotVerbHandler();
@@ -37,11 +44,14 @@
SnapshotCommand command = message.payload;
if (command.clear_snapshot)
{
- Keyspace.clearSnapshot(command.snapshot_name, command.keyspace);
+ StorageService.instance.clearSnapshot(command.snapshot_name, command.keyspace);
}
else if (DiagnosticSnapshotService.isDiagnosticSnapshotRequest(command))
{
- DiagnosticSnapshotService.snapshot(command, message.from());
+ List<Range<Token>> ranges = Collections.emptyList();
+ if (message.header.params().containsKey(SNAPSHOT_RANGES))
+ ranges = (List<Range<Token>>) message.header.params().get(SNAPSHOT_RANGES);
+ DiagnosticSnapshotService.snapshot(command, ranges, message.from());
}
else
{
diff --git a/src/java/org/apache/cassandra/service/StartupCheck.java b/src/java/org/apache/cassandra/service/StartupCheck.java
index 331b381..c3790e8 100644
--- a/src/java/org/apache/cassandra/service/StartupCheck.java
+++ b/src/java/org/apache/cassandra/service/StartupCheck.java
@@ -31,7 +31,7 @@
* misconfiguration of cluster_name in cassandra.yaml.
*
* The StartupChecks class manages a collection of these tests, which it executes
- * right at the beginning of the server settup process.
+ * right at the beginning of the server setup process.
*/
public interface StartupCheck
{
diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java
index 0aacc02..d04d559 100644
--- a/src/java/org/apache/cassandra/service/StartupChecks.java
+++ b/src/java/org/apache/cassandra/service/StartupChecks.java
@@ -73,6 +73,7 @@
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.SigarLibrary;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_VERSION;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_VM_NAME;
@@ -164,7 +165,7 @@
/**
* Run the configured tests and return a report detailing the results.
- * @throws org.apache.cassandra.exceptions.StartupException if any test determines that the
+ * @throws StartupException if any test determines that the
* system is not in an valid state to startup
* @param options options to pass to respective checks for their configration
*/
@@ -257,7 +258,7 @@
logger.warn("JMX is not enabled to receive remote connections. Please see cassandra-env.sh for more info.");
jmxPort = CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT.toString();
if (jmxPort == null)
- logger.error("cassandra.jmx.local.port missing from cassandra-env.sh, unable to start local JMX service.");
+ logger.error(CASSANDRA_JMX_LOCAL_PORT.getKey() + " missing from cassandra-env.sh, unable to start local JMX service.");
}
else
{
@@ -554,7 +555,7 @@
try
{
- Descriptor desc = Descriptor.fromFilename(file);
+ Descriptor desc = Descriptor.fromFileWithComponent(file, false).left;
if (!desc.isCompatible())
invalid.add(file.toString());
@@ -570,6 +571,40 @@
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException
{
+ String[] nameParts = FileUtils.getCanonicalPath(new File(dir)).split(java.io.File.separator);
+ if (nameParts.length >= 2)
+ {
+ String tablePart = nameParts[nameParts.length - 1];
+ String ksPart = nameParts[nameParts.length - 2];
+
+ if (tablePart.contains("-"))
+ tablePart = tablePart.split("-")[0];
+
+ // In very old versions of Cassandra, we wouldn't necessarily delete sstables from dropped system tables
+ // which were removed in various major version upgrades (e.g system.Versions in 1.2)
+ if (ksPart.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) && !SystemKeyspace.ALL_TABLE_NAMES.contains(tablePart))
+ {
+ String canonicalPath = FileUtils.getCanonicalPath(new File(dir));
+
+ // We can have snapshots of our system tables or snapshots created with a -t tag of "system" that would trigger
+ // this potential warning, so we warn more softly in the case that it's probably a snapshot.
+ if (canonicalPath.contains("snapshot"))
+ {
+ logger.info("Found unknown system directory {}.{} at {} that contains the word snapshot. " +
+ "This may be left over from a previous version of Cassandra or may be normal. " +
+ " Consider removing after inspection if determined to be unnecessary.",
+ ksPart, tablePart, canonicalPath);
+ }
+ else
+ {
+ logger.warn("Found unknown system directory {}.{} at {} - this is likely left over from a previous " +
+ "version of Cassandra and should be removed after inspection.",
+ ksPart, tablePart, canonicalPath);
+ }
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ }
+
String name = dir.getFileName().toString();
return (name.equals(Directories.SNAPSHOT_SUBDIR)
|| name.equals(Directories.BACKUPS_SUBDIR)
@@ -647,7 +682,7 @@
logger.warn(String.format("Cassandra system property flag %s is deprecated and you should " +
"use startup check configuration in cassandra.yaml",
CassandraRelevantProperties.IGNORE_DC.getKey()));
- enabled = !Boolean.getBoolean(CassandraRelevantProperties.IGNORE_DC.getKey());
+ enabled = !CassandraRelevantProperties.IGNORE_DC.getBoolean();
}
if (enabled)
{
@@ -684,7 +719,7 @@
logger.warn(String.format("Cassandra system property flag %s is deprecated and you should " +
"use startup check configuration in cassandra.yaml",
CassandraRelevantProperties.IGNORE_RACK.getKey()));
- enabled = !Boolean.getBoolean(CassandraRelevantProperties.IGNORE_RACK.getKey());
+ enabled = !CassandraRelevantProperties.IGNORE_RACK.getBoolean();
}
if (enabled)
{
diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java
index 2a0ccab..1bb40ad 100644
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@ -44,19 +44,15 @@
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.service.paxos.*;
-import org.apache.cassandra.service.paxos.Paxos;
-import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.concurrent.CountDownLatch;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.batchlog.Batch;
import org.apache.cassandra.batchlog.BatchlogManager;
+import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
@@ -88,6 +84,7 @@
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.IsBootstrappingException;
import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.exceptions.ReadAbortException;
import org.apache.cassandra.exceptions.ReadFailureException;
import org.apache.cassandra.exceptions.ReadTimeoutException;
@@ -102,6 +99,7 @@
import org.apache.cassandra.hints.Hint;
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.DynamicEndpointSnitch;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -111,6 +109,7 @@
import org.apache.cassandra.locator.ReplicaPlans;
import org.apache.cassandra.locator.Replicas;
import org.apache.cassandra.metrics.CASClientRequestMetrics;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
import org.apache.cassandra.metrics.DenylistMetrics;
import org.apache.cassandra.metrics.ReadRepairMetrics;
import org.apache.cassandra.metrics.StorageMetrics;
@@ -125,6 +124,11 @@
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.paxos.Ballot;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.ContentionStrategy;
+import org.apache.cassandra.service.paxos.Paxos;
+import org.apache.cassandra.service.paxos.PaxosState;
import org.apache.cassandra.service.paxos.v1.PrepareCallback;
import org.apache.cassandra.service.paxos.v1.ProposeCallback;
import org.apache.cassandra.service.reads.AbstractReadExecutor;
@@ -139,13 +143,17 @@
import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
-import static com.google.common.collect.Iterables.concat;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+import static com.google.common.collect.Iterables.concat;
+import static org.apache.commons.lang3.StringUtils.join;
+
import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
-import static org.apache.cassandra.net.Message.out;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
@@ -153,8 +161,15 @@
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.viewWriteMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetricsForLevel;
+import static org.apache.cassandra.net.Message.out;
import static org.apache.cassandra.net.NoPayload.noPayload;
-import static org.apache.cassandra.net.Verb.*;
+import static org.apache.cassandra.net.Verb.BATCH_STORE_REQ;
+import static org.apache.cassandra.net.Verb.MUTATION_REQ;
+import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_REQ;
+import static org.apache.cassandra.net.Verb.PAXOS_PREPARE_REQ;
+import static org.apache.cassandra.net.Verb.PAXOS_PROPOSE_REQ;
+import static org.apache.cassandra.net.Verb.SCHEMA_VERSION_REQ;
+import static org.apache.cassandra.net.Verb.TRUNCATE_REQ;
import static org.apache.cassandra.service.BatchlogResponseHandler.BatchlogCleanup;
import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL;
@@ -165,7 +180,6 @@
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
-import static org.apache.commons.lang3.StringUtils.join;
public class StorageProxy implements StorageProxyMBean
{
@@ -350,6 +364,9 @@
// Create the desired updates
PartitionUpdate updates = request.makeUpdates(current, clientState, ballot);
+ // Update the metrics before triggers potentially add mutations.
+ ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(updates);
+
long size = updates.dataSize();
casWriteMetrics.mutationSize.update(size);
writeMetricsForLevel(consistencyForPaxos).mutationSize.update(size);
@@ -829,6 +846,12 @@
}
@Override
+ public String description()
+ {
+ return "Paxos " + message.payload.toString();
+ }
+
+ @Override
protected Verb verb()
{
return PAXOS_COMMIT_REQ;
@@ -1263,7 +1286,7 @@
logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, replica, batch.size());
if (replica.isSelf())
- performLocally(Stage.MUTATION, replica, () -> BatchlogManager.store(batch), handler);
+ performLocally(Stage.MUTATION, replica, () -> BatchlogManager.store(batch), handler, "Batchlog store");
else
MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
}
@@ -1279,7 +1302,7 @@
logger.trace("Sending batchlog remove request {} to {}", uuid, target);
if (target.isSelf())
- performLocally(Stage.MUTATION, target, () -> BatchlogManager.remove(uuid));
+ performLocally(Stage.MUTATION, target, () -> BatchlogManager.remove(uuid), "Batchlog remove");
else
MessagingService.instance().send(message, target.endpoint());
}
@@ -1457,6 +1480,15 @@
List<InetAddressAndPort> backPressureHosts = null;
+ // For performance, Mutation caches serialized buffers that are computed lazily in serializedBuffer(). That
+ // computation is not synchronized however and we will potentially call that method concurrently for each
+ // dispatched message (not that concurrent calls to serializedBuffer() are "unsafe" per se, just that they
+ // may result in multiple computations, making the caching optimization moot). So forcing the serialization
+ // here to make sure it's already cached/computed when it's concurrently used later.
+ // Side note: we have one cached buffers for each used EncodingVersion and this only pre-compute the one for
+ // the current version, but it's just an optimization and we're ok not optimizing for mixed-version clusters.
+ Mutation.serializer.prepareSerializedBuffer(mutation, MessagingService.current_version);
+
for (Replica destination : plan.contacts())
{
checkHintOverload(destination);
@@ -1523,7 +1555,7 @@
if (insertLocal)
{
Preconditions.checkNotNull(localReplica);
- performLocally(stage, localReplica, mutation::apply, responseHandler);
+ performLocally(stage, localReplica, mutation::apply, responseHandler, mutation);
}
if (localDc != null)
@@ -1566,7 +1598,7 @@
if (targets.size() > 1)
{
- target = targets.get(ThreadLocalRandom.current().nextInt(0, targets.size()));
+ target = pickReplica(targets);
EndpointsForToken forwardToReplicas = targets.filter(r -> r != target, targets.size());
for (Replica replica : forwardToReplicas)
@@ -1586,11 +1618,19 @@
target = targets.get(0);
}
+ Tracing.trace("Sending mutation to remote replica {}", target);
MessagingService.instance().sendWriteWithCallback(message, target, handler);
logger.trace("Sending message to {}@{}", message.id(), target);
}
- private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable)
+ private static Replica pickReplica(EndpointsForToken targets)
+ {
+ EndpointsForToken healthy = targets.filter(r -> DynamicEndpointSnitch.getSeverity(r.endpoint()) == 0);
+ EndpointsForToken select = healthy.isEmpty() ? targets : healthy;
+ return select.get(ThreadLocalRandom.current().nextInt(0, select.size()));
+ }
+
+ private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, String description)
{
stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
{
@@ -1607,6 +1647,12 @@
}
@Override
+ public String description()
+ {
+ return description;
+ }
+
+ @Override
protected Verb verb()
{
return Verb.MUTATION_REQ;
@@ -1614,7 +1660,7 @@
});
}
- private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, final RequestCallback<?> handler)
+ private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, final RequestCallback<?> handler, Object description)
{
stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
{
@@ -1634,6 +1680,14 @@
}
@Override
+ public String description()
+ {
+ // description is an Object and toString() called so we do not have to evaluate the Mutation.toString()
+ // unless expliclitly checked
+ return description.toString();
+ }
+
+ @Override
protected Verb verb()
{
return Verb.MUTATION_REQ;
@@ -1673,8 +1727,8 @@
// we build this ONLY to perform the sufficiency check that happens on construction
ReplicaPlans.forWrite(keyspace, cm.consistency(), tk, ReplicaPlans.writeAll);
- // This host isn't a replica, so mark the request as being remote. If this host is a
- // replica, applyCounterMutationOnCoordinator() in the branch above will call performWrite(), and
+ // This host isn't a replica, so mark the request as being remote. If this host is a
+ // replica, applyCounterMutationOnCoordinator() in the branch above will call performWrite(), and
// there we'll mark a local request against the metrics.
writeMetrics.remoteRequests.mark();
@@ -1792,7 +1846,7 @@
public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException
{
- if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(group.queries))
+ if (!isSafeToPerformRead(group.queries))
{
readMetrics.unavailables.mark();
readMetricsForLevel(consistencyLevel).unavailables.mark();
@@ -1819,6 +1873,16 @@
: readRegular(group, consistencyLevel, queryStartNanoTime);
}
+ public static boolean isSafeToPerformRead(List<SinglePartitionReadCommand> queries)
+ {
+ return isSafeToPerformRead() || systemKeyspaceQuery(queries);
+ }
+
+ public static boolean isSafeToPerformRead()
+ {
+ return !StorageService.instance.isBootstrapMode();
+ }
+
private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
{
@@ -2087,7 +2151,7 @@
return concatAndBlockOnRepair(results, repairs);
}
- public static class LocalReadRunnable extends DroppableRunnable
+ public static class LocalReadRunnable extends DroppableRunnable implements RunnableDebuggableTask
{
private final ReadCommand command;
private final ReadCallback handler;
@@ -2129,6 +2193,12 @@
response = command.createEmptyResponse();
readRejected = true;
}
+ catch (QueryCancelledException e)
+ {
+ logger.debug("Query cancelled (timeout)", e);
+ response = null;
+ assert !command.isCompleted() : "Local read marked as completed despite being aborted by timeout to table " + command.metadata();
+ }
if (command.complete())
{
@@ -2157,6 +2227,24 @@
}
}
}
+
+ @Override
+ public long creationTimeNanos()
+ {
+ return approxCreationTimeNanos;
+ }
+
+ @Override
+ public long startTimeNanos()
+ {
+ return approxStartTimeNanos;
+ }
+
+ @Override
+ public String description()
+ {
+ return command.toCQLString();
+ }
}
public static PartitionIterator getRangeSlice(PartitionRangeReadCommand command,
@@ -2467,7 +2555,9 @@
*/
private static abstract class DroppableRunnable implements Runnable
{
- final long approxCreationTimeNanos;
+ protected final long approxCreationTimeNanos;
+ protected volatile long approxStartTimeNanos;
+
final Verb verb;
public DroppableRunnable(Verb verb)
@@ -2478,11 +2568,11 @@
public final void run()
{
- long approxCurrentTimeNanos = MonotonicClock.Global.approxTime.now();
+ approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
long expirationTimeNanos = verb.expiresAtNanos(approxCreationTimeNanos);
- if (approxCurrentTimeNanos > expirationTimeNanos)
+ if (approxStartTimeNanos > expirationTimeNanos)
{
- long timeTakenNanos = approxCurrentTimeNanos - approxCreationTimeNanos;
+ long timeTakenNanos = approxStartTimeNanos - approxCreationTimeNanos;
MessagingService.instance().metrics.recordSelfDroppedMessage(verb, timeTakenNanos, NANOSECONDS);
return;
}
@@ -2503,9 +2593,10 @@
* Like DroppableRunnable, but if it aborts, it will rerun (on the mutation stage) after
* marking itself as a hint in progress so that the hint backpressure mechanism can function.
*/
- private static abstract class LocalMutationRunnable implements Runnable
+ private static abstract class LocalMutationRunnable implements RunnableDebuggableTask
{
private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
+ private volatile long approxStartTimeNanos;
private final Replica localReplica;
@@ -2517,11 +2608,12 @@
public final void run()
{
final Verb verb = verb();
- long nowNanos = MonotonicClock.Global.approxTime.now();
+ approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
long expirationTimeNanos = verb.expiresAtNanos(approxCreationTimeNanos);
- if (nowNanos > expirationTimeNanos)
+
+ if (approxStartTimeNanos > expirationTimeNanos)
{
- long timeTakenNanos = nowNanos - approxCreationTimeNanos;
+ long timeTakenNanos = approxStartTimeNanos - approxCreationTimeNanos;
MessagingService.instance().metrics.recordSelfDroppedMessage(Verb.MUTATION_REQ, timeTakenNanos, NANOSECONDS);
HintRunnable runnable = new HintRunnable(EndpointsForToken.of(localReplica.range().right, localReplica))
@@ -2545,14 +2637,34 @@
}
}
+ @Override
+ public long creationTimeNanos()
+ {
+ return approxCreationTimeNanos;
+ }
+
+ @Override
+ public long startTimeNanos()
+ {
+ return approxStartTimeNanos;
+ }
+
+ @Override
+ abstract public String description();
+
abstract protected Verb verb();
abstract protected void runMayThrow() throws Exception;
}
public static void logRequestException(Exception exception, Collection<? extends ReadCommand> commands)
{
+ // Multiple different types of errors can happen, so by dedupping on the error type we can see each error
+ // case rather than just exposing the first error seen; this should make sure more rare issues are exposed
+ // rather than being hidden by more common errors such as timeout or unavailable
+ // see CASSANDRA-17754
+ String msg = exception.getClass().getSimpleName() + " \"{}\" while executing {}";
NoSpamLogger.log(logger, NoSpamLogger.Level.INFO, FAILURE_LOGGING_INTERVAL_SECONDS, TimeUnit.SECONDS,
- "\"{}\" while executing {}",
+ msg,
() -> new Object[]
{
exception.getMessage(),
@@ -3084,4 +3196,40 @@
{
return PaxosState.getDisableCoordinatorLocking();
}
+
+ @Override
+ public boolean getDumpHeapOnUncaughtException()
+ {
+ return DatabaseDescriptor.getDumpHeapOnUncaughtException();
+ }
+
+ @Override
+ public void setDumpHeapOnUncaughtException(boolean enabled)
+ {
+ DatabaseDescriptor.setDumpHeapOnUncaughtException(enabled);
+ }
+
+ @Override
+ public boolean getSStableReadRatePersistenceEnabled()
+ {
+ return DatabaseDescriptor.getSStableReadRatePersistenceEnabled();
+ }
+
+ @Override
+ public void setSStableReadRatePersistenceEnabled(boolean enabled)
+ {
+ DatabaseDescriptor.setSStableReadRatePersistenceEnabled(enabled);
+ }
+
+ @Override
+ public boolean getClientRequestSizeMetricsEnabled()
+ {
+ return DatabaseDescriptor.getClientRequestSizeMetricsEnabled();
+ }
+
+ @Override
+ public void setClientRequestSizeMetricsEnabled(boolean enabled)
+ {
+ DatabaseDescriptor.setClientRequestSizeMetricsEnabled(enabled);
+ }
}
diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java
index 546143d..4ba41d6 100644
--- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java
@@ -135,4 +135,13 @@
void setPaxosCoordinatorLockingDisabled(boolean disabled);
boolean getPaxosCoordinatorLockingDisabled();
+
+ public boolean getDumpHeapOnUncaughtException();
+ public void setDumpHeapOnUncaughtException(boolean enabled);
+
+ boolean getSStableReadRatePersistenceEnabled();
+ void setSStableReadRatePersistenceEnabled(boolean enabled);
+
+ boolean getClientRequestSizeMetricsEnabled();
+ void setClientRequestSizeMetricsEnabled(boolean enabled);
}
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index ed10f04..1d4a0b1 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.service;
-
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOError;
@@ -27,11 +26,13 @@
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.time.Instant;
+import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -61,7 +62,10 @@
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
+import javax.management.ListenerNotFoundException;
import javax.management.NotificationBroadcasterSupport;
+import javax.management.NotificationFilter;
+import javax.management.NotificationListener;
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.TabularData;
@@ -116,7 +120,7 @@
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.Verifier;
+import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.dht.BootStrapper;
@@ -142,9 +146,10 @@
import org.apache.cassandra.gms.TokenSerializer;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.hints.HintsService;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.IVerifier;
import org.apache.cassandra.io.sstable.SSTableLoader;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.VersionAndType;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.PathUtils;
@@ -165,6 +170,8 @@
import org.apache.cassandra.locator.Replicas;
import org.apache.cassandra.locator.SystemReplicas;
import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.metrics.Sampler;
+import org.apache.cassandra.metrics.SamplingManager;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.AsyncOneResponse;
import org.apache.cassandra.net.Message;
@@ -200,6 +207,7 @@
import org.apache.cassandra.tracing.TraceKeyspace;
import org.apache.cassandra.transport.ClientResourceLimits;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -220,6 +228,7 @@
import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
import org.apache.cassandra.utils.progress.jmx.JMXProgressSupport;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.collect.Iterables.tryFind;
import static java.util.Arrays.asList;
@@ -230,10 +239,24 @@
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_JOIN;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_REPLACE;
import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SCHEMA_DELAY_MS;
import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_RANGE_MOVEMENT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_SIMULTANEOUS_MOVES_ALLOW;
import static org.apache.cassandra.config.CassandraRelevantProperties.DRAIN_EXECUTOR_TIMEOUT_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOAD_RING_STATE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.OVERRIDE_DECOMMISSION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRIES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRY_DELAY_SECONDS;
import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACEMENT_ALLOW_EMPTY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS_FIRST_BOOT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.START_GOSSIP;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_WRITE_SURVEY;
import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName;
import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily;
import static org.apache.cassandra.net.NoPayload.noPayload;
@@ -301,10 +324,15 @@
private final List<Runnable> preShutdownHooks = new ArrayList<>();
private final List<Runnable> postShutdownHooks = new ArrayList<>();
- private final SnapshotManager snapshotManager = new SnapshotManager();
+ public final SnapshotManager snapshotManager = new SnapshotManager();
public static final StorageService instance = new StorageService();
+ private final SamplingManager samplingManager = new SamplingManager();
+
+ @VisibleForTesting // this is used for dtests only, see CASSANDRA-18152
+ public volatile boolean skipNotificationListeners = false;
+
@Deprecated
public boolean isInShutdownHook()
{
@@ -372,8 +400,7 @@
private volatile boolean isBootstrapMode;
/* we bootstrap but do NOT join the ring unless told to do so */
- private boolean isSurveyMode = Boolean.parseBoolean(System.getProperty
- ("cassandra.write_survey", "false"));
+ private boolean isSurveyMode = TEST_WRITE_SURVEY.getBoolean(false);
/* true if node is rebuilding and receiving data */
private final AtomicBoolean isRebuilding = new AtomicBoolean();
private final AtomicBoolean isDecommissioning = new AtomicBoolean();
@@ -387,7 +414,7 @@
/* the probability for tracing any particular request, 0 disables tracing and 1 enables for all */
private double traceProbability = 0.0;
- private enum Mode { STARTING, NORMAL, JOINING, LEAVING, DECOMMISSIONED, MOVING, DRAINING, DRAINED }
+ public enum Mode { STARTING, NORMAL, JOINING, LEAVING, DECOMMISSIONED, MOVING, DRAINING, DRAINED }
private volatile Mode operationMode = Mode.STARTING;
/* Used for tracking drain progress */
@@ -402,9 +429,9 @@
private Collection<Token> bootstrapTokens = null;
// true when keeping strict consistency while bootstrapping
- public static final boolean useStrictConsistency = Boolean.parseBoolean(System.getProperty("cassandra.consistent.rangemovement", "true"));
- private static final boolean allowSimultaneousMoves = Boolean.parseBoolean(System.getProperty("cassandra.consistent.simultaneousmoves.allow","false"));
- private static final boolean joinRing = Boolean.parseBoolean(System.getProperty("cassandra.join_ring", "true"));
+ public static final boolean useStrictConsistency = CONSISTENT_RANGE_MOVEMENT.getBoolean();
+ private static final boolean allowSimultaneousMoves = CONSISTENT_SIMULTANEOUS_MOVES_ALLOW.getBoolean();
+ private static final boolean joinRing = JOIN_RING.getBoolean();
private boolean replacing;
private final StreamStateStore streamStateStore = new StreamStateStore();
@@ -452,10 +479,14 @@
super(JMXBroadcastExecutor.executor);
jmxObjectName = "org.apache.cassandra.db:type=StorageService";
+
+ sstablesTracker = new SSTablesGlobalTracker(DatabaseDescriptor.getSelectedSSTableFormat());
+ }
+
+ private void registerMBeans()
+ {
MBeanWrapper.instance.registerMBean(this, jmxObjectName);
MBeanWrapper.instance.registerMBean(StreamManager.instance, StreamManager.OBJECT_NAME);
-
- sstablesTracker = new SSTablesGlobalTracker(SSTableFormat.Type.current());
}
public void registerDaemon(CassandraDaemon daemon)
@@ -643,11 +674,11 @@
if (!joinRing)
throw new ConfigurationException("Cannot set both join_ring=false and attempt to replace a node");
- if (!shouldBootstrap() && !Boolean.getBoolean("cassandra.allow_unsafe_replace"))
+ if (!shouldBootstrap() && !ALLOW_UNSAFE_REPLACE.getBoolean())
throw new RuntimeException("Replacing a node without bootstrapping risks invalidating consistency " +
"guarantees as the expected data may not be present until repair is run. " +
"To perform this operation, please restart with " +
- "-Dcassandra.allow_unsafe_replace=true");
+ "-D" + ALLOW_UNSAFE_REPLACE.getKey() + "=true");
InetAddressAndPort replaceAddress = DatabaseDescriptor.getReplaceAddress();
logger.info("Gathering node replacement information for {}", replaceAddress);
@@ -763,9 +794,9 @@
public synchronized void checkForEndpointCollision(UUID localHostId, Set<InetAddressAndPort> peers) throws ConfigurationException
{
- if (Boolean.getBoolean("cassandra.allow_unsafe_join"))
+ if (ALLOW_UNSAFE_JOIN.getBoolean())
{
- logger.warn("Skipping endpoint collision check as cassandra.allow_unsafe_join=true");
+ logger.warn("Skipping endpoint collision check as " + ALLOW_UNSAFE_JOIN.getKey() + "=true");
return;
}
@@ -782,8 +813,8 @@
if (!Gossiper.instance.isSafeForStartup(FBUtilities.getBroadcastAddressAndPort(), localHostId, shouldBootstrap(), epStates))
{
throw new RuntimeException(String.format("A node with address %s already exists, cancelling join. " +
- "Use cassandra.replace_address if you want to replace this node.",
- FBUtilities.getBroadcastAddressAndPort()));
+ "Use %s if you want to replace this node.",
+ FBUtilities.getBroadcastAddressAndPort(), REPLACE_ADDRESS.getKey()));
}
validateEndpointSnitch(epStates.values().iterator());
@@ -806,7 +837,7 @@
assert (pieces.length > 0);
String state = pieces[0];
if (state.equals(VersionedValue.STATUS_BOOTSTRAPPING) || state.equals(VersionedValue.STATUS_LEAVING) || state.equals(VersionedValue.STATUS_MOVING))
- throw new UnsupportedOperationException("Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while cassandra.consistent.rangemovement is true");
+ throw new UnsupportedOperationException("Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while " + CONSISTENT_RANGE_MOVEMENT.getKey() + " is true");
}
}
}
@@ -862,6 +893,7 @@
public synchronized void initServer(int schemaTimeoutMillis, int ringTimeoutMillis) throws ConfigurationException
{
logger.info("Cassandra version: {}", FBUtilities.getReleaseVersionString());
+ logger.info("Git SHA: {}", FBUtilities.getGitSHA());
logger.info("CQL version: {}", QueryProcessor.CQL_VERSION);
logger.info("Native protocol supported versions: {} (default: {})",
StringUtils.join(ProtocolVersion.supportedVersions(), ", "), ProtocolVersion.CURRENT);
@@ -871,14 +903,14 @@
// Ensure StorageProxy is initialized on start-up; see CASSANDRA-3797.
Class.forName("org.apache.cassandra.service.StorageProxy");
// also IndexSummaryManager, which is otherwise unreferenced
- Class.forName("org.apache.cassandra.io.sstable.IndexSummaryManager");
+ Class.forName("org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager");
}
catch (ClassNotFoundException e)
{
throw new AssertionError(e);
}
- if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
+ if (LOAD_RING_STATE.getBoolean())
{
logger.info("Loading persisted ring state");
populatePeerTokenMetadata();
@@ -911,10 +943,10 @@
replacing = isReplacing();
- if (!Boolean.parseBoolean(System.getProperty("cassandra.start_gossip", "true")))
+ if (!START_GOSSIP.getBoolean())
{
logger.info("Not starting gossip as requested.");
- initialized = true;
+ completeInitialization();
return;
}
@@ -952,12 +984,19 @@
logger.info("Not joining ring as requested. Use JMX (StorageService->joinRing()) to initiate ring joining");
}
+ completeInitialization();
+ }
+
+ private void completeInitialization()
+ {
+ if (!initialized)
+ registerMBeans();
initialized = true;
}
public void populateTokenMetadata()
{
- if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
+ if (LOAD_RING_STATE.getBoolean())
{
populatePeerTokenMetadata();
// if we have not completed bootstrapping, we should not add ourselves as a normal token
@@ -994,9 +1033,9 @@
if (replacing)
return true;
- if (System.getProperty("cassandra.replace_address_first_boot", null) != null && SystemKeyspace.bootstrapComplete())
+ if (REPLACE_ADDRESS_FIRST_BOOT.getString() != null && SystemKeyspace.bootstrapComplete())
{
- logger.info("Replace address on first boot requested; this node is already bootstrapped");
+ logger.info("Replace address on the first boot requested; this node is already bootstrapped");
return false;
}
@@ -1032,17 +1071,20 @@
if (SystemKeyspace.wasDecommissioned())
{
- if (Boolean.getBoolean("cassandra.override_decommission"))
+ if (OVERRIDE_DECOMMISSION.getBoolean())
{
logger.warn("This node was decommissioned, but overriding by operator request.");
SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.COMPLETED);
}
else
- throw new ConfigurationException("This node was decommissioned and will not rejoin the ring unless cassandra.override_decommission=true has been set, or all existing data is removed and the node is bootstrapped again");
+ {
+ throw new ConfigurationException("This node was decommissioned and will not rejoin the ring unless -D" + OVERRIDE_DECOMMISSION.getKey() +
+ "=true has been set, or all existing data is removed and the node is bootstrapped again");
+ }
}
if (DatabaseDescriptor.getReplaceTokens().size() > 0 || DatabaseDescriptor.getReplaceNode() != null)
- throw new RuntimeException("Replace method removed; use cassandra.replace_address instead");
+ throw new RuntimeException("Replace method removed; use " + REPLACE_ADDRESS.getKey() + " system property instead.");
MessagingService.instance().listen();
@@ -1105,7 +1147,7 @@
if (!(notification instanceof SSTablesVersionsInUseChangeNotification))
return;
- Set<VersionAndType> versions = ((SSTablesVersionsInUseChangeNotification)notification).versionsInUse;
+ Set<Version> versions = ((SSTablesVersionsInUseChangeNotification)notification).versionsInUse;
logger.debug("Updating local sstables version in Gossip to {}", versions);
Gossiper.instance.addLocalApplicationState(ApplicationState.SSTABLE_VERSIONS,
@@ -1323,6 +1365,7 @@
DatabaseDescriptor.getRoleManager().setup();
DatabaseDescriptor.getAuthenticator().setup();
+ DatabaseDescriptor.getInternodeAuthenticator().setupInternode();
DatabaseDescriptor.getAuthorizer().setup();
DatabaseDescriptor.getNetworkAuthorizer().setup();
AuthCacheService.initializeAndRegisterCaches();
@@ -1358,11 +1401,16 @@
public void rebuild(String sourceDc)
{
- rebuild(sourceDc, null, null, null);
+ rebuild(sourceDc, null, null, null, false);
}
public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources)
{
+ rebuild(sourceDc, keyspace, tokens, specificSources, false);
+ }
+
+ public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources, boolean excludeLocalDatacenterNodes)
+ {
try
{
// check ongoing rebuild
@@ -1371,6 +1419,12 @@
throw new IllegalStateException("Node is still rebuilding. Check nodetool netstats.");
}
+ // fail if source DC is local and --exclude-local-dc is set
+ if (sourceDc != null && sourceDc.equals(DatabaseDescriptor.getLocalDataCenter()) && excludeLocalDatacenterNodes)
+ {
+ throw new IllegalArgumentException("Cannot set source data center to be local data center, when excludeLocalDataCenter flag is set");
+ }
+
if (sourceDc != null)
{
TokenMetadata.Topology topology = getTokenMetadata().cloneOnlyTokenMap().getTopology();
@@ -1414,6 +1468,9 @@
if (sourceDc != null)
streamer.addSourceFilter(new RangeStreamer.SingleDatacenterFilter(DatabaseDescriptor.getEndpointSnitch(), sourceDc));
+ if (excludeLocalDatacenterNodes)
+ streamer.addSourceFilter(new RangeStreamer.ExcludeLocalDatacenterFilter(DatabaseDescriptor.getEndpointSnitch()));
+
if (keyspace == null)
{
for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
@@ -1922,7 +1979,8 @@
String bootstrapTokens = StringUtils.join(tokenMetadata.getBootstrapTokens().valueSet(), ',');
String leavingTokens = StringUtils.join(tokenMetadata.getLeavingEndpoints(), ',');
String movingTokens = StringUtils.join(tokenMetadata.getMovingEndpoints().stream().map(e -> e.right).toArray(), ',');
- throw new UnsupportedOperationException(String.format("Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while cassandra.consistent.rangemovement is true. Nodes detected, bootstrapping: %s; leaving: %s; moving: %s;", bootstrapTokens, leavingTokens, movingTokens));
+ throw new UnsupportedOperationException(String.format("Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while %s is true. Nodes detected, bootstrapping: %s; leaving: %s; moving: %s;",
+ CONSISTENT_RANGE_MOVEMENT.getKey(), bootstrapTokens, leavingTokens, movingTokens));
}
// get bootstrap tokens
@@ -1940,11 +1998,18 @@
{
if (!isReplacingSameAddress())
{
+ // Historically BROADCAST_INTERVAL was used, but this is unrelated to ring_delay, so using it to know
+ // how long to sleep only works with the default settings (ring_delay=30s, broadcast=60s). For users
+ // who are aware of this relationship, this coupling should not be broken, but for most users this
+ // relationship isn't known and instead we should rely on the ring_delay.
+ // See CASSANDRA-17776
+ long sleepDelayMillis = Math.max(LoadBroadcaster.BROADCAST_INTERVAL, ringTimeoutMillis * 2);
try
{
// Sleep additionally to make sure that the server actually is not alive
// and giving it more time to gossip if alive.
- Thread.sleep(LoadBroadcaster.BROADCAST_INTERVAL);
+ logger.info("Sleeping for {}ms waiting to make sure no new gossip updates happen for {}", sleepDelayMillis, DatabaseDescriptor.getReplaceAddress());
+ Thread.sleep(sleepDelayMillis);
}
catch (InterruptedException e)
{
@@ -1952,14 +2017,23 @@
}
// check for operator errors...
+ long nanoDelay = MILLISECONDS.toNanos(ringTimeoutMillis);
for (Token token : bootstrapTokens)
{
InetAddressAndPort existing = tokenMetadata.getEndpoint(token);
if (existing != null)
{
- long nanoDelay = ringTimeoutMillis * 1000000L;
- if (Gossiper.instance.getEndpointStateForEndpoint(existing).getUpdateTimestamp() > (nanoTime() - nanoDelay))
+ EndpointState endpointStateForExisting = Gossiper.instance.getEndpointStateForEndpoint(existing);
+ long updateTimestamp = endpointStateForExisting.getUpdateTimestamp();
+ long allowedDelay = nanoTime() - nanoDelay;
+
+ // if the node was updated within the ring delay or the node is alive, we should fail
+ if (updateTimestamp > allowedDelay || endpointStateForExisting.isAlive())
+ {
+ logger.error("Unable to replace node for token={}. The node is reporting as {}alive with updateTimestamp={}, allowedDelay={}",
+ token, endpointStateForExisting.isAlive() ? "" : "not ", updateTimestamp, allowedDelay);
throw new UnsupportedOperationException("Cannot replace a live node... ");
+ }
collisions.add(existing);
}
else
@@ -2024,10 +2098,10 @@
if (!Gossiper.instance.seenAnySeed())
throw new IllegalStateException("Unable to contact any seeds: " + Gossiper.instance.getSeeds());
- if (Boolean.getBoolean("cassandra.reset_bootstrap_progress"))
+ if (RESET_BOOTSTRAP_PROGRESS.getBoolean())
{
logger.info("Resetting bootstrap progress to start fresh");
- SystemKeyspace.resetAvailableRanges();
+ SystemKeyspace.resetAvailableStreamedRanges();
}
// Force disk boundary invalidation now that local tokens are set
@@ -3666,12 +3740,18 @@
updateNetVersion(endpoint, netVersion);
}
-
+ @Override
public String getLoadString()
{
return FileUtils.stringifyFileSize(StorageMetrics.load.getCount());
}
+ @Override
+ public String getUncompressedLoadString()
+ {
+ return FileUtils.stringifyFileSize(StorageMetrics.uncompressedLoad.getCount());
+ }
+
public Map<String, String> getLoadMapWithPort()
{
return getLoadMap(true);
@@ -3744,6 +3824,12 @@
return FBUtilities.getReleaseVersionString();
}
+ @Override
+ public String getGitSHA()
+ {
+ return FBUtilities.getGitSHA();
+ }
+
public String getSchemaVersion()
{
return Schema.instance.getVersion().toString();
@@ -3908,7 +3994,7 @@
return forceKeyspaceCleanup(0, keyspaceName, tables);
}
- public int forceKeyspaceCleanup(int jobs, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException
+ public int forceKeyspaceCleanup(int jobs, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
if (SchemaConstants.isLocalSystemKeyspace(keyspaceName))
throw new RuntimeException("Cleanup of the system keyspace is neither necessary nor wise");
@@ -3917,39 +4003,38 @@
throw new RuntimeException("Node is involved in cluster membership changes. Not safe to run cleanup.");
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
- for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, tables))
+ logger.info("Starting {} on {}.{}", OperationType.CLEANUP, keyspaceName, Arrays.toString(tableNames));
+ for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, tableNames))
{
CompactionManager.AllSSTableOpStatus oneStatus = cfStore.forceCleanup(jobs);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
+ logger.info("Completed {} with status {}", OperationType.CLEANUP, status);
return status.statusCode;
}
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException
+ public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
- return scrub(disableSnapshot, skipCorrupted, true, 0, keyspaceName, tables);
+ IScrubber.Options options = IScrubber.options()
+ .skipCorrupted(skipCorrupted)
+ .checkData(checkData)
+ .reinsertOverflowedTTLRows(reinsertOverflowedTTL)
+ .build();
+ return scrub(disableSnapshot, options, jobs, keyspaceName, tableNames);
}
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException
- {
- return scrub(disableSnapshot, skipCorrupted, checkData, 0, keyspaceName, tables);
- }
-
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException
- {
- return scrub(disableSnapshot, skipCorrupted, checkData, false, jobs, keyspaceName, tables);
- }
-
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException
+ public int scrub(boolean disableSnapshot, IScrubber.Options options, int jobs, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
- for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName, tables))
+ logger.info("Starting {} on {}.{}", OperationType.SCRUB, keyspaceName, Arrays.toString(tableNames));
+ for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName, tableNames))
{
- CompactionManager.AllSSTableOpStatus oneStatus = cfStore.scrub(disableSnapshot, skipCorrupted, reinsertOverflowedTTL, checkData, jobs);
+ CompactionManager.AllSSTableOpStatus oneStatus = cfStore.scrub(disableSnapshot, options, jobs);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
+ logger.info("Completed {} with status {}", OperationType.SCRUB, status);
return status.statusCode;
}
@@ -3962,19 +4047,20 @@
public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
- Verifier.Options options = Verifier.options().invokeDiskFailurePolicy(diskFailurePolicy)
- .extendedVerification(extendedVerify)
- .checkVersion(checkVersion)
- .mutateRepairStatus(mutateRepairStatus)
- .checkOwnsTokens(checkOwnsTokens)
- .quick(quick).build();
- logger.info("Verifying {}.{} with options = {}", keyspaceName, Arrays.toString(tableNames), options);
+ IVerifier.Options options = IVerifier.options().invokeDiskFailurePolicy(diskFailurePolicy)
+ .extendedVerification(extendedVerify)
+ .checkVersion(checkVersion)
+ .mutateRepairStatus(mutateRepairStatus)
+ .checkOwnsTokens(checkOwnsTokens)
+ .quick(quick).build();
+ logger.info("Staring {} on {}.{} with options = {}", OperationType.VERIFY, keyspaceName, Arrays.toString(tableNames), options);
for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, tableNames))
{
CompactionManager.AllSSTableOpStatus oneStatus = cfStore.verify(options);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
+ logger.info("Completed {} with status {}", OperationType.VERIFY, status);
return status.statusCode;
}
@@ -4008,12 +4094,14 @@
String... tableNames) throws IOException, ExecutionException, InterruptedException
{
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
+ logger.info("Starting {} on {}.{}", OperationType.UPGRADE_SSTABLES, keyspaceName, Arrays.toString(tableNames));
for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, true, keyspaceName, tableNames))
{
CompactionManager.AllSSTableOpStatus oneStatus = cfStore.sstablesRewrite(skipIfCurrentVersion, skipIfNewerThanTimestamp, skipIfCompressionMatches, jobs);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
+ logger.info("Completed {} with status {}", OperationType.UPGRADE_SSTABLES, status);
return status.statusCode;
}
@@ -4039,33 +4127,37 @@
}
}
- public int relocateSSTables(String keyspaceName, String ... columnFamilies) throws IOException, ExecutionException, InterruptedException
+ public int relocateSSTables(String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
- return relocateSSTables(0, keyspaceName, columnFamilies);
+ return relocateSSTables(0, keyspaceName, tableNames);
}
- public int relocateSSTables(int jobs, String keyspaceName, String ... columnFamilies) throws IOException, ExecutionException, InterruptedException
+ public int relocateSSTables(int jobs, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
- for (ColumnFamilyStore cfs : getValidColumnFamilies(false, false, keyspaceName, columnFamilies))
+ logger.info("Starting {} on {}.{}", OperationType.RELOCATE, keyspaceName, Arrays.toString(tableNames));
+ for (ColumnFamilyStore cfs : getValidColumnFamilies(false, false, keyspaceName, tableNames))
{
CompactionManager.AllSSTableOpStatus oneStatus = cfs.relocateSSTables(jobs);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
+ logger.info("Completed {} with status {}", OperationType.RELOCATE, status);
return status.statusCode;
}
- public int garbageCollect(String tombstoneOptionString, int jobs, String keyspaceName, String ... columnFamilies) throws IOException, ExecutionException, InterruptedException
+ public int garbageCollect(String tombstoneOptionString, int jobs, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
TombstoneOption tombstoneOption = TombstoneOption.valueOf(tombstoneOptionString);
CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
- for (ColumnFamilyStore cfs : getValidColumnFamilies(false, false, keyspaceName, columnFamilies))
+ logger.info("Starting {} on {}.{}", OperationType.GARBAGE_COLLECT, keyspaceName, Arrays.toString(tableNames));
+ for (ColumnFamilyStore cfs : getValidColumnFamilies(false, false, keyspaceName, tableNames))
{
CompactionManager.AllSSTableOpStatus oneStatus = cfs.garbageCollect(tombstoneOption, jobs);
if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
status = oneStatus;
}
+ logger.info("Completed {} with status {}", OperationType.GARBAGE_COLLECT, status);
return status.statusCode;
}
@@ -4153,6 +4245,24 @@
}
}
+ /***
+ * Forces compaction for a list of partition keys in a table
+ * The method will ignore the gc_grace_seconds for the partitionKeysIgnoreGcGrace during the comapction,
+ * in order to purge the tombstones and free up space quicker.
+ * @param keyspaceName keyspace name
+ * @param tableName table name
+ * @param partitionKeysIgnoreGcGrace partition keys ignoring the gc_grace_seconds
+ * @throws IOException on any I/O operation error
+ * @throws ExecutionException when attempting to retrieve the result of a task that aborted by throwing an exception
+ * @throws InterruptedException when a thread is waiting, sleeping, or otherwise occupied, and the thread is interrupted, either before or during the activity
+ */
+ public void forceCompactionKeysIgnoringGcGrace(String keyspaceName,
+ String tableName, String... partitionKeysIgnoreGcGrace) throws IOException, ExecutionException, InterruptedException
+ {
+ ColumnFamilyStore cfStore = getValidKeyspace(keyspaceName).getColumnFamilyStore(tableName);
+ cfStore.forceCompactionKeysIgnoringGcGrace(partitionKeysIgnoreGcGrace);
+ }
+
/**
* Takes the snapshot for the given keyspaces. A snapshot name must be specified.
*
@@ -4306,15 +4416,23 @@
* Remove the snapshot with the given name from the given keyspaces.
* If no tag is specified we will remove all snapshots.
*/
- public void clearSnapshot(String tag, String... keyspaceNames) throws IOException
+ public void clearSnapshot(String tag, String... keyspaceNames)
{
- if(tag == null)
+ clearSnapshot(Collections.emptyMap(), tag, keyspaceNames);
+ }
+
+ public void clearSnapshot(Map<String, Object> options, String tag, String... keyspaceNames)
+ {
+ if (tag == null)
tag = "";
+ if (options == null)
+ options = Collections.emptyMap();
+
Set<String> keyspaces = new HashSet<>();
for (String dataDir : DatabaseDescriptor.getAllDataFileLocations())
{
- for(String keyspaceDir : new File(dataDir).tryListNames())
+ for (String keyspaceDir : new File(dataDir).tryListNames())
{
// Only add a ks if it has been specified as a param, assuming params were actually provided.
if (keyspaceNames.length > 0 && !Arrays.asList(keyspaceNames).contains(keyspaceDir))
@@ -4323,16 +4441,58 @@
}
}
+ Object olderThan = options.get("older_than");
+ Object olderThanTimestamp = options.get("older_than_timestamp");
+
+ final long clearOlderThanTimestamp;
+ if (olderThan != null)
+ {
+ assert olderThan instanceof String : "it is expected that older_than is an instance of java.lang.String";
+ clearOlderThanTimestamp = Clock.Global.currentTimeMillis() - new DurationSpec.LongSecondsBound((String) olderThan).toMilliseconds();
+ }
+ else if (olderThanTimestamp != null)
+ {
+ assert olderThanTimestamp instanceof String : "it is expected that older_than_timestamp is an instance of java.lang.String";
+ try
+ {
+ clearOlderThanTimestamp = Instant.parse((String) olderThanTimestamp).toEpochMilli();
+ }
+ catch (DateTimeParseException ex)
+ {
+ throw new RuntimeException("Parameter older_than_timestamp has to be a valid instant in ISO format.");
+ }
+ }
+ else
+ clearOlderThanTimestamp = 0L;
+
for (String keyspace : keyspaces)
- Keyspace.clearSnapshot(tag, keyspace);
+ clearKeyspaceSnapshot(keyspace, tag, clearOlderThanTimestamp);
if (logger.isDebugEnabled())
logger.debug("Cleared out snapshot directories");
}
+ /**
+ * Clear snapshots for a given keyspace.
+ * @param keyspace keyspace to remove snapshots for
+ * @param tag the user supplied snapshot name. If empty or null, all the snapshots will be cleaned
+ * @param olderThanTimestamp if a snapshot was created before this timestamp, it will be cleared,
+ * if its value is 0, this parameter is effectively ignored.
+ */
+ private void clearKeyspaceSnapshot(String keyspace, String tag, long olderThanTimestamp)
+ {
+ Set<TableSnapshot> snapshotsToClear = snapshotManager.loadSnapshots(keyspace)
+ .stream()
+ .filter(TableSnapshot.shouldClearSnapshot(tag, olderThanTimestamp))
+ .collect(Collectors.toSet());
+ for (TableSnapshot snapshot : snapshotsToClear)
+ snapshotManager.clearSnapshot(snapshot);
+ }
+
public Map<String, TabularData> getSnapshotDetails(Map<String, String> options)
{
boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));
+ boolean includeEphemeral = options != null && Boolean.parseBoolean(options.getOrDefault("include_ephemeral", "false"));
Map<String, TabularData> snapshotMap = new HashMap<>();
@@ -4340,6 +4500,8 @@
{
if (skipExpiring && snapshot.isExpiring())
continue;
+ if (!includeEphemeral && snapshot.isEphemeral())
+ continue;
TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag());
if (data == null)
@@ -4576,8 +4738,8 @@
logger.info("repairing paxos for {}", reason);
int retries = 0;
- int maxRetries = Integer.getInteger("cassandra.paxos_repair_on_topology_change_retries", 10);
- int delaySec = Integer.getInteger("cassandra.paxos_repair_on_topology_change_retry_delay_seconds", 10);
+ int maxRetries = PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRIES.getInt();
+ int delaySec = PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRY_DELAY_SECONDS.getInt();
boolean completed = false;
while (!completed)
@@ -4651,16 +4813,44 @@
ImmutableList.<String>builder().add(pair.left.name()).addAll(pair.right).build();
}
+ @Deprecated
+ @Override
public void setRepairSessionMaxTreeDepth(int depth)
{
DatabaseDescriptor.setRepairSessionMaxTreeDepth(depth);
}
+ @Deprecated
+ @Override
public int getRepairSessionMaxTreeDepth()
{
return DatabaseDescriptor.getRepairSessionMaxTreeDepth();
}
+ @Override
+ public void setRepairSessionMaximumTreeDepth(int depth)
+ {
+ try
+ {
+ DatabaseDescriptor.setRepairSessionMaxTreeDepth(depth);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+
+ /*
+ * In CASSANDRA-17668, JMX setters that did not throw standard exceptions were deprecated in favor of ones that do.
+ * For consistency purposes, the respective getter "getRepairSessionMaxTreeDepth" was also deprecated and replaced
+ * by this method.
+ */
+ @Override
+ public int getRepairSessionMaximumTreeDepth()
+ {
+ return DatabaseDescriptor.getRepairSessionMaxTreeDepth();
+ }
+
/* End of MBean interface methods */
/**
@@ -4928,6 +5118,7 @@
*/
private void startLeaving()
{
+ DatabaseDescriptor.getSeverityDuringDecommission().ifPresent(DynamicEndpointSnitch::addSeverity);
Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.leaving(getLocalTokens()));
Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.leaving(getLocalTokens()));
tokenMetadata.addLeavingEndpoint(FBUtilities.getBroadcastAddressAndPort());
@@ -5077,9 +5268,20 @@
logger.debug("waiting for batch log processing.");
batchlogReplay.get();
- setMode(Mode.LEAVING, "streaming hints to other nodes", true);
+ Future<?> hintsSuccess = ImmediateFuture.success(null);
- Future hintsSuccess = streamHints();
+ if (DatabaseDescriptor.getTransferHintsOnDecommission())
+ {
+ setMode(Mode.LEAVING, "streaming hints to other nodes", true);
+ hintsSuccess = streamHints();
+ }
+ else
+ {
+ setMode(Mode.LEAVING, "pausing dispatch and deleting hints", true);
+ DatabaseDescriptor.setHintedHandoffEnabled(false);
+ HintsService.instance.pauseDispatch();
+ HintsService.instance.deleteAllHints();
+ }
// wait for the transfer runnables to signal the latch.
logger.debug("waiting for stream acks.");
@@ -6095,17 +6297,23 @@
return sampledKeys;
}
+ @Override
+ public Map<String, List<CompositeData>> samplePartitions(int duration, int capacity, int count, List<String> samplers) throws OpenDataException {
+ return samplePartitions(null, duration, capacity, count, samplers);
+ }
+
/*
* { "sampler_name": [ {table: "", count: i, error: i, value: ""}, ... ] }
*/
@Override
- public Map<String, List<CompositeData>> samplePartitions(int durationMillis, int capacity, int count,
- List<String> samplers) throws OpenDataException
+ public Map<String, List<CompositeData>> samplePartitions(String keyspace, int durationMillis, int capacity, int count,
+ List<String> samplers) throws OpenDataException
{
ConcurrentHashMap<String, List<CompositeData>> result = new ConcurrentHashMap<>();
+ Iterable<ColumnFamilyStore> tables = SamplingManager.getTables(keyspace, null);
for (String sampler : samplers)
{
- for (ColumnFamilyStore table : ColumnFamilyStore.all())
+ for (ColumnFamilyStore table : tables)
{
table.beginLocalSampling(sampler, capacity, durationMillis);
}
@@ -6115,7 +6323,7 @@
for (String sampler : samplers)
{
List<CompositeData> topk = new ArrayList<>();
- for (ColumnFamilyStore table : ColumnFamilyStore.all())
+ for (ColumnFamilyStore table : tables)
{
topk.addAll(table.finishLocalSampling(sampler, count));
}
@@ -6133,6 +6341,44 @@
return result;
}
+ @Override // Note from parent javadoc: ks and table are nullable
+ public boolean startSamplingPartitions(String ks, String table, int duration, int interval, int capacity, int count, List<String> samplers)
+ {
+ Preconditions.checkArgument(duration > 0, "Sampling duration %s must be positive.", duration);
+
+ Preconditions.checkArgument(interval <= 0 || interval >= duration,
+ "Sampling interval %s should be greater then or equals to duration %s if defined.",
+ interval, duration);
+
+ Preconditions.checkArgument(capacity > 0 && capacity <= 1024,
+ "Sampling capacity %s must be positive and the max value is 1024 (inclusive).",
+ capacity);
+
+ Preconditions.checkArgument(count > 0 && count < capacity,
+ "Sampling count %s must be positive and smaller than capacity %s.",
+ count, capacity);
+
+ Preconditions.checkArgument(!samplers.isEmpty(), "Samplers cannot be empty.");
+
+ Set<Sampler.SamplerType> available = EnumSet.allOf(Sampler.SamplerType.class);
+ samplers.forEach((x) -> checkArgument(available.contains(Sampler.SamplerType.valueOf(x)),
+ "'%s' sampler is not available from: %s",
+ x, Arrays.toString(Sampler.SamplerType.values())));
+ return samplingManager.register(ks, table, duration, interval, capacity, count, samplers);
+ }
+
+ @Override
+ public boolean stopSamplingPartitions(String ks, String table)
+ {
+ return samplingManager.unregister(ks, table);
+ }
+
+ @Override
+ public List<String> getSampleTasks()
+ {
+ return samplingManager.allJobs();
+ }
+
public void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames)
{
String[] indices = asList(idxNames).stream()
@@ -6272,29 +6518,76 @@
logger.info("updated replica_filtering_protection.cached_rows_fail_threshold to {}", threshold);
}
+ @Override
public int getColumnIndexSizeInKiB()
{
return DatabaseDescriptor.getColumnIndexSizeInKiB();
}
+ @Override
+ public void setColumnIndexSizeInKiB(int columnIndexSizeInKiB)
+ {
+ int oldValueInKiB = DatabaseDescriptor.getColumnIndexSizeInKiB();
+ try
+ {
+ DatabaseDescriptor.setColumnIndexSizeInKiB(columnIndexSizeInKiB);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ logger.info("Updated column_index_size to {} KiB (was {} KiB)", columnIndexSizeInKiB, oldValueInKiB);
+ }
+
+ @Deprecated
+ @Override
public void setColumnIndexSize(int columnIndexSizeInKB)
{
int oldValueInKiB = DatabaseDescriptor.getColumnIndexSizeInKiB();
- DatabaseDescriptor.setColumnIndexSize(columnIndexSizeInKB);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(columnIndexSizeInKB);
logger.info("Updated column_index_size to {} KiB (was {} KiB)", columnIndexSizeInKB, oldValueInKiB);
}
+ @Deprecated
+ @Override
public int getColumnIndexCacheSize()
{
return DatabaseDescriptor.getColumnIndexCacheSizeInKiB();
}
+ @Deprecated
+ @Override
public void setColumnIndexCacheSize(int cacheSizeInKB)
{
DatabaseDescriptor.setColumnIndexCacheSize(cacheSizeInKB);
logger.info("Updated column_index_cache_size to {}", cacheSizeInKB);
}
+ /*
+ * In CASSANDRA-17668, JMX setters that did not throw standard exceptions were deprecated in favor of ones that do.
+ * For consistency purposes, the respective getter "getColumnIndexCacheSize" was also deprecated and replaced by
+ * this method.
+ */
+ @Override
+ public int getColumnIndexCacheSizeInKiB()
+ {
+ return DatabaseDescriptor.getColumnIndexCacheSizeInKiB();
+ }
+
+ @Override
+ public void setColumnIndexCacheSizeInKiB(int cacheSizeInKiB)
+ {
+ try
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(cacheSizeInKiB);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ logger.info("Updated column_index_cache_size to {}", cacheSizeInKiB);
+ }
+
public int getBatchSizeFailureThreshold()
{
return DatabaseDescriptor.getBatchSizeFailThresholdInKiB();
@@ -6306,17 +6599,47 @@
logger.info("updated batch_size_fail_threshold to {}", threshold);
}
+ @Deprecated
+ @Override
public int getBatchSizeWarnThreshold()
{
return DatabaseDescriptor.getBatchSizeWarnThresholdInKiB();
}
+ @Deprecated
+ @Override
public void setBatchSizeWarnThreshold(int threshold)
{
DatabaseDescriptor.setBatchSizeWarnThresholdInKiB(threshold);
logger.info("Updated batch_size_warn_threshold to {}", threshold);
}
+ /*
+ * In CASSANDRA-17668, JMX setters that did not throw standard exceptions were deprecated in favor of ones that do.
+ * For consistency purposes, the respective getter "getBatchSizeWarnThreshold" was also deprecated and replaced by
+ * this method.
+ */
+ @Override
+ public int getBatchSizeWarnThresholdInKiB()
+ {
+ return DatabaseDescriptor.getBatchSizeWarnThresholdInKiB();
+ }
+
+ @Override
+ public void setBatchSizeWarnThresholdInKiB(int thresholdInKiB)
+ {
+ try
+ {
+ DatabaseDescriptor.setBatchSizeWarnThresholdInKiB(thresholdInKiB);
+ }
+ catch (ConfigurationException e)
+ {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+
+ logger.info("Updated batch_size_warn_threshold to {}", thresholdInKiB);
+ }
+
public int getInitialRangeTombstoneListAllocationSize()
{
return DatabaseDescriptor.getInitialRangeTombstoneListAllocationSize();
@@ -6358,6 +6681,17 @@
logger.info("updated hinted_handoff_throttle to {} KiB", throttleInKB);
}
+ public boolean getTransferHintsOnDecommission()
+ {
+ return DatabaseDescriptor.getTransferHintsOnDecommission();
+ }
+
+ public void setTransferHintsOnDecommission(boolean enabled)
+ {
+ DatabaseDescriptor.setTransferHintsOnDecommission(enabled);
+ logger.info("updated transfer_hints_on_decommission to {}", enabled);
+ }
+
@Override
public void clearConnectionHistory()
{
@@ -6953,4 +7287,39 @@
{
DatabaseDescriptor.setMinTrackedPartitionTombstoneCount(value);
}
+
+ public void setSkipStreamDiskSpaceCheck(boolean value)
+ {
+ if (value != DatabaseDescriptor.getSkipStreamDiskSpaceCheck())
+ logger.info("Changing skip_stream_disk_space_check from {} to {}", DatabaseDescriptor.getSkipStreamDiskSpaceCheck(), value);
+ DatabaseDescriptor.setSkipStreamDiskSpaceCheck(value);
+ }
+
+ public boolean getSkipStreamDiskSpaceCheck()
+ {
+ return DatabaseDescriptor.getSkipStreamDiskSpaceCheck();
+ }
+
+ @Override
+ public void removeNotificationListener(NotificationListener listener) throws ListenerNotFoundException
+ {
+ if (!skipNotificationListeners)
+ super.removeNotificationListener(listener);
+ }
+
+ @Override
+ public void removeNotificationListener(NotificationListener listener, NotificationFilter filter, Object handback) throws ListenerNotFoundException
+ {
+ if (!skipNotificationListeners)
+ super.removeNotificationListener(listener, filter, handback);
+ }
+
+ @Override
+ public void addNotificationListener(NotificationListener listener,
+ NotificationFilter filter,
+ Object handback) throws java.lang.IllegalArgumentException
+ {
+ if (!skipNotificationListeners)
+ super.addNotificationListener(listener, filter, handback);
+ }
}
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index ac2ff68..5897afc 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -103,6 +103,12 @@
public String getReleaseVersion();
/**
+ * Fetch a string representation of the Cassandra git SHA.
+ * @return A string representation of the Cassandra git SHA.
+ */
+ public String getGitSHA();
+
+ /**
* Fetch a string representation of the current Schema version.
* @return A string representation of the Schema version.
*/
@@ -209,6 +215,9 @@
/** Human-readable load value */
public String getLoadString();
+ /** Human-readable uncompressed load value */
+ public String getUncompressedLoadString();
+
/** Human-readable load value. Keys are IP addresses. */
@Deprecated public Map<String, String> getLoadMap();
public Map<String, String> getLoadMapWithPort();
@@ -267,10 +276,24 @@
/**
* Remove the snapshot with the given name from the given keyspaces.
* If no tag is specified we will remove all snapshots.
+ *
+ * @param tag name of snapshot to clear, if null or empty string, all snapshots of given keyspace will be cleared
+ * @param keyspaceNames name of keyspaces to clear snapshots for
*/
+ @Deprecated
public void clearSnapshot(String tag, String... keyspaceNames) throws IOException;
/**
+ * Remove the snapshot with the given name from the given keyspaces.
+ * If no tag is specified we will remove all snapshots.
+ *
+ * @param options map of options for cleanup operation, consult nodetool's ClearSnapshot
+ * @param tag name of snapshot to clear, if null or empty string, all snapshots of given keyspace will be cleared
+ * @param keyspaceNames name of keyspaces to clear snapshots for
+ */
+ public void clearSnapshot(Map<String, Object> options, String tag, String... keyspaceNames) throws IOException;
+
+ /**
* Get the details of all the snapshot
* @return A map of snapshotName to all its details in Tabular form.
*/
@@ -342,6 +365,13 @@
public void forceKeyspaceCompactionForPartitionKey(String keyspaceName, String partitionKey, String... tableNames) throws IOException, ExecutionException, InterruptedException;
/**
+ * Forces compaction for a list of partition keys on a table.
+ * The method will ignore the gc_grace_seconds for the partitionKeysIgnoreGcGrace during the comapction,
+ * in order to purge the tombstones and free up space quicker.
+ */
+ public void forceCompactionKeysIgnoringGcGrace(String keyspaceName, String tableName, String... partitionKeysIgnoreGcGrace) throws IOException, ExecutionException, InterruptedException;
+
+ /**
* Trigger a cleanup of keys on a single keyspace
*/
@Deprecated
@@ -355,11 +385,22 @@
* Scrubbed CFs will be snapshotted first, if disableSnapshot is false
*/
@Deprecated
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
+ default int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+ {
+ return scrub(disableSnapshot, skipCorrupted, true, keyspaceName, tableNames);
+ }
+
@Deprecated
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException;
+ default int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
+ {
+ return scrub(disableSnapshot, skipCorrupted, checkData, 0, keyspaceName, tableNames);
+ }
+
@Deprecated
- public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException;
+ default int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException
+ {
+ return scrub(disableSnapshot, skipCorrupted, checkData, false, jobs, keyspaceName, columnFamilies);
+ }
public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException;
@@ -417,10 +458,22 @@
public void forceTerminateAllRepairSessions();
+ /**
+ * @deprecated use setRepairSessionMaximumTreeDepth instead as it will not throw non-standard exceptions
+ */
+ @Deprecated
public void setRepairSessionMaxTreeDepth(int depth);
+ /**
+ * @deprecated use getRepairSessionMaximumTreeDepth instead
+ */
+ @Deprecated
public int getRepairSessionMaxTreeDepth();
+ public void setRepairSessionMaximumTreeDepth(int depth);
+
+ public int getRepairSessionMaximumTreeDepth();
+
/**
* Get the status of a given parent repair session.
* @param cmd the int reference returned when issuing the repair
@@ -740,9 +793,23 @@
* @param keyspace Name of the keyspace which to rebuild or null to rebuild all keyspaces.
* @param tokens Range of tokens to rebuild or null to rebuild all token ranges. In the format of:
* "(start_token_1,end_token_1],(start_token_2,end_token_2],...(start_token_n,end_token_n]"
+ * @param specificSources list of sources that can be used for rebuilding. Must be other nodes in the cluster.
+ * The format of the string is comma separated values.
*/
public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources);
+ /**
+ * Same as {@link #rebuild(String)}, but only for specified keyspace and ranges. It excludes local data center nodes
+ *
+ * @param sourceDc Name of DC from which to select sources for streaming or null to pick any node
+ * @param keyspace Name of the keyspace which to rebuild or null to rebuild all keyspaces.
+ * @param tokens Range of tokens to rebuild or null to rebuild all token ranges. In the format of:
+ * "(start_token_1,end_token_1],(start_token_2,end_token_2],...(start_token_n,end_token_n]"
+ * @param specificSources list of sources that can be used for rebuilding. Mostly other nodes in the cluster.
+ * @param excludeLocalDatacenterNodes Flag to indicate whether local data center nodes should be excluded as sources for streaming.
+ */
+ public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources, boolean excludeLocalDatacenterNodes);
+
/** Starts a bulk load and blocks until it completes. */
public void bulkLoad(String directory);
@@ -795,6 +862,36 @@
public Map<String, List<CompositeData>> samplePartitions(int duration, int capacity, int count, List<String> samplers) throws OpenDataException;
+ public Map<String, List<CompositeData>> samplePartitions(String keyspace, int duration, int capacity, int count, List<String> samplers) throws OpenDataException;
+
+ /**
+ * Start a scheduled sampling
+ * @param ks Keyspace. Nullable. If null, the scheduled sampling is on all keyspaces and tables
+ * @param table Nullable. If null, the scheduled sampling is on all tables of the specified keyspace
+ * @param duration Duration of each scheduled sampling job in milliseconds
+ * @param interval Interval of each scheduled sampling job in milliseconds
+ * @param capacity Capacity of the sampler, higher for more accuracy
+ * @param count Number of the top samples to list
+ * @param samplers a list of samplers to enable
+ * @return true if the scheduled sampling is started successfully. Otherwise return false
+ */
+ public boolean startSamplingPartitions(String ks, String table, int duration, int interval, int capacity, int count, List<String> samplers) throws OpenDataException;
+
+ /**
+ * Stop a scheduled sampling
+ * @param ks Keyspace. Nullable. If null, the scheduled sampling is on all keysapces and tables
+ * @param table Nullable. If null, the scheduled sampling is on all tables of the specified keyspace
+ * @return true if the scheduled sampling is stopped. False is returned if the sampling task is not found
+ */
+ public boolean stopSamplingPartitions(String ks, String table) throws OpenDataException;
+
+ /**
+ * @return a list of qualified table names that have active scheduled sampling tasks. The format of the name is `KEYSPACE.TABLE`
+ * The wild card symbol (*) indicates all keyspace/table. For example, "*.*" indicates all tables in all keyspaces. "foo.*" indicates
+ * all tables under keyspace 'foo'. "foo.bar" indicates the scheduled sampling is enabled for the table 'bar'
+ */
+ public List<String> getSampleTasks();
+
/**
* Returns the configured tracing probability.
*/
@@ -833,29 +930,74 @@
/** Sets the number of rows cached at the coordinator before filtering/index queries fail outright. */
public void setCachedReplicaRowsFailThreshold(int threshold);
- /** Returns the granularity of the collation index of rows within a partition **/
+ /**
+ * Returns the granularity of the collation index of rows within a partition.
+ * -1 stands for the SSTable format's default.
+ **/
public int getColumnIndexSizeInKiB();
- /** Sets the granularity of the collation index of rows within a partition **/
+
+ /**
+ * Sets the granularity of the collation index of rows within a partition.
+ * Use -1 to select the SSTable format's default.
+ **/
+ public void setColumnIndexSizeInKiB(int columnIndexSizeInKiB);
+
+ /**
+ * Sets the granularity of the collation index of rows within a partition
+ * @deprecated use setColumnIndexSizeInKiB instead as it will not throw non-standard exceptions
+ */
+ @Deprecated
public void setColumnIndexSize(int columnIndexSizeInKB);
- /** Returns the threshold for skipping the column index when caching partition info **/
+ /**
+ * Returns the threshold for skipping the column index when caching partition info
+ * @deprecated use getColumnIndexCacheSizeInKiB
+ */
+ @Deprecated
public int getColumnIndexCacheSize();
- /** Sets the threshold for skipping the column index when caching partition info **/
+
+ /**
+ * Sets the threshold for skipping the column index when caching partition info
+ * @deprecated use setColumnIndexCacheSizeInKiB instead as it will not throw non-standard exceptions
+ */
+ @Deprecated
public void setColumnIndexCacheSize(int cacheSizeInKB);
+ /** Returns the threshold for skipping the column index when caching partition info **/
+ public int getColumnIndexCacheSizeInKiB();
+
+ /** Sets the threshold for skipping the column index when caching partition info **/
+ public void setColumnIndexCacheSizeInKiB(int cacheSizeInKiB);
+
/** Returns the threshold for rejecting queries due to a large batch size */
public int getBatchSizeFailureThreshold();
/** Sets the threshold for rejecting queries due to a large batch size */
public void setBatchSizeFailureThreshold(int batchSizeDebugThreshold);
- /** Returns the threshold for warning queries due to a large batch size */
+ /**
+ * Returns the threshold for warning queries due to a large batch size
+ * @deprecated use getBatchSizeWarnThresholdInKiB instead
+ */
+ @Deprecated
public int getBatchSizeWarnThreshold();
- /** Sets the threshold for warning queries due to a large batch size */
+ /**
+ * Sets the threshold for warning queries due to a large batch size
+ * @deprecated use setBatchSizeWarnThresholdInKiB instead as it will not throw non-standard exceptions
+ */
+ @Deprecated
public void setBatchSizeWarnThreshold(int batchSizeDebugThreshold);
+ /** Returns the threshold for warning queries due to a large batch size */
+ public int getBatchSizeWarnThresholdInKiB();
+ /** Sets the threshold for warning queries due to a large batch size **/
+ public void setBatchSizeWarnThresholdInKiB(int batchSizeDebugThreshold);
+
/** Sets the hinted handoff throttle in KiB per second, per delivery thread. */
public void setHintedHandoffThrottleInKB(int throttleInKB);
+ public boolean getTransferHintsOnDecommission();
+ public void setTransferHintsOnDecommission(boolean enabled);
+
/**
* Resume bootstrap streaming when there is failed data streaming.
*
@@ -1046,4 +1188,7 @@
public void setMinTrackedPartitionSize(String value);
public long getMinTrackedPartitionTombstoneCount();
public void setMinTrackedPartitionTombstoneCount(long value);
-}
+
+ public void setSkipStreamDiskSpaceCheck(boolean value);
+ public boolean getSkipStreamDiskSpaceCheck();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/service/TruncateResponseHandler.java b/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
index 984ba5a..54b1241 100644
--- a/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/TruncateResponseHandler.java
@@ -17,7 +17,9 @@
*/
package org.apache.cassandra.service;
-import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -46,7 +48,7 @@
private final int responseCount;
protected final AtomicInteger responses = new AtomicInteger(0);
private final long start;
- private volatile InetAddress truncateFailingReplica;
+ private final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint = new ConcurrentHashMap<>();
public TruncateResponseHandler(int responseCount)
{
@@ -61,24 +63,31 @@
public void get() throws TimeoutException
{
long timeoutNanos = getTruncateRpcTimeout(NANOSECONDS) - (nanoTime() - start);
- boolean completedInTime;
+ boolean signaled;
try
{
- completedInTime = condition.await(timeoutNanos, NANOSECONDS); // TODO truncate needs a much longer timeout
+ signaled = condition.await(timeoutNanos, NANOSECONDS); // TODO truncate needs a much longer timeout
}
catch (InterruptedException e)
{
throw new UncheckedInterruptedException(e);
}
- if (!completedInTime)
- {
+ if (!signaled)
throw new TimeoutException("Truncate timed out - received only " + responses.get() + " responses");
- }
- if (truncateFailingReplica != null)
+ if (!failureReasonByEndpoint.isEmpty())
{
- throw new TruncateException("Truncate failed on replica " + truncateFailingReplica);
+ // clone to make sure no race condition happens
+ Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint = new HashMap<>(this.failureReasonByEndpoint);
+ if (RequestCallback.isTimeout(failureReasonByEndpoint))
+ throw new TimeoutException("Truncate timed out - received only " + responses.get() + " responses");
+
+ StringBuilder sb = new StringBuilder("Truncate failed on ");
+ for (Map.Entry<InetAddressAndPort, RequestFailureReason> e : failureReasonByEndpoint.entrySet())
+ sb.append("replica ").append(e.getKey()).append(" -> ").append(e.getValue()).append(", ");
+ sb.setLength(sb.length() - 2);
+ throw new TruncateException(sb.toString());
}
}
@@ -94,7 +103,7 @@
public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
// If the truncation hasn't succeeded on some replica, abort and indicate this back to the client.
- truncateFailingReplica = from.getAddress();
+ failureReasonByEndpoint.put(from, failureReason);
condition.signalAll();
}
diff --git a/src/java/org/apache/cassandra/service/pager/PagingState.java b/src/java/org/apache/cassandra/service/pager/PagingState.java
index 6e1a52f..3691d14 100644
--- a/src/java/org/apache/cassandra/service/pager/PagingState.java
+++ b/src/java/org/apache/cassandra/service/pager/PagingState.java
@@ -22,7 +22,6 @@
import java.util.*;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.primitives.Ints;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -130,8 +129,8 @@
DataOutputBuffer out = new DataOutputBufferFixed(modernSerializedSize());
writeWithVIntLength(null == partitionKey ? EMPTY_BYTE_BUFFER : partitionKey, out);
writeWithVIntLength(null == rowMark ? EMPTY_BYTE_BUFFER : rowMark.mark, out);
- out.writeUnsignedVInt(remaining);
- out.writeUnsignedVInt(remainingInPartition);
+ out.writeUnsignedVInt32(remaining);
+ out.writeUnsignedVInt32(remainingInPartition);
return out.buffer(false);
}
@@ -207,8 +206,8 @@
ByteBuffer partitionKey = readWithVIntLength(in);
ByteBuffer rawMark = readWithVIntLength(in);
- int remaining = Ints.checkedCast(in.readUnsignedVInt());
- int remainingInPartition = Ints.checkedCast(in.readUnsignedVInt());
+ int remaining = in.readUnsignedVInt32();
+ int remainingInPartition = in.readUnsignedVInt32();
return new PagingState(partitionKey.hasRemaining() ? partitionKey : null,
rawMark.hasRemaining() ? new RowMark(rawMark, protocolVersion) : null,
diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java
index bf5f90e..acc6267 100644
--- a/src/java/org/apache/cassandra/service/paxos/Paxos.java
+++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java
@@ -48,6 +48,7 @@
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.locator.ReplicaLayout.ForTokenWrite;
import org.apache.cassandra.locator.ReplicaPlan.ForRead;
+import org.apache.cassandra.metrics.ClientRequestSizeMetrics;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -99,6 +100,8 @@
import static java.util.Collections.emptyMap;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_LOG_TTL_LINEARIZABILITY_VIOLATIONS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_MODERN_RELEASE;
import static org.apache.cassandra.config.Config.PaxosVariant.v2_without_linearizable_reads_or_rejected_writes;
import static org.apache.cassandra.db.Keyspace.openAndGetStore;
import static org.apache.cassandra.exceptions.RequestFailureReason.TIMEOUT;
@@ -213,8 +216,8 @@
private static final Logger logger = LoggerFactory.getLogger(Paxos.class);
private static volatile Config.PaxosVariant PAXOS_VARIANT = DatabaseDescriptor.getPaxosVariant();
- private static final CassandraVersion MODERN_PAXOS_RELEASE = new CassandraVersion(System.getProperty("cassandra.paxos.modern_release", "4.1"));
- static final boolean LOG_TTL_LINEARIZABILITY_VIOLATIONS = Boolean.parseBoolean(System.getProperty("cassandra.paxos.log_ttl_linearizability_violations", "true"));
+ private static final CassandraVersion MODERN_PAXOS_RELEASE = new CassandraVersion(PAXOS_MODERN_RELEASE.getString());
+ static final boolean LOG_TTL_LINEARIZABILITY_VIOLATIONS = PAXOS_LOG_TTL_LINEARIZABILITY_VIOLATIONS.getBoolean();
static class Electorate implements Iterable<InetAddressAndPort>
{
@@ -709,6 +712,9 @@
// TODO "turn null updates into delete?" - what does this TODO even mean?
PartitionUpdate updates = request.makeUpdates(current, clientState, begin.ballot);
+ // Update the metrics before triggers potentially add mutations.
+ ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(updates);
+
// Apply triggers to cas updates. A consideration here is that
// triggers emit Mutations, and so a given trigger implementation
// may generate mutations for partitions other than the one this
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java
index 4321fc9..43f9dd1 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java
@@ -25,6 +25,7 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Mutation;
@@ -57,7 +58,7 @@
public static final RequestHandler requestHandler = new RequestHandler();
private static final Logger logger = LoggerFactory.getLogger(PaxosCommit.class);
- private static volatile boolean ENABLE_DC_LOCAL_COMMIT = Boolean.parseBoolean(System.getProperty("cassandra.enable_dc_local_commit", "true"));
+ private static volatile boolean ENABLE_DC_LOCAL_COMMIT = CassandraRelevantProperties.ENABLE_DC_LOCAL_COMMIT.getBoolean();
public static boolean getEnableDcLocalCommit()
{
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java
index d88323c..8ae8ebd 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java
@@ -65,6 +65,7 @@
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_VERSION_VALIDATION;
import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN;
import static org.apache.cassandra.net.Verb.PAXOS2_REPAIR_REQ;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -634,7 +635,7 @@
}
}
- private static volatile boolean SKIP_VERSION_VALIDATION = Boolean.getBoolean("cassandra.skip_paxos_repair_version_validation");
+ private static volatile boolean SKIP_VERSION_VALIDATION = SKIP_PAXOS_REPAIR_VERSION_VALIDATION.getBoolean();
public static void setSkipPaxosRepairCompatibilityCheck(boolean v)
{
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java
index 1627fdb..d9546d4 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java
@@ -29,6 +29,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.dht.Range;
@@ -180,7 +181,7 @@
Ballot[] ballotLowBounds = new Ballot[tuples.size()];
for (int i = 0 ; i < tuples.size() ; ++i)
{
- ByteBuffer[] split = TYPE.split(tuples.get(i));
+ ByteBuffer[] split = TYPE.split(ByteBufferAccessor.instance, tuples.get(i));
if (i < tokenInclusiveUpperBounds.length)
tokenInclusiveUpperBounds[i] = TOKEN_FACTORY.fromByteArray(split[0]);
ballotLowBounds[i] = Ballot.deserialize(split[1]);
@@ -314,7 +315,7 @@
{
public void serialize(PaxosRepairHistory history, DataOutputPlus out, int version) throws IOException
{
- out.writeUnsignedVInt(history.size());
+ out.writeUnsignedVInt32(history.size());
for (int i = 0; i < history.size() ; ++i)
{
Token.serializer.serialize(history.tokenInclusiveUpperBound[i], out, version);
@@ -325,7 +326,7 @@
public PaxosRepairHistory deserialize(DataInputPlus in, int version) throws IOException
{
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
Token[] tokenInclusiveUpperBounds = new Token[size];
Ballot[] ballotLowBounds = new Ballot[size + 1];
for (int i = 0; i < size; i++)
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java b/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java
index 282aeb2..aad32ac 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java
@@ -37,7 +37,7 @@
public abstract class PaxosRequestCallback<T> extends FailureRecordingCallback<T>
{
private static final Logger logger = LoggerFactory.getLogger(PaxosRequestCallback.class);
- private static final boolean USE_SELF_EXECUTION = CassandraRelevantProperties.PAXOS_EXECUTE_ON_SELF.getBoolean();
+ private static final boolean USE_SELF_EXECUTION = CassandraRelevantProperties.PAXOS_USE_SELF_EXECUTION.getBoolean();
protected abstract void onResponse(T response, InetAddressAndPort from);
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
index e802cd0..22f064d 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
@@ -50,6 +50,7 @@
import org.apache.cassandra.utils.Nemesis;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_DISABLE_COORDINATOR_LOCKING;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.config.Config.PaxosStatePurging.gc_grace;
import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
@@ -66,7 +67,7 @@
*/
public class PaxosState implements PaxosOperationLock
{
- private static volatile boolean DISABLE_COORDINATOR_LOCKING = Boolean.getBoolean("cassandra.paxos.disable_coordinator_locking");
+ private static volatile boolean DISABLE_COORDINATOR_LOCKING = PAXOS_DISABLE_COORDINATOR_LOCKING.getBoolean();
public static final ConcurrentHashMap<Key, PaxosState> ACTIVE = new ConcurrentHashMap<>();
public static final Map<Key, Snapshot> RECENT = Caffeine.newBuilder()
.maximumWeight(DatabaseDescriptor.getPaxosCacheSizeInMiB() << 20)
diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java
index 3d765ea..5f1eea6 100644
--- a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java
+++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java
@@ -43,6 +43,7 @@
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_CLEANUP_SESSION_TIMEOUT_SECONDS;
import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@@ -56,7 +57,7 @@
static final long TIMEOUT_NANOS;
static
{
- long timeoutSeconds = Integer.getInteger("cassandra.paxos_cleanup_session_timeout_seconds", (int) TimeUnit.HOURS.toSeconds(2));
+ long timeoutSeconds = PAXOS_CLEANUP_SESSION_TIMEOUT_SECONDS.getLong();
TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(timeoutSeconds);
}
diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
index d3594b3..514ce15 100644
--- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
+++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
@@ -53,6 +53,9 @@
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FORCE_PAXOS_STATE_REBUILD;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_STATE_REBUILD;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TRUNCATE_BALLOT_METADATA;
import static org.apache.cassandra.db.SystemKeyspace.PAXOS_REPAIR_HISTORY;
import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME;
@@ -63,27 +66,23 @@
{
private static final Logger logger = LoggerFactory.getLogger(PaxosStateTracker.class);
- // when starting with no data, skip rebuilding uncommitted data from the paxos table
- static final String SKIP_REBUILD_PROP = "cassandra.skip_paxos_state_rebuild";
- static final String FORCE_REBUILD_PROP = "cassandra.force_paxos_state_rebuild";
- static final String TRUNCATE_BALLOT_METADATA_PROP = "cassandra.truncate_ballot_metadata";
-
+ /** when starting with no data, skip rebuilding uncommitted data from the paxos table. */
private static boolean skipRebuild()
{
- return Boolean.getBoolean(SKIP_REBUILD_PROP);
+ return SKIP_PAXOS_STATE_REBUILD.getBoolean();
}
private static boolean forceRebuild()
{
- return Boolean.getBoolean(FORCE_REBUILD_PROP);
+ return FORCE_PAXOS_STATE_REBUILD.getBoolean();
}
private static boolean truncateBallotMetadata()
{
- return Boolean.getBoolean(TRUNCATE_BALLOT_METADATA_PROP);
+ return TRUNCATE_BALLOT_METADATA.getBoolean();
}
- private static final String DIRECTORY = "system/_paxos_repair_state";
+ private static final String DIRECTORY = "system/" + SystemKeyspace.PAXOS_REPAIR_STATE;
private final PaxosUncommittedTracker uncommitted;
private final PaxosBallotTracker ballots;
@@ -130,8 +129,8 @@
boolean rebuildNeeded = !hasExistingData || forceRebuild();
if (truncateBallotMetadata() && !rebuildNeeded)
- logger.warn("{} was set, but {} was not and no rebuild is required. Ballot data will not be truncated",
- TRUNCATE_BALLOT_METADATA_PROP, FORCE_REBUILD_PROP);
+ logger.warn("{} was set to true, but {} was not and no rebuild is required. Ballot data will not be truncated",
+ TRUNCATE_BALLOT_METADATA.getKey(), FORCE_PAXOS_STATE_REBUILD.getKey());
if (rebuildNeeded)
{
@@ -164,7 +163,7 @@
@SuppressWarnings("resource")
private void rebuildUncommittedData() throws IOException
{
- logger.info("Beginning uncommitted paxos data rebuild. Set -Dcassandra.skip_paxos_state_rebuild=true and restart to skip");
+ logger.info("Beginning uncommitted paxos data rebuild. Set -D{}=true and restart to skip", SKIP_PAXOS_STATE_REBUILD.getKey());
String queryStr = "SELECT * FROM " + SYSTEM_KEYSPACE_NAME + '.' + SystemKeyspace.PAXOS;
SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(queryStr).prepare(ClientState.forInternalCalls());
diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java
index 5e3b540..b03701d 100644
--- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java
+++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java
@@ -18,7 +18,12 @@
package org.apache.cassandra.service.paxos.uncommitted;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
@@ -29,13 +34,24 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.WriteContext;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@@ -43,14 +59,14 @@
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.index.transactions.IndexTransaction;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Indexes;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.CloseableIterator;
-import static java.util.Collections.*;
+import static java.util.Collections.singletonList;
import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME;
import static org.apache.cassandra.service.paxos.PaxosState.ballotTracker;
import static org.apache.cassandra.service.paxos.PaxosState.uncommittedTracker;
diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java
index 7712c79..ae4662f 100644
--- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java
+++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java
@@ -49,6 +49,9 @@
import org.apache.cassandra.service.paxos.cleanup.PaxosTableRepairs;
import org.apache.cassandra.utils.CloseableIterator;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTO_REPAIR_FREQUENCY_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_PAXOS_AUTO_REPAIRS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_PAXOS_STATE_FLUSH;
import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled;
import static org.apache.cassandra.service.paxos.uncommitted.PaxosKeyState.mergeUncommitted;
@@ -69,8 +72,8 @@
private static volatile UpdateSupplier updateSupplier;
- private volatile boolean autoRepairsEnabled = !Boolean.getBoolean("cassandra.disable_paxos_auto_repairs");
- private volatile boolean stateFlushEnabled = !Boolean.getBoolean("cassandra.disable_paxos_state_flush");
+ private volatile boolean autoRepairsEnabled = !DISABLE_PAXOS_AUTO_REPAIRS.getBoolean();
+ private volatile boolean stateFlushEnabled = !DISABLE_PAXOS_STATE_FLUSH.getBoolean();
private boolean started = false;
private boolean autoRepairStarted = false;
@@ -327,7 +330,7 @@
{
if (autoRepairStarted)
return;
- int seconds = Integer.getInteger("cassandra.auto_repair_frequency_seconds", (int) TimeUnit.MINUTES.toSeconds(5));
+ int seconds = AUTO_REPAIR_FREQUENCY_SECONDS.getInt();
ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(this::maintenance, seconds, seconds, TimeUnit.SECONDS);
autoRepairStarted = true;
}
diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java
index e69e6bd..c25b1f0 100644
--- a/src/java/org/apache/cassandra/service/reads/ReadCallback.java
+++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.service.reads;
+import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -120,6 +121,12 @@
*/
int received = resolver.responses.size();
boolean failed = failures > 0 && (blockFor > received || !resolver.isDataPresent());
+ // If all messages came back as a TIMEOUT then signaled=true and failed=true.
+ // Need to distinguish between a timeout and a failure (network, bad data, etc.), so store an extra field.
+ // see CASSANDRA-17828
+ boolean timedout = !signaled;
+ if (failed)
+ timedout = RequestCallback.isTimeout(new HashMap<>(failureReasonByEndpoint));
WarningContext warnings = warningContext;
// save the snapshot so abort state is not changed between now and when mayAbort gets called
WarningsSnapshot snapshot = null;
@@ -138,19 +145,19 @@
if (isTracing())
{
String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : "";
- Tracing.trace("{}; received {} of {} responses{}", failed ? "Failed" : "Timed out", received, blockFor, gotData);
+ Tracing.trace("{}; received {} of {} responses{}", !timedout ? "Failed" : "Timed out", received, blockFor, gotData);
}
else if (logger.isDebugEnabled())
{
String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : "";
- logger.debug("{}; received {} of {} responses{}", failed ? "Failed" : "Timed out", received, blockFor, gotData);
+ logger.debug("{}; received {} of {} responses{}", !timedout ? "Failed" : "Timed out", received, blockFor, gotData);
}
if (snapshot != null)
snapshot.maybeAbort(command, replicaPlan().consistencyLevel(), received, blockFor, resolver.isDataPresent(), failureReasonByEndpoint);
// Same as for writes, see AbstractWriteResponseHandler
- throw failed
+ throw !timedout
? new ReadFailureException(replicaPlan().consistencyLevel(), received, blockFor, resolver.isDataPresent(), failureReasonByEndpoint)
: new ReadTimeoutException(replicaPlan().consistencyLevel(), received, blockFor, resolver.isDataPresent());
}
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
index 5b656d7..1c5ce7f 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
@@ -21,6 +21,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
@@ -49,8 +50,7 @@
* don't want a burst of range requests that will back up, hurting all other queries. At the same time,
* we want to give range queries a chance to run if resources are available.
*/
- private static final int MAX_CONCURRENT_RANGE_REQUESTS = Math.max(1, Integer.getInteger("cassandra.max_concurrent_range_requests",
- FBUtilities.getAvailableProcessors() * 10));
+ private static final int MAX_CONCURRENT_RANGE_REQUESTS = Math.max(1, CassandraRelevantProperties.MAX_CONCURRENT_RANGE_REQUESTS.getInt(FBUtilities.getAvailableProcessors() * 10));
@SuppressWarnings("resource") // created iterators will be closed in CQL layer through the chain of transformations
public static PartitionIterator partitions(PartitionRangeReadCommand command,
diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java
index 7a4882b..de49f5a 100644
--- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java
+++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java
@@ -33,14 +33,14 @@
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.tracing.Tracing;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DROP_OVERSIZED_READ_REPAIR_MUTATIONS;
import static org.apache.cassandra.db.IMutation.MAX_MUTATION_SIZE;
public class BlockingReadRepairs
{
private static final Logger logger = LoggerFactory.getLogger(BlockingReadRepairs.class);
- private static final boolean DROP_OVERSIZED_READ_REPAIR_MUTATIONS =
- Boolean.getBoolean("cassandra.drop_oversized_readrepair_mutations");
+ private static final boolean SHOULD_DROP_OVERSIZED_READ_REPAIR_MUTATIONS = DROP_OVERSIZED_READ_REPAIR_MUTATIONS.getBoolean();
/**
* Create a read repair mutation from the given update, if the mutation is not larger than the maximum
@@ -65,7 +65,7 @@
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
TableMetadata metadata = update.metadata();
- if (DROP_OVERSIZED_READ_REPAIR_MUTATIONS)
+ if (SHOULD_DROP_OVERSIZED_READ_REPAIR_MUTATIONS)
{
logger.debug("Encountered an oversized ({}/{}) read repair mutation for table {}, key {}, node {}",
e.mutationSize,
diff --git a/src/java/org/apache/cassandra/service/reads/thresholds/CoordinatorWarnings.java b/src/java/org/apache/cassandra/service/reads/thresholds/CoordinatorWarnings.java
index ff5bd78..1b317db 100644
--- a/src/java/org/apache/cassandra/service/reads/thresholds/CoordinatorWarnings.java
+++ b/src/java/org/apache/cassandra/service/reads/thresholds/CoordinatorWarnings.java
@@ -33,10 +33,12 @@
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.service.ClientWarn;
+import static org.apache.cassandra.config.CassandraRelevantProperties.READS_THRESHOLDS_COORDINATOR_DEFENSIVE_CHECKS_ENABLED;
+
public class CoordinatorWarnings
{
private static final Logger logger = LoggerFactory.getLogger(CoordinatorWarnings.class);
- private static final boolean ENABLE_DEFENSIVE_CHECKS = Boolean.getBoolean("cassandra.reads.thresholds.coordinator.defensive_checks_enabled");
+ private static final boolean ENABLE_DEFENSIVE_CHECKS = READS_THRESHOLDS_COORDINATOR_DEFENSIVE_CHECKS_ENABLED.getBoolean();
// when .init() is called set the STATE to be INIT; this is to lazy allocate the map only when warnings are generated
private static final Map<ReadCommand, WarningsSnapshot> INIT = Collections.emptyMap();
diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
index 523e1e5..d31df36 100644
--- a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
+++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
@@ -73,6 +73,11 @@
this.dataDirectories = dataDirs;
}
+ public SnapshotLoader(Directories directories)
+ {
+ this(directories.getCFDirectories().stream().map(File::toPath).collect(Collectors.toList()));
+ }
+
@VisibleForTesting
static class Visitor extends SimpleFileVisitor<Path>
{
@@ -152,17 +157,24 @@
}
}
- public Set<TableSnapshot> loadSnapshots()
+ public Set<TableSnapshot> loadSnapshots(String keyspace)
{
+ // if we supply a keyspace, the walking max depth will be suddenly shorther
+ // because we are one level down in the directory structure
+ int maxDepth = keyspace == null ? 5 : 4;
+
Map<String, TableSnapshot.Builder> snapshots = new HashMap<>();
Visitor visitor = new Visitor(snapshots);
for (Path dataDir : dataDirectories)
{
+ if (keyspace != null)
+ dataDir = dataDir.resolve(keyspace);
+
try
{
if (new File(dataDir).exists())
- Files.walkFileTree(dataDir, Collections.emptySet(), 5, visitor);
+ Files.walkFileTree(dataDir, Collections.emptySet(), maxDepth, visitor);
else
logger.debug("Skipping non-existing data directory {}", dataDir);
}
@@ -174,4 +186,9 @@
return snapshots.values().stream().map(TableSnapshot.Builder::build).collect(Collectors.toSet());
}
+
+ public Set<TableSnapshot> loadSnapshots()
+ {
+ return loadSnapshots(null);
+ }
}
diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
index 8e630c7..3925f3f 100644
--- a/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
+++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
@@ -109,6 +109,11 @@
}
}
+ public synchronized Set<TableSnapshot> loadSnapshots(String keyspace)
+ {
+ return snapshotLoader.loadSnapshots(keyspace);
+ }
+
public synchronized Set<TableSnapshot> loadSnapshots()
{
return snapshotLoader.loadSnapshots();
@@ -148,6 +153,9 @@
}
}
+ /**
+ * Deletes snapshot and remove it from manager
+ */
public synchronized void clearSnapshot(TableSnapshot snapshot)
{
for (File snapshotDir : snapshot.getDirectories())
diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java
index ba840ef..8ee737c 100644
--- a/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java
+++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java
@@ -28,7 +28,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JsonUtils;
// Only serialize fields
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY,
@@ -46,19 +46,25 @@
@JsonProperty("expires_at")
public final Instant expiresAt;
+ @JsonProperty("ephemeral")
+ public final boolean ephemeral;
+
/** needed for jackson serialization */
@SuppressWarnings("unused")
- private SnapshotManifest() {
+ private SnapshotManifest()
+ {
this.files = null;
this.createdAt = null;
this.expiresAt = null;
+ this.ephemeral = false;
}
- public SnapshotManifest(List<String> files, DurationSpec.IntSecondsBound ttl, Instant creationTime)
+ public SnapshotManifest(List<String> files, DurationSpec.IntSecondsBound ttl, Instant creationTime, boolean ephemeral)
{
this.files = files;
this.createdAt = creationTime;
this.expiresAt = ttl == null ? null : createdAt.plusSeconds(ttl.toSeconds());
+ this.ephemeral = ephemeral;
}
public List<String> getFiles()
@@ -76,14 +82,19 @@
return expiresAt;
}
+ public boolean isEphemeral()
+ {
+ return ephemeral;
+ }
+
public void serializeToJsonFile(File outputFile) throws IOException
{
- FBUtilities.serializeToJsonFile(this, outputFile);
+ JsonUtils.serializeToJsonFile(this, outputFile);
}
public static SnapshotManifest deserializeFromJsonFile(File file) throws IOException
{
- return FBUtilities.deserializeFromJsonFile(SnapshotManifest.class, file);
+ return JsonUtils.deserializeFromJsonFile(SnapshotManifest.class, file);
}
@Override
@@ -92,12 +103,15 @@
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SnapshotManifest manifest = (SnapshotManifest) o;
- return Objects.equals(files, manifest.files) && Objects.equals(createdAt, manifest.createdAt) && Objects.equals(expiresAt, manifest.expiresAt);
+ return Objects.equals(files, manifest.files)
+ && Objects.equals(createdAt, manifest.createdAt)
+ && Objects.equals(expiresAt, manifest.expiresAt)
+ && Objects.equals(ephemeral, manifest.ephemeral);
}
@Override
public int hashCode()
{
- return Objects.hash(files, createdAt, expiresAt);
+ return Objects.hash(files, createdAt, expiresAt, ephemeral);
}
}
diff --git a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
index 0cfcfea..d10092a 100644
--- a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
+++ b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
@@ -27,6 +27,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Predicate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +45,7 @@
private final String tableName;
private final UUID tableId;
private final String tag;
+ private final boolean ephemeral;
private final Instant createdAt;
private final Instant expiresAt;
@@ -52,7 +54,7 @@
public TableSnapshot(String keyspaceName, String tableName, UUID tableId,
String tag, Instant createdAt, Instant expiresAt,
- Set<File> snapshotDirs)
+ Set<File> snapshotDirs, boolean ephemeral)
{
this.keyspaceName = keyspaceName;
this.tableName = tableName;
@@ -61,6 +63,7 @@
this.createdAt = createdAt;
this.expiresAt = expiresAt;
this.snapshotDirs = snapshotDirs;
+ this.ephemeral = ephemeral;
}
/**
@@ -123,6 +126,11 @@
return snapshotDirs.stream().anyMatch(File::exists);
}
+ public boolean isEphemeral()
+ {
+ return ephemeral;
+ }
+
public boolean isExpiring()
{
return expiresAt != null;
@@ -192,13 +200,13 @@
return Objects.equals(keyspaceName, snapshot.keyspaceName) && Objects.equals(tableName, snapshot.tableName) &&
Objects.equals(tableId, snapshot.tableId) && Objects.equals(tag, snapshot.tag) &&
Objects.equals(createdAt, snapshot.createdAt) && Objects.equals(expiresAt, snapshot.expiresAt) &&
- Objects.equals(snapshotDirs, snapshot.snapshotDirs);
+ Objects.equals(snapshotDirs, snapshot.snapshotDirs) && Objects.equals(ephemeral, snapshot.ephemeral);
}
@Override
public int hashCode()
{
- return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs);
+ return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral);
}
@Override
@@ -212,6 +220,7 @@
", createdAt=" + createdAt +
", expiresAt=" + expiresAt +
", snapshotDirs=" + snapshotDirs +
+ ", ephemeral=" + ephemeral +
'}';
}
@@ -223,6 +232,7 @@
private Instant createdAt = null;
private Instant expiresAt = null;
+ private boolean ephemeral;
private final Set<File> snapshotDirs = new HashSet<>();
@@ -238,12 +248,17 @@
{
snapshotDirs.add(snapshotDir);
File manifestFile = new File(snapshotDir, "manifest.json");
- if (manifestFile.exists() && createdAt == null && expiresAt == null) {
- loadTimestampsFromManifest(manifestFile);
- }
+ if (manifestFile.exists() && createdAt == null && expiresAt == null)
+ loadMetadataFromManifest(manifestFile);
+
+ // check if an ephemeral marker file exists only in case it is not already ephemeral
+ // by reading it from manifest
+ // TODO remove this on Cassandra 4.3 release, see CASSANDRA-16911
+ if (!ephemeral && new File(snapshotDir, "ephemeral.snapshot").exists())
+ ephemeral = true;
}
- private void loadTimestampsFromManifest(File manifestFile)
+ private void loadMetadataFromManifest(File manifestFile)
{
try
{
@@ -251,6 +266,9 @@
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(manifestFile);
createdAt = manifest.createdAt;
expiresAt = manifest.expiresAt;
+ // a snapshot may be ephemeral when it has a marker file (old way) or flag in manifest (new way)
+ if (!ephemeral)
+ ephemeral = manifest.ephemeral;
}
catch (IOException e)
{
@@ -260,7 +278,7 @@
TableSnapshot build()
{
- return new TableSnapshot(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs);
+ return new TableSnapshot(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral);
}
}
@@ -305,4 +323,30 @@
}
return new File(liveDir.toString(), snapshotFilePath.getFileName().toString());
}
+
+ public static Predicate<TableSnapshot> shouldClearSnapshot(String tag, long olderThanTimestamp)
+ {
+ return ts ->
+ {
+ // When no tag is supplied, all snapshots must be cleared
+ boolean clearAll = tag == null || tag.isEmpty();
+ if (!clearAll && ts.isEphemeral())
+ logger.info("Skipping deletion of ephemeral snapshot '{}' in keyspace {}. " +
+ "Ephemeral snapshots are not removable by a user.",
+ tag, ts.keyspaceName);
+ boolean notEphemeral = !ts.isEphemeral();
+ boolean shouldClearTag = clearAll || ts.tag.equals(tag);
+ boolean byTimestamp = true;
+
+ if (olderThanTimestamp > 0L)
+ {
+ Instant createdAt = ts.getCreatedAt();
+ if (createdAt != null)
+ byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp));
+ }
+
+ return notEphemeral && shouldClearTag && byTimestamp;
+ };
+ }
+
}
diff --git a/src/java/org/apache/cassandra/streaming/ProgressInfo.java b/src/java/org/apache/cassandra/streaming/ProgressInfo.java
index 2ed78ac..159775c 100644
--- a/src/java/org/apache/cassandra/streaming/ProgressInfo.java
+++ b/src/java/org/apache/cassandra/streaming/ProgressInfo.java
@@ -60,7 +60,6 @@
public ProgressInfo(InetAddressAndPort peer, int sessionIndex, String fileName, Direction direction,
long currentBytes, long deltaBytes, long totalBytes)
{
- assert totalBytes > 0;
this.peer = peer;
this.sessionIndex = sessionIndex;
@@ -79,6 +78,11 @@
return currentBytes >= totalBytes;
}
+ public int progressPercentage()
+ {
+ return totalBytes == 0 ? 100 : (int) ((100 * currentBytes) / totalBytes);
+ }
+
/**
* ProgressInfo is considered to be equal only when all attributes except currentBytes are equal.
*/
@@ -114,10 +118,10 @@
StringBuilder sb = new StringBuilder(fileName);
sb.append(" ").append(currentBytes);
sb.append("/").append(totalBytes).append(" bytes ");
- sb.append("(").append(currentBytes*100/totalBytes).append("%) ");
+ sb.append("(").append(progressPercentage()).append("%) ");
sb.append(direction == Direction.OUT ? "sent to " : "received from ");
sb.append("idx:").append(sessionIndex);
sb.append(peer.toString(withPorts));
return sb.toString();
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/streaming/SessionInfo.java b/src/java/org/apache/cassandra/streaming/SessionInfo.java
index d95d85b..92af245 100644
--- a/src/java/org/apache/cassandra/streaming/SessionInfo.java
+++ b/src/java/org/apache/cassandra/streaming/SessionInfo.java
@@ -47,12 +47,15 @@
private final Map<String, ProgressInfo> receivingFiles = new ConcurrentHashMap<>();
private final Map<String, ProgressInfo> sendingFiles = new ConcurrentHashMap<>();
+ public final String failureReason;
+
public SessionInfo(InetSocketAddress peer,
int sessionIndex,
InetSocketAddress connecting,
Collection<StreamSummary> receivingSummaries,
Collection<StreamSummary> sendingSummaries,
- StreamSession.State state)
+ StreamSession.State state,
+ String failureReason)
{
this.peer = peer;
this.sessionIndex = sessionIndex;
@@ -60,11 +63,12 @@
this.receivingSummaries = ImmutableSet.copyOf(receivingSummaries);
this.sendingSummaries = ImmutableSet.copyOf(sendingSummaries);
this.state = state;
+ this.failureReason = failureReason;
}
public SessionInfo(SessionInfo other)
{
- this(other.peer, other.sessionIndex, other.connecting, other.receivingSummaries, other.sendingSummaries, other.state);
+ this(other.peer, other.sessionIndex, other.connecting, other.receivingSummaries, other.sendingSummaries, other.state, other.failureReason);
}
public boolean isFailed()
@@ -205,4 +209,9 @@
{
return new SessionSummary(FBUtilities.getBroadcastAddressAndPort(), peer, receivingSummaries, sendingSummaries);
}
+
+ public String getFailureReason()
+ {
+ return failureReason;
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamHook.java b/src/java/org/apache/cassandra/streaming/StreamHook.java
index 86b5182..84db420 100644
--- a/src/java/org/apache/cassandra/streaming/StreamHook.java
+++ b/src/java/org/apache/cassandra/streaming/StreamHook.java
@@ -22,6 +22,8 @@
import org.apache.cassandra.streaming.messages.OutgoingStreamMessage;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.STREAM_HOOK;
+
public interface StreamHook
{
public static final StreamHook instance = createHook();
@@ -32,7 +34,7 @@
static StreamHook createHook()
{
- String className = System.getProperty("cassandra.stream_hook");
+ String className = STREAM_HOOK.getString();
if (className != null)
{
return FBUtilities.construct(className, StreamHook.class.getSimpleName());
diff --git a/src/java/org/apache/cassandra/streaming/StreamManager.java b/src/java/org/apache/cassandra/streaming/StreamManager.java
index 408b6f4..fec8b2d 100644
--- a/src/java/org/apache/cassandra/streaming/StreamManager.java
+++ b/src/java/org/apache/cassandra/streaming/StreamManager.java
@@ -31,6 +31,7 @@
import com.google.common.base.Function;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
@@ -252,18 +253,35 @@
}
};
+ protected void addStreamingStateAgain(StreamingState state)
+ {
+ if (!DatabaseDescriptor.getStreamingStatsEnabled())
+ return;
+ states.put(state.id(), state);
+ }
+
public StreamManager()
{
DurationSpec.LongNanosecondsBound duration = DatabaseDescriptor.getStreamingStateExpires();
long sizeBytes = DatabaseDescriptor.getStreamingStateSize().toBytes();
- long numElements = sizeBytes / StreamingState.ELEMENT_SIZE;
- logger.info("Storing streaming state for {} or for {} elements", duration, numElements);
+ logger.info("Storing streaming state for {} or for size {}", duration, sizeBytes);
states = CacheBuilder.newBuilder()
.expireAfterWrite(duration.quantity(), duration.unit())
- .maximumSize(numElements)
+ .maximumWeight(sizeBytes)
+ .weigher(new StreamingStateWeigher())
.build();
}
+ private static class StreamingStateWeigher implements Weigher<TimeUUID,StreamingState>
+ {
+ @Override
+ public int weigh(TimeUUID key, StreamingState val)
+ {
+ long costOfStreamingState = val.unsharedHeapSize() + TimeUUID.TIMEUUID_SIZE;
+ return Math.toIntExact(costOfStreamingState);
+ }
+ }
+
public void start()
{
addListener(listener);
@@ -447,6 +465,17 @@
return streamResultFuture.getSession(peer, sessionIndex);
}
+ public long getTotalRemainingOngoingBytes()
+ {
+ long total = 0;
+ for (StreamResultFuture fut : Iterables.concat(initiatorStreams.values(), followerStreams.values()))
+ {
+ for (SessionInfo sessionInfo : fut.getCurrentState().sessions)
+ total += sessionInfo.getTotalSizeToReceive() - sessionInfo.getTotalSizeReceived();
+ }
+ return total;
+ }
+
public interface StreamListener
{
default void onRegister(StreamResultFuture result) {}
diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java
index 8151b47..98a4070 100644
--- a/src/java/org/apache/cassandra/streaming/StreamOperation.java
+++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java
@@ -19,43 +19,43 @@
public enum StreamOperation
{
- OTHER("Other"), // Fallback to avoid null types when deserializing from string
- RESTORE_REPLICA_COUNT("Restore replica count", false), // Handles removeNode
- DECOMMISSION("Unbootstrap", false),
- RELOCATION("Relocation", false),
- BOOTSTRAP("Bootstrap", false),
- REBUILD("Rebuild", false),
- BULK_LOAD("Bulk Load"),
- REPAIR("Repair");
+ OTHER("Other", true, false), // Fallback to avoid null types when deserializing from string
+ RESTORE_REPLICA_COUNT("Restore replica count", false, false), // Handles removeNode
+ DECOMMISSION("Unbootstrap", false, true),
+ RELOCATION("Relocation", false, true),
+ BOOTSTRAP("Bootstrap", false, true),
+ REBUILD("Rebuild", false, true),
+ BULK_LOAD("Bulk Load", true, false),
+ REPAIR("Repair", true, false);
private final String description;
private final boolean requiresViewBuild;
-
-
- StreamOperation(String description) {
- this(description, true);
- }
+ private final boolean keepSSTableLevel;
/**
* @param description The operation description
* @param requiresViewBuild Whether this operation requires views to be updated if it involves a base table
*/
- StreamOperation(String description, boolean requiresViewBuild) {
+ StreamOperation(String description, boolean requiresViewBuild, boolean keepSSTableLevel)
+ {
this.description = description;
this.requiresViewBuild = requiresViewBuild;
+ this.keepSSTableLevel = keepSSTableLevel;
}
- public static StreamOperation fromString(String text) {
- for (StreamOperation b : StreamOperation.values()) {
- if (b.description.equalsIgnoreCase(text)) {
+ public static StreamOperation fromString(String text)
+ {
+ for (StreamOperation b : StreamOperation.values())
+ {
+ if (b.description.equalsIgnoreCase(text))
return b;
- }
}
return OTHER;
}
- public String getDescription() {
+ public String getDescription()
+ {
return description;
}
@@ -66,4 +66,9 @@
{
return this.requiresViewBuild;
}
+
+ public boolean keepSSTableLevel()
+ {
+ return keepSSTableLevel;
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamResultFuture.java b/src/java/org/apache/cassandra/streaming/StreamResultFuture.java
index b43203d..5277b9d 100644
--- a/src/java/org/apache/cassandra/streaming/StreamResultFuture.java
+++ b/src/java/org/apache/cassandra/streaming/StreamResultFuture.java
@@ -239,8 +239,16 @@
StreamState finalState = getCurrentState();
if (finalState.hasFailedSession())
{
- logger.warn("[Stream #{}] Stream failed", planId);
- tryFailure(new StreamException(finalState, "Stream failed"));
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("Stream failed: ");
+ for (SessionInfo info : finalState.sessions())
+ {
+ if (info.isFailed())
+ stringBuilder.append("\nSession peer ").append(info.peer).append(' ').append(info.failureReason);
+ }
+ String message = stringBuilder.toString();
+ logger.warn("[Stream #{}] {}", planId, message);
+ tryFailure(new StreamException(finalState, message));
}
else if (finalState.hasAbortedSession())
{
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 811717f..e170ca6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -20,16 +20,22 @@
import java.io.EOFException;
import java.net.SocketTimeoutException;
import java.nio.channels.ClosedChannelException;
+import java.nio.file.FileStore;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
@@ -40,31 +46,36 @@
import io.netty.channel.Channel;
import io.netty.util.concurrent.Future; //checkstyle: permit this import
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.locator.RangesAtEndpoint;
-import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.concurrent.FutureCombiner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionStrategyManager;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.gms.*;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.StreamingMetrics;
import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.async.StreamingMultiplexedChannel;
import org.apache.cassandra.streaming.messages.*;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.FutureCombiner;
import static com.google.common.collect.Iterables.all;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_STREAMING_DEBUG_STACKTRACE_LIMIT;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.locator.InetAddressAndPort.hostAddressAndPort;
import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
@@ -144,9 +155,10 @@
* (via {@link org.apache.cassandra.net.MessagingService}, while the actual files themselves are sent by a special
* "streaming" connection type. See {@link StreamingMultiplexedChannel} for details. Because of the asynchronous
*/
-public class StreamSession implements IEndpointStateChangeSubscriber
+public class StreamSession
{
private static final Logger logger = LoggerFactory.getLogger(StreamSession.class);
+ private static final int DEBUG_STACKTRACE_LIMIT = CASSANDRA_STREAMING_DEBUG_STACKTRACE_LIMIT.getInt();
public enum PrepareDirection { SEND, ACK }
@@ -193,6 +205,8 @@
private final TimeUUID pendingRepair;
private final PreviewKind previewKind;
+ public String failureReason;
+
/**
* State Transition:
*
@@ -275,7 +289,15 @@
public StreamOperation streamOperation()
{
- return streamResult == null ? null : streamResult.streamOperation;
+ if (streamResult == null)
+ {
+ logger.warn("StreamResultFuture not initialized {} {}", channel.connectedTo(), isFollower ? "follower" : "initiator");
+ return null;
+ }
+ else
+ {
+ return streamResult.streamOperation;
+ }
}
public StreamOperation getStreamOperation()
@@ -496,13 +518,20 @@
}
}
- private synchronized Future<?> closeSession(State finalState)
+ private Future<?> closeSession(State finalState)
+ {
+ return closeSession(finalState, null);
+ }
+
+ private synchronized Future<?> closeSession(State finalState, String failureReason)
{
// it's session is already closed
if (closeFuture != null)
return closeFuture;
state(finalState);
+ //this refers to StreamInfo
+ this.failureReason = failureReason;
List<Future<?>> futures = new ArrayList<>();
@@ -655,7 +684,6 @@
if (state.finalState)
{
logger.debug("[Stream #{}] Socket closed after session completed with state {}", planId(), state);
-
return null;
}
else
@@ -664,8 +692,7 @@
planId(),
peer.getHostAddressAndPort(),
e);
-
- return closeSession(State.FAILED);
+ return closeSession(State.FAILED, "Failed because there was an " + e.getClass().getCanonicalName() + " with state=" + state.name());
}
}
@@ -676,8 +703,9 @@
state(State.FAILED); // make sure subsequent error handling sees the session in a final state
channel.sendControlMessage(new SessionFailedMessage()).awaitUninterruptibly();
}
-
- return closeSession(State.FAILED);
+ StringBuilder failureReason = new StringBuilder("Failed because of an unknown exception\n");
+ boundStackTrace(e, DEBUG_STACKTRACE_LIMIT, failureReason);
+ return closeSession(State.FAILED, failureReason.toString());
}
private void logError(Throwable e)
@@ -730,6 +758,8 @@
*/
private void prepareAsync(Collection<StreamRequest> requests, Collection<StreamSummary> summaries)
{
+ if (StreamOperation.REPAIR == streamOperation())
+ checkAvailableDiskSpaceAndCompactions(summaries);
for (StreamRequest request : requests)
addTransferRanges(request.keyspace, RangesAtEndpoint.concat(request.full, request.transientReplicas), request.columnFamilies, true); // always flush on stream request
for (StreamSummary summary : summaries)
@@ -758,6 +788,8 @@
private void prepareSynAck(PrepareSynAckMessage msg)
{
+ if (StreamOperation.REPAIR == streamOperation())
+ checkAvailableDiskSpaceAndCompactions(msg.summaries);
if (!msg.summaries.isEmpty())
{
for (StreamSummary summary : msg.summaries)
@@ -782,6 +814,163 @@
}
/**
+ * In the case where we have an error checking disk space we allow the Operation to continue.
+ * In the case where we do _not_ have available space, this method raises a RTE.
+ * TODO: Consider revising this to returning a boolean and allowing callers upstream to handle that.
+ */
+ private void checkAvailableDiskSpaceAndCompactions(Collection<StreamSummary> summaries)
+ {
+ if (DatabaseDescriptor.getSkipStreamDiskSpaceCheck())
+ return;
+
+ boolean hasAvailableSpace = true;
+
+ try
+ {
+ hasAvailableSpace = checkAvailableDiskSpaceAndCompactions(summaries, planId(), peer.getHostAddress(true), pendingRepair != null);
+ }
+ catch (Exception e)
+ {
+ logger.error("[Stream #{}] Could not check available disk space and compactions for {}, summaries = {}", planId(), this, summaries, e);
+ }
+ if (!hasAvailableSpace)
+ throw new RuntimeException(String.format("Not enough disk space for stream %s), summaries=%s", this, summaries));
+ }
+
+ /**
+ * Makes sure that we expect to have enough disk space available for the new streams, taking into consideration
+ * the ongoing compactions and streams.
+ */
+ @VisibleForTesting
+ public static boolean checkAvailableDiskSpaceAndCompactions(Collection<StreamSummary> summaries,
+ @Nullable TimeUUID planId,
+ @Nullable String remoteAddress,
+ boolean isForIncremental)
+ {
+ Map<TableId, Long> perTableIdIncomingBytes = new HashMap<>();
+ Map<TableId, Integer> perTableIdIncomingFiles = new HashMap<>();
+ long newStreamTotal = 0;
+ for (StreamSummary summary : summaries)
+ {
+ perTableIdIncomingFiles.merge(summary.tableId, summary.files, Integer::sum);
+ perTableIdIncomingBytes.merge(summary.tableId, summary.totalSize, Long::sum);
+ newStreamTotal += summary.totalSize;
+ }
+ if (perTableIdIncomingBytes.isEmpty() || newStreamTotal == 0)
+ return true;
+
+ return checkDiskSpace(perTableIdIncomingBytes, planId, Directories::getFileStore) &&
+ checkPendingCompactions(perTableIdIncomingBytes, perTableIdIncomingFiles, planId, remoteAddress, isForIncremental, newStreamTotal);
+ }
+
+ @VisibleForTesting
+ static boolean checkDiskSpace(Map<TableId, Long> perTableIdIncomingBytes,
+ TimeUUID planId,
+ Function<File, FileStore> fileStoreMapper)
+ {
+ Map<FileStore, Long> newStreamBytesToWritePerFileStore = new HashMap<>();
+ Set<FileStore> allFileStores = new HashSet<>();
+ // Sum up the incoming bytes per file store - we assume that the stream is evenly distributed over the writable
+ // file stores for the table.
+ for (Map.Entry<TableId, Long> entry : perTableIdIncomingBytes.entrySet())
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(entry.getKey());
+ if (cfs == null || perTableIdIncomingBytes.get(entry.getKey()) == 0)
+ continue;
+
+ Set<FileStore> allWriteableFileStores = cfs.getDirectories().allFileStores(fileStoreMapper);
+ if (allWriteableFileStores.isEmpty())
+ {
+ logger.error("[Stream #{}] Could not get any writeable FileStores for {}.{}", planId, cfs.keyspace.getName(), cfs.getTableName());
+ continue;
+ }
+ allFileStores.addAll(allWriteableFileStores);
+ long totalBytesInPerFileStore = entry.getValue() / allWriteableFileStores.size();
+ for (FileStore fs : allWriteableFileStores)
+ newStreamBytesToWritePerFileStore.merge(fs, totalBytesInPerFileStore, Long::sum);
+ }
+ Map<FileStore, Long> totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteBytes(),
+ fileStoreMapper);
+ long totalStreamRemaining = StreamManager.instance.getTotalRemainingOngoingBytes();
+ long totalBytesStreamRemainingPerFileStore = totalStreamRemaining / Math.max(1, allFileStores.size());
+ Map<FileStore, Long> allWriteData = new HashMap<>();
+ for (Map.Entry<FileStore, Long> fsBytes : newStreamBytesToWritePerFileStore.entrySet())
+ allWriteData.put(fsBytes.getKey(), fsBytes.getValue() +
+ totalBytesStreamRemainingPerFileStore +
+ totalCompactionWriteRemaining.getOrDefault(fsBytes.getKey(), 0L));
+
+ if (!Directories.hasDiskSpaceForCompactionsAndStreams(allWriteData))
+ {
+ logger.error("[Stream #{}] Not enough disk space to stream {} to {} (stream ongoing remaining={}, compaction ongoing remaining={}, all ongoing writes={})",
+ planId,
+ newStreamBytesToWritePerFileStore,
+ perTableIdIncomingBytes.keySet().stream()
+ .map(ColumnFamilyStore::getIfExists).filter(Objects::nonNull)
+ .map(cfs -> cfs.keyspace.getName() + '.' + cfs.name)
+ .collect(Collectors.joining(",")),
+ totalStreamRemaining,
+ totalCompactionWriteRemaining,
+ allWriteData);
+ return false;
+ }
+ return true;
+ }
+
+ @VisibleForTesting
+ static boolean checkPendingCompactions(Map<TableId, Long> perTableIdIncomingBytes,
+ Map<TableId, Integer> perTableIdIncomingFiles,
+ TimeUUID planId, String remoteAddress,
+ boolean isForIncremental,
+ long newStreamTotal)
+ {
+
+ int pendingCompactionsBeforeStreaming = 0;
+ int pendingCompactionsAfterStreaming = 0;
+ List<String> tables = new ArrayList<>(perTableIdIncomingFiles.size());
+ for (Keyspace ks : Keyspace.all())
+ {
+ Map<ColumnFamilyStore, TableId> cfStreamed = perTableIdIncomingBytes.keySet().stream()
+ .filter(ks::hasColumnFamilyStore)
+ .collect(Collectors.toMap(ks::getColumnFamilyStore, Function.identity()));
+ for (ColumnFamilyStore cfs : ks.getColumnFamilyStores())
+ {
+ CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
+ int tasksOther = csm.getEstimatedRemainingTasks();
+ int tasksStreamed = tasksOther;
+ if (cfStreamed.containsKey(cfs))
+ {
+ TableId tableId = cfStreamed.get(cfs);
+ tasksStreamed = csm.getEstimatedRemainingTasks(perTableIdIncomingFiles.get(tableId),
+ perTableIdIncomingBytes.get(tableId),
+ isForIncremental);
+ tables.add(String.format("%s.%s", cfs.keyspace.getName(), cfs.name));
+ }
+ pendingCompactionsBeforeStreaming += tasksOther;
+ pendingCompactionsAfterStreaming += tasksStreamed;
+ }
+ }
+ Collections.sort(tables);
+ int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
+ if (pendingCompactionsAfterStreaming > pendingThreshold)
+ {
+ logger.error("[Stream #{}] Rejecting incoming files based on pending compactions calculation " +
+ "pendingCompactionsBeforeStreaming={} pendingCompactionsAfterStreaming={} pendingThreshold={} remoteAddress={}",
+ planId, pendingCompactionsBeforeStreaming, pendingCompactionsAfterStreaming, pendingThreshold, remoteAddress);
+ return false;
+ }
+
+ long newStreamFiles = perTableIdIncomingFiles.values().stream().mapToInt(i -> i).sum();
+
+ logger.info("[Stream #{}] Accepting incoming files newStreamTotalSSTables={} newStreamTotalBytes={} " +
+ "pendingCompactionsBeforeStreaming={} pendingCompactionsAfterStreaming={} pendingThreshold={} remoteAddress={} " +
+ "streamedTables=\"{}\"",
+ planId, newStreamFiles, newStreamTotal,
+ pendingCompactionsBeforeStreaming, pendingCompactionsAfterStreaming, pendingThreshold, remoteAddress,
+ String.join(",", tables));
+ return true;
+ }
+
+ /**
* Call back after sending StreamMessageHeader.
*
* @param message sent stream message
@@ -802,7 +991,7 @@
StreamTransferTask task = transfers.get(message.header.tableId);
if (task != null)
{
- task.scheduleTimeout(message.header.sequenceNumber, 12, TimeUnit.HOURS);
+ task.scheduleTimeout(message.header.sequenceNumber, DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds(), TimeUnit.MILLISECONDS);
}
}
@@ -928,7 +1117,9 @@
public synchronized void sessionFailed()
{
logger.error("[Stream #{}] Remote peer {} failed stream session.", planId(), peer.toString());
- closeSession(State.FAILED);
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("Remote peer ").append(peer).append(" failed stream session");
+ closeSession(State.FAILED, stringBuilder.toString());
}
/**
@@ -937,7 +1128,7 @@
public synchronized void sessionTimeout()
{
logger.error("[Stream #{}] timeout with {}.", planId(), peer.toString());
- closeSession(State.FAILED);
+ closeSession(State.FAILED, "Session timed out");
}
/**
@@ -951,7 +1142,7 @@
List<StreamSummary> transferSummaries = Lists.newArrayList();
for (StreamTask transfer : transfers.values())
transferSummaries.add(transfer.getSummary());
- return new SessionInfo(channel.peer(), index, channel.connectedTo(), receivingSummaries, transferSummaries, state);
+ return new SessionInfo(channel.peer(), index, channel.connectedTo(), receivingSummaries, transferSummaries, state, failureReason);
}
public synchronized void taskCompleted(StreamReceiveTask completedTask)
@@ -966,18 +1157,6 @@
maybeCompleted();
}
- public void onRemove(InetAddressAndPort endpoint)
- {
- logger.error("[Stream #{}] Session failed because remote peer {} has left.", planId(), peer.toString());
- closeSession(State.FAILED);
- }
-
- public void onRestart(InetAddressAndPort endpoint, EndpointState epState)
- {
- logger.error("[Stream #{}] Session failed because remote peer {} was restarted.", planId(), peer.toString());
- closeSession(State.FAILED);
- }
-
private void completePreview()
{
try
@@ -1130,7 +1309,7 @@
logger.debug("[Stream #{}] Stream session with peer {} is already in a final state on abort.", planId(), peer);
return;
}
-
+
logger.info("[Stream #{}] Aborting stream session with peer {}...", planId(), peer);
if (channel.connected())
@@ -1145,4 +1324,53 @@
logger.error("[Stream #{}] Error aborting stream session with peer {}", planId(), peer);
}
}
+
+ @Override
+ public String toString()
+ {
+ return "StreamSession{" +
+ "streamOperation=" + streamOperation +
+ ", peer=" + peer +
+ ", channel=" + channel +
+ ", requests=" + requests +
+ ", transfers=" + transfers +
+ ", isFollower=" + isFollower +
+ ", pendingRepair=" + pendingRepair +
+ ", previewKind=" + previewKind +
+ ", state=" + state +
+ '}';
+ }
+
+ public static StringBuilder boundStackTrace(Throwable e, int limit, StringBuilder out)
+ {
+ Set<Throwable> visited = Collections.newSetFromMap(new IdentityHashMap<>());
+ return boundStackTrace(e, limit, limit, visited, out);
+ }
+
+ public static StringBuilder boundStackTrace(Throwable e, int limit, int counter, Set<Throwable> visited, StringBuilder out)
+ {
+ if (e == null)
+ return out;
+
+ if (!visited.add(e))
+ return out.append("[CIRCULAR REFERENCE: ").append(e.getClass().getName()).append(": ").append(e.getMessage()).append("]").append('\n');
+ visited.add(e);
+
+ StackTraceElement[] stackTrace = e.getStackTrace();
+ out.append(e.getClass().getName() + ": " + e.getMessage()).append('\n');
+
+ // When dealing with the leaf, ignore how many stack traces were already written, and allow the max.
+ // This is here as the leaf tends to show where the issue started, so tends to be impactful for debugging
+ if (e.getCause() == null)
+ counter = limit;
+
+ for (int i = 0, size = Math.min(e.getStackTrace().length, limit); i < size && counter > 0; i++)
+ {
+ out.append('\t').append(stackTrace[i]).append('\n');
+ counter--;
+ }
+
+ boundStackTrace(e.getCause(), limit, counter, visited, out);
+ return out;
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamState.java b/src/java/org/apache/cassandra/streaming/StreamState.java
index 88eb76d..69ba698 100644
--- a/src/java/org/apache/cassandra/streaming/StreamState.java
+++ b/src/java/org/apache/cassandra/streaming/StreamState.java
@@ -56,4 +56,8 @@
{
return Lists.newArrayList(Iterables.transform(sessions, SessionInfo::createSummary));
}
+
+ public Set<SessionInfo> sessions() {
+ return sessions;
+ }
}
diff --git a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
index 45fbcc6..0721316 100644
--- a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
@@ -143,8 +143,10 @@
}
}
streams.clear();
- if (fail != null)
- Throwables.propagate(fail);
+ if (fail != null) {
+ Throwables.throwIfUnchecked(fail);
+ throw new RuntimeException(fail);
+ }
}
public synchronized int getTotalNumberOfFiles()
diff --git a/src/java/org/apache/cassandra/streaming/StreamingState.java b/src/java/org/apache/cassandra/streaming/StreamingState.java
index c2eed1e..aaeceb2 100644
--- a/src/java/org/apache/cassandra/streaming/StreamingState.java
+++ b/src/java/org/apache/cassandra/streaming/StreamingState.java
@@ -34,6 +34,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.cache.IMeasurableMemory;
import org.apache.cassandra.db.virtual.SimpleDataSet;
import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
import org.apache.cassandra.utils.Clock;
@@ -43,11 +44,11 @@
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-public class StreamingState implements StreamEventHandler
+public class StreamingState implements StreamEventHandler, IMeasurableMemory
{
private static final Logger logger = LoggerFactory.getLogger(StreamingState.class);
- public static final long ELEMENT_SIZE = ObjectSizes.measureDeep(new StreamingState(nextTimeUUID(), StreamOperation.OTHER, false));
+ public static final long EMPTY = ObjectSizes.measureDeep(new StreamingState(nextTimeUUID(), StreamOperation.OTHER, false));
public enum Status
{INIT, START, SUCCESS, FAILURE}
@@ -70,6 +71,14 @@
// API for state changes
public final Phase phase = new Phase();
+ @Override
+ public long unsharedHeapSize()
+ {
+ long costOfPeers = peers().size() * (ObjectSizes.IPV6_SOCKET_ADDRESS_SIZE + 48); // 48 represents the datastructure cost computed by the JOL
+ long costOfCompleteMessage = ObjectSizes.sizeOf(completeMessage());
+ return costOfPeers + costOfCompleteMessage + EMPTY;
+ }
+
public StreamingState(StreamResultFuture result)
{
this(result.planId, result.streamOperation, result.getCoordinator().isFollower());
@@ -104,6 +113,11 @@
return this.peers;
}
+ public String completeMessage()
+ {
+ return this.completeMessage;
+ }
+
public Status status()
{
return status;
@@ -283,6 +297,8 @@
{
completeMessage = Throwables.getStackTraceAsString(throwable);
updateState(Status.FAILURE);
+ //we know the size is now very different from the estimate so recompute by adding again
+ StreamManager.instance.addStreamingStateAgain(this);
}
private synchronized void updateState(Status state)
diff --git a/src/java/org/apache/cassandra/streaming/async/NettyStreamingConnectionFactory.java b/src/java/org/apache/cassandra/streaming/async/NettyStreamingConnectionFactory.java
index 6a57e39..529b396 100644
--- a/src/java/org/apache/cassandra/streaming/async/NettyStreamingConnectionFactory.java
+++ b/src/java/org/apache/cassandra/streaming/async/NettyStreamingConnectionFactory.java
@@ -20,6 +20,9 @@
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import com.google.common.annotations.VisibleForTesting;
@@ -35,7 +38,10 @@
import org.apache.cassandra.streaming.StreamingChannel;
import static org.apache.cassandra.locator.InetAddressAndPort.getByAddress;
+import static org.apache.cassandra.net.InternodeConnectionUtils.isSSLError;
import static org.apache.cassandra.net.OutboundConnectionInitiator.initiateStreaming;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.SslFallbackConnectionType;
+import static org.apache.cassandra.net.OutboundConnectionInitiator.SslFallbackConnectionType.SERVER_CONFIG;
public class NettyStreamingConnectionFactory implements StreamingChannel.Factory
{
@@ -45,27 +51,38 @@
public static NettyStreamingChannel connect(OutboundConnectionSettings template, int messagingVersion, StreamingChannel.Kind kind) throws IOException
{
EventLoop eventLoop = MessagingService.instance().socketFactory.outboundStreamingGroup().next();
+ OutboundConnectionSettings settings = template.withDefaults(ConnectionCategory.STREAMING);
+ List<SslFallbackConnectionType> sslFallbacks = settings.withEncryption() && settings.encryption.getOptional()
+ ? Arrays.asList(SslFallbackConnectionType.values())
+ : Collections.singletonList(SERVER_CONFIG);
- int attempts = 0;
- while (true)
+ Throwable cause = null;
+ for (final SslFallbackConnectionType sslFallbackConnectionType : sslFallbacks)
{
- Future<Result<StreamingSuccess>> result = initiateStreaming(eventLoop, template.withDefaults(ConnectionCategory.STREAMING), messagingVersion);
- result.awaitUninterruptibly(); // initiate has its own timeout, so this is "guaranteed" to return relatively promptly
- if (result.isSuccess())
+ for (int i = 0; i < MAX_CONNECT_ATTEMPTS; i++)
{
- Channel channel = result.getNow().success().channel;
- NettyStreamingChannel streamingChannel = new NettyStreamingChannel(messagingVersion, channel, kind);
- if (kind == StreamingChannel.Kind.CONTROL)
+ Future<Result<StreamingSuccess>> result = initiateStreaming(eventLoop, settings, sslFallbackConnectionType, messagingVersion);
+ result.awaitUninterruptibly(); // initiate has its own timeout, so this is "guaranteed" to return relatively promptly
+ if (result.isSuccess())
{
- ChannelPipeline pipeline = channel.pipeline();
- pipeline.addLast("stream", streamingChannel);
+ Channel channel = result.getNow().success().channel;
+ NettyStreamingChannel streamingChannel = new NettyStreamingChannel(messagingVersion, channel, kind);
+ if (kind == StreamingChannel.Kind.CONTROL)
+ {
+ ChannelPipeline pipeline = channel.pipeline();
+ pipeline.addLast("stream", streamingChannel);
+ }
+ return streamingChannel;
}
- return streamingChannel;
+ cause = result.cause();
}
-
- if (++attempts == MAX_CONNECT_ATTEMPTS)
- throw new IOException("failed to connect to " + template.to + " for streaming data", result.cause());
+ if (!isSSLError(cause))
+ {
+ // Fallback only when the error is SSL related, otherwise retries are exhausted, so fail
+ break;
+ }
}
+ throw new IOException("failed to connect to " + template.to + " for streaming data", cause);
}
@Override
diff --git a/src/java/org/apache/cassandra/streaming/async/StreamingMultiplexedChannel.java b/src/java/org/apache/cassandra/streaming/async/StreamingMultiplexedChannel.java
index 560fee9..4ebb391 100644
--- a/src/java/org/apache/cassandra/streaming/async/StreamingMultiplexedChannel.java
+++ b/src/java/org/apache/cassandra/streaming/async/StreamingMultiplexedChannel.java
@@ -53,14 +53,12 @@
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static com.google.common.base.Throwables.getRootCause;
-import static java.lang.Integer.parseInt;
import static java.lang.String.format;
-import static java.lang.System.getProperty;
import static java.lang.Thread.currentThread;
import static java.util.concurrent.TimeUnit.*;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.config.Config.PROPERTY_PREFIX;
+import static org.apache.cassandra.config.CassandraRelevantProperties.STREAMING_SESSION_PARALLELTRANSFERS;
import static org.apache.cassandra.streaming.StreamSession.createLogTag;
import static org.apache.cassandra.streaming.messages.StreamMessage.serialize;
import static org.apache.cassandra.streaming.messages.StreamMessage.serializedSize;
@@ -91,7 +89,7 @@
private static final Logger logger = LoggerFactory.getLogger(StreamingMultiplexedChannel.class);
private static final int DEFAULT_MAX_PARALLEL_TRANSFERS = getAvailableProcessors();
- private static final int MAX_PARALLEL_TRANSFERS = parseInt(getProperty(PROPERTY_PREFIX + "streaming.session.parallelTransfers", Integer.toString(DEFAULT_MAX_PARALLEL_TRANSFERS)));
+ private static final int MAX_PARALLEL_TRANSFERS = STREAMING_SESSION_PARALLELTRANSFERS.getInt(DEFAULT_MAX_PARALLEL_TRANSFERS);
// a simple mechansim for allowing a degree of fairness across multiple sessions
private static final Semaphore fileTransferSemaphore = newFairSemaphore(DEFAULT_MAX_PARALLEL_TRANSFERS);
diff --git a/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java b/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java
index 72ab844..38ff31e 100644
--- a/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/ProgressInfoCompositeData.java
@@ -22,8 +22,6 @@
import java.util.Map;
import javax.management.openmbean.*;
-import com.google.common.base.Throwables;
-
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.utils.TimeUUID;
@@ -68,7 +66,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -91,7 +89,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -110,7 +108,7 @@
}
catch (UnknownHostException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
}
diff --git a/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java b/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java
index 665b4cd..1a6fea5 100644
--- a/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/SessionCompleteEventCompositeData.java
@@ -21,8 +21,6 @@
import java.util.Map;
import javax.management.openmbean.*;
-import com.google.common.base.Throwables;
-
import org.apache.cassandra.streaming.StreamEvent;
public class SessionCompleteEventCompositeData
@@ -53,7 +51,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -70,7 +68,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
}
diff --git a/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java b/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java
index f39b321..eaa37bb 100644
--- a/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/SessionInfoCompositeData.java
@@ -22,7 +22,6 @@
import javax.management.openmbean.*;
import com.google.common.base.Function;
-import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -83,7 +82,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -121,7 +120,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -138,7 +137,7 @@
}
catch (UnknownHostException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
Function<CompositeData, StreamSummary> toStreamSummary = new Function<CompositeData, StreamSummary>()
{
@@ -152,7 +151,7 @@
connecting,
fromArrayOfCompositeData((CompositeData[]) values[5], toStreamSummary),
fromArrayOfCompositeData((CompositeData[]) values[6], toStreamSummary),
- StreamSession.State.valueOf((String) values[7]));
+ StreamSession.State.valueOf((String) values[7]), null); // null is here to maintain backwards compatibility
Function<CompositeData, ProgressInfo> toProgressInfo = new Function<CompositeData, ProgressInfo>()
{
public ProgressInfo apply(CompositeData input)
diff --git a/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java b/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java
index 5ee4f32..c04205f 100644
--- a/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/StreamStateCompositeData.java
@@ -21,7 +21,6 @@
import javax.management.openmbean.*;
import com.google.common.base.Function;
-import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -67,7 +66,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -114,7 +113,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java b/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java
index a1f2496..05a0afc 100644
--- a/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java
+++ b/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java
@@ -21,8 +21,6 @@
import java.util.Map;
import javax.management.openmbean.*;
-import com.google.common.base.Throwables;
-
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.StreamSummary;
@@ -53,7 +51,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -69,7 +67,7 @@
}
catch (OpenDataException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/tools/BootstrapMonitor.java b/src/java/org/apache/cassandra/tools/BootstrapMonitor.java
index 4d58638..12da56d 100644
--- a/src/java/org/apache/cassandra/tools/BootstrapMonitor.java
+++ b/src/java/org/apache/cassandra/tools/BootstrapMonitor.java
@@ -34,6 +34,7 @@
private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
private final PrintStream out;
private final Condition condition = newOneTimeCondition();
+ private volatile Exception error;
public BootstrapMonitor(PrintStream out)
{
@@ -66,7 +67,7 @@
@Override
public void handleConnectionFailed(long timestamp, String message)
{
- Exception error = new IOException(String.format("[%s] JMX connection closed. (%s)",
+ error = new IOException(String.format("[%s] JMX connection closed. (%s)",
format.format(timestamp), message));
out.println(error.getMessage());
condition.signalAll();
@@ -82,9 +83,19 @@
message = message + " (progress: " + (int)event.getProgressPercentage() + "%)";
}
out.println(message);
+ if (type == ProgressEventType.ERROR)
+ {
+ error = new RuntimeException(String.format("Bootstrap resume has failed with error: %s", message));
+ condition.signalAll();
+ }
if (type == ProgressEventType.COMPLETE)
{
condition.signalAll();
}
}
+
+ public Exception getError()
+ {
+ return error;
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java b/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java
index eef0ef4..b282932 100644
--- a/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java
+++ b/src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java
@@ -53,7 +53,7 @@
int messagingVersion,
StreamingChannel.Kind kind) throws IOException
{
- // The preferred address is always overwritten in create(). This method override only exists so we can avoid
+ // The preferred address is always overwritten in create(). This method override only exists so we can avoid
// falling back to the NettyStreamingConnectionFactory implementation.
OutboundConnectionSettings template = new OutboundConnectionSettings(getByAddress(to), getByAddress(preferred));
return create(template, messagingVersion, kind);
diff --git a/src/java/org/apache/cassandra/tools/BulkLoader.java b/src/java/org/apache/cassandra/tools/BulkLoader.java
index ebdd072..8d5a1d4 100644
--- a/src/java/org/apache/cassandra/tools/BulkLoader.java
+++ b/src/java/org/apache/cassandra/tools/BulkLoader.java
@@ -72,7 +72,8 @@
buildSSLOptions(options.clientEncOptions)),
handler,
options.connectionsPerHost,
- options.targetKeyspace);
+ options.targetKeyspace,
+ options.targetTable);
DatabaseDescriptor.setStreamThroughputOutboundBytesPerSec(options.throttleBytes);
DatabaseDescriptor.setInterDCStreamThroughputOutboundBytesPerSec(options.interDcThrottleBytes);
DatabaseDescriptor.setEntireSSTableStreamThroughputOutboundMebibytesPerSec(options.entireSSTableThrottleMebibytes);
@@ -90,7 +91,6 @@
{
future = loader.stream(options.ignores, indicator);
}
-
}
catch (Exception e)
{
diff --git a/src/java/org/apache/cassandra/tools/GenerateTokens.java b/src/java/org/apache/cassandra/tools/GenerateTokens.java
index a6888d7..560a53f 100644
--- a/src/java/org/apache/cassandra/tools/GenerateTokens.java
+++ b/src/java/org/apache/cassandra/tools/GenerateTokens.java
@@ -26,7 +26,6 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.Level;
@@ -107,7 +106,7 @@
}
catch (Throwable t)
{
- logger.warn("Error running tool.", t);
+ logger.warn(t, "Error running tool.");
System.exit(1);
}
}
@@ -152,5 +151,4 @@
"Options are:";
new HelpFormatter().printHelp(usage, header, options, "");
}
-}
-
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/HashPassword.java b/src/java/org/apache/cassandra/tools/HashPassword.java
index ddc0a72..f229858 100644
--- a/src/java/org/apache/cassandra/tools/HashPassword.java
+++ b/src/java/org/apache/cassandra/tools/HashPassword.java
@@ -57,7 +57,7 @@
String password = null;
if (cmd.hasOption(ENV_VAR))
{
- password = System.getenv(cmd.getOptionValue(ENV_VAR));
+ password = System.getenv(cmd.getOptionValue(ENV_VAR)); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
if (password == null)
{
System.err.println(String.format("Environment variable '%s' is undefined.", cmd.getOptionValue(ENV_VAR)));
diff --git a/src/java/org/apache/cassandra/tools/JMXTool.java b/src/java/org/apache/cassandra/tools/JMXTool.java
index d054716..dcc9f6a 100644
--- a/src/java/org/apache/cassandra/tools/JMXTool.java
+++ b/src/java/org/apache/cassandra/tools/JMXTool.java
@@ -62,7 +62,6 @@
import com.google.common.collect.Sets.SetView;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import io.airlift.airline.Arguments;
import io.airlift.airline.Cli;
import io.airlift.airline.Command;
@@ -71,6 +70,7 @@
import io.airlift.airline.Option;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.utils.JsonUtils;
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.Constructor;
@@ -157,8 +157,7 @@
{
void dump(OutputStream output, Map<String, Info> map) throws IOException
{
- ObjectMapper mapper = new ObjectMapper();
- mapper.writeValue(output, map);
+ JsonUtils.JSON_OBJECT_PRETTY_WRITER.writeValue(output, map);
}
},
yaml
@@ -373,8 +372,7 @@
{
Map<String, Info> load(InputStream input) throws IOException
{
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(input, new TypeReference<Map<String, Info>>() {});
+ return JsonUtils.JSON_OBJECT_MAPPER.readValue(input, new TypeReference<Map<String, Info>>() {});
}
},
yaml
diff --git a/src/java/org/apache/cassandra/tools/LoaderOptions.java b/src/java/org/apache/cassandra/tools/LoaderOptions.java
index 7729955..e9a1f8b 100644
--- a/src/java/org/apache/cassandra/tools/LoaderOptions.java
+++ b/src/java/org/apache/cassandra/tools/LoaderOptions.java
@@ -29,7 +29,6 @@
import java.util.HashSet;
import java.util.Set;
-import com.google.common.base.Throwables;
import com.google.common.net.HostAndPort;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -93,6 +92,7 @@
public static final String ENTIRE_SSTABLE_INTER_DC_THROTTLE_MEBIBYTES = "entire-sstable-inter-dc-throttle-mib";
public static final String TOOL_NAME = "sstableloader";
public static final String TARGET_KEYSPACE = "target-keyspace";
+ public static final String TARGET_TABLE = "target-table";
/* client encryption options */
public static final String SSL_TRUSTSTORE = "truststore";
@@ -124,6 +124,7 @@
public final Set<InetSocketAddress> hosts;
public final Set<InetAddressAndPort> ignores;
public final String targetKeyspace;
+ public final String targetTable;
LoaderOptions(Builder builder)
{
@@ -147,6 +148,7 @@
hosts = builder.hosts;
ignores = builder.ignores;
targetKeyspace = builder.targetKeyspace;
+ targetTable = builder.targetTable;
}
static class Builder
@@ -175,6 +177,7 @@
Set<InetSocketAddress> hosts = new HashSet<>();
Set<InetAddressAndPort> ignores = new HashSet<>();
String targetKeyspace;
+ String targetTable;
Builder()
{
@@ -198,7 +201,7 @@
}
catch (UnknownHostException e)
{
- Throwables.propagate(e);
+ throw new RuntimeException(e);
}
return new LoaderOptions(this);
@@ -389,6 +392,18 @@
return this;
}
+ public Builder targetKeyspace(String keyspace)
+ {
+ this.targetKeyspace = keyspace;
+ return this;
+ }
+
+ public Builder targetTable(String table)
+ {
+ this.targetKeyspace = table;
+ return this;
+ }
+
public Builder parseArgs(String cmdArgs[])
{
CommandLineParser parser = new GnuParser();
@@ -657,10 +672,16 @@
{
targetKeyspace = cmd.getOptionValue(TARGET_KEYSPACE);
if (StringUtils.isBlank(targetKeyspace))
- {
errorMsg("Empty keyspace is not supported.", options);
- }
}
+
+ if (cmd.hasOption(TARGET_TABLE))
+ {
+ targetTable = cmd.getOptionValue(TARGET_TABLE);
+ if (StringUtils.isBlank(targetTable))
+ errorMsg("Empty table is not supported.", options);
+ }
+
return this;
}
catch (ParseException | ConfigurationException | MalformedURLException e)
@@ -771,6 +792,7 @@
options.addOption("ciphers", SSL_CIPHER_SUITES, "CIPHER-SUITES", "Client SSL: comma-separated list of encryption suites to use");
options.addOption("f", CONFIG_PATH, "path to config file", "cassandra.yaml file path for streaming throughput and client/server SSL.");
options.addOption("k", TARGET_KEYSPACE, "target keyspace name", "target keyspace name");
+ options.addOption("tb", TARGET_TABLE, "target table name", "target table name");
return options;
}
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 4a2f43d..e30ebd5 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -115,6 +115,9 @@
import org.apache.cassandra.tools.nodetool.GetTimeout;
import org.apache.cassandra.utils.NativeLibrary;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SSL_ENABLE;
+
/**
* JMX client operations for Cassandra.
*/
@@ -124,7 +127,7 @@
private static final String ssObjName = "org.apache.cassandra.db:type=StorageService";
private static final int defaultPort = 7199;
- static long JMX_NOTIFICATION_POLL_INTERVAL_SECONDS = Long.getLong("cassandra.nodetool.jmx_notification_poll_interval_seconds", TimeUnit.SECONDS.convert(5, TimeUnit.MINUTES));
+ static long JMX_NOTIFICATION_POLL_INTERVAL_SECONDS = NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS.getLong();
final String host;
final int port;
@@ -293,7 +296,7 @@
private RMIClientSocketFactory getRMIClientSocketFactory()
{
- if (Boolean.parseBoolean(System.getProperty("ssl.enable")))
+ if (SSL_ENABLE.getBoolean())
return new SslRMIClientSocketFactory();
else
return RMISocketFactory.getDefaultSocketFactory();
@@ -460,6 +463,11 @@
ssProxy.forceKeyspaceCompactionForPartitionKey(keyspaceName, partitionKey, tableNames);
}
+ public void forceCompactionKeysIgnoringGcGrace(String keyspaceName, String tableName, String... partitionKeysIgnoreGcGrace) throws IOException, ExecutionException, InterruptedException
+ {
+ ssProxy.forceCompactionKeysIgnoringGcGrace(keyspaceName, tableName, partitionKeysIgnoreGcGrace);
+ }
+
public void forceKeyspaceFlush(String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
ssProxy.forceKeyspaceFlush(keyspaceName, tableNames);
@@ -498,9 +506,29 @@
}
}
}
- public Map<String, List<CompositeData>> getPartitionSample(int capacity, int durationMillis, int count, List<String> samplers) throws OpenDataException
+
+ public boolean handleScheduledSampling(String ks,
+ String table,
+ int capacity,
+ int count,
+ int durationMillis,
+ int intervalMillis,
+ List<String> samplers,
+ boolean shouldStop) throws OpenDataException
{
- return ssProxy.samplePartitions(durationMillis, capacity, count, samplers);
+ return shouldStop ?
+ ssProxy.stopSamplingPartitions(ks, table) :
+ ssProxy.startSamplingPartitions(ks, table, durationMillis, intervalMillis, capacity, count, samplers);
+ }
+
+ public List<String> getSampleTasks()
+ {
+ return ssProxy.getSampleTasks();
+ }
+
+ public Map<String, List<CompositeData>> getPartitionSample(String ks, int capacity, int durationMillis, int count, List<String> samplers) throws OpenDataException
+ {
+ return ssProxy.samplePartitions(ks, durationMillis, capacity, count, samplers);
}
public Map<String, List<CompositeData>> getPartitionSample(String ks, String cf, int capacity, int durationMillis, int count, List<String> samplers) throws OpenDataException
@@ -750,11 +778,21 @@
return ssProxy.getLoadString();
}
+ public String getUncompressedLoadString()
+ {
+ return ssProxy.getUncompressedLoadString();
+ }
+
public String getReleaseVersion()
{
return ssProxy.getReleaseVersion();
}
+ public String getGitSHA()
+ {
+ return ssProxy.getGitSHA();
+ }
+
public int getCurrentGenerationNumber()
{
return ssProxy.getCurrentGenerationNumber();
@@ -827,11 +865,31 @@
}
/**
- * Remove all the existing snapshots.
+ * Remove all the existing snapshots of given tag for provided keyspaces.
+ * When no keyspaces are specified, take all keyspaces into account. When tag is not specified (null or empty string),
+ * take all tags into account.
+ *
+ * @param tag tag of snapshot to clear
+ * @param keyspaces keyspaces to clear snapshots for
*/
+ @Deprecated
public void clearSnapshot(String tag, String... keyspaces) throws IOException
{
- ssProxy.clearSnapshot(tag, keyspaces);
+ clearSnapshot(Collections.emptyMap(), tag, keyspaces);
+ }
+
+ /**
+ * Remove all the existing snapshots of given tag for provided keyspaces.
+ * When no keyspaces are specified, take all keyspaces into account. When tag is not specified (null or empty string),
+ * take all tags into account.
+ *
+ * @param options options to supply for snapshot clearing
+ * @param tag tag of snapshot to clear
+ * @param keyspaces keyspaces to clear snapshots for
+ */
+ public void clearSnapshot(Map<String, Object> options, String tag, String... keyspaces) throws IOException
+ {
+ ssProxy.clearSnapshot(options, tag, keyspaces);
}
public Map<String, TabularData> getSnapshotDetails(Map<String, String> options)
@@ -989,6 +1047,18 @@
return cfsProxy.getSSTablesForKey(key, hexFormat);
}
+ public Map<Integer, Set<String>> getSSTablesWithLevel(String keyspace, String cf, String key, boolean hexFormat)
+ {
+ ColumnFamilyStoreMBean cfsProxy = getCfsProxy(keyspace, cf);
+ return cfsProxy.getSSTablesForKeyWithLevel(key, hexFormat);
+ }
+
+ public boolean isLeveledCompaction(String keyspace, String cf)
+ {
+ ColumnFamilyStoreMBean cfsProxy = getCfsProxy(keyspace, cf);
+ return cfsProxy.isLeveledCompaction();
+ }
+
public Set<StreamState> getStreamStatus()
{
return Sets.newHashSet(Iterables.transform(streamProxy.getCurrentStreams(), new Function<CompositeData, StreamState>()
@@ -1534,9 +1604,9 @@
return withPort ? ssProxy.describeRingWithPortJMX(keyspaceName) : ssProxy.describeRingJMX(keyspaceName);
}
- public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources)
+ public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources, boolean excludeLocalDatacenterNodes)
{
- ssProxy.rebuild(sourceDc, keyspace, tokens, specificSources);
+ ssProxy.rebuild(sourceDc, keyspace, tokens, specificSources, excludeLocalDatacenterNodes);
}
public List<String> sampleKeyRange()
@@ -1769,6 +1839,8 @@
case "EstimatedPartitionCount":
case "KeyCacheHitRate":
case "LiveSSTableCount":
+ case "MaxSSTableDuration":
+ case "MaxSSTableSize":
case "OldVersionSSTableCount":
case "MaxPartitionSize":
case "MeanPartitionSize":
@@ -1989,6 +2061,8 @@
{
out.println("Resuming bootstrap");
monitor.awaitCompletion();
+ if (monitor.getError() != null)
+ throw monitor.getError();
}
else
{
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index 8d87c88..5500240 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -45,6 +45,8 @@
import java.util.Scanner;
import java.util.SortedMap;
+import javax.management.InstanceNotFoundException;
+
import com.google.common.base.Joiner;
import com.google.common.base.Throwables;
@@ -225,7 +227,8 @@
UpgradeSSTable.class,
Verify.class,
Version.class,
- ViewBuildStatus.class
+ ViewBuildStatus.class,
+ ForceCompact.class
);
Cli.CliBuilder<NodeToolCmdRunnable> builder = Cli.builder("nodetool");
@@ -306,6 +309,10 @@
protected void err(Throwable e)
{
+ // CASSANDRA-11537: friendly error message when server is not ready
+ if (e instanceof InstanceNotFoundException)
+ throw new IllegalArgumentException("Server is not initialized yet, cannot run nodetool.");
+
output.err.println("error: " + e.getMessage());
output.err.println("-- StackTrace --");
output.err.println(getStackTraceAsString(e));
@@ -484,6 +491,11 @@
{
return cmdArgs.size() <= 1 ? EMPTY_STRING_ARRAY : toArray(cmdArgs.subList(1, cmdArgs.size()), String.class);
}
+
+ protected String[] parsePartitionKeys(List<String> cmdArgs)
+ {
+ return cmdArgs.size() <= 2 ? EMPTY_STRING_ARRAY : toArray(cmdArgs.subList(2, cmdArgs.size()), String.class);
+ }
}
public static SortedMap<String, SetHostStatWithPort> getOwnershipByDcWithPort(NodeProbe probe, boolean resolveIp,
diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index f5d24ed..04d77a9 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@ -26,14 +26,15 @@
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -74,12 +75,12 @@
{
try
{
- SSTableReader reader = SSTableReader.open(sstable.getKey());
+ SSTableReader reader = SSTableReader.open(cfs, sstable.getKey());
sstables.add(reader);
}
catch (Throwable t)
{
- out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA)+" ("+t.getMessage()+")");
+ out.println("Couldn't open sstable: " + sstable.getKey().fileFor(Components.DATA) + " (" + t.getMessage() + ")");
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index 771380b..da9298a 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -25,8 +25,14 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.TableMetadata;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
@@ -39,13 +45,8 @@
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.KeyIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.FBUtilities;
@@ -71,7 +72,7 @@
static
{
- DatabaseDescriptor.clientInitialization();
+ DatabaseDescriptor.toolInitialization();
Option optKey = new Option(KEY_OPTION, true, "List of included partition keys");
// Number of times -k <key> can be passed on the command line.
@@ -135,20 +136,21 @@
printUsage();
System.exit(1);
}
- String ssTableFileName = new File(cmd.getArgs()[0]).absolutePath();
+ File ssTableFile = new File(cmd.getArgs()[0]);
- if (!new File(ssTableFileName).exists())
+ if (!ssTableFile.exists())
{
- System.err.println("Cannot find file " + ssTableFileName);
+ System.err.println("Cannot find file " + ssTableFile.absolutePath());
System.exit(1);
}
- Descriptor desc = Descriptor.fromFilename(ssTableFileName);
+ Descriptor desc = Descriptor.fromFileWithComponent(ssTableFile, false).left;
try
{
TableMetadata metadata = Util.metadataFromSSTable(desc);
+ SSTableReader sstable = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata));
if (cmd.hasOption(ENUMERATE_KEYS_OPTION))
{
- try (KeyIterator iter = new KeyIterator(desc, metadata))
+ try (KeyIterator iter = sstable.keyIterator())
{
JsonTransformer.keysToJson(null, Util.iterToStream(iter),
cmd.hasOption(RAW_TIMESTAMPS),
@@ -158,7 +160,6 @@
}
else
{
- SSTableReader sstable = SSTableReader.openNoValidation(desc, TableMetadataRef.forOfflineTools(metadata));
IPartitioner partitioner = sstable.getPartitioner();
final ISSTableScanner currentScanner;
if ((keys != null) && (keys.length > 0))
diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index 9618d362..0aa9e10 100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@ -21,15 +21,16 @@
import java.util.Map;
import java.util.Set;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.JVMStabilityInspector;
/**
@@ -90,19 +91,19 @@
boolean foundSSTable = false;
for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
{
- if (sstable.getValue().contains(Component.STATS))
+ if (sstable.getValue().contains(Components.STATS))
{
foundSSTable = true;
Descriptor descriptor = sstable.getKey();
- StatsMetadata metadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS);
+ StatsMetadata metadata = StatsComponent.load(descriptor).statsMetadata();
if (metadata.sstableLevel > 0)
{
- out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.filenameFor(Component.DATA));
+ out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.fileFor(Components.DATA));
descriptor.getMetadataSerializer().mutateLevel(descriptor, 0);
}
else
{
- out.println("Skipped " + descriptor.filenameFor(Component.DATA) + " since it is already on level 0");
+ out.println("Skipped " + descriptor.fileFor(Components.DATA) + " since it is already on level 0");
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index c4bef35..1d9dd91 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -17,15 +17,11 @@
*/
package org.apache.cassandra.tools;
-import java.io.DataInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.Arrays;
import java.util.Comparator;
-import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -41,6 +37,7 @@
import org.apache.commons.cli.PosixParser;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -51,14 +48,13 @@
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.IndexSummary;
+import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
import org.apache.cassandra.io.util.File;
@@ -91,7 +87,7 @@
static
{
- DatabaseDescriptor.clientInitialization();
+ DatabaseDescriptor.toolInitialization();
}
boolean color;
@@ -175,7 +171,7 @@
private void printScannedOverview(Descriptor descriptor, StatsMetadata stats) throws IOException
{
TableMetadata cfm = Util.metadataFromSSTable(descriptor);
- SSTableReader reader = SSTableReader.openNoValidation(descriptor, TableMetadataRef.forOfflineTools(cfm));
+ SSTableReader reader = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(cfm));
try (ISSTableScanner scanner = reader.getScanner())
{
long bytes = scanner.getLengthInBytes();
@@ -316,23 +312,22 @@
}
}
- private void printSStableMetadata(String fname, boolean scan) throws IOException
+ private void printSStableMetadata(File file, boolean scan) throws IOException
{
- Descriptor descriptor = Descriptor.fromFilename(fname);
- Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer()
- .deserialize(descriptor, EnumSet.allOf(MetadataType.class));
- ValidationMetadata validation = (ValidationMetadata) metadata.get(MetadataType.VALIDATION);
- StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
- CompactionMetadata compaction = (CompactionMetadata) metadata.get(MetadataType.COMPACTION);
- CompressionMetadata compression = null;
- File compressionFile = new File(descriptor.filenameFor(Component.COMPRESSION_INFO));
- if (compressionFile.exists())
- compression = CompressionMetadata.create(fname);
- SerializationHeader.Component header = (SerializationHeader.Component) metadata
- .get(MetadataType.HEADER);
+ Descriptor descriptor = Descriptor.fromFileWithComponent(file, false).left;
+ StatsComponent statsComponent = StatsComponent.load(descriptor);
+ ValidationMetadata validation = statsComponent.validationMetadata();
+ StatsMetadata stats = statsComponent.statsMetadata();
+ CompactionMetadata compaction = statsComponent.compactionMetadata();
+ SerializationHeader.Component header = statsComponent.serializationHeader();
+ Class<? extends ICompressor> compressorClass = null;
+ try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor))
+ {
+ compressorClass = compression != null ? compression.compressor().getClass() : null;
+ }
field("SSTable", descriptor);
- if (scan && descriptor.version.getVersion().compareTo("ma") >= 0)
+ if (scan && descriptor.version.version.compareTo("ma") >= 0)
{
printScannedOverview(descriptor, stats);
}
@@ -347,29 +342,19 @@
field("Maximum timestamp", stats.maxTimestamp, toDateString(stats.maxTimestamp, tsUnit));
field("SSTable min local deletion time", stats.minLocalDeletionTime, deletion(stats.minLocalDeletionTime));
field("SSTable max local deletion time", stats.maxLocalDeletionTime, deletion(stats.maxLocalDeletionTime));
- field("Compressor", compression != null ? compression.compressor().getClass().getName() : "-");
- if (compression != null)
+ field("Compressor", compressorClass != null ? compressorClass.getName() : "-");
+ if (compressorClass != null)
field("Compression ratio", stats.compressionRatio);
field("TTL min", stats.minTTL, toDurationString(stats.minTTL, TimeUnit.SECONDS));
field("TTL max", stats.maxTTL, toDurationString(stats.maxTTL, TimeUnit.SECONDS));
if (validation != null && header != null)
- printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType());
+ printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType(), stats);
- if (header != null && header.getClusteringTypes().size() == stats.minClusteringValues.size())
+ if (header != null)
{
- List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
- List<ByteBuffer> minClusteringValues = stats.minClusteringValues;
- List<ByteBuffer> maxClusteringValues = stats.maxClusteringValues;
- String[] minValues = new String[clusteringTypes.size()];
- String[] maxValues = new String[clusteringTypes.size()];
- for (int i = 0; i < clusteringTypes.size(); i++)
- {
- minValues[i] = clusteringTypes.get(i).getString(minClusteringValues.get(i));
- maxValues[i] = clusteringTypes.get(i).getString(maxClusteringValues.get(i));
- }
- field("minClusteringValues", Arrays.toString(minValues));
- field("maxClusteringValues", Arrays.toString(maxValues));
+ ClusteringComparator comparator = new ClusteringComparator(header.getClusteringTypes());
+ field("Covered clusterings", stats.coveredClustering.toString(comparator));
}
field("Estimated droppable tombstones",
stats.getEstimatedDroppableTombstoneRatio((int) (currentTimeMillis() / 1000) - this.gc));
@@ -475,17 +460,23 @@
}
}
- private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType<?> keyType)
- throws IOException
+ private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType<?> keyType, StatsMetadata statsMetadata)
+ throws IOException
{
- File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
- if (!summariesFile.exists())
- return;
-
- try (DataInputStream iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath())))
+ if (descriptor.version.hasKeyRange())
{
- Pair<DecoratedKey, DecoratedKey> firstLast = new IndexSummary.IndexSummarySerializer()
- .deserializeFirstLastKey(iStream, partitioner);
+ if (statsMetadata.firstKey == null || statsMetadata.lastKey == null)
+ return;
+
+ field("First token", partitioner.getToken(statsMetadata.firstKey), keyType.getString(statsMetadata.firstKey));
+ field("Last token", partitioner.getToken(statsMetadata.lastKey), keyType.getString(statsMetadata.lastKey));
+ }
+ else
+ {
+ Pair<DecoratedKey, DecoratedKey> firstLast = descriptor.getFormat().getReaderFactory().readKeyRange(descriptor, partitioner);
+ if (firstLast == null)
+ return;
+
field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey()));
field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey()));
}
@@ -545,7 +536,7 @@
File sstable = new File(fname);
if (sstable.exists())
{
- metawriter.printSStableMetadata(sstable.absolutePath(), fullScan);
+ metawriter.printSStableMetadata(sstable, fullScan);
}
else
{
diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
index e4e343f..16faa92 100644
--- a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@ -33,16 +33,17 @@
import com.google.common.collect.Multimap;
import com.google.common.collect.SetMultimap;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.Schema;
/**
* Create a decent leveling for the given keyspace/column family
@@ -116,13 +117,14 @@
{
try
{
- SSTableReader reader = SSTableReader.open(sstable.getKey());
+ SSTableReader reader = SSTableReader.open(cfs, sstable.getKey());
sstableMultimap.put(reader.descriptor.directory, reader);
}
catch (Throwable t)
{
- out.println("Couldn't open sstable: "+sstable.getKey().filenameFor(Component.DATA));
- Throwables.propagate(t);
+ out.println("Couldn't open sstable: "+sstable.getKey().fileFor(Components.DATA));
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/SSTablePartitions.java b/src/java/org/apache/cassandra/tools/SSTablePartitions.java
new file mode 100644
index 0000000..c513853
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/SSTablePartitions.java
@@ -0,0 +1,889 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.ToLongFunction;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+
+public class SSTablePartitions
+{
+ private static final String KEY_OPTION = "k";
+ private static final String EXCLUDE_KEY_OPTION = "x";
+ private static final String RECURSIVE_OPTION = "r";
+ private static final String SNAPSHOTS_OPTION = "s";
+ private static final String BACKUPS_OPTION = "b";
+ private static final String PARTITIONS_ONLY_OPTION = "y";
+ private static final String SIZE_THRESHOLD_OPTION = "t";
+ private static final String TOMBSTONE_THRESHOLD_OPTION = "o";
+ private static final String CELL_THRESHOLD_OPTION = "c";
+ private static final String ROW_THRESHOLD_OPTION = "w";
+ private static final String CSV_OPTION = "m";
+ private static final String CURRENT_TIMESTAMP_OPTION = "u";
+
+ private static final Options options = new Options();
+
+ private static final TableId EMPTY_TABLE_ID = TableId.fromUUID(new UUID(0L, 0L));
+
+ static
+ {
+ DatabaseDescriptor.clientInitialization();
+
+ Option optKey = new Option(KEY_OPTION, "key", true, "Partition keys to include");
+ // Number of times -k <key> can be passed on the command line.
+ optKey.setArgs(Option.UNLIMITED_VALUES);
+ options.addOption(optKey);
+
+ Option excludeKey = new Option(EXCLUDE_KEY_OPTION, "exclude-key", true,
+ "Excluded partition key(s) from partition detailed row/cell/tombstone " +
+ "information (irrelevant, if --partitions-only is given)");
+ excludeKey.setArgs(Option.UNLIMITED_VALUES); // Number of times -x <key> can be passed on the command line.
+ options.addOption(excludeKey);
+
+ Option thresholdKey = new Option(SIZE_THRESHOLD_OPTION, "min-size", true,
+ "partition size threshold, expressed as either the number of bytes or a " +
+ "size with unit of the form 10KiB, 20MiB, 30GiB, etc.");
+ options.addOption(thresholdKey);
+
+ Option tombstoneKey = new Option(TOMBSTONE_THRESHOLD_OPTION, "min-tombstones", true,
+ "partition tombstone count threshold");
+ options.addOption(tombstoneKey);
+
+ Option cellKey = new Option(CELL_THRESHOLD_OPTION, "min-cells", true, "partition cell count threshold");
+ options.addOption(cellKey);
+
+ Option rowKey = new Option(ROW_THRESHOLD_OPTION, "min-rows", true, "partition row count threshold");
+ options.addOption(rowKey);
+
+ Option currentTimestampKey = new Option(CURRENT_TIMESTAMP_OPTION, "current-timestamp", true,
+ "timestamp (seconds since epoch, unit time) for TTL expired calculation");
+ options.addOption(currentTimestampKey);
+
+ Option recursiveKey = new Option(RECURSIVE_OPTION, "recursive", false, "scan for sstables recursively");
+ options.addOption(recursiveKey);
+
+ Option snapshotsKey = new Option(SNAPSHOTS_OPTION, "snapshots", false,
+ "include snapshots present in data directories (recursive scans)");
+ options.addOption(snapshotsKey);
+
+ Option backupsKey = new Option(BACKUPS_OPTION, "backups", false,
+ "include backups present in data directories (recursive scans)");
+ options.addOption(backupsKey);
+
+ Option partitionsOnlyKey = new Option(PARTITIONS_ONLY_OPTION, "partitions-only", false,
+ "Do not process per-partition detailed row/cell/tombstone information, " +
+ "only brief information");
+ options.addOption(partitionsOnlyKey);
+
+ Option csvKey = new Option(CSV_OPTION, "csv", false, "CSV output (machine readable)");
+ options.addOption(csvKey);
+ }
+
+ /**
+ * Given arguments specifying a list of SSTables or directories, print information about SSTable partitions.
+ *
+ * @param args command lines arguments
+ * @throws ConfigurationException on configuration failure (wrong params given)
+ */
+ public static void main(String[] args) throws ConfigurationException, IOException
+ {
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmd;
+ try
+ {
+ cmd = parser.parse(options, args);
+ }
+ catch (ParseException e)
+ {
+ System.err.println(e.getMessage());
+ printUsage();
+ System.exit(1);
+ return;
+ }
+
+ if (cmd.getArgs().length == 0)
+ {
+ System.err.println("You must supply at least one sstable or directory");
+ printUsage();
+ System.exit(1);
+ }
+
+ int ec = processArguments(cmd);
+
+ System.exit(ec);
+ }
+
+ private static void printUsage()
+ {
+ String usage = String.format("sstablepartitions <options> <sstable files or directories>%n");
+ String header = "Print partition statistics of one or more sstables.";
+ new HelpFormatter().printHelp(usage, header, options, "");
+ }
+
+ private static int processArguments(CommandLine cmd) throws IOException
+ {
+ String[] keys = cmd.getOptionValues(KEY_OPTION);
+ Set<String> excludes = cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null
+ ? Collections.emptySet()
+ : ImmutableSet.copyOf(cmd.getOptionValues(EXCLUDE_KEY_OPTION));
+
+ boolean scanRecursive = cmd.hasOption(RECURSIVE_OPTION);
+ boolean withSnapshots = cmd.hasOption(SNAPSHOTS_OPTION);
+ boolean withBackups = cmd.hasOption(BACKUPS_OPTION);
+ boolean csv = cmd.hasOption(CSV_OPTION);
+ boolean partitionsOnly = cmd.hasOption(PARTITIONS_ONLY_OPTION);
+
+ long sizeThreshold = Long.MAX_VALUE;
+ int cellCountThreshold = Integer.MAX_VALUE;
+ int rowCountThreshold = Integer.MAX_VALUE;
+ int tombstoneCountThreshold = Integer.MAX_VALUE;
+ long currentTime = Clock.Global.currentTimeMillis() / 1000L;
+
+ try
+ {
+ if (cmd.hasOption(SIZE_THRESHOLD_OPTION))
+ {
+ String threshold = cmd.getOptionValue(SIZE_THRESHOLD_OPTION);
+ sizeThreshold = NumberUtils.isParsable(threshold)
+ ? Long.parseLong(threshold)
+ : new DataStorageSpec.LongBytesBound(threshold).toBytes();
+ }
+ if (cmd.hasOption(CELL_THRESHOLD_OPTION))
+ cellCountThreshold = Integer.parseInt(cmd.getOptionValue(CELL_THRESHOLD_OPTION));
+ if (cmd.hasOption(ROW_THRESHOLD_OPTION))
+ rowCountThreshold = Integer.parseInt(cmd.getOptionValue(ROW_THRESHOLD_OPTION));
+ if (cmd.hasOption(TOMBSTONE_THRESHOLD_OPTION))
+ tombstoneCountThreshold = Integer.parseInt(cmd.getOptionValue(TOMBSTONE_THRESHOLD_OPTION));
+ if (cmd.hasOption(CURRENT_TIMESTAMP_OPTION))
+ currentTime = Integer.parseInt(cmd.getOptionValue(CURRENT_TIMESTAMP_OPTION));
+ }
+ catch (NumberFormatException e)
+ {
+ System.err.printf("Invalid threshold argument: %s%n", e.getMessage());
+ return 1;
+ }
+
+ if (sizeThreshold < 0 || cellCountThreshold < 0 || tombstoneCountThreshold < 0 || currentTime < 0)
+ {
+ System.err.println("Negative values are not allowed");
+ return 1;
+ }
+
+ List<File> directories = new ArrayList<>();
+ List<ExtendedDescriptor> descriptors = new ArrayList<>();
+
+ if (!argumentsToFiles(cmd.getArgs(), descriptors, directories))
+ return 1;
+
+ for (File directory : directories)
+ {
+ processDirectory(scanRecursive, withSnapshots, withBackups, directory, descriptors);
+ }
+
+ if (csv)
+ System.out.println("key,keyBinary,live,offset,size,rowCount,cellCount," +
+ "tombstoneCount,rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount," +
+ "cellTombstoneCount,rowTtlExpired,cellTtlExpired," +
+ "directory,keyspace,table,index," +
+ "snapshot,backup,generation,format,version");
+
+ Collections.sort(descriptors);
+
+ for (ExtendedDescriptor desc : descriptors)
+ {
+ processSSTable(keys, excludes, desc,
+ sizeThreshold, cellCountThreshold, rowCountThreshold, tombstoneCountThreshold, partitionsOnly,
+ csv, currentTime);
+ }
+
+ return 0;
+ }
+
+ private static void processDirectory(boolean scanRecursive,
+ boolean withSnapshots,
+ boolean withBackups,
+ File dir,
+ List<ExtendedDescriptor> descriptors)
+ {
+ File[] files = dir.tryList();
+ if (files == null)
+ return;
+
+ for (File file : files)
+ {
+ if (file.isFile())
+ {
+ try
+ {
+ if (Descriptor.componentFromFile(file) != BigFormat.Components.DATA)
+ continue;
+
+ ExtendedDescriptor desc = ExtendedDescriptor.guessFromFile(file);
+ if (desc.snapshot != null && !withSnapshots)
+ continue;
+ if (desc.backup != null && !withBackups)
+ continue;
+
+ descriptors.add(desc);
+ }
+ catch (IllegalArgumentException e)
+ {
+ // ignore that error when scanning directories
+ }
+ }
+ if (scanRecursive && file.isDirectory())
+ {
+ processDirectory(true,
+ withSnapshots, withBackups,
+ file,
+ descriptors);
+ }
+ }
+ }
+
+ private static boolean argumentsToFiles(String[] args, List<ExtendedDescriptor> descriptors, List<File> directories)
+ {
+ boolean err = false;
+ for (String arg : args)
+ {
+ File file = new File(arg);
+ if (!file.exists())
+ {
+ System.err.printf("Argument '%s' does not resolve to a file or directory%n", arg);
+ err = true;
+ }
+
+ if (!file.isReadable())
+ {
+ System.err.printf("Argument '%s' is not a readable file or directory (check permissions)%n", arg);
+ err = true;
+ continue;
+ }
+
+ if (file.isFile())
+ {
+ try
+ {
+ descriptors.add(ExtendedDescriptor.guessFromFile(file));
+ }
+ catch (IllegalArgumentException e)
+ {
+ System.err.printf("Argument '%s' is not an sstable%n", arg);
+ err = true;
+ }
+ }
+ if (file.isDirectory())
+ directories.add(file);
+ }
+ return !err;
+ }
+
+ private static void processSSTable(String[] keys,
+ Set<String> excludedKeys,
+ ExtendedDescriptor desc,
+ long sizeThreshold,
+ int cellCountThreshold,
+ int rowCountThreshold,
+ int tombstoneCountThreshold,
+ boolean partitionsOnly,
+ boolean csv,
+ long currentTime) throws IOException
+ {
+ TableMetadata metadata = Util.metadataFromSSTable(desc.descriptor);
+ SSTableReader sstable = SSTableReader.openNoValidation(null, desc.descriptor, TableMetadataRef.forOfflineTools(metadata));
+
+ if (!csv)
+ System.out.printf("%nProcessing %s (%s uncompressed, %s on disk)%n",
+ desc,
+ prettyPrintMemory(sstable.uncompressedLength()),
+ prettyPrintMemory(sstable.onDiskLength()));
+
+ List<PartitionStats> matches = new ArrayList<>();
+ SSTableStats sstableStats = new SSTableStats();
+
+ try (ISSTableScanner scanner = buildScanner(sstable, metadata, keys, excludedKeys))
+ {
+ while (scanner.hasNext())
+ {
+ try (UnfilteredRowIterator partition = scanner.next())
+ {
+ ByteBuffer key = partition.partitionKey().getKey();
+ boolean isExcluded = excludedKeys.contains(metadata.partitionKeyType.getString(key));
+
+ PartitionStats partitionStats = new PartitionStats(key,
+ scanner.getCurrentPosition(),
+ partition.partitionLevelDeletion().isLive());
+
+ // Consume the partition to populate the stats.
+ while (partition.hasNext())
+ {
+ Unfiltered unfiltered = partition.next();
+
+ // We don't need any details if we are only interested on its size or if it's excluded.
+ if (!partitionsOnly && !isExcluded)
+ partitionStats.addUnfiltered(desc, currentTime, unfiltered);
+ }
+
+ // record the partiton size
+ partitionStats.endOfPartition(scanner.getCurrentPosition());
+
+ if (isExcluded)
+ continue;
+
+ sstableStats.addPartition(partitionStats);
+
+ if (partitionStats.size < sizeThreshold &&
+ partitionStats.rowCount < rowCountThreshold &&
+ partitionStats.cellCount < cellCountThreshold &&
+ partitionStats.tombstoneCount() < tombstoneCountThreshold)
+ continue;
+
+ matches.add(partitionStats);
+ if (csv)
+ partitionStats.printPartitionInfoCSV(metadata, desc);
+ else
+ partitionStats.printPartitionInfo(metadata, partitionsOnly);
+ }
+ }
+ }
+ catch (RuntimeException e)
+ {
+ System.err.printf("Failure processing sstable %s: %s%n", desc.descriptor, e);
+ }
+ finally
+ {
+ sstable.selfRef().release();
+ }
+
+ if (!csv)
+ {
+ printSummary(metadata, desc, sstableStats, matches, partitionsOnly);
+ }
+ }
+
+ private static String prettyPrintMemory(long bytes)
+ {
+ return FBUtilities.prettyPrintMemory(bytes, true);
+ }
+
+ private static ISSTableScanner buildScanner(SSTableReader sstable,
+ TableMetadata metadata,
+ String[] keys,
+ Set<String> excludedKeys)
+ {
+ if (keys != null && keys.length > 0)
+ {
+ try
+ {
+ return sstable.getScanner(Arrays.stream(keys)
+ .filter(key -> !excludedKeys.contains(key))
+ .map(metadata.partitionKeyType::fromString)
+ .map(k -> sstable.getPartitioner().decorateKey(k))
+ .sorted()
+ .map(DecoratedKey::getToken)
+ .map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound()))
+ .collect(Collectors.<AbstractBounds<PartitionPosition>>toList())
+ .iterator());
+ }
+ catch (RuntimeException e)
+ {
+ System.err.printf("Cannot use one or more partition keys in %s for the partition key type ('%s') " +
+ "of the underlying table: %s%n",
+ Arrays.toString(keys),
+ metadata.partitionKeyType.asCQL3Type(), e);
+ }
+ }
+ return sstable.getScanner();
+ }
+
+ private static void printSummary(TableMetadata metadata,
+ ExtendedDescriptor desc,
+ SSTableStats stats,
+ List<PartitionStats> matches,
+ boolean partitionsOnly)
+ {
+ // Print header
+ if (!matches.isEmpty())
+ {
+ System.out.printf("Summary of %s:%n" +
+ " File: %s%n" +
+ " %d partitions match%n" +
+ " Keys:", desc, desc.descriptor.fileFor(BigFormat.Components.DATA), matches.size());
+
+ for (PartitionStats match : matches)
+ System.out.print(" " + maybeEscapeKeyForSummary(metadata, match.key));
+
+ System.out.println();
+ }
+
+ // Print stats table columns
+ String format;
+ if (partitionsOnly)
+ {
+ System.out.printf(" %20s%n", "Partition size");
+ format = " %-5s %20s%n";
+ }
+ else
+ {
+ System.out.printf(" %20s %20s %20s %20s%n", "Partition size", "Row count", "Cell count", "Tombstone count");
+ format = " %-5s %20s %20d %20d %20d%n";
+ }
+
+ // Print approximate percentiles from the histograms
+ printPercentile(partitionsOnly, stats, format, "~p50", h -> h.percentile(.5d));
+ printPercentile(partitionsOnly, stats, format, "~p75", h -> h.percentile(.75d));
+ printPercentile(partitionsOnly, stats, format, "~p90", h -> h.percentile(.90d));
+ printPercentile(partitionsOnly, stats, format, "~p95", h -> h.percentile(.95d));
+ printPercentile(partitionsOnly, stats, format, "~p99", h -> h.percentile(.99d));
+ printPercentile(partitionsOnly, stats, format, "~p999", h -> h.percentile(.999d));
+
+ // Print accurate metrics (min/max/count)
+ if (partitionsOnly)
+ {
+ System.out.printf(format, "min", prettyPrintMemory(stats.minSize));
+ System.out.printf(format, "max", prettyPrintMemory(stats.maxSize));
+ }
+ else
+ {
+ System.out.printf(format,
+ "min",
+ prettyPrintMemory(stats.minSize),
+ stats.minRowCount,
+ stats.minCellCount,
+ stats.minTombstoneCount);
+ System.out.printf(format,
+ "max",
+ prettyPrintMemory(stats.maxSize),
+ stats.maxRowCount,
+ stats.maxCellCount,
+ stats.maxTombstoneCount);
+ }
+ System.out.printf(" count %20d%n", stats.partitionSizeHistogram.count());
+ }
+
+ private static void printPercentile(boolean partitionsOnly,
+ SSTableStats stats,
+ String format,
+ String header,
+ ToLongFunction<EstimatedHistogram> value)
+ {
+ if (partitionsOnly)
+ {
+ System.out.printf(format,
+ header,
+ prettyPrintMemory(value.applyAsLong(stats.partitionSizeHistogram)));
+ }
+ else
+ {
+ System.out.printf(format,
+ header,
+ prettyPrintMemory(value.applyAsLong(stats.partitionSizeHistogram)),
+ value.applyAsLong(stats.rowCountHistogram),
+ value.applyAsLong(stats.cellCountHistogram),
+ value.applyAsLong(stats.tombstoneCountHistogram));
+ }
+ }
+
+ private static String maybeEscapeKeyForSummary(TableMetadata metadata, ByteBuffer key)
+ {
+ String s = metadata.partitionKeyType.getString(key);
+ if (s.indexOf(' ') == -1)
+ return s;
+ return "\"" + StringUtils.replace(s, "\"", "\"\"") + "\"";
+ }
+
+ static final class SSTableStats
+ {
+ // EH of 155 can track a max value of 3520571548412 i.e. 3.5TB
+ EstimatedHistogram partitionSizeHistogram = new EstimatedHistogram(155, true);
+
+ // EH of 118 can track a max value of 4139110981, i.e., > 4B rows, cells or tombstones
+ EstimatedHistogram rowCountHistogram = new EstimatedHistogram(118, true);
+ EstimatedHistogram cellCountHistogram = new EstimatedHistogram(118, true);
+ EstimatedHistogram tombstoneCountHistogram = new EstimatedHistogram(118, true);
+
+ long minSize = 0;
+ long maxSize = 0;
+
+ int minRowCount = 0;
+ int maxRowCount = 0;
+
+ int minCellCount = 0;
+ int maxCellCount = 0;
+
+ int minTombstoneCount = 0;
+ int maxTombstoneCount = 0;
+
+ void addPartition(PartitionStats stats)
+ {
+ partitionSizeHistogram.add(stats.size);
+ rowCountHistogram.add(stats.rowCount);
+ cellCountHistogram.add(stats.cellCount);
+ tombstoneCountHistogram.add(stats.tombstoneCount());
+
+ if (minSize == 0 || stats.size < minSize)
+ minSize = stats.size;
+ if (stats.size > maxSize)
+ maxSize = stats.size;
+
+ if (minRowCount == 0 || stats.rowCount < minRowCount)
+ minRowCount = stats.rowCount;
+ if (stats.rowCount > maxRowCount)
+ maxRowCount = stats.rowCount;
+
+ if (minCellCount == 0 || stats.cellCount < minCellCount)
+ minCellCount = stats.cellCount;
+ if (stats.cellCount > maxCellCount)
+ maxCellCount = stats.cellCount;
+
+ if (minTombstoneCount == 0 || stats.tombstoneCount() < minTombstoneCount)
+ minTombstoneCount = stats.tombstoneCount();
+ if (stats.tombstoneCount() > maxTombstoneCount)
+ maxTombstoneCount = stats.tombstoneCount();
+ }
+ }
+
+ static final class PartitionStats
+ {
+ final ByteBuffer key;
+ final long offset;
+ final boolean live;
+
+ long size = -1;
+ int rowCount = 0;
+ int cellCount = 0;
+ int rowTombstoneCount = 0;
+ int rangeTombstoneCount = 0;
+ int complexTombstoneCount = 0;
+ int cellTombstoneCount = 0;
+ int rowTtlExpired = 0;
+ int cellTtlExpired = 0;
+
+ PartitionStats(ByteBuffer key, long offset, boolean live)
+ {
+ this.key = key;
+ this.offset = offset;
+ this.live = live;
+ }
+
+ void endOfPartition(long position)
+ {
+ size = position - offset;
+ }
+
+ int tombstoneCount()
+ {
+ return rowTombstoneCount + rangeTombstoneCount + complexTombstoneCount + cellTombstoneCount + rowTtlExpired + cellTtlExpired;
+ }
+
+ void addUnfiltered(ExtendedDescriptor desc, long currentTime, Unfiltered unfiltered)
+ {
+ if (unfiltered instanceof Row)
+ {
+ Row row = (Row) unfiltered;
+ rowCount++;
+
+ if (!row.deletion().isLive())
+ rowTombstoneCount++;
+
+ LivenessInfo liveInfo = row.primaryKeyLivenessInfo();
+ if (!liveInfo.isEmpty() && liveInfo.isExpiring() && liveInfo.localExpirationTime() < currentTime)
+ rowTtlExpired++;
+
+ for (ColumnData cd : row)
+ {
+
+ if (cd.column().isSimple())
+ {
+ addCell((int) currentTime, liveInfo, (Cell<?>) cd);
+ }
+ else
+ {
+ ComplexColumnData complexData = (ComplexColumnData) cd;
+ if (!complexData.complexDeletion().isLive())
+ complexTombstoneCount++;
+
+ for (Cell<?> cell : complexData)
+ addCell((int) currentTime, liveInfo, cell);
+ }
+ }
+ }
+ else if (unfiltered instanceof RangeTombstoneMarker)
+ {
+ rangeTombstoneCount++;
+ }
+ else
+ {
+ throw new UnsupportedOperationException("Unknown kind " + unfiltered.kind() + " in sstable " + desc.descriptor);
+ }
+ }
+
+ private void addCell(int currentTime, LivenessInfo liveInfo, Cell<?> cell)
+ {
+ cellCount++;
+ if (cell.isTombstone())
+ cellTombstoneCount++;
+ if (cell.isExpiring() && (liveInfo.isEmpty() || cell.ttl() != liveInfo.ttl()) && !cell.isLive(currentTime))
+ cellTtlExpired++;
+ }
+
+ void printPartitionInfo(TableMetadata metadata, boolean partitionsOnly)
+ {
+ String key = metadata.partitionKeyType.getString(this.key);
+ if (partitionsOnly)
+ System.out.printf(" Partition: '%s' (%s) %s, size: %s%n",
+ key,
+ ByteBufferUtil.bytesToHex(this.key), live ? "live" : "not live",
+ prettyPrintMemory(size));
+ else
+ System.out.printf(" Partition: '%s' (%s) %s, size: %s, rows: %d, cells: %d, " +
+ "tombstones: %d (row:%d, range:%d, complex:%d, cell:%d, row-TTLd:%d, cell-TTLd:%d)%n",
+ key,
+ ByteBufferUtil.bytesToHex(this.key),
+ live ? "live" : "not live",
+ prettyPrintMemory(size),
+ rowCount,
+ cellCount,
+ tombstoneCount(),
+ rowTombstoneCount,
+ rangeTombstoneCount,
+ complexTombstoneCount,
+ cellTombstoneCount,
+ rowTtlExpired,
+ cellTtlExpired);
+ }
+
+ void printPartitionInfoCSV(TableMetadata metadata, ExtendedDescriptor desc)
+ {
+ System.out.printf("\"%s\",%s,%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%s,%s,%s,%s,%s,%s,%s,%s,%s%n",
+ maybeEscapeKeyForSummary(metadata, key),
+ ByteBufferUtil.bytesToHex(key),
+ live ? "true" : "false",
+ offset, size,
+ rowCount, cellCount, tombstoneCount(),
+ rowTombstoneCount, rangeTombstoneCount, complexTombstoneCount, cellTombstoneCount,
+ rowTtlExpired, cellTtlExpired,
+ desc.descriptor.fileFor(BigFormat.Components.DATA),
+ notNull(desc.keyspace),
+ notNull(desc.table),
+ notNull(desc.index),
+ notNull(desc.snapshot),
+ notNull(desc.backup),
+ desc.descriptor.id,
+ desc.descriptor.version.format.name(),
+ desc.descriptor.version.version);
+ }
+ }
+
+ static final class ExtendedDescriptor implements Comparable<ExtendedDescriptor>
+ {
+ final String keyspace;
+ final String table;
+ final String index;
+ final String snapshot;
+ final String backup;
+ final TableId tableId;
+ final Descriptor descriptor;
+
+ ExtendedDescriptor(String keyspace, String table, TableId tableId, String index, String snapshot, String backup, Descriptor descriptor)
+ {
+ this.keyspace = keyspace;
+ this.table = table;
+ this.tableId = tableId;
+ this.index = index;
+ this.snapshot = snapshot;
+ this.backup = backup;
+ this.descriptor = descriptor;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ if (backup != null)
+ sb.append("Backup:").append(backup).append(' ');
+ if (snapshot != null)
+ sb.append("Snapshot:").append(snapshot).append(' ');
+ if (keyspace != null)
+ sb.append(keyspace).append('.');
+ if (table != null)
+ sb.append(table);
+ if (index != null)
+ sb.append('.').append(index);
+ if (tableId != null)
+ sb.append('-').append(tableId.toHexString());
+ return sb.append(" #")
+ .append(descriptor.id)
+ .append(" (")
+ .append(descriptor.version.format.name())
+ .append('-')
+ .append(descriptor.version.version)
+ .append(')')
+ .toString();
+ }
+
+ static ExtendedDescriptor guessFromFile(File fArg)
+ {
+ Descriptor desc = Descriptor.fromFile(fArg);
+
+ String snapshot = null;
+ String backup = null;
+ String index = null;
+
+ File parent = fArg.parent();
+ File grandparent = parent.parent();
+
+ if (parent.name().length() > 1 && parent.name().startsWith(".") && parent.name().charAt(1) != '.')
+ {
+ index = parent.name().substring(1);
+ parent = parent.parent();
+ grandparent = parent.parent();
+ }
+
+ if (parent.name().equals(Directories.BACKUPS_SUBDIR))
+ {
+ backup = parent.name();
+ parent = parent.parent();
+ grandparent = parent.parent();
+ }
+
+ if (grandparent.name().equals(Directories.SNAPSHOT_SUBDIR))
+ {
+ snapshot = parent.name();
+ parent = grandparent.parent();
+ grandparent = parent.parent();
+ }
+
+ try
+ {
+ Pair<String, TableId> tableNameAndId = TableId.tableNameAndIdFromFilename(parent.name());
+ if (tableNameAndId != null)
+ {
+ return new ExtendedDescriptor(grandparent.name(),
+ tableNameAndId.left,
+ tableNameAndId.right,
+ index,
+ snapshot,
+ backup,
+ desc);
+ }
+ }
+ catch (NumberFormatException e)
+ {
+ // ignore non-parseable table-IDs
+ }
+
+ return new ExtendedDescriptor(null,
+ null,
+ null,
+ index,
+ snapshot,
+ backup,
+ desc);
+ }
+
+ @Override
+ public int compareTo(ExtendedDescriptor o)
+ {
+ int c = descriptor.directory.toString().compareTo(o.descriptor.directory.toString());
+ if (c != 0)
+ return c;
+ c = notNull(keyspace).compareTo(notNull(o.keyspace));
+ if (c != 0)
+ return c;
+ c = notNull(table).compareTo(notNull(o.table));
+ if (c != 0)
+ return c;
+ c = notNull(tableId).toString().compareTo(notNull(o.tableId).toString());
+ if (c != 0)
+ return c;
+ c = notNull(index).compareTo(notNull(o.index));
+ if (c != 0)
+ return c;
+ c = notNull(snapshot).compareTo(notNull(o.snapshot));
+ if (c != 0)
+ return c;
+ c = notNull(backup).compareTo(notNull(o.backup));
+ if (c != 0)
+ return c;
+ c = notNull(descriptor.id.toString()).compareTo(notNull(o.descriptor.id.toString()));
+ if (c != 0)
+ return c;
+ return Integer.compare(System.identityHashCode(this), System.identityHashCode(o));
+ }
+ }
+
+ private static String notNull(String s)
+ {
+ return s != null ? s : "";
+ }
+
+ private static TableId notNull(TableId s)
+ {
+ return s != null ? s : EMPTY_TABLE_ID;
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
index 1289e7e..f5ee3df 100644
--- a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
@@ -26,20 +26,8 @@
import java.util.Arrays;
import java.util.List;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-
-/**
- * Set repairedAt status on a given set of sstables.
- *
- * If you pass --is-repaired, it will set the repairedAt time to the last modified time.
- *
- * If you know you ran repair 2 weeks ago, you can do something like
- *
- * {@code
- * sstablerepairset --is-repaired -f <(find /var/lib/cassandra/data/.../ -iname "*Data.db*" -mtime +14)
- * }
- */
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.util.File;
public class SSTableRepairedAtSetter
@@ -81,7 +69,7 @@
for (String fname: fileNames)
{
- Descriptor descriptor = Descriptor.fromFilename(fname);
+ Descriptor descriptor = Descriptor.fromFileWithComponent(new File(fname), false).left;
if (!descriptor.version.isCompatible())
{
System.err.println("SSTable " + fname + " is in a old and unsupported format");
@@ -90,7 +78,7 @@
if (setIsRepaired)
{
- FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath());
+ FileTime f = Files.getLastModifiedTime(descriptor.fileFor(Components.DATA).toPath());
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, f.toMillis(), null, false);
}
else
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 4484b69..26cc363 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -25,7 +25,6 @@
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.io.util.File;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -42,18 +41,22 @@
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.Scrubber;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IScrubber;
import org.apache.cassandra.io.sstable.SSTableHeaderFix;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Pair;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class StandaloneScrubber
@@ -78,7 +81,7 @@
{
Options options = Options.parseArgs(args);
- if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
+ if (TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean())
DatabaseDescriptor.toolInitialization(false); //Necessary for testing
else
Util.initDatabaseDescriptor();
@@ -120,7 +123,7 @@
{
Descriptor descriptor = entry.getKey();
Set<Component> components = entry.getValue();
- if (!components.contains(Component.DATA))
+ if (!components.contains(Components.DATA))
continue;
listResult.add(Pair.create(descriptor, components));
@@ -143,7 +146,7 @@
headerFixBuilder = headerFixBuilder.dryRun();
for (Pair<Descriptor, Set<Component>> p : listResult)
- headerFixBuilder.withPath(File.getPath(p.left.filenameFor(Component.DATA)));
+ headerFixBuilder.withPath(p.left.fileFor(Components.DATA).toPath());
SSTableHeaderFix headerFix = headerFixBuilder.build();
try
@@ -197,7 +200,7 @@
{
Descriptor descriptor = pair.left;
Set<Component> components = pair.right;
- if (!components.contains(Component.DATA))
+ if (!components.contains(Components.DATA))
continue;
try
@@ -221,7 +224,9 @@
try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable))
{
txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591
- try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, !options.noValidate, options.reinserOverflowedTTL))
+
+ SSTableFormat format = sstable.descriptor.getFormat();
+ try (IScrubber scrubber = format.getScrubber(cfs, txn, handler, options.build()))
{
scrubber.scrub();
}
@@ -275,7 +280,7 @@
}
}
- private static class Options
+ private static class Options extends IScrubber.Options.Builder
{
public final String keyspaceName;
public final String cfName;
@@ -283,9 +288,6 @@
public boolean debug;
public boolean verbose;
public boolean manifestCheckOnly;
- public boolean skipCorrupted;
- public boolean noValidate;
- public boolean reinserOverflowedTTL;
public HeaderFixMode headerFixMode = HeaderFixMode.VALIDATE;
enum HeaderFixMode
@@ -344,9 +346,9 @@
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION);
- opts.skipCorrupted = cmd.hasOption(SKIP_CORRUPTED_OPTION);
- opts.noValidate = cmd.hasOption(NO_VALIDATE_OPTION);
- opts.reinserOverflowedTTL = cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION);
+ opts.skipCorrupted(cmd.hasOption(SKIP_CORRUPTED_OPTION));
+ opts.checkData(!cmd.hasOption(NO_VALIDATE_OPTION));
+ opts.reinsertOverflowedTTLRows(cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION));
if (cmd.hasOption(HEADERFIX_OPTION))
{
try
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index efadb56..f9f248a 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -18,25 +18,38 @@
*/
package org.apache.cassandra.tools;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.ParseException;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.compaction.SSTableSplitter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -50,10 +63,10 @@
private static final String NO_SNAPSHOT_OPTION = "no-snapshot";
private static final String SIZE_OPTION = "size";
- public static void main(String args[])
+ public static void main(String[] args)
{
Options options = Options.parseArgs(args);
- if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
+ if (TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean())
DatabaseDescriptor.toolInitialization(false); //Necessary for testing
else
Util.initDatabaseDescriptor();
@@ -74,7 +87,7 @@
continue;
}
- Descriptor desc = SSTable.tryDescriptorFromFilename(file);
+ Descriptor desc = SSTable.tryDescriptorFromFile(file);
if (desc == null) {
System.out.println("Skipping non sstable file " + file);
continue;
@@ -90,21 +103,7 @@
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same table");
- Set<Component> components = new HashSet<Component>(Arrays.asList(new Component[]{
- Component.DATA,
- Component.PRIMARY_INDEX,
- Component.FILTER,
- Component.COMPRESSION_INFO,
- Component.STATS
- }));
-
- Iterator<Component> iter = components.iterator();
- while (iter.hasNext()) {
- Component component = iter.next();
- if (!(new File(desc.filenameFor(component)).exists()))
- iter.remove();
- }
- parsedFilenames.put(desc, components);
+ parsedFilenames.put(desc, desc.getComponents(Collections.emptySet(), desc.getFormat().batchComponents()));
}
if (ksName == null || cfName == null)
@@ -125,8 +124,8 @@
{
SSTableReader sstable = SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs);
if (!isSSTableLargerEnough(sstable, options.sizeInMB)) {
- System.out.println(String.format("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)",
- sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB));
+ System.out.printf("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)%n",
+ sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB);
continue;
}
sstables.add(sstable);
@@ -140,7 +139,7 @@
catch (Exception e)
{
JVMStabilityInspector.inspectThrowable(e);
- System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
+ System.err.printf("Error Loading %s: %s%n", fn.getKey(), e.getMessage());
if (options.debug)
e.printStackTrace(System.err);
}
@@ -150,7 +149,7 @@
System.exit(0);
}
if (options.snapshot)
- System.out.println(String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
+ System.out.printf("Pre-split sstables snapshotted into snapshot %s%n", snapshotName);
for (SSTableReader sstable : sstables)
{
@@ -160,7 +159,7 @@
}
catch (Exception e)
{
- System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
+ System.err.printf("Error splitting %s: %s%n", sstable, e.getMessage());
if (options.debug)
e.printStackTrace(System.err);
@@ -200,7 +199,7 @@
this.filenames = filenames;
}
- public static Options parseArgs(String cmdArgs[])
+ public static Options parseArgs(String[] cmdArgs)
{
CommandLineParser parser = new GnuParser();
CmdLineOptions options = getCmdLineOptions();
@@ -251,19 +250,18 @@
options.addOption(null, DEBUG_OPTION, "display stack traces");
options.addOption("h", HELP_OPTION, "display this help message");
options.addOption(null, NO_SNAPSHOT_OPTION, "don't snapshot the sstables before splitting");
- options.addOption("s", SIZE_OPTION, "size", "maximum size in MB for the output sstables (default: " + DEFAULT_SSTABLE_SIZE + ")");
+ options.addOption("s", SIZE_OPTION, "size", "maximum size in MB for the output sstables (default: " + DEFAULT_SSTABLE_SIZE + ')');
return options;
}
public static void printUsage(CmdLineOptions options)
{
String usage = String.format("%s [options] <filename> [<filename>]*", TOOL_NAME);
- StringBuilder header = new StringBuilder();
- header.append("--\n");
- header.append("Split the provided sstables files in sstables of maximum provided file size (see option --" + SIZE_OPTION + ")." );
- header.append("\n--\n");
- header.append("Options are:");
- new HelpFormatter().printHelp(usage, header.toString(), options, "");
+ String header = "--\n" +
+ "Split the provided sstables files in sstables of maximum provided file size (see option --" + SIZE_OPTION + ")." +
+ "\n--\n" +
+ "Options are:";
+ new HelpFormatter().printHelp(usage, header, options, "");
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index a7c099c..1009c3f 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -17,10 +17,17 @@
*/
package org.apache.cassandra.tools;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.ParseException;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -30,13 +37,14 @@
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.compaction.Upgrader;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
public class StandaloneUpgrader
@@ -49,7 +57,7 @@
public static void main(String args[])
{
Options options = Options.parseArgs(args);
- if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
+ if (TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean())
DatabaseDescriptor.toolInitialization(false); //Necessary for testing
else
Util.initDatabaseDescriptor();
@@ -80,13 +88,13 @@
for (Map.Entry<Descriptor, Set<Component>> entry : lister.sortedList())
{
Set<Component> components = entry.getValue();
- if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX))
+ if (!components.containsAll(entry.getKey().getFormat().primaryComponents()))
continue;
try
{
SSTableReader sstable = SSTableReader.openNoValidation(entry.getKey(), components, cfs);
- if (sstable.descriptor.version.equals(SSTableFormat.Type.current().info.getLatestVersion()))
+ if (sstable.descriptor.version.equals(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()))
{
sstable.selfRef().release();
continue;
diff --git a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
index d7554dd..547a1e0 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
@@ -39,17 +39,19 @@
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IVerifier;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Throwables;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
public class StandaloneVerifier
@@ -78,6 +80,8 @@
System.out.println("sstableverify using the following options: " + options);
+ List<SSTableReader> sstables = new ArrayList<>();
+ int exitCode = 0;
try
{
// load keyspace descriptions.
@@ -97,13 +101,11 @@
OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
- List<SSTableReader> sstables = new ArrayList<>();
-
// Verify sstables
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
{
Set<Component> components = entry.getValue();
- if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX))
+ if (!components.containsAll(entry.getKey().getFormat().primaryComponents()))
continue;
try
@@ -117,45 +119,56 @@
System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
if (options.debug)
e.printStackTrace(System.err);
- System.exit(1);
+ exitCode = 1;
+ return;
}
}
- Verifier.Options verifyOptions = Verifier.options().invokeDiskFailurePolicy(false)
- .extendedVerification(options.extended)
- .checkVersion(options.checkVersion)
- .mutateRepairStatus(options.mutateRepairStatus)
- .checkOwnsTokens(!options.tokens.isEmpty())
- .tokenLookup(ignore -> options.tokens)
- .build();
+ IVerifier.Options verifyOptions = IVerifier.options().invokeDiskFailurePolicy(false)
+ .extendedVerification(options.extended)
+ .checkVersion(options.checkVersion)
+ .mutateRepairStatus(options.mutateRepairStatus)
+ .checkOwnsTokens(!options.tokens.isEmpty())
+ .tokenLookup(ignore -> options.tokens)
+ .build();
handler.output("Running verifier with the following options: " + verifyOptions);
for (SSTableReader sstable : sstables)
{
- try (Verifier verifier = new Verifier(cfs, sstable, handler, true, verifyOptions))
+ try (IVerifier verifier = sstable.getVerifier(cfs, handler, true, verifyOptions))
{
verifier.verify();
}
catch (Exception e)
{
- handler.warn(String.format("Error verifying %s: %s", sstable, e.getMessage()), e);
+ handler.warn(e, String.format("Error verifying %s: %s", sstable, e.getMessage()));
hasFailed = true;
}
}
CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
- System.exit( hasFailed ? 1 : 0 ); // We need that to stop non daemonized threads
+ for (SSTableReader reader : sstables)
+ Throwables.perform((Throwable) null, () -> reader.selfRef().close());
+
+ exitCode = hasFailed ? 1 : 0; // We need that to stop non daemonized threads
}
catch (Exception e)
{
System.err.println(e.getMessage());
if (options.debug)
e.printStackTrace(System.err);
- System.exit(1);
+ exitCode = 1;
+ }
+ finally
+ {
+ for (SSTableReader reader : sstables)
+ Throwables.perform((Throwable) null, () -> reader.selfRef().close());
+
+ System.exit(exitCode);
}
}
private static void initDatabaseDescriptorForTool() {
- if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
+ if (TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean())
DatabaseDescriptor.toolInitialization(false); //Necessary for testing
else
Util.initDatabaseDescriptor();
diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java
index 3757754..8a254e2 100644
--- a/src/java/org/apache/cassandra/tools/Util.java
+++ b/src/java/org/apache/cassandra/tools/Util.java
@@ -18,22 +18,22 @@
package org.apache.cassandra.tools;
-import static java.lang.String.format;
-
import java.io.IOException;
import java.io.PrintStream;
-import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.TreeMap;
-import java.util.Map.Entry;
import java.util.function.LongFunction;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.SerializationHeader;
@@ -41,20 +41,18 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
+import static java.lang.String.format;
@SuppressWarnings("serial")
public final class Util
{
- public static final String ALLOW_TOOL_REINIT_FOR_TEST = Util.class.getName() + "ALLOW_TOOL_REINIT_FOR_TEST"; // Necessary for testing
static final String RESET = "\u001B[0m";
static final String BLUE = "\u001B[34m";
static final String CYAN = "\u001B[36m";
@@ -310,12 +308,11 @@
*/
public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOException
{
- if (desc.version.getVersion().compareTo("ma") < 0)
- throw new IOException("pre-3.0 SSTable is not supported.");
+ if (!desc.version.isCompatible())
+ throw new IOException("Unsupported SSTable version " + desc.getFormat().name() + "/" + desc.version);
- EnumSet<MetadataType> types = EnumSet.of(MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
- SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
+ StatsComponent statsComponent = StatsComponent.load(desc, MetadataType.STATS, MetadataType.HEADER);
+ SerializationHeader.Component header = statsComponent.serializationHeader();
IPartitioner partitioner = FBUtilities.newPartitioner(desc);
@@ -337,4 +334,4 @@
}
return builder.build();
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java b/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
index b005818..a55cded 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
@@ -22,17 +22,27 @@
import java.io.IOError;
import java.io.IOException;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
+
@Command(name = "resume", description = "Resume bootstrap streaming")
public class BootstrapResume extends NodeToolCmd
{
+ @Option(title = "force",
+ name = { "-f", "--force"},
+ description = "Use --force to resume bootstrap regardless of cassandra.reset_bootstrap_progress environment variable. WARNING: This is potentially dangerous, see CASSANDRA-17679")
+ boolean force = false;
+
@Override
protected void execute(NodeProbe probe)
{
try
{
+ if ((!RESET_BOOTSTRAP_PROGRESS.isPresent() || RESET_BOOTSTRAP_PROGRESS.getBoolean()) && !force)
+ throw new RuntimeException("'nodetool bootstrap resume' is disabled.");
probe.resumeBootstrap(probe.output().out);
}
catch (IOException e)
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java b/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java
index fb69b25..866b85d 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java
@@ -25,9 +25,14 @@
import io.airlift.airline.Option;
import java.io.IOException;
+import java.time.Instant;
+import java.time.format.DateTimeParseException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@@ -43,14 +48,45 @@
@Option(title = "clear_all_snapshots", name = "--all", description = "Removes all snapshots")
private boolean clearAllSnapshots = false;
+ @Option(title = "older_than", name = "--older-than", description = "Clear snapshots older than specified time period.")
+ private String olderThan;
+
+ @Option(title = "older_than_timestamp", name = "--older-than-timestamp",
+ description = "Clear snapshots older than specified timestamp. It has to be a string in ISO format, for example '2022-12-03T10:15:30Z'")
+ private String olderThanTimestamp;
+
@Override
public void execute(NodeProbe probe)
{
- if(snapshotName.isEmpty() && !clearAllSnapshots)
- throw new RuntimeException("Specify snapshot name or --all");
+ if (snapshotName.isEmpty() && !clearAllSnapshots)
+ throw new IllegalArgumentException("Specify snapshot name or --all");
- if(!snapshotName.isEmpty() && clearAllSnapshots)
- throw new RuntimeException("Specify only one of snapshot name or --all");
+ if (!snapshotName.isEmpty() && clearAllSnapshots)
+ throw new IllegalArgumentException("Specify only one of snapshot name or --all");
+
+ if (olderThan != null && olderThanTimestamp != null)
+ throw new IllegalArgumentException("Specify only one of --older-than or --older-than-timestamp");
+
+ if (!snapshotName.isEmpty() && olderThan != null)
+ throw new IllegalArgumentException("Specifying snapshot name together with --older-than flag is not allowed");
+
+ if (!snapshotName.isEmpty() && olderThanTimestamp != null)
+ throw new IllegalArgumentException("Specifying snapshot name together with --older-than-timestamp flag is not allowed");
+
+ if (olderThanTimestamp != null)
+ try
+ {
+ Instant.parse(olderThanTimestamp);
+ }
+ catch (DateTimeParseException ex)
+ {
+ throw new IllegalArgumentException("Parameter --older-than-timestamp has to be a valid instant in ISO format.");
+ }
+
+ Long olderThanInSeconds = null;
+ if (olderThan != null)
+ // fail fast when it is not valid
+ olderThanInSeconds = new DurationSpec.LongSecondsBound(olderThan).toSeconds();
StringBuilder sb = new StringBuilder();
@@ -59,18 +95,32 @@
if (keyspaces.isEmpty())
sb.append("[all keyspaces]");
else
- sb.append("[").append(join(keyspaces, ", ")).append("]");
+ sb.append('[').append(join(keyspaces, ", ")).append(']');
if (snapshotName.isEmpty())
sb.append(" with [all snapshots]");
else
- sb.append(" with snapshot name [").append(snapshotName).append("]");
+ sb.append(" with snapshot name [").append(snapshotName).append(']');
- probe.output().out.println(sb.toString());
+ if (olderThanInSeconds != null)
+ sb.append(" older than ")
+ .append(olderThanInSeconds)
+ .append(" seconds.");
+
+ if (olderThanTimestamp != null)
+ sb.append(" older than timestamp ").append(olderThanTimestamp);
+
+ probe.output().out.println(sb);
try
{
- probe.clearSnapshot(snapshotName, toArray(keyspaces, String.class));
+ Map<String, Object> parameters = new HashMap<>();
+ if (olderThan != null)
+ parameters.put("older_than", olderThan);
+ if (olderThanTimestamp != null)
+ parameters.put("older_than_timestamp", olderThanTimestamp);
+
+ probe.clearSnapshot(parameters, snapshotName, toArray(keyspaces, String.class));
} catch (IOException e)
{
throw new RuntimeException("Error during clearing snapshots", e);
diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
index 799ef56..aedc8f2 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
@@ -91,7 +91,7 @@
TableBuilder table = new TableBuilder();
if (vtableOutput)
- table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit");
+ table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit", "target directory");
else
table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress");
@@ -110,7 +110,10 @@
String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + "%";
String id = c.get(CompactionInfo.COMPACTION_ID);
if (vtableOutput)
- table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit);
+ {
+ String targetDirectory = c.get(CompactionInfo.TARGET_DIRECTORY);
+ table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit, targetDirectory);
+ }
else
table.add(id, taskType, keyspace, columnFamily, progressStr, totalStr, unit, percentComplete);
@@ -128,4 +131,4 @@
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java b/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java
index 33a0a4d..f855e62 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java
@@ -32,8 +32,6 @@
import org.apache.cassandra.tools.NodeTool;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-import static java.lang.String.format;
-
@Command(name = "describecluster", description = "Print the name, snitch, partitioner and schema version of a cluster")
public class DescribeCluster extends NodeToolCmd
{
@@ -62,9 +60,9 @@
// display schema version for each node
out.println("\tSchema versions:");
Map<String, List<String>> schemaVersions = printPort ? probe.getSpProxy().getSchemaVersionsWithPort() : probe.getSpProxy().getSchemaVersions();
- for (String version : schemaVersions.keySet())
+ for (Map.Entry<String, List<String>> entry : schemaVersions.entrySet())
{
- out.println(format("\t\t%s: %s%n", version, schemaVersions.get(version)));
+ out.printf("\t\t%s: %s%n%n", entry.getKey(), entry.getValue());
}
// Collect status information of all nodes
@@ -86,6 +84,7 @@
out.println("\tUnreachable: " + unreachableNodes.size());
Map<String, String> tokensToEndpoints = probe.getTokenToEndpointMap(withPort);
+ StringBuilder errors = new StringBuilder();
Map<String, Float> ownerships = null;
try
{
@@ -93,12 +92,20 @@
}
catch (IllegalStateException ex)
{
- ownerships = probe.getOwnershipWithPort();
- out.println("Error: " + ex.getMessage());
+ try
+ {
+ ownerships = probe.getOwnershipWithPort();
+ errors.append("Note: ").append(ex.getMessage()).append("%n");
+ }
+ catch (Exception e)
+ {
+ out.printf("%nError: %s%n", e.getMessage());
+ System.exit(1);
+ }
}
catch (IllegalArgumentException ex)
{
- out.println("%nError: " + ex.getMessage());
+ out.printf("%nError: %s%n", ex.getMessage());
System.exit(1);
}
@@ -107,7 +114,7 @@
out.println("\nData Centers: ");
for (Map.Entry<String, SetHostStatWithPort> dc : dcs.entrySet())
{
- out.print("\t" + dc.getKey());
+ out.print('\t' + dc.getKey());
ArrayListMultimap<InetAddressAndPort, HostStatWithPort> hostToTokens = ArrayListMultimap.create();
for (HostStatWithPort stat : dc.getValue())
@@ -129,9 +136,9 @@
// display database version for each node
out.println("\nDatabase versions:");
Map<String, List<String>> databaseVersions = probe.getGossProxy().getReleaseVersionsWithPort();
- for (String version : databaseVersions.keySet())
+ for (Map.Entry<String, List<String>> entry : databaseVersions.entrySet())
{
- out.println(format("\t%s: %s%n", version, databaseVersions.get(version)));
+ out.printf("\t%s: %s%n%n", entry.getKey(), entry.getValue());
}
out.println("Keyspaces:");
@@ -142,7 +149,10 @@
{
out.println("something went wrong for keyspace: " + keyspaceName);
}
- out.println("\t" + keyspaceName + " -> Replication class: " + replicationInfo);
+ out.printf("\t%s -> Replication class: %s%n", keyspaceName, replicationInfo);
}
+
+ if (errors.length() != 0)
+ out.printf("%n" + errors);
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ForceCompact.java b/src/java/org/apache/cassandra/tools/nodetool/ForceCompact.java
new file mode 100644
index 0000000..99265e7
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ForceCompact.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+@Command(name = "forcecompact", description = "Force a (major) compaction on a table")
+public class ForceCompact extends NodeToolCmd
+{
+ @Arguments(usage = "[<keyspace> <table> <keys>]", description = "The keyspace, table, and a list of partition keys ignoring the gc_grace_seconds")
+ private List<String> args = new ArrayList<>();
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ // Check if the input has valid size
+ checkArgument(args.size() >= 3, "forcecompact requires keyspace, table and keys args");
+
+ // We rely on lower-level APIs to check and throw exceptions if the input keyspace or table name are invalid
+ String keyspaceName = args.get(0);
+ String tableName = args.get(1);
+ String[] partitionKeysIgnoreGcGrace = parsePartitionKeys(args);
+
+ try
+ {
+ probe.forceCompactionKeysIgnoringGcGrace(keyspaceName, tableName, partitionKeysIgnoreGcGrace);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException("Error occurred during compaction keys", e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java b/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java
index f1e2117..657e0ec 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetSSTables.java
@@ -23,6 +23,8 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
@@ -36,6 +38,9 @@
description = "Specify the key in hexadecimal string format")
private boolean hexFormat = false;
+ @Option(name={"-l", "--show-levels"}, description="If the table is using leveled compaction the level of each sstable will be included in the output (Default: false)")
+ private boolean showLevels = false;
+
@Arguments(usage = "<keyspace> <cfname> <key>", description = "The keyspace, the column family, and the key")
private List<String> args = new ArrayList<>();
@@ -47,10 +52,19 @@
String cf = args.get(1);
String key = args.get(2);
- List<String> sstables = probe.getSSTables(ks, cf, key, hexFormat);
- for (String sstable : sstables)
+ if (showLevels && probe.isLeveledCompaction(ks, cf))
{
- probe.output().out.println(sstable);
+ Map<Integer, Set<String>> sstables = probe.getSSTablesWithLevel(ks, cf, key, hexFormat);
+ for (Integer level : sstables.keySet())
+ for (String sstable : sstables.get(level))
+ probe.output().out.println(level + ": " + sstable);
+ } else
+ {
+ List<String> sstables = probe.getSSTables(ks, cf, key, hexFormat);
+ for (String sstable : sstables)
+ {
+ probe.output().out.println(sstable);
+ }
}
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Info.java b/src/java/org/apache/cassandra/tools/nodetool/Info.java
index db7277e..5e0d87c 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Info.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Info.java
@@ -51,6 +51,8 @@
out.printf("%-23s: %s%n", "Gossip active", gossipInitialized);
out.printf("%-23s: %s%n", "Native Transport active", probe.isNativeTransportRunning());
out.printf("%-23s: %s%n", "Load", probe.getLoadString());
+ out.printf("%-23s: %s%n", "Uncompressed load", probe.getUncompressedLoadString());
+
if (gossipInitialized)
out.printf("%-23s: %s%n", "Generation No", probe.getCurrentGenerationNumber());
else
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java b/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
index b70a7a9..803fe5a 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
@@ -40,6 +40,11 @@
description = "Skip snapshots with TTL")
private boolean noTTL = false;
+ @Option(title = "ephemeral",
+ name = { "-e", "--ephemeral" },
+ description = "Include ephememeral snapshots")
+ private boolean includeEphemeral = false;
+
@Override
public void execute(NodeProbe probe)
{
@@ -50,6 +55,7 @@
Map<String, String> options = new HashMap<>();
options.put("no_ttl", Boolean.toString(noTTL));
+ options.put("include_ephemeral", Boolean.toString(includeEphemeral));
final Map<String, TabularData> snapshotDetails = probe.getSnapshotDetails(options);
if (snapshotDetails.isEmpty())
@@ -62,7 +68,11 @@
TableBuilder table = new TableBuilder();
// display column names only once
final List<String> indexNames = snapshotDetails.entrySet().iterator().next().getValue().getTabularType().getIndexNames();
- table.add(indexNames.toArray(new String[indexNames.size()]));
+
+ if (includeEphemeral)
+ table.add(indexNames.toArray(new String[indexNames.size()]));
+ else
+ table.add(indexNames.subList(0, indexNames.size() - 1).toArray(new String[indexNames.size() - 1]));
for (final Map.Entry<String, TabularData> snapshotDetail : snapshotDetails.entrySet())
{
@@ -70,12 +80,15 @@
for (Object eachValue : values)
{
final List<?> value = (List<?>) eachValue;
- table.add(value.toArray(new String[value.size()]));
+ if (includeEphemeral)
+ table.add(value.toArray(new String[value.size()]));
+ else
+ table.add(value.subList(0, value.size() - 1).toArray(new String[value.size() - 1]));
}
}
table.printTo(out);
- out.println("\nTotal TrueDiskSpaceUsed: " + FileUtils.stringifyFileSize(trueSnapshotsSize) + "\n");
+ out.println("\nTotal TrueDiskSpaceUsed: " + FileUtils.stringifyFileSize(trueSnapshotsSize) + '\n');
}
catch (Exception e)
{
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java b/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java
index 487f14a..45cade7 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/ProfileLoad.java
@@ -17,36 +17,36 @@
*/
package org.apache.cassandra.tools.nodetool;
-import static com.google.common.base.Preconditions.checkArgument;
-import static org.apache.commons.lang3.StringUtils.join;
-
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
-
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.OpenDataException;
-import org.apache.cassandra.metrics.Sampler.SamplerType;
-import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
-import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
-import org.apache.cassandra.utils.Pair;
-
import com.google.common.collect.Lists;
import io.airlift.airline.Arguments;
import io.airlift.airline.Command;
import io.airlift.airline.Option;
+import org.apache.cassandra.metrics.Sampler.SamplerType;
+import org.apache.cassandra.metrics.SamplingManager;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import org.apache.cassandra.utils.Pair;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.commons.lang3.StringUtils.join;
@Command(name = "profileload", description = "Low footprint profiling of activity for a period of time")
public class ProfileLoad extends NodeToolCmd
{
- @Arguments(usage = "<keyspace> <cfname> <duration>", description = "The keyspace, column family name, and duration in milliseconds")
+ @Arguments(usage = "<keyspace> <cfname> <duration>", description = "The keyspace, column family name, and duration in milliseconds (Default: 10000)")
private List<String> args = new ArrayList<>();
@Option(name = "-s", description = "Capacity of the sampler, higher for more accuracy (Default: 256)")
@@ -58,27 +58,59 @@
@Option(name = "-a", description = "Comma separated list of samplers to use (Default: all)")
private String samplers = join(SamplerType.values(), ',');
+ @Option(name = {"-i", "--interval"}, description = "Schedule a new job that samples every interval milliseconds (Default: disabled) in the background")
+ private int intervalMillis = -1; // -1 for disabled.
+
+ @Option(name = {"-t", "--stop"}, description = "Stop the scheduled sampling job identified by <keyspace> and <cfname>. Jobs are stopped until the last schedules complete.")
+ private boolean shouldStop = false;
+
+ @Option(name = {"-l", "--list"}, description = "List the scheduled sampling jobs")
+ private boolean shouldList = false;
+
@Override
public void execute(NodeProbe probe)
{
- checkArgument(args.size() == 3 || args.size() == 1 || args.size() == 0, "Invalid arguments, either [keyspace table duration] or [duration] or no args");
- checkArgument(topCount < capacity, "TopK count (-k) option must be smaller then the summary capacity (-s)");
+ checkArgument(args.size() == 3 || args.size() == 2 || args.size() == 1 || args.size() == 0,
+ "Invalid arguments, either [keyspace table/* duration] or [keyspace table/*] or [duration] or no args.\n" +
+ "Optionally, use * to represent all tables under the keyspace.");
+ checkArgument(topCount > 0, "TopK count (-k) option must have positive value");
+ checkArgument(topCount < capacity,
+ "TopK count (-k) option must be smaller then the summary capacity (-s)");
+ checkArgument(capacity <= 1024, "Capacity (-s) cannot exceed 1024.");
String keyspace = null;
String table = null;
- Integer durationMillis = 10000;
- if(args.size() == 3)
+ int durationMillis = 10000;
+ /* There are 3 possible outcomes after processing the args.
+ * - keyspace == null && table == null. We need to sample all tables
+ * - keyspace == KEYSPACE && table == *. We need to sample all tables under the specified KEYSPACE
+ * - keyspace = KEYSPACE && table == TABLE. Sample the specific KEYSPACE.table combination
+ */
+ if (args.size() == 3)
{
keyspace = args.get(0);
table = args.get(1);
- durationMillis = Integer.valueOf(args.get(2));
+ durationMillis = Integer.parseInt(args.get(2));
+ }
+ else if (args.size() == 2)
+ {
+ keyspace = args.get(0);
+ table = args.get(1);
}
else if (args.size() == 1)
{
- durationMillis = Integer.valueOf(args.get(0));
+ durationMillis = Integer.parseInt(args.get(0));
}
+ keyspace = nullifyWildcard(keyspace);
+ table = nullifyWildcard(table);
+
+ checkArgument(durationMillis > 0, "Duration: %s must be positive", durationMillis);
+
+ checkArgument(!hasInterval() || intervalMillis >= durationMillis,
+ "Invalid scheduled sampling interval. Expecting interval >= duration, but interval: %s ms; duration: %s ms",
+ intervalMillis, durationMillis);
// generate the list of samplers
List<String> targets = Lists.newArrayList();
- List<String> available = Arrays.stream(SamplerType.values()).map(Enum::toString).collect(Collectors.toList());
+ Set<String> available = Arrays.stream(SamplerType.values()).map(Enum::toString).collect(Collectors.toSet());
for (String s : samplers.split(","))
{
String sampler = s.trim().toUpperCase();
@@ -86,108 +118,70 @@
targets.add(sampler);
}
+ PrintStream out = probe.output().out;
+
Map<String, List<CompositeData>> results;
try
{
- if (keyspace == null)
- results = probe.getPartitionSample(capacity, durationMillis, topCount, targets);
+ // handle scheduled samplings, i.e. start or stop
+ if (hasInterval() || shouldStop)
+ {
+ // keyspace and table are nullable
+ boolean opSuccess = probe.handleScheduledSampling(keyspace, table, capacity, topCount, durationMillis, intervalMillis, targets, shouldStop);
+ if (!opSuccess)
+ {
+ if (shouldStop)
+ out.printf("Unable to stop the non-existent scheduled sampling for keyspace: %s, table: %s%n", keyspace, table);
+ else
+ out.printf("Unable to schedule sampling for keyspace: %s, table: %s due to existing samplings. " +
+ "Stop the existing sampling jobs first.%n", keyspace, table);
+ }
+ return;
+ }
+ else if (shouldList)
+ {
+ List<Pair<String, String>> sampleTasks = new ArrayList<>();
+ int maxKsLength = "KEYSPACE".length();
+ int maxTblLength = "TABLE".length();
+ for (String fullTableName : probe.getSampleTasks())
+ {
+ String[] parts = fullTableName.split("\\.");
+ checkState(parts.length == 2, "Unable to parse the full table name: %s", fullTableName);
+ sampleTasks.add(Pair.create(parts[0], parts[1]));
+ maxKsLength = Math.max(maxKsLength, parts[0].length());
+ }
+ // print the header line and put enough space between KEYSPACE AND TABLE.
+ String lineFormat = "%" + maxKsLength + "s %" + maxTblLength + "s%n";
+ out.printf(lineFormat, "KEYSPACE", "TABLE");
+ sampleTasks.forEach(pair -> out.printf(lineFormat, pair.left, pair.right));
+ return;
+ }
else
- results = probe.getPartitionSample(keyspace, table, capacity, durationMillis, topCount, targets);
-
- } catch (OpenDataException e)
+ {
+ // blocking sample all the tables or all the tables under a keyspace
+ if (keyspace == null || table == null)
+ results = probe.getPartitionSample(keyspace, capacity, durationMillis, topCount, targets);
+ else // blocking sample the specific table
+ results = probe.getPartitionSample(keyspace, table, capacity, durationMillis, topCount, targets);
+ }
+ }
+ catch (OpenDataException e)
{
throw new RuntimeException(e);
}
AtomicBoolean first = new AtomicBoolean(true);
- ResultBuilder rb = new ResultBuilder(first, results, targets);
-
- for(String sampler : Lists.newArrayList("READS", "WRITES", "CAS_CONTENTIONS"))
- {
- rb.forType(SamplerType.valueOf(sampler), "Frequency of " + sampler.toLowerCase().replaceAll("_", " ") + " by partition")
- .addColumn("Table", "table")
- .addColumn("Partition", "value")
- .addColumn("Count", "count")
- .addColumn("+/-", "error")
- .print(probe.output().out);
- }
-
- rb.forType(SamplerType.WRITE_SIZE, "Max mutation size by partition")
- .addColumn("Table", "table")
- .addColumn("Partition", "value")
- .addColumn("Bytes", "count")
- .print(probe.output().out);
-
- rb.forType(SamplerType.LOCAL_READ_TIME, "Longest read query times")
- .addColumn("Query", "value")
- .addColumn("Microseconds", "count")
- .print(probe.output().out);
+ SamplingManager.ResultBuilder rb = new SamplingManager.ResultBuilder(first, results, targets);
+ out.println(SamplingManager.formatResult(rb));
}
- private class ResultBuilder
+ private boolean hasInterval()
{
- private SamplerType type;
- private String description;
- private AtomicBoolean first;
- private Map<String, List<CompositeData>> results;
- private List<String> targets;
- private List<Pair<String, String>> dataKeys;
+ return intervalMillis != -1;
+ }
- public ResultBuilder(AtomicBoolean first, Map<String, List<CompositeData>> results, List<String> targets)
- {
- super();
- this.first = first;
- this.results = results;
- this.targets = targets;
- this.dataKeys = new ArrayList<>();
- this.dataKeys.add(Pair.create(" ", " "));
- }
-
- public ResultBuilder forType(SamplerType type, String description)
- {
- ResultBuilder rb = new ResultBuilder(first, results, targets);
- rb.type = type;
- rb.description = description;
- return rb;
- }
-
- public ResultBuilder addColumn(String title, String key)
- {
- this.dataKeys.add(Pair.create(title, key));
- return this;
- }
-
- private String get(CompositeData cd, String key)
- {
- if (cd.containsKey(key))
- return cd.get(key).toString();
- return key;
- }
-
- public void print(PrintStream outStream)
- {
- if (targets.contains(type.toString()))
- {
- if (!first.get())
- outStream.println();
- first.set(false);
- outStream.println(description + ':');
- TableBuilder out = new TableBuilder();
- out.add(dataKeys.stream().map(p -> p.left).collect(Collectors.toList()).toArray(new String[] {}));
- List<CompositeData> topk = results.get(type.toString());
- for (CompositeData cd : topk)
- {
- out.add(dataKeys.stream().map(p -> get(cd, p.right)).collect(Collectors.toList()).toArray(new String[] {}));
- }
- if (topk.size() == 0)
- {
- outStream.println(" Nothing recorded during sampling period...");
- }
- else
- {
- out.printTo(outStream);
- }
- }
- }
+ private String nullifyWildcard(String input)
+ {
+ return input != null && input.equals("*") ? null : input;
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java b/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java
index a16e8f2..ed4e97c 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Rebuild.java
@@ -28,7 +28,7 @@
public class Rebuild extends NodeToolCmd
{
@Arguments(usage = "<src-dc-name>",
- description = "Name of DC from which to select sources for streaming. By default, pick any DC")
+ description = "Name of DC from which to select sources for streaming. By default, pick any DC (except local DC when --exclude-local-dc is set)")
private String sourceDataCenterName = null;
@Option(title = "specific_keyspace",
@@ -46,6 +46,11 @@
description = "Use -s to specify hosts that this node should stream from when -ts is used. Multiple hosts should be separated using commas (e.g. 127.0.0.1,127.0.0.2,...)")
private String specificSources = null;
+ @Option(title = "exclude_local_dc",
+ name = {"--exclude-local-dc"},
+ description = "Use --exclude-local-dc to exclude nodes in local data center as source for streaming.")
+ private boolean excludeLocalDatacenterNodes = false;
+
@Override
public void execute(NodeProbe probe)
{
@@ -55,6 +60,6 @@
throw new IllegalArgumentException("Cannot specify tokens without keyspace.");
}
- probe.rebuild(sourceDataCenterName, keyspace, tokens, specificSources);
+ probe.rebuild(sourceDataCenterName, keyspace, tokens, specificSources, excludeLocalDatacenterNodes);
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Ring.java b/src/java/org/apache/cassandra/tools/nodetool/Ring.java
index ccc7912..eb7645d6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Ring.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Ring.java
@@ -85,21 +85,29 @@
boolean showEffectiveOwnership = true;
// Calculate per-token ownership of the ring
- Map<String, Float> ownerships;
+ Map<String, Float> ownerships = null;
try
{
ownerships = probe.effectiveOwnershipWithPort(keyspace);
}
catch (IllegalStateException ex)
{
- ownerships = probe.getOwnershipWithPort();
- errors.append("Note: ").append(ex.getMessage()).append("%n");
- showEffectiveOwnership = false;
+ try
+ {
+ ownerships = probe.getOwnershipWithPort();
+ errors.append("Note: ").append(ex.getMessage()).append("%n");
+ showEffectiveOwnership = false;
+ }
+ catch (Exception e)
+ {
+ out.printf("%nError: %s%n", ex.getMessage());
+ System.exit(1);
+ }
}
catch (IllegalArgumentException ex)
{
out.printf("%nError: %s%n", ex.getMessage());
- return;
+ System.exit(1);
}
out.println();
@@ -112,7 +120,7 @@
out.println(" To view status related info of a node use \"nodetool status\" instead.\n");
}
- out.printf("%n " + errors.toString());
+ out.printf("%n " + errors);
}
private void printDc(String format, String dc,
@@ -167,9 +175,7 @@
else if (movingNodes.contains(endpoint))
state = "Moving";
- String load = loadMap.containsKey(endpoint)
- ? loadMap.get(endpoint)
- : "?";
+ String load = loadMap.getOrDefault(endpoint, "?");
String owns = stat.owns != null && showEffectiveOwnership? new DecimalFormat("##0.00%").format(stat.owns) : "?";
out.printf(format, stat.ipOrDns(printPort), rack, status, state, load, owns, stat.token);
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Status.java b/src/java/org/apache/cassandra/tools/nodetool/Status.java
index 369affc..a89ca64 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Status.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Status.java
@@ -79,8 +79,16 @@
}
catch (IllegalStateException e)
{
- ownerships = probe.getOwnershipWithPort();
- errors.append("Note: ").append(e.getMessage()).append("%n");
+ try
+ {
+ ownerships = probe.getOwnershipWithPort();
+ errors.append("Note: ").append(e.getMessage()).append("%n");
+ }
+ catch (Exception ex)
+ {
+ out.printf("%nError: %s%n", ex.getMessage());
+ System.exit(1);
+ }
}
catch (IllegalArgumentException ex)
{
diff --git a/src/java/org/apache/cassandra/tools/nodetool/TableStats.java b/src/java/org/apache/cassandra/tools/nodetool/TableStats.java
index 4a33c64..c1a76a0 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/TableStats.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/TableStats.java
@@ -60,7 +60,8 @@
+ "memtable_off_heap_memory_used, memtable_switch_count, number_of_partitions_estimate, "
+ "off_heap_memory_used_total, pending_flushes, percent_repaired, read_latency, reads, "
+ "space_used_by_snapshots_total, space_used_live, space_used_total, "
- + "sstable_compression_ratio, sstable_count, table_name, write_latency, writes)")
+ + "sstable_compression_ratio, sstable_count, table_name, write_latency, writes, " +
+ "max_sstable_size, local_read_write_ratio, twcs_max_duration)")
private String sortKey = "";
@Option(title = "top",
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Version.java b/src/java/org/apache/cassandra/tools/nodetool/Version.java
index f95907a..6556a04 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Version.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Version.java
@@ -18,16 +18,23 @@
package org.apache.cassandra.tools.nodetool;
import io.airlift.airline.Command;
-
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "version", description = "Print cassandra version")
public class Version extends NodeToolCmd
{
+ @Option(title = "verbose",
+ name = {"-v", "--verbose"},
+ description = "Include additional information")
+ private boolean verbose = false;
+
@Override
public void execute(NodeProbe probe)
{
probe.output().out.println("ReleaseVersion: " + probe.getReleaseVersion());
+ if (verbose)
+ probe.output().out.println("GitSHA: " + probe.getGitSHA());
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryHolder.java
index 8799ffb..362bc67 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryHolder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryHolder.java
@@ -54,8 +54,9 @@
private final long bytesIn;
private final long bytesOut;
private final String rowMerged;
+ private final String compactionProperties;
- CompactionHistoryRow(String id, String ksName, String cfName, long compactedAt, long bytesIn, long bytesOut, String rowMerged)
+ CompactionHistoryRow(String id, String ksName, String cfName, long compactedAt, long bytesIn, long bytesOut, String rowMerged, String compactionProperties)
{
this.id = id;
this.ksName = ksName;
@@ -64,6 +65,7 @@
this.bytesIn = bytesIn;
this.bytesOut = bytesOut;
this.rowMerged = rowMerged;
+ this.compactionProperties = compactionProperties;
}
public int compareTo(CompactionHistoryHolder.CompactionHistoryRow chr)
@@ -83,6 +85,7 @@
compaction.put("bytes_in", this.bytesIn);
compaction.put("bytes_out", this.bytesOut);
compaction.put("rows_merged", this.rowMerged);
+ compaction.put("compaction_properties", this.compactionProperties);
return compaction;
}
}
@@ -110,7 +113,8 @@
(Long)value.get(3),
(Long)value.get(4),
(Long)value.get(5),
- (String)value.get(6)
+ (String)value.get(6),
+ (String)value.get(7)
);
chrList.add(chr);
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryPrinter.java
index 97ed1c4..c2b62ee 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryPrinter.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/CompactionHistoryPrinter.java
@@ -65,7 +65,7 @@
for (Object chr : compactionHistories)
{
Map value = chr instanceof Map<?, ?> ? (Map)chr : Collections.emptyMap();
- String[] obj = new String[7];
+ String[] obj = new String[8];
obj[0] = (String)value.get("id");
obj[1] = (String)value.get("keyspace_name");
obj[2] = (String)value.get("columnfamily_name");
@@ -73,6 +73,7 @@
obj[4] = value.get("bytes_in").toString();
obj[5] = value.get("bytes_out").toString();
obj[6] = (String)value.get("rows_merged");
+ obj[7] = (String)value.get("compaction_properties");
table.add(obj);
}
table.printTo(out);
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsPrinter.java
index 037227b..86d7af7 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsPrinter.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsPrinter.java
@@ -18,10 +18,10 @@
package org.apache.cassandra.tools.nodetool.stats;
-import java.io.IOException;
import java.io.PrintStream;
+import java.util.Map;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.cassandra.utils.JsonUtils;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.Yaml;
@@ -40,17 +40,17 @@
@Override
public void print(T data, PrintStream out)
{
- ObjectMapper mapper = new ObjectMapper();
- try
- {
- String json = mapper.writerWithDefaultPrettyPrinter()
- .writeValueAsString(data.convert2Map());
- out.println(json);
- }
- catch (IOException e)
- {
- out.println(e);
- }
+ // First need to get a Map representation of stats
+ final Map<String, Object> stats = data.convert2Map();
+ // but then also need slight massaging to coerce NaN values into nulls
+ for (Object statEntry : stats.values())
+ if (statEntry instanceof Map<?,?>)
+ for (Map.Entry<String, Object> entry : ((Map<String, Object>) statEntry).entrySet())
+ if (entry.getValue() instanceof Double && !Double.isFinite((Double) entry.getValue()))
+ entry.setValue(null);
+
+ // and then we can serialize
+ out.println(JsonUtils.writeAsPrettyJsonString(stats));
}
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
index 8b5090b..2484570 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
@@ -31,6 +31,7 @@
public boolean isLeveledSstable = false;
public Object sstableCount;
public Object oldSSTableCount;
+ public Long maxSSTableSize;
public String spaceUsedLive;
public String spaceUsedTotal;
public String spaceUsedBySnapshotsTotal;
@@ -71,10 +72,14 @@
public String droppedMutations;
public List<String> sstablesInEachLevel = new ArrayList<>();
public List<String> sstableBytesInEachLevel = new ArrayList<>();
+ public int[] sstableCountPerTWCSBucket = null;
public Boolean isInCorrectLocation = null; // null: option not active
public double droppableTombstoneRatio;
public Map<String, String> topSizePartitions;
public Map<String, Long> topTombstonePartitions;
public String topSizePartitionsLastUpdate;
public String topTombstonePartitionsLastUpdate;
+ public double localReadWriteRatio;
+ public Long twcsDurationInMillis;
+ public String twcs;
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java
index 17b5c61..a7dabb2 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java
@@ -65,7 +65,8 @@
"pending_flushes", "percent_repaired", "read_latency", "reads",
"space_used_by_snapshots_total", "space_used_live",
"space_used_total", "sstable_compression_ratio", "sstable_count",
- "table_name", "write_latency", "writes" };
+ "table_name", "write_latency", "writes", "max_sstable_size",
+ "local_read_write_ratio", "twcs_max_duration"};
public StatsTableComparator(String sortKey, boolean humanReadable)
{
@@ -229,6 +230,10 @@
{
result = compareDoubles(stx.localWriteLatencyMs, sty.localWriteLatencyMs);
}
+ else if (sortKey.equals("local_read_write_ratio"))
+ {
+ result = compareDoubles(stx.localReadWriteRatio, sty.localReadWriteRatio);
+ }
else if (sortKey.equals("maximum_live_cells_per_slice_last_five_minutes"))
{
result = sign * Long.valueOf(stx.maximumLiveCellsPerSliceLastFiveMinutes)
@@ -295,6 +300,21 @@
{
result = compareDoubles(stx.percentRepaired, sty.percentRepaired);
}
+ else if (sortKey.equals("max_sstable_size"))
+ {
+ result = sign * stx.maxSSTableSize.compareTo(sty.maxSSTableSize);
+ }
+ else if (sortKey.equals("twcs_max_duration"))
+ {
+ if (stx.twcsDurationInMillis != null && sty.twcsDurationInMillis == null)
+ return sign;
+ else if (stx.twcsDurationInMillis == null && sty.twcsDurationInMillis != null)
+ return sign * -1;
+ else if (stx.twcsDurationInMillis == null)
+ return 0;
+ else
+ result = sign * stx.twcsDurationInMillis.compareTo(sty.twcsDurationInMillis);
+ }
else if (sortKey.equals("space_used_by_snapshots_total"))
{
result = compareFileSizes(stx.spaceUsedBySnapshotsTotal,
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
index 60132d1..80660611 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
@@ -25,7 +25,12 @@
import com.google.common.collect.ArrayListMultimap;
import javax.management.InstanceNotFoundException;
+
+import org.apache.commons.lang3.time.DurationFormatUtils;
+
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
+import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategyOptions;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.metrics.*;
import org.apache.cassandra.tools.*;
@@ -33,7 +38,7 @@
public class TableStatsHolder implements StatsHolder
{
public final List<StatsKeyspace> keyspaces;
- public final int numberOfTables;
+ public int numberOfTables = 0;
public final boolean humanReadable;
public final String sortKey;
public final int top;
@@ -48,14 +53,7 @@
this.locationCheck = locationCheck;
if (!this.isTestTableStatsHolder())
- {
- this.numberOfTables = probe.getNumberOfTables();
this.initializeKeyspaces(probe, ignore, tableNames);
- }
- else
- {
- this.numberOfTables = 0;
- }
}
@Override
@@ -123,6 +121,8 @@
mpTable.put("old_sstable_count", table.oldSSTableCount);
mpTable.put("sstables_in_each_level", table.sstablesInEachLevel);
mpTable.put("sstable_bytes_in_each_level", table.sstableBytesInEachLevel);
+ mpTable.put("max_sstable_size", table.maxSSTableSize);
+ mpTable.put("twcs", table.twcs);
mpTable.put("space_used_live", table.spaceUsedLive);
mpTable.put("space_used_total", table.spaceUsedTotal);
mpTable.put("space_used_by_snapshots_total", table.spaceUsedBySnapshotsTotal);
@@ -139,6 +139,7 @@
mpTable.put("local_read_latency_ms", String.format("%01.3f", table.localReadLatencyMs));
mpTable.put("local_write_count", table.localWriteCount);
mpTable.put("local_write_latency_ms", String.format("%01.3f", table.localWriteLatencyMs));
+ mpTable.put("local_read_write_ratio", String.format("%01.5f", table.localReadWriteRatio));
mpTable.put("pending_flushes", table.pendingFlushes);
mpTable.put("percent_repaired", table.percentRepaired);
mpTable.put("bytes_repaired", table.bytesRepaired);
@@ -205,6 +206,8 @@
}
}
+ numberOfTables = selectedTableMbeans.size();
+
// make sure all specified keyspace and tables exist
filter.verifyKeyspaces(probe.getKeyspaces());
filter.verifyTables();
@@ -227,6 +230,8 @@
statsTable.isIndex = tableName.contains(".");
statsTable.sstableCount = probe.getColumnFamilyMetric(keyspaceName, tableName, "LiveSSTableCount");
statsTable.oldSSTableCount = probe.getColumnFamilyMetric(keyspaceName, tableName, "OldVersionSSTableCount");
+ Long sstableSize = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "MaxSSTableSize");
+ statsTable.maxSSTableSize = sstableSize == null ? 0 : sstableSize;
int[] leveledSStables = table.getSSTableCountPerLevel();
if (leveledSStables != null)
@@ -243,6 +248,7 @@
statsTable.sstablesInEachLevel.add(count + ((count > maxCount) ? "/" + maxCount : ""));
}
}
+ statsTable.sstableCountPerTWCSBucket = table.getSSTableCountPerTWCSBucket();
long[] leveledSSTablesBytes = table.getPerLevelSizeBytes();
if (leveledSSTablesBytes != null)
@@ -290,6 +296,9 @@
statsTable.spaceUsedLive = format((Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "LiveDiskSpaceUsed"), humanReadable);
statsTable.spaceUsedTotal = format((Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "TotalDiskSpaceUsed"), humanReadable);
statsTable.spaceUsedBySnapshotsTotal = format((Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "SnapshotsSize"), humanReadable);
+
+ maybeAddTWCSWindowWithMaxDuration(statsTable, probe, keyspaceName, tableName);
+
if (offHeapSize != null)
{
statsTable.offHeapUsed = true;
@@ -330,6 +339,9 @@
double localWriteLatency = ((CassandraMetricsRegistry.JmxTimerMBean) probe.getColumnFamilyMetric(keyspaceName, tableName, "WriteLatency")).getMean() / 1000;
double localWLatency = localWriteLatency > 0 ? localWriteLatency : Double.NaN;
+
+ statsTable.localReadWriteRatio = statsTable.localWriteCount > 0 ? statsTable.localReadCount / (double) statsTable.localWriteCount : 0;
+
statsTable.localWriteLatencyMs = localWLatency;
statsTable.pendingFlushes = probe.getColumnFamilyMetric(keyspaceName, tableName, "PendingFlushes");
@@ -379,6 +391,27 @@
}
}
+ private void maybeAddTWCSWindowWithMaxDuration(StatsTable statsTable, NodeProbe probe, String keyspaceName, String tableName)
+ {
+ Map<String, String> compactionParameters = probe.getCfsProxy(statsTable.keyspaceName, statsTable.tableName)
+ .getCompactionParameters();
+
+ if (compactionParameters == null)
+ return;
+
+ String compactor = compactionParameters.get("class");
+
+ if (compactor == null || !compactor.endsWith(TimeWindowCompactionStrategy.class.getSimpleName()))
+ return;
+
+ String unit = compactionParameters.get(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_UNIT_KEY);
+ String size = compactionParameters.get(TimeWindowCompactionStrategyOptions.COMPACTION_WINDOW_SIZE_KEY);
+
+ statsTable.twcsDurationInMillis = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "MaxSSTableDuration");
+ String maxDuration = millisToDuration(statsTable.twcsDurationInMillis);
+ statsTable.twcs = String.format("%s %s, max duration: %s", size, unit, maxDuration);
+ }
+
private String format(long bytes, boolean humanReadable)
{
return humanReadable ? FileUtils.stringifyFileSize(bytes) : Long.toString(bytes);
@@ -400,6 +433,11 @@
return df.format(new Date(millis));
}
+ private String millisToDuration(long millis)
+ {
+ return DurationFormatUtils.formatDurationWords(millis, true, true);
+ }
+
/**
* Sort and filter this TableStatHolder's tables as specified by its sortKey and top attributes.
*/
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
index 7f17c21..c30ab6d 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
@@ -50,6 +50,9 @@
@Override
public void print(TableStatsHolder data, PrintStream out)
{
+ if (data.numberOfTables == 0)
+ return;
+
out.println("Total number of tables: " + data.numberOfTables);
out.println("----------------");
@@ -57,7 +60,7 @@
for (StatsKeyspace keyspace : keyspaces)
{
// print each keyspace's information
- out.println("Keyspace : " + keyspace.name);
+ out.println("Keyspace: " + keyspace.name);
out.println("\tRead Count: " + keyspace.readCount);
out.println("\tRead Latency: " + keyspace.readLatency() + " ms");
out.println("\tWrite Count: " + keyspace.writeCount);
@@ -66,6 +69,9 @@
// print each table's information
List<StatsTable> tables = keyspace.tables;
+ if (tables.size() == 0)
+ continue;
+
for (StatsTable table : tables)
{
printStatsTable(table, table.tableName, "\t\t", out);
@@ -79,6 +85,9 @@
out.println(indent + "Table" + (table.isIndex ? " (index): " : ": ") + tableDisplayName);
out.println(indent + "SSTable count: " + table.sstableCount);
out.println(indent + "Old SSTable count: " + table.oldSSTableCount);
+ out.println(indent + "Max SSTable size: " + FBUtilities.prettyPrintMemory(table.maxSSTableSize));
+ if (table.twcs != null)
+ out.println(indent + "SSTables Time Window: " + table.twcs);
if (table.isLeveledSstable)
{
out.println(indent + "SSTables in each level: [" + String.join(", ",
@@ -93,7 +102,7 @@
if (table.offHeapUsed)
out.println(indent + "Off heap memory used (total): " + table.offHeapMemoryUsedTotal);
- out.println(indent + "SSTable Compression Ratio: " + table.sstableCompressionRatio);
+ out.printf(indent + "SSTable Compression Ratio: %01.5f%n", table.sstableCompressionRatio);
out.println(indent + "Number of partitions (estimate): " + table.numberOfPartitionsEstimate);
out.println(indent + "Memtable cell count: " + table.memtableCellCount);
out.println(indent + "Memtable data size: " + table.memtableDataSize);
@@ -105,6 +114,9 @@
out.printf(indent + "Local read latency: %01.3f ms%n", table.localReadLatencyMs);
out.println(indent + "Local write count: " + table.localWriteCount);
out.printf(indent + "Local write latency: %01.3f ms%n", table.localWriteLatencyMs);
+
+ out.printf(indent + "Local read/write ratio: %01.5f%n", table.localReadWriteRatio);
+
out.println(indent + "Pending flushes: " + table.pendingFlushes);
out.println(indent + "Percent repaired: " + table.percentRepaired);
@@ -164,6 +176,10 @@
public void print(TableStatsHolder data, PrintStream out)
{
List<StatsTable> tables = data.getSortedFilteredTables();
+
+ if (tables.size() == 0)
+ return;
+
String totalTablesSummary = String.format("Total number of tables: %d", data.numberOfTables);
if (data.top > 0)
{
diff --git a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
index f8691ee..113420d 100644
--- a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
+++ b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java
@@ -28,6 +28,7 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.OverloadedException;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -50,8 +51,7 @@
private static final Logger logger = LoggerFactory.getLogger(TraceStateImpl.class);
@VisibleForTesting
- public static int WAIT_FOR_PENDING_EVENTS_TIMEOUT_SECS =
- Integer.parseInt(System.getProperty("cassandra.wait_for_tracing_events_timeout_secs", "0"));
+ public static int WAIT_FOR_PENDING_EVENTS_TIMEOUT_SECS = CassandraRelevantProperties.WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS.getInt();
private final Set<Future<?>> pendingFutures = ConcurrentHashMap.newKeySet();
diff --git a/src/java/org/apache/cassandra/tracing/Tracing.java b/src/java/org/apache/cassandra/tracing/Tracing.java
index 5c820db..73cf2eb 100644
--- a/src/java/org/apache/cassandra/tracing/Tracing.java
+++ b/src/java/org/apache/cassandra/tracing/Tracing.java
@@ -42,6 +42,7 @@
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.TimeUUID;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_TRACING_CLASS;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
/**
@@ -110,13 +111,14 @@
static
{
Tracing tracing = null;
- String customTracingClass = System.getProperty("cassandra.custom_tracing_class");
+ String customTracingClass = CUSTOM_TRACING_CLASS.getString();
if (null != customTracingClass)
{
try
{
tracing = FBUtilities.construct(customTracingClass, "Tracing");
- logger.info("Using {} as tracing queries (as requested with -Dcassandra.custom_tracing_class)", customTracingClass);
+ logger.info("Using the {} class to trace queries (as requested by the {} system property)",
+ customTracingClass, CUSTOM_TRACING_CLASS.getKey());
}
catch (Exception e)
{
diff --git a/src/java/org/apache/cassandra/transport/CBUtil.java b/src/java/org/apache/cassandra/transport/CBUtil.java
index 6cab638..d47f0ce 100644
--- a/src/java/org/apache/cassandra/transport/CBUtil.java
+++ b/src/java/org/apache/cassandra/transport/CBUtil.java
@@ -32,20 +32,21 @@
import java.util.List;
import java.util.Map;
import java.util.UUID;
-
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.util.concurrent.FastThreadLocal;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_NETTY_USE_HEAP_ALLOCATOR;
/**
* ByteBuf utility methods.
@@ -56,7 +57,7 @@
*/
public abstract class CBUtil
{
- public static final boolean USE_HEAP_ALLOCATOR = Boolean.getBoolean(Config.PROPERTY_PREFIX + "netty_use_heap_allocator");
+ public static final boolean USE_HEAP_ALLOCATOR = CASSANDRA_NETTY_USE_HEAP_ALLOCATOR.getBoolean();
public static final ByteBufAllocator allocator = USE_HEAP_ALLOCATOR ? new UnpooledByteBufAllocator(false) : new PooledByteBufAllocator(true);
private static final int UUID_SIZE = 16;
@@ -69,6 +70,15 @@
}
};
+ private final static FastThreadLocal<ByteBuffer> localDirectBuffer = new FastThreadLocal<ByteBuffer>()
+ {
+ @Override
+ protected ByteBuffer initialValue()
+ {
+ return MemoryUtil.getHollowDirectByteBuffer();
+ }
+ };
+
private final static FastThreadLocal<CharBuffer> TL_CHAR_BUFFER = new FastThreadLocal<>();
private CBUtil() {}
@@ -478,7 +488,35 @@
cb.writeInt(remaining);
if (remaining > 0)
- cb.writeBytes(bytes.duplicate());
+ addBytes(bytes, cb);
+ }
+
+ public static void addBytes(ByteBuffer src, ByteBuf dest)
+ {
+ if (src.remaining() == 0)
+ return;
+
+ int length = src.remaining();
+
+ if (src.hasArray())
+ {
+ // Heap buffers are copied using a raw array instead of shared heap buffer and MemoryUtil.unsafe to avoid a CMS bug, which causes the JVM to crash with the follwing:
+ // # Problematic frame:
+ // # V [libjvm.dylib+0x63e858] void ParScanClosure::do_oop_work<unsigned int>(unsigned int*, bool, bool)+0x94
+ // More details can be found here: https://bugs.openjdk.org/browse/JDK-8222798
+ byte[] array = src.array();
+ dest.writeBytes(array, src.arrayOffset() + src.position(), length);
+ }
+ else if (src.isDirect())
+ {
+ ByteBuffer local = getLocalDirectBuffer();
+ MemoryUtil.duplicateDirectByteBuffer(src, local);
+ dest.writeBytes(local);
+ }
+ else
+ {
+ dest.writeBytes(src.duplicate());
+ }
}
public static int sizeOfValue(byte[] bytes)
@@ -614,4 +652,8 @@
return bytes;
}
+ private static ByteBuffer getLocalDirectBuffer()
+ {
+ return localDirectBuffer.get();
+ }
}
diff --git a/src/java/org/apache/cassandra/transport/Dispatcher.java b/src/java/org/apache/cassandra/transport/Dispatcher.java
index da79c3d..f21acc2 100644
--- a/src/java/org/apache/cassandra/transport/Dispatcher.java
+++ b/src/java/org/apache/cassandra/transport/Dispatcher.java
@@ -23,16 +23,18 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
-import org.apache.cassandra.metrics.ClientMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.channel.Channel;
import io.netty.channel.EventLoop;
import io.netty.util.AttributeKey;
+import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.apache.cassandra.concurrent.LocalAwareExecutorPlus;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.metrics.ClientMetrics;
import org.apache.cassandra.net.FrameEncoder;
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.service.QueryState;
@@ -42,19 +44,39 @@
import org.apache.cassandra.transport.messages.ErrorMessage;
import org.apache.cassandra.transport.messages.EventMessage;
import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.NoSpamLogger;
import static org.apache.cassandra.concurrent.SharedExecutorPool.SHARED;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public class Dispatcher
{
private static final Logger logger = LoggerFactory.getLogger(Dispatcher.class);
-
- private static final LocalAwareExecutorPlus requestExecutor = SHARED.newExecutor(DatabaseDescriptor.getNativeTransportMaxThreads(),
- DatabaseDescriptor::setNativeTransportMaxThreads,
- "transport",
- "Native-Transport-Requests");
+
+ @VisibleForTesting
+ static final LocalAwareExecutorPlus requestExecutor = SHARED.newExecutor(DatabaseDescriptor.getNativeTransportMaxThreads(),
+ DatabaseDescriptor::setNativeTransportMaxThreads,
+ "transport",
+ "Native-Transport-Requests");
+
+ /** CASSANDRA-17812: Rate-limit new client connection setup to avoid overwhelming during bcrypt
+ *
+ * authExecutor is a separate thread pool for handling requests on connections that need to be authenticated.
+ * Calls to AUTHENTICATE can be expensive if the number of rounds for bcrypt is configured to a high value,
+ * so during a connection storm checking the password hash would starve existing connected clients for CPU and
+ * trigger timeouts if on the same thread pool as standard requests.
+ *
+ * Moving authentication requests to a small, separate pool prevents starvation handling all other
+ * requests. If the authExecutor pool backs up, it may cause authentication timeouts but the clients should
+ * back off and retry while the rest of the system continues to make progress.
+ *
+ * Setting less than 1 will service auth requests on the standard {@link Dispatcher#requestExecutor}
+ */
+ @VisibleForTesting
+ static final LocalAwareExecutorPlus authExecutor = SHARED.newExecutor(Math.max(1, DatabaseDescriptor.getNativeTransportMaxAuthThreads()),
+ DatabaseDescriptor::setNativeTransportMaxAuthThreads,
+ "transport",
+ "Native-Transport-Auth-Requests");
private static final ConcurrentMap<EventLoop, Flusher> flusherLookup = new ConcurrentHashMap<>();
private final boolean useLegacyFlusher;
@@ -79,17 +101,67 @@
public void dispatch(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
{
- requestExecutor.submit(() -> processRequest(channel, request, forFlusher, backpressure));
+ // if native_transport_max_auth_threads is < 1, don't delegate to new pool on auth messages
+ boolean isAuthQuery = DatabaseDescriptor.getNativeTransportMaxAuthThreads() > 0 &&
+ (request.type == Message.Type.AUTH_RESPONSE || request.type == Message.Type.CREDENTIALS);
+
+ // Importantly, the authExecutor will handle the AUTHENTICATE message which may be CPU intensive.
+ LocalAwareExecutorPlus executor = isAuthQuery ? authExecutor : requestExecutor;
+
+ executor.submit(new RequestProcessor(channel, request, forFlusher, backpressure));
ClientMetrics.instance.markRequestDispatched();
}
+ public class RequestProcessor implements RunnableDebuggableTask
+ {
+ private final Channel channel;
+ private final Message.Request request;
+ private final FlushItemConverter forFlusher;
+ private final Overload backpressure;
+
+ private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
+ private volatile long approxStartTimeNanos;
+
+ public RequestProcessor(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
+ {
+ this.channel = channel;
+ this.request = request;
+ this.forFlusher = forFlusher;
+ this.backpressure = backpressure;
+ }
+
+ @Override
+ public void run()
+ {
+ approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
+ processRequest(channel, request, forFlusher, backpressure, approxStartTimeNanos);
+ }
+
+ @Override
+ public long creationTimeNanos()
+ {
+ return approxCreationTimeNanos;
+ }
+
+ @Override
+ public long startTimeNanos()
+ {
+ return approxStartTimeNanos;
+ }
+
+ @Override
+ public String description()
+ {
+ return request.toString();
+ }
+ }
+
/**
* Note: this method may be executed on the netty event loop, during initial protocol negotiation; the caller is
* responsible for cleaning up any global or thread-local state. (ex. tracing, client warnings, etc.).
*/
- private static Message.Response processRequest(ServerConnection connection, Message.Request request, Overload backpressure)
+ private static Message.Response processRequest(ServerConnection connection, Message.Request request, Overload backpressure, long startTimeNanos)
{
- long queryStartNanoTime = nanoTime();
if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4))
ClientWarn.instance.captureWarnings();
@@ -119,7 +191,7 @@
Message.logger.trace("Received: {}, v={}", request, connection.getVersion());
connection.requests.inc();
- Message.Response response = request.execute(qstate, queryStartNanoTime);
+ Message.Response response = request.execute(qstate, startTimeNanos);
if (request.isTrackable())
CoordinatorWarnings.done();
@@ -130,15 +202,15 @@
connection.applyStateTransition(request.type, response.type);
return response;
}
-
+
/**
* Note: this method may be executed on the netty event loop.
*/
- static Message.Response processRequest(Channel channel, Message.Request request, Overload backpressure)
+ static Message.Response processRequest(Channel channel, Message.Request request, Overload backpressure, long approxStartTimeNanos)
{
try
{
- return processRequest((ServerConnection) request.connection(), request, backpressure);
+ return processRequest((ServerConnection) request.connection(), request, backpressure, approxStartTimeNanos);
}
catch (Throwable t)
{
@@ -163,9 +235,9 @@
/**
* Note: this method is not expected to execute on the netty event loop.
*/
- void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
+ void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure, long approxStartTimeNanos)
{
- Message.Response response = processRequest(channel, request, backpressure);
+ Message.Response response = processRequest(channel, request, backpressure, approxStartTimeNanos);
FlushItem<?> toFlush = forFlusher.toFlushItem(channel, request, response);
Message.logger.trace("Responding: {}, v={}", response, request.connection().getVersion());
flush(toFlush);
@@ -189,19 +261,16 @@
public static void shutdown()
{
- if (requestExecutor != null)
- {
- requestExecutor.shutdown();
- }
+ requestExecutor.shutdown();
+ authExecutor.shutdown();
}
-
/**
* Dispatcher for EventMessages. In {@link Server.ConnectionTracker#send(Event)}, the strategy
* for delivering events to registered clients is dependent on protocol version and the configuration
* of the pipeline. For v5 and newer connections, the event message is encoded into an Envelope,
* wrapped in a FlushItem and then delivered via the pipeline's flusher, in a similar way to
- * a Response returned from {@link #processRequest(Channel, Message.Request, FlushItemConverter, Overload)}.
+ * a Response returned from {@link #processRequest(Channel, Message.Request, FlushItemConverter, Overload, long)}.
* It's worth noting that events are not generally fired as a direct response to a client request,
* so this flush item has a null request attribute. The dispatcher itself is created when the
* pipeline is first configured during protocol negotiation and is attached to the channel for
diff --git a/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java b/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java
index 75cb72e..e4cff99 100644
--- a/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java
+++ b/src/java/org/apache/cassandra/transport/InitialConnectionHandler.java
@@ -26,6 +26,7 @@
import java.util.Map;
import org.apache.cassandra.transport.ClientResourceLimits.Overload;
+import org.apache.cassandra.utils.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -148,7 +149,9 @@
promise = new VoidChannelPromise(ctx.channel(), false);
}
- final Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE);
+ long approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
+ final Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE, approxStartTimeNanos);
+
outbound = response.encode(inbound.header.version);
ctx.writeAndFlush(outbound, promise);
logger.trace("Configured pipeline: {}", ctx.pipeline());
diff --git a/src/java/org/apache/cassandra/transport/Message.java b/src/java/org/apache/cassandra/transport/Message.java
index 75c997e..82015c1 100644
--- a/src/java/org/apache/cassandra/transport/Message.java
+++ b/src/java/org/apache/cassandra/transport/Message.java
@@ -36,6 +36,7 @@
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.messages.*;
import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.ReflectionUtils;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -132,7 +133,7 @@
Codec<?> original = this.codec;
Field field = Type.class.getDeclaredField("codec");
field.setAccessible(true);
- Field modifiers = Field.class.getDeclaredField("modifiers");
+ Field modifiers = ReflectionUtils.getModifiersField();
modifiers.setAccessible(true);
modifiers.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(this, codec);
@@ -193,7 +194,8 @@
this.customPayload = customPayload;
}
- public String debugString()
+ @Override
+ public String toString()
{
return String.format("(%s:%s:%s)", type, streamId, connection == null ? "null" : connection.getVersion().asInt());
}
diff --git a/src/java/org/apache/cassandra/transport/PipelineConfigurator.java b/src/java/org/apache/cassandra/transport/PipelineConfigurator.java
index 81ff136..ff52478 100644
--- a/src/java/org/apache/cassandra/transport/PipelineConfigurator.java
+++ b/src/java/org/apache/cassandra/transport/PipelineConfigurator.java
@@ -48,6 +48,9 @@
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.transport.messages.StartupMessage;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UNSAFE_VERBOSE_DEBUG_CLIENT_PROTOCOL;
+import static org.apache.cassandra.net.SocketFactory.newSslHandler;
+
/**
* Takes care of intializing a Netty Channel and Pipeline for client protocol connections.
* The pipeline is first set up with some common handlers for connection limiting, dropping
@@ -61,7 +64,7 @@
// Not to be used in production, this causes a Netty logging handler to be added to the pipeline,
// which will throttle a system under any normal load.
- private static final boolean DEBUG = Boolean.getBoolean("cassandra.unsafe_verbose_debug_client_protocol");
+ private static final boolean DEBUG = TEST_UNSAFE_VERBOSE_DEBUG_CLIENT_PROTOCOL.getBoolean();
// Stateless handlers
private static final ConnectionLimitHandler connectionLimitHandler = new ConnectionLimitHandler();
@@ -181,7 +184,8 @@
{
// Connection uses SSL/TLS, replace the detection handler with a SslHandler and so use
// encryption.
- SslHandler sslHandler = sslContext.newHandler(channel.alloc());
+ InetSocketAddress peer = encryptionOptions.require_endpoint_verification ? (InetSocketAddress) channel.remoteAddress() : null;
+ SslHandler sslHandler = newSslHandler(channel, sslContext, peer);
channelHandlerContext.pipeline().replace(SSL_HANDLER, SSL_HANDLER, sslHandler);
}
else
@@ -199,7 +203,8 @@
SslContext sslContext = SSLFactory.getOrCreateSslContext(encryptionOptions,
encryptionOptions.require_client_auth,
ISslContextFactory.SocketType.SERVER);
- channel.pipeline().addFirst(SSL_HANDLER, sslContext.newHandler(channel.alloc()));
+ InetSocketAddress peer = encryptionOptions.require_endpoint_verification ? (InetSocketAddress) channel.remoteAddress() : null;
+ channel.pipeline().addFirst(SSL_HANDLER, newSslHandler(channel, sslContext, peer));
};
default:
throw new IllegalStateException("Unrecognized TLS encryption policy: " + this.tlsEncryptionPolicy);
diff --git a/src/java/org/apache/cassandra/transport/ProtocolVersion.java b/src/java/org/apache/cassandra/transport/ProtocolVersion.java
index fc97cc8..6078fb2 100644
--- a/src/java/org/apache/cassandra/transport/ProtocolVersion.java
+++ b/src/java/org/apache/cassandra/transport/ProtocolVersion.java
@@ -75,7 +75,10 @@
/** Old unsupported versions, this is OK as long as we never add newer unsupported versions */
public final static EnumSet<ProtocolVersion> UNSUPPORTED = EnumSet.complementOf(SUPPORTED);
- /** The preferred versions */
+ /** The preferred version.
+ *
+ * When updating this remember to also update the MULTI_UPGRADES in cassandra-dtest/upgrade_tests/upgrade_through_versions_test.py
+ */
public final static ProtocolVersion CURRENT = V5;
public final static Optional<ProtocolVersion> BETA = Optional.of(V6);
diff --git a/src/java/org/apache/cassandra/transport/ServerConnection.java b/src/java/org/apache/cassandra/transport/ServerConnection.java
index 06e7842..dd6fa31 100644
--- a/src/java/org/apache/cassandra/transport/ServerConnection.java
+++ b/src/java/org/apache/cassandra/transport/ServerConnection.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.transport;
+import java.security.cert.Certificate;
import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.security.cert.X509Certificate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,11 +121,11 @@
return saslNegotiator;
}
- private X509Certificate[] certificates()
+ private Certificate[] certificates()
{
SslHandler sslHandler = (SslHandler) channel().pipeline()
.get("ssl");
- X509Certificate[] certificates = null;
+ Certificate[] certificates = null;
if (sslHandler != null)
{
@@ -133,7 +133,7 @@
{
certificates = sslHandler.engine()
.getSession()
- .getPeerCertificateChain();
+ .getPeerCertificates();
}
catch (SSLPeerUnverifiedException e)
{
diff --git a/src/java/org/apache/cassandra/transport/SimpleClient.java b/src/java/org/apache/cassandra/transport/SimpleClient.java
index 43bb8ad..2b57d10 100644
--- a/src/java/org/apache/cassandra/transport/SimpleClient.java
+++ b/src/java/org/apache/cassandra/transport/SimpleClient.java
@@ -52,6 +52,7 @@
import org.apache.cassandra.transport.messages.*;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+import static org.apache.cassandra.net.SocketFactory.newSslHandler;
import static org.apache.cassandra.transport.CQLMessageHandler.envelopeSize;
import static org.apache.cassandra.transport.Flusher.MAX_FRAMED_PAYLOAD_SIZE;
import static org.apache.cassandra.utils.concurrent.NonBlockingRateLimiter.NO_OP_LIMITER;
@@ -624,7 +625,8 @@
super.initChannel(channel);
SslContext sslContext = SSLFactory.getOrCreateSslContext(encryptionOptions, encryptionOptions.require_client_auth,
ISslContextFactory.SocketType.CLIENT);
- channel.pipeline().addFirst("ssl", sslContext.newHandler(channel.alloc()));
+ InetSocketAddress peer = encryptionOptions.require_endpoint_verification ? new InetSocketAddress(host, port) : null;
+ channel.pipeline().addFirst("ssl", newSslHandler(channel, sslContext, peer));
}
}
diff --git a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
index 9a296e4..c295216 100644
--- a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
@@ -148,6 +148,7 @@
@Override
public String toString()
{
- return String.format("QUERY %s [pageSize = %d]", query, options.getPageSize());
+ return String.format("QUERY %s [pageSize = %d] at consistency %s",
+ query, options.getPageSize(), options.getConsistency());
}
}
diff --git a/src/java/org/apache/cassandra/utils/AlwaysPresentFilter.java b/src/java/org/apache/cassandra/utils/AlwaysPresentFilter.java
deleted file mode 100644
index b046e84..0000000
--- a/src/java/org/apache/cassandra/utils/AlwaysPresentFilter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils;
-
-import org.apache.cassandra.utils.concurrent.Ref;
-
-public class AlwaysPresentFilter implements IFilter
-{
- public boolean isPresent(FilterKey key)
- {
- return true;
- }
-
- public void add(FilterKey key) { }
-
- public void clear() { }
-
- public void close() { }
-
- public IFilter sharedCopy()
- {
- return this;
- }
-
- public Throwable close(Throwable accumulate)
- {
- return accumulate;
- }
-
- public void addTo(Ref.IdentityCollection identities)
- {
- }
-
- public long serializedSize() { return 0; }
-
- @Override
- public long offHeapSize()
- {
- return 0;
- }
-}
diff --git a/src/java/org/apache/cassandra/utils/BloomFilter.java b/src/java/org/apache/cassandra/utils/BloomFilter.java
index bf48d43..a95d131 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilter.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilter.java
@@ -17,10 +17,13 @@
*/
package org.apache.cassandra.utils;
+import java.io.IOException;
+
import com.google.common.annotations.VisibleForTesting;
import io.netty.util.concurrent.FastThreadLocal;
import net.nicoulaj.compilecommand.annotations.Inline;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable;
import org.apache.cassandra.utils.obs.IBitSet;
@@ -29,6 +32,7 @@
{
private final static FastThreadLocal<long[]> reusableIndexes = new FastThreadLocal<long[]>()
{
+ @Override
protected long[] initialValue()
{
return new long[21];
@@ -52,9 +56,15 @@
this.bitset = copy.bitset;
}
- public long serializedSize()
+ public long serializedSize(boolean old)
{
- return BloomFilterSerializer.serializedSize(this);
+ return BloomFilterSerializer.forVersion(old).serializedSize(this);
+ }
+
+ @Override
+ public void serialize(DataOutputStreamPlus out, boolean old) throws IOException
+ {
+ BloomFilterSerializer.forVersion(old).serialize(this, out);
}
// Murmur is faster than an SHA-based approach and provides as-good collision
@@ -101,6 +111,7 @@
}
}
+ @Override
public void add(FilterKey key)
{
long[] indexes = indexes(key);
@@ -110,6 +121,7 @@
}
}
+ @Override
public final boolean isPresent(FilterKey key)
{
long[] indexes = indexes(key);
@@ -123,12 +135,14 @@
return true;
}
+ @Override
public void clear()
{
bitset.clear();
}
- public IFilter sharedCopy()
+ @Override
+ public BloomFilter sharedCopy()
{
return new BloomFilter(this);
}
@@ -139,11 +153,19 @@
return bitset.offHeapSize();
}
+ @Override
+ public boolean isInformative()
+ {
+ return bitset.offHeapSize() > 0;
+ }
+
+ @Override
public String toString()
{
return "BloomFilter[hashCount=" + hashCount + ";capacity=" + bitset.capacity() + ']';
}
+ @Override
public void addTo(Ref.IdentityCollection identities)
{
super.addTo(identities);
diff --git a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
index 3df4314..e4b34a4 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
@@ -17,47 +17,65 @@
*/
package org.apache.cassandra.utils;
-import java.io.DataInput;
import java.io.IOException;
-import java.io.InputStream;
import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.IGenericSerializer;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.utils.obs.IBitSet;
import org.apache.cassandra.utils.obs.OffHeapBitSet;
-public final class BloomFilterSerializer
+public final class BloomFilterSerializer implements IGenericSerializer<BloomFilter, DataInputStreamPlus, DataOutputStreamPlus>
{
- private BloomFilterSerializer()
+ public final static BloomFilterSerializer newFormatInstance = new BloomFilterSerializer(false);
+ public final static BloomFilterSerializer oldFormatInstance = new BloomFilterSerializer(true);
+
+ private final boolean oldFormat;
+
+ private <T> BloomFilterSerializer(boolean oldFormat)
{
+ this.oldFormat = oldFormat;
}
- public static void serialize(BloomFilter bf, DataOutputPlus out) throws IOException
+ public static BloomFilterSerializer forVersion(boolean oldSerializationFormat)
{
+ if (oldSerializationFormat)
+ return oldFormatInstance;
+
+ return newFormatInstance;
+ }
+
+ @Override
+ public void serialize(BloomFilter bf, DataOutputStreamPlus out) throws IOException
+ {
+ assert !oldFormat : "Filter should not be serialized in old format";
out.writeInt(bf.hashCount);
bf.bitset.serialize(out);
}
- @SuppressWarnings("resource")
- public static <I extends InputStream & DataInput> BloomFilter deserialize(I in, boolean oldBfFormat) throws IOException
- {
- int hashes = in.readInt();
- IBitSet bs = OffHeapBitSet.deserialize(in, oldBfFormat);
-
- return new BloomFilter(hashes, bs);
- }
-
/**
* Calculates a serialized size of the given Bloom Filter
- * @param bf Bloom filter to calculate serialized size
- * @see org.apache.cassandra.io.ISerializer#serialize(Object, org.apache.cassandra.io.util.DataOutputPlus)
*
+ * @param bf Bloom filter to calculate serialized size
* @return serialized size of the given bloom filter
+ * @see org.apache.cassandra.io.ISerializer#serialize(Object, org.apache.cassandra.io.util.DataOutputPlus)
*/
- public static long serializedSize(BloomFilter bf)
+ @Override
+ public long serializedSize(BloomFilter bf)
{
int size = TypeSizes.sizeof(bf.hashCount); // hash count
size += bf.bitset.serializedSize();
return size;
}
+
+ @Override
+ @SuppressWarnings("resource")
+ public BloomFilter deserialize(DataInputStreamPlus in) throws IOException
+ {
+ int hashes = in.readInt();
+ IBitSet bs = OffHeapBitSet.deserialize(in, oldFormat);
+
+ return new BloomFilter(hashes, bs);
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/ByteArrayUtil.java b/src/java/org/apache/cassandra/utils/ByteArrayUtil.java
index 8c84ee5..f0e797c 100644
--- a/src/java/org/apache/cassandra/utils/ByteArrayUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteArrayUtil.java
@@ -234,7 +234,7 @@
public static void writeWithVIntLength(byte[] bytes, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(bytes.length);
+ out.writeUnsignedVInt32(bytes.length);
out.write(bytes);
}
@@ -259,7 +259,7 @@
public static byte[] readWithVIntLength(DataInputPlus in) throws IOException
{
- int length = (int)in.readUnsignedVInt();
+ int length = in.readUnsignedVInt32();
if (length < 0)
throw new IOException("Corrupt (negative) value length encountered");
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index ba7d1be..790ff2a 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -23,9 +23,13 @@
* afterward, and ensure the tests still pass.
*/
-import java.io.*;
+import java.io.DataInput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.InetAddress;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -34,8 +38,8 @@
import net.nicoulaj.compilecommand.annotations.Inline;
import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileUtils;
@@ -338,7 +342,7 @@
public static void writeWithVIntLength(ByteBuffer bytes, DataOutputPlus out) throws IOException
{
- out.writeUnsignedVInt(bytes.remaining());
+ out.writeUnsignedVInt32(bytes.remaining());
out.write(bytes);
}
@@ -364,7 +368,7 @@
public static ByteBuffer readWithVIntLength(DataInputPlus in) throws IOException
{
- int length = (int)in.readUnsignedVInt();
+ int length = in.readUnsignedVInt32();
if (length < 0)
throw new IOException("Corrupt (negative) value length encountered");
@@ -385,7 +389,7 @@
public static void skipWithVIntLength(DataInputPlus in) throws IOException
{
- int length = (int)in.readUnsignedVInt();
+ int length = in.readUnsignedVInt32();
if (length < 0)
throw new IOException("Corrupt (negative) value length encountered");
@@ -681,13 +685,14 @@
public static boolean canMinimize(ByteBuffer buf)
{
- return buf != null && (buf.capacity() > buf.remaining() || !buf.hasArray());
+ return buf != null && (!buf.hasArray() || buf.array().length > buf.remaining());
+ // Note: buf.array().length is different from buf.capacity() for sliced buffers.
}
/** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */
public static ByteBuffer minimalBufferFor(ByteBuffer buf)
{
- return buf.capacity() > buf.remaining() || !buf.hasArray() ? ByteBuffer.wrap(getArray(buf)) : buf;
+ return !buf.hasArray() || buf.array().length > buf.remaining() ? ByteBuffer.wrap(getArray(buf)) : buf;
}
public static ByteBuffer[] minimizeBuffers(ByteBuffer[] src)
@@ -865,4 +870,35 @@
return true;
}
-}
+
+ /**
+ * Returns true if the buffer at the current position in the input matches given buffer.
+ * If true, the input is positioned at the end of the consumed buffer.
+ * If false, the position of the input is undefined.
+ * <p>
+ * The matched buffer is unchanged
+ */
+ public static boolean equalsWithShortLength(DataInput in, ByteBuffer toMatch) throws IOException
+ {
+ int length = readShortLength(in);
+ if (length != toMatch.remaining())
+ return false;
+ int limit = toMatch.limit();
+ for (int i = toMatch.position(); i < limit; ++i)
+ if (toMatch.get(i) != in.readByte())
+ return false;
+
+ return true;
+ }
+
+ public static void readFully(FileChannel channel, ByteBuffer dst, long position) throws IOException
+ {
+ while (dst.hasRemaining())
+ {
+ int read = channel.read(dst, position);
+ if (read == -1)
+ throw new EOFException();
+ position += read;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/CollectionSerializer.java b/src/java/org/apache/cassandra/utils/CollectionSerializer.java
index 4f8e8b0..9de6450 100644
--- a/src/java/org/apache/cassandra/utils/CollectionSerializer.java
+++ b/src/java/org/apache/cassandra/utils/CollectionSerializer.java
@@ -40,7 +40,7 @@
public static <V> void serializeCollection(IVersionedSerializer<V> valueSerializer, Collection<V> values, DataOutputPlus out, int version) throws IOException
{
- out.writeUnsignedVInt(values.size());
+ out.writeUnsignedVInt32(values.size());
for (V value : values)
valueSerializer.serialize(value, out, version);
}
@@ -48,14 +48,14 @@
public static <V, L extends List<V> & RandomAccess> void serializeList(IVersionedSerializer<V> valueSerializer, L values, DataOutputPlus out, int version) throws IOException
{
int size = values.size();
- out.writeUnsignedVInt(size);
+ out.writeUnsignedVInt32(size);
for (int i = 0 ; i < size ; ++i)
valueSerializer.serialize(values.get(i), out, version);
}
public static <K, V> void serializeMap(IVersionedSerializer<K> keySerializer, IVersionedSerializer<V> valueSerializer, Map<K, V> map, DataOutputPlus out, int version) throws IOException
{
- out.writeUnsignedVInt(map.size());
+ out.writeUnsignedVInt32(map.size());
for (Map.Entry<K, V> e : map.entrySet())
{
keySerializer.serialize(e.getKey(), out, version);
@@ -65,7 +65,7 @@
public static <V, C extends Collection<? super V>> C deserializeCollection(IVersionedSerializer<V> serializer, IntFunction<C> factory, DataInputPlus in, int version) throws IOException
{
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
C result = factory.apply(size);
while (size-- > 0)
result.add(serializer.deserialize(in, version));
@@ -74,7 +74,7 @@
public static <K, V, M extends Map<K, V>> M deserializeMap(IVersionedSerializer<K> keySerializer, IVersionedSerializer<V> valueSerializer, IntFunction<M> factory, DataInputPlus in, int version) throws IOException
{
- int size = (int) in.readUnsignedVInt();
+ int size = in.readUnsignedVInt32();
M result = factory.apply(size);
while (size-- > 0)
{
diff --git a/src/java/org/apache/cassandra/utils/Comparables.java b/src/java/org/apache/cassandra/utils/Comparables.java
new file mode 100644
index 0000000..612ade8
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Comparables.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+
+/**
+ * Utility methods linked to comparing comparable values.
+ */
+public class Comparables
+{
+ /**
+ * Returns the maximum of 2 comparable values. On ties, returns the first argument.
+ */
+ public static <T extends Comparable<? super T>> T max(T a, T b)
+ {
+ return a.compareTo(b) < 0 ? b : a;
+ }
+
+ /**
+ * Returns the maximum of 2 values given a comparator of those values. On ties, returns the first argument.
+ */
+ public static <C, T extends C> T max(T a, T b, Comparator<C> comparator)
+ {
+ return comparator.compare(a, b) < 0 ? b : a;
+ }
+
+ /**
+ * Returns the minimum of 2 comparable values. On ties, returns the first argument.
+ */
+ public static <T extends Comparable<? super T>> T min(T a, T b)
+ {
+ return a.compareTo(b) > 0 ? b : a;
+ }
+
+ /**
+ * Returns the minimum of 2 values given a comparator of those values. On ties, returns the first argument.
+ */
+ public static <C, T extends C> T min(T a, T b, Comparator<C> comparator)
+ {
+ return comparator.compare(a, b) > 0 ? b : a;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java b/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
index ab2d67e..168285d 100644
--- a/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
+++ b/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
@@ -20,6 +20,8 @@
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
@@ -29,6 +31,9 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
@@ -36,8 +41,10 @@
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.net.ParamType.SNAPSHOT_RANGES;
/**
* Provides a means to take snapshots when triggered by anomalous events or when the breaking of invariants is
@@ -68,6 +75,7 @@
public static final String REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX = "RepairedDataMismatch-";
public static final String DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX = "DuplicateRows-";
+ private static final int MAX_SNAPSHOT_RANGE_COUNT = 100; // otherwise, snapshot everything
private final Executor executor;
@@ -76,34 +84,34 @@
this.executor = executor;
}
- // Issue at most 1 snapshot request per minute for any given table.
- // Replicas will only create one snapshot per day, but this stops us
- // from swamping the network.
- // Overridable via system property for testing.
- private static final long SNAPSHOT_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(1);
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.BASIC_ISO_DATE;
private final ConcurrentHashMap<TableId, AtomicLong> lastSnapshotTimes = new ConcurrentHashMap<>();
+ public static void repairedDataMismatch(TableMetadata metadata, Iterable<InetAddressAndPort> replicas)
+ {
+ repairedDataMismatch(metadata, replicas, Collections.emptyList());
+ }
+
+ public static void repairedDataMismatch(TableMetadata metadata, Iterable<InetAddressAndPort> replicas, List<Range<Token>> ranges)
+ {
+ instance.maybeTriggerSnapshot(metadata, REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX, replicas, ranges);
+ }
+
public static void duplicateRows(TableMetadata metadata, Iterable<InetAddressAndPort> replicas)
{
instance.maybeTriggerSnapshot(metadata, DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX, replicas);
}
- public static void repairedDataMismatch(TableMetadata metadata, Iterable<InetAddressAndPort> replicas)
- {
- instance.maybeTriggerSnapshot(metadata, REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX, replicas);
- }
-
public static boolean isDiagnosticSnapshotRequest(SnapshotCommand command)
{
return command.snapshot_name.startsWith(REPAIRED_DATA_MISMATCH_SNAPSHOT_PREFIX)
|| command.snapshot_name.startsWith(DUPLICATE_ROWS_DETECTED_SNAPSHOT_PREFIX);
}
- public static void snapshot(SnapshotCommand command, InetAddressAndPort initiator)
+ public static void snapshot(SnapshotCommand command, List<Range<Token>> ranges, InetAddressAndPort initiator)
{
Preconditions.checkArgument(isDiagnosticSnapshotRequest(command));
- instance.maybeSnapshot(command, initiator);
+ instance.maybeSnapshot(command, ranges, initiator);
}
public static String getSnapshotName(String prefix)
@@ -119,10 +127,15 @@
private void maybeTriggerSnapshot(TableMetadata metadata, String prefix, Iterable<InetAddressAndPort> endpoints)
{
+ maybeTriggerSnapshot(metadata, prefix, endpoints, Collections.emptyList());
+ }
+
+ private void maybeTriggerSnapshot(TableMetadata metadata, String prefix, Iterable<InetAddressAndPort> endpoints, List<Range<Token>> ranges)
+ {
long now = nanoTime();
AtomicLong cached = lastSnapshotTimes.computeIfAbsent(metadata.id, u -> new AtomicLong(0));
long last = cached.get();
- long interval = Long.getLong("cassandra.diagnostic_snapshot_interval_nanos", SNAPSHOT_INTERVAL_NANOS);
+ long interval = DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS.getLong();
if (now - last > interval && cached.compareAndSet(last, now))
{
Message<SnapshotCommand> msg = Message.out(Verb.SNAPSHOT_REQ,
@@ -130,6 +143,9 @@
metadata.name,
getSnapshotName(prefix),
false));
+
+ if (!ranges.isEmpty() && ranges.size() < MAX_SNAPSHOT_RANGE_COUNT)
+ msg = msg.withParam(SNAPSHOT_RANGES, ranges);
for (InetAddressAndPort replica : endpoints)
MessagingService.instance().send(msg, replica);
}
@@ -139,19 +155,21 @@
}
}
- private void maybeSnapshot(SnapshotCommand command, InetAddressAndPort initiator)
+ private void maybeSnapshot(SnapshotCommand command, List<Range<Token>> ranges, InetAddressAndPort initiator)
{
- executor.execute(new DiagnosticSnapshotTask(command, initiator));
+ executor.execute(new DiagnosticSnapshotTask(command, ranges, initiator));
}
private static class DiagnosticSnapshotTask implements Runnable
{
final SnapshotCommand command;
final InetAddressAndPort from;
+ final List<Range<Token>> ranges;
- DiagnosticSnapshotTask(SnapshotCommand command, InetAddressAndPort from)
+ DiagnosticSnapshotTask(SnapshotCommand command, List<Range<Token>> ranges, InetAddressAndPort from)
{
this.command = command;
+ this.ranges = ranges;
this.from = from;
}
@@ -185,7 +203,17 @@
command.keyspace,
command.column_family,
command.snapshot_name);
- cfs.snapshot(command.snapshot_name);
+
+ if (ranges.isEmpty())
+ cfs.snapshot(command.snapshot_name);
+ else
+ {
+ cfs.snapshot(command.snapshot_name,
+ (sstable) -> checkIntersection(ranges,
+ sstable.first.getToken(),
+ sstable.last.getToken()),
+ false, false);
+ }
}
catch (IllegalArgumentException e)
{
@@ -196,4 +224,11 @@
}
}
}
+
+ private static boolean checkIntersection(List<Range<Token>> normalizedRanges, Token first, Token last)
+ {
+ Bounds<Token> bounds = new Bounds<>(first, last);
+ return normalizedRanges.stream().anyMatch(range -> range.intersects(bounds));
+ }
+
}
diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java
index 6d210ce..9086d6b 100644
--- a/src/java/org/apache/cassandra/utils/FBUtilities.java
+++ b/src/java/org/apache/cassandra/utils/FBUtilities.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.utils;
-
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
@@ -27,37 +26,42 @@
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.math.BigInteger;
-import java.net.*;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.net.URL;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
-
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
-
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
-import org.apache.cassandra.utils.concurrent.*;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cassandra.audit.IAuditLogger;
import org.apache.cassandra.auth.AllowAllNetworkAuthorizer;
import org.apache.cassandra.auth.IAuthenticator;
@@ -66,7 +70,6 @@
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.LocalPartitioner;
@@ -75,41 +78,43 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.security.ISslContextFactory;
+import org.apache.cassandra.utils.concurrent.FutureCombiner;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+import org.objectweb.asm.Opcodes;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_AVAILABLE_PROCESSORS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GIT_SHA;
import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.OS_NAME;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RELEASE_VERSION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TRIGGERS_DIR;
import static org.apache.cassandra.config.CassandraRelevantProperties.USER_HOME;
-import static org.apache.cassandra.io.util.File.WriteMode.OVERWRITE;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
public class FBUtilities
{
- private static final ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory());
-
static
{
preventIllegalAccessWarnings();
- jsonMapper.registerModule(new JavaTimeModule());
- jsonMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
}
private static final Logger logger = LoggerFactory.getLogger(FBUtilities.class);
public static final String UNKNOWN_RELEASE_VERSION = "Unknown";
+ public static final String UNKNOWN_GIT_SHA = "Unknown";
public static final BigInteger TWO = new BigInteger("2");
private static final String DEFAULT_TRIGGER_DIR = "triggers";
- private static final String OPERATING_SYSTEM = System.getProperty("os.name").toLowerCase();
+ private static final String OPERATING_SYSTEM = OS_NAME.getString().toLowerCase();
public static final boolean isLinux = OPERATING_SYSTEM.contains("linux");
private static volatile InetAddress localInetAddress;
@@ -121,7 +126,7 @@
private static volatile String previousReleaseVersionString;
- private static int availableProcessors = Integer.getInteger("cassandra.available_processors", DatabaseDescriptor.getAvailableProcessors());
+ private static int availableProcessors = CASSANDRA_AVAILABLE_PROCESSORS.getInt(DatabaseDescriptor.getAvailableProcessors());
public static void setAvailableProcessors(int value)
{
@@ -138,6 +143,8 @@
public static final int MAX_UNSIGNED_SHORT = 0xFFFF;
+ public static final int ASM_BYTECODE_VERSION = Opcodes.ASM9;
+
public static MessageDigest newMessageDigest(String algorithm)
{
try
@@ -391,9 +398,9 @@
public static File cassandraTriggerDir()
{
File triggerDir = null;
- if (System.getProperty("cassandra.triggers_dir") != null)
+ if (TRIGGERS_DIR.getString() != null)
{
- triggerDir = new File(System.getProperty("cassandra.triggers_dir"));
+ triggerDir = new File(TRIGGERS_DIR.getString());
}
else
{
@@ -419,26 +426,42 @@
return previousReleaseVersionString;
}
- public static String getReleaseVersionString()
+ private static Properties getVersionProperties()
{
try (InputStream in = FBUtilities.class.getClassLoader().getResourceAsStream("org/apache/cassandra/config/version.properties"))
{
if (in == null)
{
- return System.getProperty("cassandra.releaseVersion", UNKNOWN_RELEASE_VERSION);
+ return null;
}
Properties props = new Properties();
props.load(in);
- return props.getProperty("CassandraVersion");
+ return props;
}
catch (Exception e)
{
JVMStabilityInspector.inspectThrowable(e);
logger.warn("Unable to load version.properties", e);
- return "debug version";
+ return null;
}
}
+ public static String getReleaseVersionString()
+ {
+ Properties props = getVersionProperties();
+ if (props == null)
+ return RELEASE_VERSION.getString(UNKNOWN_RELEASE_VERSION);
+ return props.getProperty("CassandraVersion");
+ }
+
+ public static String getGitSHA()
+ {
+ Properties props = getVersionProperties();
+ if (props == null)
+ return GIT_SHA.getString(UNKNOWN_GIT_SHA);
+ return props.getProperty("GitSHA", UNKNOWN_GIT_SHA);
+ }
+
public static String getReleaseVersionMajor()
{
String releaseVersion = FBUtilities.getReleaseVersionString();
@@ -588,11 +611,8 @@
*/
public static IPartitioner newPartitioner(Descriptor desc) throws IOException
{
- EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
- ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
- SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
- return newPartitioner(validationMetadata.partitioner, Optional.of(header.getKeyType()));
+ StatsComponent statsComponent = StatsComponent.load(desc, MetadataType.VALIDATION, MetadataType.HEADER);
+ return newPartitioner(statsComponent.validationMetadata().partitioner, Optional.of(statsComponent.serializationHeader().getKeyType()));
}
public static IPartitioner newPartitioner(String partitionerClassName) throws ConfigurationException
@@ -807,58 +827,6 @@
return new WrappedCloseableIterator<T>(iterator);
}
- public static Map<String, String> fromJsonMap(String json)
- {
- try
- {
- return jsonMapper.readValue(json, Map.class);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public static List<String> fromJsonList(String json)
- {
- try
- {
- return jsonMapper.readValue(json, List.class);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public static String json(Object object)
- {
- try
- {
- return jsonMapper.writeValueAsString(object);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public static void serializeToJsonFile(Object object, File outputFile) throws IOException
- {
- try (FileOutputStreamPlus out = outputFile.newOutputStream(OVERWRITE))
- {
- jsonMapper.writeValue((OutputStream) out, object);
- }
- }
-
- public static <T> T deserializeFromJsonFile(Class<T> tClass, File file) throws IOException
- {
- try (FileInputStreamPlus in = file.newInputStream())
- {
- return jsonMapper.readValue((InputStream) in, tClass);
- }
- }
-
public static String prettyPrintMemory(long size)
{
return prettyPrintMemory(size, false);
@@ -1141,4 +1109,30 @@
}
return sb.toString();
}
-}
+
+ @SafeVarargs
+ public static <T> ImmutableList<T> immutableListWithFilteredNulls(T... values)
+ {
+ ImmutableList.Builder<T> builder = ImmutableList.builderWithExpectedSize(values.length);
+ for (int i = 0; i < values.length; i++)
+ {
+ if (values[i] != null)
+ builder.add(values[i]);
+ }
+ return builder.build();
+ }
+
+ public static void closeQuietly(Object o)
+ {
+ if (!(o instanceof AutoCloseable))
+ return;
+ try
+ {
+ ((AutoCloseable) o).close();
+ }
+ catch (Exception e)
+ {
+ logger.warn("Closing {} had an unexpected exception", o, e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/FilterFactory.java b/src/java/org/apache/cassandra/utils/FilterFactory.java
index 4cf0cbf..6276a3a 100644
--- a/src/java/org/apache/cassandra/utils/FilterFactory.java
+++ b/src/java/org/apache/cassandra/utils/FilterFactory.java
@@ -17,15 +17,19 @@
*/
package org.apache.cassandra.utils;
+import java.io.IOException;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.obs.IBitSet;
import org.apache.cassandra.utils.obs.OffHeapBitSet;
public class FilterFactory
{
- public static final IFilter AlwaysPresent = new AlwaysPresentFilter();
+ public static final IFilter AlwaysPresent = AlwaysPresentFilter.instance;
private static final Logger logger = LoggerFactory.getLogger(FilterFactory.class);
private static final long BITSET_EXCESS = 20;
@@ -57,7 +61,7 @@
{
assert maxFalsePosProbability <= 1.0 : "Invalid probability";
if (maxFalsePosProbability == 1.0)
- return new AlwaysPresentFilter();
+ return FilterFactory.AlwaysPresent;
int bucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement, maxFalsePosProbability);
return createFilter(spec.K, numElements, spec.bucketsPerElement);
@@ -70,4 +74,56 @@
IBitSet bitset = new OffHeapBitSet(numBits);
return new BloomFilter(hash, bitset);
}
+
+ private static class AlwaysPresentFilter implements IFilter
+ {
+ public static final AlwaysPresentFilter instance = new AlwaysPresentFilter();
+
+ private AlwaysPresentFilter() { }
+
+ public boolean isPresent(FilterKey key)
+ {
+ return true;
+ }
+
+ public void add(FilterKey key) { }
+
+ public void clear() { }
+
+ public void close() { }
+
+ public IFilter sharedCopy()
+ {
+ return this;
+ }
+
+ public Throwable close(Throwable accumulate)
+ {
+ return accumulate;
+ }
+
+ public void addTo(Ref.IdentityCollection identities)
+ {
+ }
+
+ public long serializedSize(boolean oldSerializationFormat) { return 0; }
+
+ @Override
+ public void serialize(DataOutputStreamPlus out, boolean oldSerializationFormat) throws IOException
+ {
+ // no-op
+ }
+
+ @Override
+ public long offHeapSize()
+ {
+ return 0;
+ }
+
+ @Override
+ public boolean isInformative()
+ {
+ return false;
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/GuidGenerator.java b/src/java/org/apache/cassandra/utils/GuidGenerator.java
index 46843b4..e06270f 100644
--- a/src/java/org/apache/cassandra/utils/GuidGenerator.java
+++ b/src/java/org/apache/cassandra/utils/GuidGenerator.java
@@ -34,7 +34,7 @@
{
if (!JAVA_SECURITY_EGD.isPresent())
{
- System.setProperty("java.security.egd", "file:/dev/urandom");
+ JAVA_SECURITY_EGD.setString("file:/dev/urandom");
}
mySecureRand = new SecureRandom();
long secureInitializer = mySecureRand.nextLong();
diff --git a/src/java/org/apache/cassandra/utils/HeapUtils.java b/src/java/org/apache/cassandra/utils/HeapUtils.java
index c0910d8..38a6969 100644
--- a/src/java/org/apache/cassandra/utils/HeapUtils.java
+++ b/src/java/org/apache/cassandra/utils/HeapUtils.java
@@ -22,15 +22,24 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
+import java.nio.file.FileStore;
+import java.nio.file.Path;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import javax.management.MBeanServer;
-import org.apache.cassandra.io.util.File;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.text.StrBuilder;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.sun.management.HotSpotDiagnosticMXBean;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
+
import static org.apache.cassandra.config.CassandraRelevantEnv.JAVA_HOME;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
* Utility to log heap histogram.
@@ -40,6 +49,8 @@
{
private static final Logger logger = LoggerFactory.getLogger(HeapUtils.class);
+ private static final Lock DUMP_LOCK = new ReentrantLock();
+
/**
* Generates a HEAP histogram in the log file.
*/
@@ -74,6 +85,72 @@
}
/**
+ * @return full path to the created heap dump file
+ */
+ public static String maybeCreateHeapDump()
+ {
+ // Make sure that only one heap dump can be in progress across all threads, and abort for
+ // threads that cannot immediately acquire the lock, allowing them to fail normally.
+ if (DUMP_LOCK.tryLock())
+ {
+ try
+ {
+ if (DatabaseDescriptor.getDumpHeapOnUncaughtException())
+ {
+ MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+
+ Path absoluteBasePath = DatabaseDescriptor.getHeapDumpPath();
+ // We should never reach this point with this value null as we initialize the bool only after confirming
+ // the -XX param / .yaml conf is present on initial init and the JMX entry point, but still worth checking.
+ if (absoluteBasePath == null)
+ {
+ DatabaseDescriptor.setDumpHeapOnUncaughtException(false);
+ throw new RuntimeException("Cannot create heap dump unless -XX:HeapDumpPath or cassandra.yaml:heap_dump_path is specified.");
+ }
+
+ long maxMemoryBytes = Runtime.getRuntime().maxMemory();
+ long freeSpaceBytes = PathUtils.tryGetSpace(absoluteBasePath, FileStore::getUnallocatedSpace);
+
+ // Abort if there isn't enough room on the target disk to dump the entire heap and then copy it.
+ if (freeSpaceBytes < 2 * maxMemoryBytes)
+ throw new RuntimeException("Cannot allocated space for a heap dump snapshot. There are only " + freeSpaceBytes + " bytes free at " + absoluteBasePath + '.');
+
+ HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class);
+ String filename = String.format("pid%s-epoch%s.hprof", HeapUtils.getProcessId().toString(), currentTimeMillis());
+ String fullPath = File.getPath(absoluteBasePath.toString(), filename).toString();
+
+ logger.info("Writing heap dump to {} on partition w/ {} free bytes...", absoluteBasePath, freeSpaceBytes);
+ mxBean.dumpHeap(fullPath, false);
+ logger.info("Heap dump written to {}", fullPath);
+
+ // Disable further heap dump creations until explicitly re-enabled.
+ DatabaseDescriptor.setDumpHeapOnUncaughtException(false);
+
+ return fullPath;
+ }
+ else
+ {
+ logger.debug("Heap dump creation on uncaught exceptions is disabled.");
+ }
+ }
+ catch (Throwable e)
+ {
+ logger.warn("Unable to create heap dump.", e);
+ }
+ finally
+ {
+ DUMP_LOCK.unlock();
+ }
+ }
+ else
+ {
+ logger.debug("Heap dump creation is already in progress. Request aborted.");
+ }
+
+ return null;
+ }
+
+ /**
* Retrieve the path to the JCMD executable.
* @return the path to the JCMD executable or null if it cannot be found.
*/
diff --git a/src/java/org/apache/cassandra/utils/IFilter.java b/src/java/org/apache/cassandra/utils/IFilter.java
index b5eb2c4..f06ae6e 100644
--- a/src/java/org/apache/cassandra/utils/IFilter.java
+++ b/src/java/org/apache/cassandra/utils/IFilter.java
@@ -17,14 +17,26 @@
*/
package org.apache.cassandra.utils;
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.utils.concurrent.SharedCloseable;
public interface IFilter extends SharedCloseable
{
interface FilterKey
{
- /** Places the murmur3 hash of the key in the given long array of size at least two. */
+ /**
+ * Places the murmur3 hash of the key in the given long array of size at least two.
+ */
void filterHash(long[] dest);
+
+ default short filterHashLowerBits()
+ {
+ long[] dest = new long[2];
+ filterHash(dest);
+ return (short) dest[1];
+ }
}
void add(FilterKey key);
@@ -33,7 +45,9 @@
void clear();
- long serializedSize();
+ long serializedSize(boolean oldSerializationFormat);
+
+ void serialize(DataOutputStreamPlus out, boolean oldSerializationFormat) throws IOException;
void close();
@@ -41,7 +55,10 @@
/**
* Returns the amount of memory in bytes used off heap.
+ *
* @return the amount of memory in bytes used off heap
*/
long offHeapSize();
+
+ boolean isInformative();
}
diff --git a/src/java/org/apache/cassandra/utils/JMXServerUtils.java b/src/java/org/apache/cassandra/utils/JMXServerUtils.java
index 5557fda..ba9c704 100644
--- a/src/java/org/apache/cassandra/utils/JMXServerUtils.java
+++ b/src/java/org/apache/cassandra/utils/JMXServerUtils.java
@@ -54,6 +54,8 @@
import org.apache.cassandra.auth.jmx.AuthenticationProxy;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_AUTHORIZER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_REMOTE_LOGIN_CONFIG;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_ACCESS_FILE;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_AUTHENTICATE;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PASSWORD_FILE;
@@ -62,6 +64,9 @@
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_CIPHER_SUITES;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_PROTOCOLS;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_SSL_NEED_CLIENT_AUTH;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JAVAX_RMI_SSL_CLIENT_ENABLED_CIPHER_SUITES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JAVAX_RMI_SSL_CLIENT_ENABLED_PROTOCOLS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_RMI_SERVER_HOSTNAME;
public class JMXServerUtils
{
@@ -82,7 +87,7 @@
if (local)
{
serverAddress = InetAddress.getLoopbackAddress();
- System.setProperty("java.rmi.server.hostname", serverAddress.getHostAddress());
+ JAVA_RMI_SERVER_HOSTNAME.setString(serverAddress.getHostAddress());
}
// Configure the RMI client & server socket factories, including SSL config.
@@ -166,7 +171,7 @@
// before creating the authenticator. If no password file has been
// explicitly set, it's read from the default location
// $JAVA_HOME/lib/management/jmxremote.password
- String configEntry = System.getProperty("cassandra.jmx.remote.login.config");
+ String configEntry = CASSANDRA_JMX_REMOTE_LOGIN_CONFIG.getString();
if (configEntry != null)
{
env.put(JMXConnectorServer.AUTHENTICATOR, new AuthenticationProxy(configEntry));
@@ -194,7 +199,7 @@
// can be set as the JMXConnectorServer's MBeanServerForwarder.
// If no custom proxy is supplied, check system properties for the location of the
// standard access file & stash it in env
- String authzProxyClass = System.getProperty("cassandra.jmx.authorizer");
+ String authzProxyClass = CASSANDRA_JMX_AUTHORIZER.getString();
if (authzProxyClass != null)
{
final InvocationHandler handler = FBUtilities.construct(authzProxyClass, "JMX authz proxy");
@@ -224,7 +229,7 @@
String protocolList = COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_PROTOCOLS.getString();
if (protocolList != null)
{
- System.setProperty("javax.rmi.ssl.client.enabledProtocols", protocolList);
+ JAVAX_RMI_SSL_CLIENT_ENABLED_PROTOCOLS.setString(protocolList);
protocols = StringUtils.split(protocolList, ',');
}
@@ -232,7 +237,7 @@
String cipherList = COM_SUN_MANAGEMENT_JMXREMOTE_SSL_ENABLED_CIPHER_SUITES.getString();
if (cipherList != null)
{
- System.setProperty("javax.rmi.ssl.client.enabledCipherSuites", cipherList);
+ JAVAX_RMI_SSL_CLIENT_ENABLED_CIPHER_SUITES.setString(cipherList);
ciphers = StringUtils.split(cipherList, ',');
}
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index 1d3c09f..a396ef9 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -46,6 +46,8 @@
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PRINT_HEAP_HISTOGRAM_ON_OUT_OF_MEMORY_ERROR;
+
/**
* Responsible for deciding whether to kill the JVM if it gets in an "unstable" state (think OOM).
*/
@@ -105,7 +107,7 @@
boolean isUnstable = false;
if (t instanceof OutOfMemoryError)
{
- if (Boolean.getBoolean("cassandra.printHeapHistogramOnOutOfMemoryError"))
+ if (PRINT_HEAP_HISTOGRAM_ON_OUT_OF_MEMORY_ERROR.getBoolean())
{
// We want to avoid printing multiple time the heap histogram if multiple OOM errors happen in a short
// time span.
@@ -133,6 +135,9 @@
isUnstable = true;
}
+ // Anything other than an OOM, we should try and heap dump to capture what's going on if configured to do so
+ HeapUtils.maybeCreateHeapDump();
+
if (t instanceof InterruptedException)
throw new UncheckedInterruptedException((InterruptedException) t);
diff --git a/src/java/org/apache/cassandra/utils/JsonUtils.java b/src/java/org/apache/cassandra/utils/JsonUtils.java
new file mode 100644
index 0000000..1cdc13c
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/JsonUtils.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.util.BufferRecyclers;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.serializers.MarshalException;
+
+import static org.apache.cassandra.io.util.File.WriteMode.OVERWRITE;
+
+public final class JsonUtils
+{
+ public static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(new JsonFactory()); // checkstyle: permit this instantiation
+ public static final ObjectWriter JSON_OBJECT_PRETTY_WRITER;
+
+ static
+ {
+ JSON_OBJECT_MAPPER.registerModule(new JavaTimeModule());
+ JSON_OBJECT_MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+ JSON_OBJECT_PRETTY_WRITER = JSON_OBJECT_MAPPER.writerWithDefaultPrettyPrinter();
+ }
+
+ private JsonUtils()
+ {
+ }
+
+ /**
+ * Quotes string contents using standard JSON quoting.
+ */
+ public static String quoteAsJsonString(String s)
+ {
+ // In future should update to directly use `JsonStringEncoder.getInstance()` but for now:
+ return new String(BufferRecyclers.getJsonStringEncoder().quoteAsString(s));
+ }
+
+ public static Object decodeJson(byte[] json)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.readValue(json, Object.class);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error decoding JSON bytes: " + ex.getMessage());
+ }
+ }
+
+ public static Object decodeJson(String json)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.readValue(json, Object.class);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error decoding JSON string: " + ex.getMessage());
+ }
+ }
+
+ public static byte[] writeAsJsonBytes(Object value)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.writeValueAsBytes(value);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error writing as JSON: " + ex.getMessage());
+ }
+ }
+
+ public static String writeAsJsonString(Object value)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.writeValueAsString(value);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error writing as JSON: " + ex.getMessage());
+ }
+ }
+
+ public static String writeAsPrettyJsonString(Object value) throws MarshalException
+ {
+ try
+ {
+ return JSON_OBJECT_PRETTY_WRITER.writeValueAsString(value);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error writing as JSON: " + ex.getMessage());
+ }
+ }
+
+ public static <T> Map<String, T> fromJsonMap(String json)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.readValue(json, Map.class);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error decoding JSON string: " + ex.getMessage());
+ }
+ }
+
+ public static <T> Map<String, T> fromJsonMap(byte[] bytes)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.readValue(bytes, Map.class);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error decoding JSON: " + ex.getMessage());
+ }
+ }
+
+ public static List<String> fromJsonList(byte[] bytes)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.readValue(bytes, List.class);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error decoding JSON: " + ex.getMessage());
+ }
+ }
+
+ public static List<String> fromJsonList(String json)
+ {
+ try
+ {
+ return JSON_OBJECT_MAPPER.readValue(json, List.class);
+ }
+ catch (IOException ex)
+ {
+ throw new MarshalException("Error decoding JSON: " + ex.getMessage());
+ }
+ }
+
+ public static void serializeToJsonFile(Object object, File outputFile) throws IOException
+ {
+ try (FileOutputStreamPlus out = outputFile.newOutputStream(OVERWRITE))
+ {
+ JSON_OBJECT_PRETTY_WRITER.writeValue((OutputStream) out, object);
+ }
+ }
+
+ public static <T> T deserializeFromJsonFile(Class<T> tClass, File file) throws IOException
+ {
+ try (FileInputStreamPlus in = file.newInputStream())
+ {
+ return JSON_OBJECT_MAPPER.readValue((InputStream) in, tClass);
+ }
+ }
+
+ /**
+ * Handles unquoting and case-insensitivity in map keys.
+ */
+ public static void handleCaseSensitivity(Map<String, Object> valueMap)
+ {
+ for (String mapKey : new ArrayList<>(valueMap.keySet()))
+ {
+ // if it's surrounded by quotes, remove them and preserve the case
+ if (mapKey.startsWith("\"") && mapKey.endsWith("\""))
+ {
+ valueMap.put(mapKey.substring(1, mapKey.length() - 1), valueMap.remove(mapKey));
+ continue;
+ }
+
+ // otherwise, lowercase it if needed
+ String lowered = mapKey.toLowerCase(Locale.US);
+ if (!mapKey.equals(lowered))
+ valueMap.put(lowered, valueMap.remove(mapKey));
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/MBeanWrapper.java b/src/java/org/apache/cassandra/utils/MBeanWrapper.java
index f8bc439..f003e1a 100644
--- a/src/java/org/apache/cassandra/utils/MBeanWrapper.java
+++ b/src/java/org/apache/cassandra/utils/MBeanWrapper.java
@@ -27,7 +27,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.cassandra.config.CassandraRelevantProperties.IS_DISABLED_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
import static org.apache.cassandra.config.CassandraRelevantProperties.MBEAN_REGISTRATION_CLASS;
/**
@@ -42,7 +42,7 @@
static MBeanWrapper create()
{
- if (IS_DISABLED_MBEAN_REGISTRATION.getBoolean())
+ if (ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.getBoolean())
return new NoOpMBeanWrapper();
String klass = MBEAN_REGISTRATION_CLASS.getString();
diff --git a/src/java/org/apache/cassandra/utils/MergeIterator.java b/src/java/org/apache/cassandra/utils/MergeIterator.java
index 6713dd0..2723896 100644
--- a/src/java/org/apache/cassandra/utils/MergeIterator.java
+++ b/src/java/org/apache/cassandra/utils/MergeIterator.java
@@ -395,10 +395,11 @@
private boolean isLowerBound()
{
+ assert item != null;
return item == lowerBound;
}
- public void consume(Reducer reducer)
+ public <Out> void consume(Reducer<In, Out> reducer)
{
if (isLowerBound())
{
@@ -488,4 +489,4 @@
return (Out) source.next();
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/MonotonicClock.java b/src/java/org/apache/cassandra/utils/MonotonicClock.java
index ad9ee81..7be54c0 100644
--- a/src/java/org/apache/cassandra/utils/MonotonicClock.java
+++ b/src/java/org/apache/cassandra/utils/MonotonicClock.java
@@ -27,11 +27,12 @@
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.Config;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.APPROXIMATE_TIME_PRECISION_MS;
import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_APPROX;
import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_PRECISE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
@@ -139,8 +140,7 @@
static abstract class AbstractEpochSamplingClock implements MonotonicClock
{
private static final Logger logger = LoggerFactory.getLogger(AbstractEpochSamplingClock.class);
- private static final String UPDATE_INTERVAL_PROPERTY = Config.PROPERTY_PREFIX + "NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL";
- private static final long UPDATE_INTERVAL_MS = Long.getLong(UPDATE_INTERVAL_PROPERTY, 10000);
+ private static final long UPDATE_INTERVAL_MS = NANOTIMETOMILLIS_TIMESTAMP_UPDATE_INTERVAL.getLong();
@VisibleForTesting
public static class AlmostSameTime implements MonotonicClockTranslation
@@ -281,7 +281,7 @@
public static class SampledClock implements MonotonicClock
{
private static final Logger logger = LoggerFactory.getLogger(SampledClock.class);
- private static final int UPDATE_INTERVAL_MS = Math.max(1, Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "approximate_time_precision_ms", "2")));
+ private static final int UPDATE_INTERVAL_MS = Math.max(1, APPROXIMATE_TIME_PRECISION_MS.getInt());
private static final long ERROR_NANOS = MILLISECONDS.toNanos(UPDATE_INTERVAL_MS);
private final MonotonicClock precise;
diff --git a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
index ddb3ea6..19dcc23 100644
--- a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
+++ b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
@@ -212,7 +212,7 @@
int position = row.getInt("position");
org.apache.cassandra.schema.ColumnMetadata.Kind kind = ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase());
- return new ColumnMetadata(keyspace, table, name, type, position, kind);
+ return new ColumnMetadata(keyspace, table, name, type, position, kind, null);
}
private static DroppedColumn createDroppedColumnFromRow(Row row, String keyspace, String table)
@@ -220,7 +220,7 @@
String name = row.getString("column_name");
AbstractType<?> type = CQLTypeParser.parse(keyspace, row.getString("type"), Types.none());
ColumnMetadata.Kind kind = ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase());
- ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind);
+ ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind, null);
long droppedTime = row.getTimestamp("dropped_time").getTime();
return new DroppedColumn(column, droppedTime);
}
diff --git a/src/java/org/apache/cassandra/utils/ObjectSizes.java b/src/java/org/apache/cassandra/utils/ObjectSizes.java
index 07066cf..2d94983 100644
--- a/src/java/org/apache/cassandra/utils/ObjectSizes.java
+++ b/src/java/org/apache/cassandra/utils/ObjectSizes.java
@@ -19,6 +19,9 @@
package org.apache.cassandra.utils;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import org.github.jamm.MemoryLayoutSpecification;
@@ -31,6 +34,7 @@
{
private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE)
.ignoreKnownSingletons();
+ private static final MemoryMeter omitSharedMeter = meter.omitSharedBufferOverhead();
private static final long EMPTY_HEAP_BUFFER_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER);
private static final long EMPTY_BYTE_ARRAY_SIZE = measure(new byte[0]);
@@ -38,6 +42,8 @@
private static final long DIRECT_BUFFER_HEAP_SIZE = measure(ByteBuffer.allocateDirect(0));
+ public static final long IPV6_SOCKET_ADDRESS_SIZE = ObjectSizes.measureDeep(new InetSocketAddress(getIpvAddress(16), 42));
+
/**
* Memory a byte array consumes
*
@@ -217,10 +223,38 @@
/**
* @param pojo the object to measure
+ * @return The size on the heap of the instance and all retained heap referenced by it, excluding portions of
+ * ByteBuffer that are not directly referenced by it but including any other referenced that may also be retained
+ * by other objects. This also includes bytes referenced in direct byte buffers, and may double-count memory if
+ * it is referenced by multiple ByteBuffer copies.
+ */
+ public static long measureDeepOmitShared(Object pojo)
+ {
+ return omitSharedMeter.measureDeep(pojo);
+ }
+
+ /**
+ * @param pojo the object to measure
* @return the size on the heap of the instance only, excluding any referenced objects
*/
public static long measure(Object pojo)
{
return meter.measure(pojo);
}
+
+ private static InetAddress getIpvAddress(int size)
+ {
+ if (size == 16 || size ==4)
+ {
+ try
+ {
+ return InetAddress.getByAddress(new byte[size]);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new IllegalArgumentException("Invalid size of a byte array when getting and ipv address: " + size);
+ }
+ }
+ else throw new IllegalArgumentException("Excpected a byte array size of 4 or 16 for an ipv address but got: " + size);
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/OutputHandler.java b/src/java/org/apache/cassandra/utils/OutputHandler.java
index 820160d..76eb345 100644
--- a/src/java/org/apache/cassandra/utils/OutputHandler.java
+++ b/src/java/org/apache/cassandra/utils/OutputHandler.java
@@ -26,20 +26,36 @@
public interface OutputHandler
{
// called when an important info need to be displayed
- public void output(String msg);
-
- // called when a less important info need to be displayed
- public void debug(String msg);
-
- // called when the user needs to be warn
- public void warn(String msg);
- public void warn(String msg, Throwable th);
- public default void warn(Throwable th)
+ void output(String msg);
+ default void output(String msg, Object ... args)
{
- warn(th.getMessage(), th);
+ output(String.format(msg, args));
}
- public static class LogOutput implements OutputHandler
+ // called when a less important info need to be displayed
+ void debug(String msg);
+ default void debug(String msg, Object ... args)
+ {
+ debug(String.format(msg, args));
+ }
+
+ // called when the user needs to be warn
+ void warn(String msg);
+ void warn(Throwable th, String msg);
+ default void warn(Throwable th)
+ {
+ warn(th, th.getMessage());
+ }
+ default void warn(Throwable th, String msg, Object... args)
+ {
+ warn(th, String.format(msg, args));
+ }
+ default void warn(String msg, Object ... args)
+ {
+ warn(String.format(msg, args));
+ }
+
+ class LogOutput implements OutputHandler
{
private static Logger logger = LoggerFactory.getLogger(LogOutput.class);
@@ -58,13 +74,13 @@
logger.warn(msg);
}
- public void warn(String msg, Throwable th)
+ public void warn(Throwable th, String msg)
{
logger.warn(msg, th);
}
}
- public static class SystemOutput implements OutputHandler
+ class SystemOutput implements OutputHandler
{
public final boolean debug;
public final boolean printStack;
@@ -95,14 +111,14 @@
public void warn(String msg)
{
- warn(msg, null);
+ warn((Throwable) null, msg);
}
- public void warn(String msg, Throwable th)
+ public void warn(Throwable th, String msg)
{
warnOut.println("WARNING: " + msg);
if (printStack && th != null)
th.printStackTrace(warnOut);
}
}
-}
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/RangesSerializer.java b/src/java/org/apache/cassandra/utils/RangesSerializer.java
new file mode 100644
index 0000000..5707503
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/RangesSerializer.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class RangesSerializer implements IVersionedSerializer<Collection<Range<Token>>>
+{
+ public static final RangesSerializer serializer = new RangesSerializer();
+
+ @Override
+ public void serialize(Collection<Range<Token>> ranges, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeInt(ranges.size());
+ for (Range<Token> r : ranges)
+ {
+ Token.serializer.serialize(r.left, out, version);
+ Token.serializer.serialize(r.right, out, version);
+ }
+ }
+
+ @Override
+ public Collection<Range<Token>> deserialize(DataInputPlus in, int version) throws IOException
+ {
+ int count = in.readInt();
+ List<Range<Token>> ranges = new ArrayList<>(count);
+ IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+ for (int i = 0; i < count; i++)
+ {
+ Token start = Token.serializer.deserialize(in, partitioner, version);
+ Token end = Token.serializer.deserialize(in, partitioner, version);
+ ranges.add(new Range<>(start, end));
+ }
+ return ranges;
+ }
+
+ @Override
+ public long serializedSize(Collection<Range<Token>> ranges, int version)
+ {
+ int size = TypeSizes.sizeof(ranges.size());
+ if (ranges.size() > 0)
+ size += ranges.size() * 2 * Token.serializer.serializedSize(ranges.iterator().next().left, version);
+ return size;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/ReflectionUtils.java b/src/java/org/apache/cassandra/utils/ReflectionUtils.java
new file mode 100644
index 0000000..801256d
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/ReflectionUtils.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+
+public class ReflectionUtils
+{
+ private ReflectionUtils()
+ {
+
+ }
+
+ public static Field getModifiersField() throws NoSuchFieldException
+ {
+ // below code works before Java 12
+ try
+ {
+ return Field.class.getDeclaredField("modifiers");
+ }
+ catch (NoSuchFieldException e)
+ {
+ // this is mitigation for JDK 17 (https://bugs.openjdk.org/browse/JDK-8210522)
+ try
+ {
+ Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class);
+ getDeclaredFields0.setAccessible(true);
+ Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false);
+ for (Field field : fields)
+ {
+ if ("modifiers".equals(field.getName()))
+ {
+ return field;
+ }
+ }
+ }
+ catch (ReflectiveOperationException ex)
+ {
+ e.addSuppressed(ex);
+ }
+ throw e;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/Shared.java b/src/java/org/apache/cassandra/utils/Shared.java
index e576c86..6433624 100644
--- a/src/java/org/apache/cassandra/utils/Shared.java
+++ b/src/java/org/apache/cassandra/utils/Shared.java
@@ -24,7 +24,7 @@
import java.lang.annotation.Target;
/**
- * Tells jvm-dtest that a class should be shared accross all {@link ClassLoader}s.
+ * Tells jvm-dtest that a class should be shared across all {@link ClassLoader}s.
*
* Jvm-dtest relies on classloader isolation to run multiple cassandra instances in the same JVM, this makes it
* so some classes do not get shared (outside a blesssed set of classes/packages). When the default behavior
diff --git a/src/java/org/apache/cassandra/utils/StatusLogger.java b/src/java/org/apache/cassandra/utils/StatusLogger.java
index dcb1135..0850224 100644
--- a/src/java/org/apache/cassandra/utils/StatusLogger.java
+++ b/src/java/org/apache/cassandra/utils/StatusLogger.java
@@ -19,17 +19,19 @@
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.cassandra.cache.*;
-import org.apache.cassandra.metrics.CassandraMetricsRegistry;
-import org.apache.cassandra.metrics.ThreadPoolMetrics;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.cache.AutoSavingCache;
+import org.apache.cassandra.cache.IRowCacheEntry;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.cache.RowCacheKey;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.ThreadPoolMetrics;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.CacheService;
@@ -92,7 +94,7 @@
"MessagingService", "n/a", pendingLargeMessages + "/" + pendingSmallMessages));
// Global key/row cache information
- AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = CacheService.instance.keyCache;
+ AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = CacheService.instance.keyCache;
AutoSavingCache<RowCacheKey, IRowCacheEntry> rowCache = CacheService.instance.rowCache;
int keyCacheKeysToSave = DatabaseDescriptor.getKeyCacheKeysToSave();
diff --git a/src/java/org/apache/cassandra/utils/SyncUtil.java b/src/java/org/apache/cassandra/utils/SyncUtil.java
index 6055859..1adc4fc 100644
--- a/src/java/org/apache/cassandra/utils/SyncUtil.java
+++ b/src/java/org/apache/cassandra/utils/SyncUtil.java
@@ -27,10 +27,13 @@
import java.nio.channels.FileChannel;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.cassandra.config.Config;
-
import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.config.CassandraRelevantEnv;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,8 +90,8 @@
fdUseCountField = fdUseCountTemp;
//If skipping syncing is requested by any means then skip them.
- boolean skipSyncProperty = Boolean.getBoolean(Config.PROPERTY_PREFIX + "skip_sync");
- boolean skipSyncEnv = Boolean.valueOf(System.getenv().getOrDefault("CASSANDRA_SKIP_SYNC", "false"));
+ boolean skipSyncProperty = CassandraRelevantProperties.TEST_CASSANDRA_SKIP_SYNC.getBoolean();
+ boolean skipSyncEnv = CassandraRelevantEnv.CASSANDRA_SKIP_SYNC.getBoolean();
SKIP_SYNC = skipSyncProperty || skipSyncEnv;
if (SKIP_SYNC)
{
@@ -99,6 +102,12 @@
public static MappedByteBuffer force(MappedByteBuffer buf)
{
Preconditions.checkNotNull(buf);
+ Object attachment = MemoryUtil.getAttachment(buf);
+ if (attachment instanceof Runnable)
+ {
+ ((Runnable) attachment).run();
+ return buf;
+ }
if (SKIP_SYNC)
{
Object fd = null;
diff --git a/src/java/org/apache/cassandra/utils/Throwables.java b/src/java/org/apache/cassandra/utils/Throwables.java
index 8337a56..d636711 100644
--- a/src/java/org/apache/cassandra/utils/Throwables.java
+++ b/src/java/org/apache/cassandra/utils/Throwables.java
@@ -18,19 +18,25 @@
*/
package org.apache.cassandra.utils;
-import org.apache.cassandra.io.util.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Iterator;
+import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.function.Predicate;
import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
public final class Throwables
@@ -184,6 +190,59 @@
}));
}
+ /**
+ * @see {@link #closeAndAddSuppressed(Throwable, Iterable)}
+ */
+ public static void closeAndAddSuppressed(@Nonnull Throwable t, AutoCloseable... closeables)
+ {
+ closeAndAddSuppressed(t, Arrays.asList(closeables));
+ }
+
+ /**
+ * Do what {@link #closeAndAddSuppressed(Throwable, Iterable)} does, additionally filtering out all null closables.
+ */
+ public static void closeNonNullAndAddSuppressed(@Nonnull Throwable t, AutoCloseable... closeables)
+ {
+ closeAndAddSuppressed(t, Iterables.filter(Arrays.asList(closeables), Objects::nonNull));
+ }
+
+ /**
+ * Closes all closables in the provided collections and accumulates the possible exceptions thrown when closing.
+ *
+ * @param accumulate non-null exception to accumulate errors thrown when closing the provided resources
+ * @param closeables closeables to be closed
+ */
+ public static void closeAndAddSuppressed(@Nonnull Throwable accumulate, Iterable<AutoCloseable> closeables)
+ {
+ Preconditions.checkNotNull(accumulate);
+ for (AutoCloseable closeable : closeables)
+ {
+ try
+ {
+ closeable.close();
+ }
+ catch (Throwable ex)
+ {
+ accumulate.addSuppressed(ex);
+ }
+ }
+ }
+
+ /**
+ * See {@link #close(Throwable, Iterable)}
+ */
+ public static Throwable close(Throwable accumulate, AutoCloseable ... closeables)
+ {
+ return close(accumulate, Arrays.asList(closeables));
+ }
+
+ /**
+ * Closes all the resources in the provided collections and accumulates the possible exceptions thrown when closing.
+ *
+ * @param accumulate the initial value for the exception accumulator, can be {@code null}
+ * @param closeables closeables to be closed
+ * @return {@code null}, {@param accumulate} or the first exception thrown when closing the provided resources
+ */
public static Throwable close(Throwable accumulate, Iterable<? extends AutoCloseable> closeables)
{
for (AutoCloseable closeable : closeables)
@@ -267,4 +326,11 @@
{
return unchecked(unwrapped(t));
}
+
+ @VisibleForTesting
+ public static void assertAnyCause(Throwable err, Class<? extends Throwable> cause)
+ {
+ if (!anyCauseMatches(err, cause::isInstance))
+ throw new AssertionError("The exception is not caused by " + cause.getName(), err);
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/TimeUUID.java b/src/java/org/apache/cassandra/utils/TimeUUID.java
index 8d79096..d993f17 100644
--- a/src/java/org/apache/cassandra/utils/TimeUUID.java
+++ b/src/java/org/apache/cassandra/utils/TimeUUID.java
@@ -53,6 +53,7 @@
import org.apache.cassandra.serializers.TypeSerializer;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_UNSAFE_TIME_UUID_NODE;
import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_UNSAFE_UUID_NODE;
import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -82,6 +83,7 @@
private static final long MIN_CLOCK_SEQ_AND_NODE = 0x8080808080808080L;
private static final long MAX_CLOCK_SEQ_AND_NODE = 0x7f7f7f7f7f7f7f7fL;
+ public static final long TIMEUUID_SIZE = ObjectSizes.measureDeep(new TimeUUID(10, 10));
final long uuidTimestamp, lsb;
@@ -438,9 +440,8 @@
if (DETERMINISM_UNSAFE_UUID_NODE.getBoolean())
return FBUtilities.getBroadcastAddressAndPort().addressBytes[3];
- Long specified = Long.getLong("cassandra.unsafe.timeuuidnode");
- if (specified != null)
- return specified
+ if (CASSANDRA_UNSAFE_TIME_UUID_NODE.isPresent())
+ return CASSANDRA_UNSAFE_TIME_UUID_NODE.getLong()
^ FBUtilities.getBroadcastAddressAndPort().addressBytes[3]
^ (FBUtilities.getBroadcastAddressAndPort().addressBytes[2] << 8);
diff --git a/src/java/org/apache/cassandra/utils/WrappedRunnable.java b/src/java/org/apache/cassandra/utils/WrappedRunnable.java
index 1de7a46..981e027 100644
--- a/src/java/org/apache/cassandra/utils/WrappedRunnable.java
+++ b/src/java/org/apache/cassandra/utils/WrappedRunnable.java
@@ -17,8 +17,6 @@
*/
package org.apache.cassandra.utils;
-import com.google.common.base.Throwables;
-
public abstract class WrappedRunnable implements Runnable
{
public final void run()
@@ -27,9 +25,13 @@
{
runMayThrow();
}
+ catch (RuntimeException e)
+ {
+ throw e;
+ }
catch (Exception e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
diff --git a/src/java/org/apache/cassandra/utils/binlog/BinLog.java b/src/java/org/apache/cassandra/utils/binlog/BinLog.java
index 43ff67e..8b0e02570 100644
--- a/src/java/org/apache/cassandra/utils/binlog/BinLog.java
+++ b/src/java/org/apache/cassandra/utils/binlog/BinLog.java
@@ -35,6 +35,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import net.openhft.chronicle.core.io.BackgroundResourceReleaser;
import net.openhft.chronicle.queue.ChronicleQueue;
import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder;
@@ -42,6 +43,7 @@
import net.openhft.chronicle.queue.RollCycles;
import net.openhft.chronicle.wire.WireOut;
import net.openhft.chronicle.wire.WriteMarshallable;
+import net.openhft.posix.PosixAPI;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -51,6 +53,7 @@
import org.apache.cassandra.utils.concurrent.WeightedQueue;
import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CHRONICLE_ANNOUNCER_DISABLE;
/**
* Bin log is a is quick and dirty binary log that is kind of a NIH version of binary logging with a traditional logging
@@ -75,6 +78,13 @@
public static final String VERSION = "version";
public static final String TYPE = "type";
+ static
+ {
+ // Avoid the chronicle announcement which is commercial advertisement, and debug info we already print at startup
+ // https://github.com/OpenHFT/Chronicle-Core/blob/chronicle-core-2.23.36/src/main/java/net/openhft/chronicle/core/announcer/Announcer.java#L32-L33
+ CHRONICLE_ANNOUNCER_DISABLE.setBoolean(true);
+ }
+
private ChronicleQueue queue;
private ExcerptAppender appender;
@VisibleForTesting
@@ -122,6 +132,7 @@
private BinLog(Path path, BinLogOptions options, BinLogArchiver archiver)
{
+ Preconditions.checkNotNull(PosixAPI.posix(), "Cannot initialize OpenHFT Posix");
Preconditions.checkNotNull(path, "path was null");
Preconditions.checkNotNull(options.roll_cycle, "roll_cycle was null");
Preconditions.checkArgument(options.max_queue_weight > 0, "max_queue_weight must be > 0");
@@ -135,7 +146,6 @@
appender = queue.acquireAppender();
this.blocking = options.block;
this.path = path;
-
this.options = options;
}
@@ -170,6 +180,7 @@
shouldContinue = false;
sampleQueue.put(NO_OP);
+ BackgroundResourceReleaser.stop();
binLogThread.join();
appender.close();
appender = null;
diff --git a/src/java/org/apache/cassandra/utils/btree/BTree.java b/src/java/org/apache/cassandra/utils/btree/BTree.java
index a026f70..2d8fefb 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTree.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTree.java
@@ -37,6 +37,7 @@
import static java.lang.Math.max;
import static java.lang.Math.min;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BTREE_BRANCH_SHIFT;
public class BTree
{
@@ -62,7 +63,7 @@
* subtrees when modifying the tree, since the modified tree would need new parent references).
* Instead, we store these references in a Path as needed when navigating the tree.
*/
- public static final int BRANCH_SHIFT = Integer.getInteger("cassandra.btree.branchshift", 5);
+ public static final int BRANCH_SHIFT = BTREE_BRANCH_SHIFT.getInt();
private static final int BRANCH_FACTOR = 1 << BRANCH_SHIFT;
public static final int MIN_KEYS = BRANCH_FACTOR / 2 - 1;
@@ -949,6 +950,9 @@
public static long sizeOfStructureOnHeap(Object[] tree)
{
+ if (tree == EMPTY_LEAF)
+ return 0;
+
long size = ObjectSizes.sizeOfArray(tree);
if (isLeaf(tree))
return size;
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java
new file mode 100644
index 0000000..6df425c
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Interface indicating a value can be represented/identified by a comparable {@link ByteSource}.
+ *
+ * All Cassandra types that can be used as part of a primary key have a corresponding byte-comparable translation,
+ * detailed in ByteComparable.md. Byte-comparable representations are used in some memtable as well as primary and
+ * secondary index implementations.
+ */
+public interface ByteComparable
+{
+ /**
+ * Returns a source that generates the byte-comparable representation of the value byte by byte.
+ */
+ ByteSource asComparableBytes(Version version);
+
+ enum Version
+ {
+ LEGACY, // Encoding used in legacy sstable format; forward (value to byte-comparable) translation only
+ OSS50, // CASSANDRA 5.0 encoding
+ }
+
+ ByteComparable EMPTY = (Version version) -> ByteSource.EMPTY;
+
+ /**
+ * Construct a human-readable string from the byte-comparable representation. Used for debugging.
+ */
+ default String byteComparableAsString(Version version)
+ {
+ StringBuilder builder = new StringBuilder();
+ ByteSource stream = asComparableBytes(version);
+ if (stream == null)
+ return "null";
+ for (int b = stream.next(); b != ByteSource.END_OF_STREAM; b = stream.next())
+ builder.append(Integer.toHexString((b >> 4) & 0xF)).append(Integer.toHexString(b & 0xF));
+ return builder.toString();
+ }
+
+ // Simple factories used for testing
+
+ static ByteComparable of(String s)
+ {
+ return v -> ByteSource.of(s, v);
+ }
+
+ static ByteComparable of(long value)
+ {
+ return v -> ByteSource.of(value);
+ }
+
+ static ByteComparable of(int value)
+ {
+ return v -> ByteSource.of(value);
+ }
+
+ static ByteComparable fixedLength(ByteBuffer bytes)
+ {
+ return v -> ByteSource.fixedLength(bytes);
+ }
+
+ static ByteComparable fixedLength(byte[] bytes)
+ {
+ return v -> ByteSource.fixedLength(bytes);
+ }
+
+ /**
+ * Returns a separator for two byte sources, i.e. something that is definitely > prevMax, and <= currMin, assuming
+ * prevMax < currMin.
+ * This returns the shortest prefix of currMin that is greater than prevMax.
+ */
+ static ByteComparable separatorPrefix(ByteComparable prevMax, ByteComparable currMin)
+ {
+ return version -> ByteSource.separatorPrefix(prevMax.asComparableBytes(version), currMin.asComparableBytes(version));
+ }
+
+ /**
+ * Returns a separator for two byte comparable, i.e. something that is definitely > prevMax, and <= currMin, assuming
+ * prevMax < currMin.
+ * This is a stream of length 1 longer than the common prefix of the two streams, with last byte one higher than the
+ * prevMax stream.
+ */
+ static ByteComparable separatorGt(ByteComparable prevMax, ByteComparable currMin)
+ {
+ return version -> ByteSource.separatorGt(prevMax.asComparableBytes(version), currMin.asComparableBytes(version));
+ }
+
+ static ByteComparable cut(ByteComparable src, int cutoff)
+ {
+ return version -> ByteSource.cut(src.asComparableBytes(version), cutoff);
+ }
+
+ /**
+ * Return the length of a byte comparable, not including the terminator byte.
+ */
+ static int length(ByteComparable src, Version version)
+ {
+ int l = 0;
+ ByteSource s = src.asComparableBytes(version);
+ while (s.next() != ByteSource.END_OF_STREAM)
+ ++l;
+ return l;
+ }
+
+ /**
+ * Compare two byte-comparable values by their byte-comparable representation. Used for tests.
+ *
+ * @return the result of the lexicographic unsigned byte comparison of the byte-comparable representations of the
+ * two arguments
+ */
+ static int compare(ByteComparable bytes1, ByteComparable bytes2, Version version)
+ {
+ ByteSource s1 = bytes1.asComparableBytes(version);
+ ByteSource s2 = bytes2.asComparableBytes(version);
+
+ if (s1 == null || s2 == null)
+ return Boolean.compare(s1 != null, s2 != null);
+
+ while (true)
+ {
+ int b1 = s1.next();
+ int b2 = s2.next();
+ int cmp = Integer.compare(b1, b2);
+ if (cmp != 0)
+ return cmp;
+ if (b1 == ByteSource.END_OF_STREAM)
+ return 0;
+ }
+ }
+
+ /**
+ * Returns the length of the minimum prefix that differentiates the two given byte-comparable representations.
+ */
+ static int diffPoint(ByteComparable bytes1, ByteComparable bytes2, Version version)
+ {
+ ByteSource s1 = bytes1.asComparableBytes(version);
+ ByteSource s2 = bytes2.asComparableBytes(version);
+ int pos = 1;
+ int b;
+ while ((b = s1.next()) == s2.next() && b != ByteSource.END_OF_STREAM)
+ ++pos;
+ return pos;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md
new file mode 100644
index 0000000..8012e27
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.md
@@ -0,0 +1,712 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Byte-comparable translation of types (ByteComparable/ByteSource)
+
+## Problem / Motivation
+
+Cassandra has a very heavy reliance on comparisons — they are used throughout read and write paths, coordination,
+compaction, etc. to be able to order and merge results. It also supports a range of types which often require the
+compared object to be completely in memory to order correctly, which in turn has necessitated interfaces where
+comparisons can only be applied if the compared objects are completely loaded.
+
+This has some rather painful implications on the performance of the database, both in terms of the time it takes to load,
+compare and garbage collect, as well as in terms of the space required to hold complete keys in on-disk indices and
+deserialized versions in in-memory data structures. In addition to this, the reliance on comparisons forces Cassandra to
+use only comparison-based structures, which aren’t the most efficient.
+
+There is no way to escape the need to compare and order objects in Cassandra, but the machinery for doing this can be
+done much more smartly if we impose some simple structure in the objects we deal with — byte ordering.
+
+The term “byte order” as used in this document refers to the property of being ordered via lexicographic compare on the
+unsigned values of the byte contents. Some of the types in Cassandra already have this property (e.g. strings, blobs),
+but other most heavily used ones (e.g. integers, uuids) don’t.
+
+When byte order is universally available for the types used for keys, several key advantages can be put to use:
+
+- Comparisons can be done using a single simple method, core machinery doesn’t need to know anything about types.
+- Prefix differences are enough to define order; unique prefixes can be used instead of complete keys.
+- Tries can be used to store, query and iterate over ranges of keys, providing fast lookup and prefix compression.
+- Merging can be performed by merging tries, significantly reducing the number of necessary comparisons.
+
+## Ordering the types
+
+As we want to keep all existing functionality in Cassandra, we need to be able to deal with existing
+non-byte-order-comparable types. This requires some form of conversion of each value to a sequence of bytes that can be
+byte-order compared (also called "byte-comparable"), as well as the inverse conversion from byte-comparable to value.
+
+As one of the main advantages of byte order is the ability to decide comparisons early, without having to read the whole
+of the input sequence, byte-ordered interpretations of values are represented as sources of bytes with unknown length,
+using the interface `ByteSource`. The interface declares one method, `next()` which produces the next byte of the
+stream, or `ByteSource.END_OF_STREAM` if the stream is exhausted.
+
+`END_OF_STREAM` is chosen as `-1` (`(int) -1`, which is outside the range of possible byte values), to make comparing
+two byte sources as trivial (and thus fast) as possible.
+
+To be able to completely abstract type information away from the storage machinery, we also flatten complex types into
+single byte sequences. To do this, we add separator bytes in front, between components, and at the end and do some
+encoding of variable-length sequences.
+
+The other interface provided by this package `ByteComparable`, is an entity whose byte-ordered interpretation can be
+requested. The interface is implemented by `DecoratedKey`, and can be requested for clustering keys and bounds using
+`ClusteringComparator.asByteComparable`. The inverse translation is provided by
+`Buffer/NativeDecoratedKey.fromByteComparable` and `ClusteringComparator.clustering/bound/boundaryFromByteComparable`.
+
+The (rather technical) paragraphs below detail the encoding we have chosen for the various types. For simplicity we
+only discuss the bidirectional `OSS50` version of the translation. The implementations in code of the various mappings
+are in the releavant `AbstractType` subclass.
+
+### Desired properties
+
+Generally, we desire the following two properties from the byte-ordered translations of values we use in the database:
+
+- Comparison equivalence (1):
+ <math xmlns="http://www.w3.org/1998/Math/MathML">
+ <semantics>
+ <mstyle displaystyle="true">
+ <mo>∀</mo>
+ <mi>x</mi>
+ <mo>,</mo>
+ <mi>y</mi>
+ <mo>∈</mo>
+ <mi>T</mi>
+ <mo>,</mo>
+ <mrow>
+ <mtext>compareBytesUnsigned</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>x</mi>
+ <mo>)</mo>
+ </mrow>
+ <mo>,</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>y</mi>
+ <mo>)</mo>
+ </mrow>
+ <mo>)</mo>
+ </mrow>
+ <mo>=</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>compare</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>x</mi>
+ <mo>,</mo>
+ <mi>y</mi>
+ <mo>)</mo>
+ </mrow>
+ </mstyle>
+ <!-- <annotation encoding="text/x-asciimath">forall x,y in T, "compareBytesUnsigned"(T."byteOrdered"(x), T."byteOrdered"(y))=T."compare"(x, y)</annotation> -->
+ </semantics>
+ </math>
+- Prefix-freedom (2):
+ <math xmlns="http://www.w3.org/1998/Math/MathML">
+ <semantics>
+ <mstyle displaystyle="true">
+ <mo>∀</mo>
+ <mi>x</mi>
+ <mo>,</mo>
+ <mi>y</mi>
+ <mo>∈</mo>
+ <mi>T</mi>
+ <mo>,</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>x</mi>
+ <mo>)</mo>
+ </mrow>
+ <mrow>
+ <mspace width="1ex" />
+ <mtext> is not a prefix of </mtext>
+ <mspace width="1ex" />
+ </mrow>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>y</mi>
+ <mo>)</mo>
+ </mrow>
+ </mstyle>
+ <!-- <annotation encoding="text/x-asciimath">forall x,y in T, T."byteOrdered"(x) " is not a prefix of " T."byteOrdered"(y)</annotation> -->
+ </semantics>
+ </math>
+
+The former is the essential requirement, and the latter allows construction of encodings of sequences of multiple
+values, as well as a little more efficiency in the data structures.
+
+To more efficiently encode byte-ordered blobs, however, we use a slightly tweaked version of the above requirements:
+
+- Comparison equivalence (3):
+ <math xmlns="http://www.w3.org/1998/Math/MathML">
+ <semantics>
+ <mstyle displaystyle="true">
+ <mo>∀</mo>
+ <mi>x</mi>
+ <mo>,</mo>
+ <mi>y</mi>
+ <mo>∈</mo>
+ <mi>T</mi>
+ <mo>,</mo>
+ <mo>∀</mo>
+ <msub>
+ <mi>b</mi>
+ <mn>1</mn>
+ </msub>
+ <mo>,</mo>
+ <msub>
+ <mi>b</mi>
+ <mn>2</mn>
+ </msub>
+ <mo>∈</mo>
+ <mrow>
+ <mo>[</mo>
+ <mn>0x10</mn>
+ <mo>-</mo>
+ <mn>0xEF</mn>
+ <mo>]</mo>
+ </mrow>
+ <mo>,</mo>
+ <mtext><br/></mtext>
+ <mrow>
+ <mtext>compareBytesUnsigned</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>x</mi>
+ <mo>)</mo>
+ </mrow>
+ <mo>+</mo>
+ <msub>
+ <mi>b</mi>
+ <mn>1</mn>
+ </msub>
+ <mo>,</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>y</mi>
+ <mo>)</mo>
+ </mrow>
+ <mo>+</mo>
+ <msub>
+ <mi>b</mi>
+ <mn>2</mn>
+ </msub>
+ <mo>)</mo>
+ </mrow>
+ <mo>=</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>compare</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>x</mi>
+ <mo>,</mo>
+ <mi>y</mi>
+ <mo>)</mo>
+ </mrow>
+ </mstyle>
+ <!-- <annotation encoding="text/x-asciimath">forall x,y in T, forall b_1, b_2 in [0x10-0xEF],
+ "compareBytesUnsigned"(T."byteOrdered"(x)+b_1, T."byteOrdered"(y)+b_2)=T."compare"(x, y)</annotation> -->
+ </semantics>
+ </math>
+- Weak prefix-freedom (4):
+ <math xmlns="http://www.w3.org/1998/Math/MathML">
+ <semantics>
+ <mstyle displaystyle="true">
+ <mo>∀</mo>
+ <mi>x</mi>
+ <mo>,</mo>
+ <mi>y</mi>
+ <mo>∈</mo>
+ <mi>T</mi>
+ <mo>,</mo>
+ <mo>∀</mo>
+ <mi>b</mi>
+ <mo>∈</mo>
+ <mrow>
+ <mo>[</mo>
+ <mn>0x10</mn>
+ <mo>-</mo>
+ <mn>0xEF</mn>
+ <mo>]</mo>
+ </mrow>
+ <mo>,</mo>
+ <mtext><br/></mtext>
+ <mrow>
+ <mo>(</mo>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>x</mi>
+ <mo>)</mo>
+ </mrow>
+ <mo>+</mo>
+ <mi>b</mi>
+ <mo>)</mo>
+ </mrow>
+ <mrow>
+ <mspace width="1ex" />
+ <mtext> is not a prefix of </mtext>
+ <mspace width="1ex" />
+ </mrow>
+ <mi>T</mi>
+ <mo>.</mo>
+ <mrow>
+ <mtext>byteOrdered</mtext>
+ </mrow>
+ <mrow>
+ <mo>(</mo>
+ <mi>y</mi>
+ <mo>)</mo>
+ </mrow>
+ </mstyle>
+ <!-- <annotation encoding="text/x-asciimath">forall x,y in T, forall b in [0x10-0xEF],
+ (T."byteOrdered"(x)+b) " is not a prefix of " T."byteOrdered"(y)</annotation> -->
+ </semantics>
+ </math>
+
+These versions allow the addition of a separator byte after each value, and guarantee that the combination with
+separator fulfills the original requirements. (3) is somewhat stronger than (1) but is necessarily true if (2) is also
+in force, while (4) trivially follows from (2).
+
+## Fixed length unsigned integers (Murmur token, date/time)
+
+This is the trivial case, as we can simply use the input bytes in big-endian order. The comparison result is the same,
+and fixed length values are trivially prefix free, i.e. (1) and (2) are satisfied, and thus (3) and (4) follow from the
+observation above.
+
+## Fixed-length signed integers (byte, short, int, legacy bigint)
+
+As above, but we need to invert the sign bit of the number to put negative numbers before positives. This maps
+`MIN_VALUE` to `0x00`..., `-1` to `0x7F…`, `0` to `0x80…`, and `MAX_VALUE` to `0xFF…`; comparing the resulting number
+as an unsigned integer has the same effect as comparing the source signed.
+
+Examples:
+
+
+| Type and value | bytes | encodes as |
+| -------------- | ----------------------- | ----------------------- |
+| int 1 | 00 00 00 01 | 80 00 00 01 |
+| short -1 | FF FF | 7F FF |
+| byte 0 | 00 | 80 |
+| byte -2 | FE | 7E |
+| int MAX_VALUE | 7F FF FF FF | FF FF FF FF |
+| long MIN_VALUE | 80 00 00 00 00 00 00 00 | 00 00 00 00 00 00 00 00 |
+
+## Variable-length encoding of integers (current bigint)
+
+Another way to encode integers that may save significant amounts of space when smaller numbers are often in use, but
+still permits large values to be efficiently encoded, is to use an encoding scheme similar to UTF-8.
+
+For unsigned numbers this can be done by starting the number with as many 1s in most significant bits as there are
+additional bytes in the encoding, followed by a 0, and the bits of the number. Numbers between 0 and 127 are encoded
+in one byte, and each additional byte adds 7 more bits. Values that use all 8 bytes do not need a 9th bit of 0 and can
+thus fit 9 bytes. Because longer numbers have more 1s in their MSBs, they compare
+higher than shorter ones (and we always use the shortest representation). Because the length is specified through these
+initial bits, no value can be a prefix of another.
+
+
+| Value | bytes | encodes as |
+| ---------------- | ----------------------- | -------------------------- |
+| 0 | 00 00 00 00 00 00 00 00 | 00 |
+| 1 | 00 00 00 00 00 00 00 01 | 01 |
+| 127 (2^7-1) | 00 00 00 00 00 00 00 7F | 7F |
+| 128 (2^7) | 00 00 00 00 00 00 00 80 | 80 80 |
+| 16383 (2^14 - 1) | 00 00 00 00 00 00 3F FF | BF FF |
+| 16384 (2^14) | 00 00 00 00 00 00 40 00 | C0 40 00 |
+| 2^31 - 1 | 00 00 00 00 7F FF FF FF | F0 7F FF FF FF |
+| 2^31 | 00 00 00 00 80 00 00 00 | F0 80 00 00 00 |
+| 2^56 - 1 | 00 FF FF FF FF FF FF FF | FE FF FF FF FF FF FF FF |
+| 2^56 | 01 00 00 00 00 00 00 00 | FF 01 00 00 00 00 00 00 00 |
+| 2^64- 1 | FF FF FF FF FF FF FF FF | FF FF FF FF FF FF FF FF FF |
+
+To encode signed numbers, we must start with the sign bit, and must also ensure that longer negative numbers sort
+smaller than shorter ones. The first bit of the encoding is the inverted sign (i.e. 1 for positive, 0 for negative),
+followed by the length encoded as a sequence of bits that matches the inverted sign, followed by a bit that differs
+(like above, not necessary for 9-byte encodings) and the bits of the number's two's complement.
+
+
+| Value | bytes | encodes as |
+| ----------------- | ----------------------- | -------------------------- |
+| 1 | 00 00 00 00 00 00 00 01 | 81 |
+| -1 | FF FF FF FF FF FF FF FF | 7F |
+| 0 | 00 00 00 00 00 00 00 00 | 80 |
+| 63 | 00 00 00 00 00 00 00 3F | BF |
+| -64 | FF FF FF FF FF FF FF C0 | 40 |
+| 64 | 00 00 00 00 00 00 00 40 | C0 40 |
+| -65 | FF FF FF FF FF FF FF BF | 3F BF |
+| 8191 | 00 00 00 00 00 00 1F FF | DF FF |
+| 8192 | 00 00 00 00 00 00 20 00 | E0 20 00 |
+| Integer.MAX_VALUE | 00 00 00 00 7F FF FF FF | F8 7F FF FF FF |
+| Long.MIN_VALUE | 80 00 00 00 00 00 00 00 | 00 00 00 00 00 00 00 00 00 |
+
+## Fixed-size floating-point numbers (float, double)
+
+IEEE-754 was designed with byte-by-byte comparisons in mind, and provides an important guarantee about the bytes of a
+floating point number:
+* If x and y are of the same sign, bytes(x) ≥ bytes(y) ⇔ |x| ≥ |y|.
+
+Thus, to be able to order floating point numbers as unsigned integers, we can:
+
+* Flip the sign bit so negatives are smaller than positive numbers.
+* If the number was negative, also flip all the other bits so larger magnitudes become smaller integers.
+
+This matches exactly the behaviour of `Double.compare`, which doesn’t fully agree with numerical comparisons (see spec)
+in order to define a natural order over the floating point numbers.
+
+Examples:
+
+
+| Type and value | bytes | encodes as |
+| -------------- | ----------------------- | ----------------------- |
+| float +1.0 | 3F 80 00 00 | BF 80 00 00 |
+| float +0.0 | 00 00 00 00 | 80 00 00 00 |
+| float -0.0 | 80 00 00 00 | 7F FF FF FF |
+| float -1.0 | BF 80 00 00 | 40 7F FF FF |
+| double +1.0 | 3F F0 00 00 00 00 00 00 | BF F0 00 00 00 00 00 00 |
+| double +Inf | 7F F0 00 00 00 00 00 00 | FF F0 00 00 00 00 00 00 |
+| double -Inf | FF F0 00 00 00 00 00 00 | 00 0F FF FF FF FF FF FF |
+| double NaN | 7F F8 00 00 00 00 00 00 | FF F8 00 00 00 00 00 00 |
+
+## UUIDs
+
+UUIDs are fixed-length unsigned integers, where the UUID version/type is compared first, and where bits need to be
+reordered for the time UUIDs. To create a byte-ordered representation, we reorder the bytes: pull the version digit
+first, then the rest of the digits, using the special time order if the version is equal to one.
+
+Examples:
+
+
+| Type and value | bytes | encodes as |
+| -------------- | ------------------------------------ | -------------------------------- |
+| Random (v4) | cc520882-9507-44fb-8fc9-b349ecdee658 | 4cc52088295074fb8fc9b349ecdee658 |
+| Time (v1) | 2a92d750-d8dc-11e6-a2de-cf8ecd4cf053 | 11e6d8dc2a92d750a2decf8ecd4cf053 |
+
+## Multi-component sequences (Partition or Clustering keys, tuples), bounds and nulls
+
+As mentioned above, we encode sequences by adding separator bytes in front, between components, and a terminator at the
+end. The values we chose for the separator and terminator are `0x40` and `0x38`, and they serve several purposes:
+
+- Permits partially specified bounds, with strict/exclusive or non-strict/inclusive semantics. This is done by finishing
+ a bound with a terminator value that is smaller/greater than the separator and terminator. We can use `0x20` for `<`/`≥`
+ and `0x60` for `≤`/`>`.
+- Permits encoding of `null` and `empty` values. We use `0x3E` as the separator for nulls and `0x3F` for empty,
+ followed by no value bytes. This is always smaller than a sequence with non-null value for this component, but not
+ smaller than a sequence that ends in this component.
+- Helps identify the ending of variable-length components (see below).
+
+Examples:
+
+
+| Types and values | bytes | encodes as |
+| ------------------------ | ---------------------- | ------------------------------ |
+| (short 1, float 1.0) | 00 01, 3F 80 00 00 | 40·80 01·40·BF 80 00 00·38 |
+| (short -1, null) | FF FF, — | 40·7F FF·3E·38 |
+| ≥ (short 0, float -Inf) | 00 00, FF 80 00 00, >= | 40·80 00·40·00 7F FF FF·20 |
+| < (short MIN) | 80 00, <= | 40·00 00·20 |
+| \> (null) | | 3E·60 |
+| BOTTOM | | 20 |
+| TOP | | 60 |
+
+(The middle dot · doesn't exist in the encoding, it’s just a visualisation of the boundaries in the examples.)
+
+Since:
+
+- all separators in use are within `0x10`-`0xEF`, and
+- we use the same separator for internal components, with the exception of nulls which we encode with a smaller
+ separator
+- the sequence has a fixed number of components or we use a different trailing value whenever it can be shorter
+
+the properties (3) and (4) guarantee that the byte comparison of the encoding goes in the same direction as the
+lexicographical comparison of the sequence. In combination with the third point above, (4) also ensures that no encoding
+is a prefix of another. Since we have (1) and (2), (3) and (4) are also satisfied.
+
+Note that this means that the encodings of all partition and clustering keys used in the database will be prefix-free.
+
+## Variable-length byte comparables (ASCII, UTF-8 strings, blobs, InetAddress)
+
+In isolation, these can be compared directly without reinterpretation. However, once we place these inside a flattened
+sequence of values we need to clearly define the boundaries between values while maintaining order. To do this we use an
+end-of-value marker; since shorter values must be smaller than longer, this marker must be 0 and we need to find a way
+to encode/escape actual 0s in the input sequence.
+
+The method we chose for this is the following:
+
+- If the input does not end on `00`, a `00` byte is appended at the end.
+- If the input contains a `00` byte, it is encoded as `00 FF`.
+- If the input contains a sequence of *n* `00` bytes, they are encoded as `00` `FE` (*n*-1 times) `FF`
+ (so that we don’t double the size of `00` blobs).
+- If the input ends in `00`, the last `FF` is changed to `FE`
+ (to ensure it’s smaller than the same value with `00` appended).
+
+Examples:
+
+
+| bytes/sequence | encodes as |
+| ------------------ | ------------------------ |
+| 22 00 | 22 00 FE |
+| 22 00 00 33 | 22 00 FE FF 33 00 |
+| 22 00 11 | 22 00 FF 11 00 |
+| (blob 22, short 0) | 40·22 00·40·80 00·40 |
+| ≥ (blob 22 00) | 40·22 00 FE·20 |
+| ≤ (blob 22 00 00) | 40·22 00 FE FE·60 |
+
+Within the encoding, a `00` byte can only be followed by a `FE` or `FF` byte, and hence if an encoding is a prefix of
+another, the latter has to have a `FE` or `FF` as the next byte, which ensures both (4) (adding `10`-`EF` to the former
+makes it no longer a prefix of the latter) and (3) (adding `10`-`EF` to the former makes it smaller than the latter; in
+this case the original value of the former is a prefix of the original value of the latter).
+
+## Variable-length integers (varint, RandomPartitioner token), legacy encoding
+
+If integers of unbounded length are guaranteed to start with a non-zero digit, to compare them we can first use a signed
+length, as numbers with longer representations have higher magnitudes. Only if the lengths match we need to compare the
+sequence of digits, which now has a known length.
+
+(Note: The meaning of “digit” here is not the same as “decimal digit”. We operate with numbers stored as bytes, thus it
+makes most sense to treat the numbers as encoded in base-256, where each digit is a byte.)
+
+This translates to the following encoding of varints:
+
+- Strip any leading zeros. Note that for negative numbers, `BigInteger` encodes leading 0 as `0xFF`.
+- If the length is 128 or greater, lead with a byte of `0xFF` (positive) or `0x00` (negative) for every 128 until there
+ are less than 128 left.
+- Encode the sign and (remaining) length of the number as a byte:
+ - `0x80 + (length - 1)` for positive numbers (so that greater magnitude is higher);
+ - `0x7F - (length - 1)` for negative numbers (so that greater magnitude is lower, and all negatives are lower than
+ positives).
+- Paste the bytes of the number, 2’s complement encoded for negative numbers (`BigInteger` already applies the 2’s
+ complement).
+
+Since when comparing two numbers we either have a difference in the length prefix, or the lengths are the same if we
+need to compare the content bytes, there is no risk that a longer number can be confused with a shorter combined in a
+multi-component sequence. In other words, no value can be a prefix of another, thus we have (1) and (2) and thus (3) and (4)
+as well.
+
+Examples:
+
+
+| value | bytes | encodes as |
+| ------: | ---------------- | ----------------------- |
+| 0 | 00 | 80·00 |
+| 1 | 01 | 80·01 |
+| -1 | FF | 7F·FF |
+| 255 | 00 FF | 80·FF |
+| -256 | FF 00 | 7F·00 |
+| 256 | 01 00 | 81·01 00 |
+| 2^16 | 01 00 00 | 82·01 00 00 |
+| -2^32 | FF 00 00 00 00 | 7C·00 00 00 00 |
+| 2^1024 | 01 00(128 times) | FF 80·01 00(128 times) |
+| -2^2048 | FF 00(256 times) | 00 00 80·00(256 times) |
+
+(Middle dot · shows the transition point between length and digits.)
+
+## Variable-length integers, current encoding
+
+Because variable-length integers are also often used to store smaller range integers, it makes sense to also apply
+the variable-length integer encoding. Thus, the current varint scheme chooses to:
+
+- Strip any leading zeros. Note that for negative numbers, `BigInteger` encodes leading 0 as `0xFF`.
+- Map numbers directly to their [variable-length integer encoding](#variable-length-encoding-of-integers-current-bigint),
+ if they have 6 bytes or less.
+- Otherwise, encode as:
+ - a sign byte (00 for negative numbers, FF for positive, distinct from the leading byte of the variable-length
+ encoding above)
+ - a variable-length encoded number of bytes adjusted by -7 (so that the smallest length this encoding uses maps to
+ 0), inverted for negative numbers (so that greater length compares smaller)
+ - the bytes of the number, two's complement encoded.
+ We never use a longer encoding (e.g. using the second method if variable-length suffices or with added 00 leading
+ bytes) if a shorter one suffices.
+
+By the same reasoning as above, and the fact that the sign byte cannot be confused with a variable-length encoding
+first byte, no value can be a prefix of another. As the sign byte compares smaller for negative (respectively bigger
+for positive numbers) than any variable-length encoded integer, the comparison order is maintained when one number
+uses variable-length encoding, and the other doesn't. Longer numbers compare smaller when negative (because of the
+inverted length bytes), and bigger when positive.
+
+Examples:
+
+
+| value | bytes | encodes as |
+| ------: | ----------------------- | ------------------------------- |
+| 0 | 00 | 80 |
+| 1 | 01 | 81 |
+| -1 | FF | 7F |
+| 255 | 00 FF | C0 FF |
+| -256 | FF 00 | 3F 00 |
+| 256 | 01 00 | C1 00 |
+| 2^16 | 01 00 00 | E1 00 00 |
+| -2^32 | FF 00 00 00 00 | 07 00 00 00 00 |
+| 2^56-1 | 00 FF FF FF FF FF FF FF | FE FF FF FF FF FF FF FF |
+| -2^56 | FF 00 00 00 00 00 00 00 | 01 00 00 00 00 00 00 00 |
+| 2^56 | 01 00 00 00 00 00 00 00 | FF·00·01 00 00 00 00 00 00 00 |
+| -2^56-1 | FE FF FF FF FF FF FF FF | 00·FF·FE FF FF FF FF FF FF FF |
+| 2^1024 | 01 00(128 times) | FF·7A·01 00(128 times) |
+| -2^2048 | FF 00(256 times) | 00·7F 06·00(256 times) |
+
+(Middle dot · shows the transition point between length and digits.)
+
+## Variable-length floating-point decimals (decimal)
+
+Variable-length floats are more complicated, but we can treat them similarly to IEEE-754 floating point numbers, by
+normalizing them by splitting them into sign, mantissa and signed exponent such that the mantissa is a number below 1
+with a non-zero leading digit. We can then compare sign, exponent and mantissa in sequence (where the comparison of
+exponent and mantissa are with reversed meaning if the sign is negative) and that gives us the decimal ordering.
+
+A bit of extra care must be exercised when encoding decimals. Since fractions like `0.1` cannot be perfectly encoded in
+binary, decimals (and mantissas) cannot be encoded in binary or base-256 correctly. A decimal base must be used; since
+we deal with bytes, it makes most sense to make things a little more efficient by using base-100. Floating-point
+encoding and the comparison idea from the previous paragraph work in any number base.
+
+`BigDecimal` presents a further challenge, as it encodes decimals using a mixture of bases: numbers have a binary-
+encoded integer part and a decimal power-of-ten scale. The bytes produced by a `BigDecimal` are thus not suitable for
+direct conversion to byte comparable and we must first instantiate the bytes as a `BigDecimal`, and then apply the
+class’s methods to operate on it as a number.
+
+We then use the following encoding:
+
+- If the number is 0, the encoding is a single `0x80` byte.
+- Convert the input to signed mantissa and signed exponent in base-100. If the value is negative, invert the sign of the
+ exponent to form the "modulated exponent".
+- Output a byte encoding:
+ - the sign of the number encoded as `0x80` if positive and `0x00` if negative,
+ - the exponent length (stripping leading 0s) in bytes as `0x40 + modulated_exponent_length`, where the length is given
+ with the sign of the modulated exponent.
+- Output `exponent_length` bytes of modulated exponent, 2’s complement encoded so that negative values are correctly
+ ordered.
+- Output `0x80 + leading signed byte of mantissa`, which is obtained by multiplying the mantissa by 100 and rounding to
+ -∞. The rounding is done so that the remainder of the mantissa becomes positive, and thus every new byte adds some
+ value to it, making shorter sequences lower in value.
+- Update the mantissa to be the remainder after the rounding above. The result is guaranteed to be 0 or greater.
+- While the mantissa is non-zero, output `0x80 + leading byte` as above and update the mantissa to be the remainder.
+- Output `0x00`.
+
+As a description of how this produces the correct ordering, consider the result of comparison in the first differing
+byte:
+
+- Difference in the first byte can be caused by:
+ - Difference in sign of the number or being zero, which yields the correct ordering because
+ - Negative numbers start with `0x3c` - `0x44`
+ - Zero starts with `0x80`
+ - Positive numbers start with `0xbc` - `0xc4`
+ - Difference in sign of the exponent modulated with the sign of the number. In a positive number negative exponents
+ mean smaller values, while in a negative number it’s the opposite, thus the modulation with the number’s sign
+ ensures the correct ordering.
+ - Difference in modulated length of the exponent: again, since we gave the length a sign that is formed from both
+ the sign of the exponent and the sign of the number, smaller numbers mean smaller exponent in the positive number
+ case, and bigger exponent in the negative number case. In either case this provides the correct ordering.
+- Difference in one of the bytes of the modulated exponent (whose length and sign are now equal for both compared
+ numbers):
+ - Smaller byte means a smaller modulated exponent. In the positive case this means a smaller exponent, thus a smaller
+ number. In the negative case this means the exponent is bigger, the absolute value of the number as well, and thus
+ the number is smaller.
+- It is not possible for the difference to mix one number’s exponent with another’s mantissa (as such numbers would have
+ different leading bytes).
+- Difference in a mantissa byte present in both inputs:
+ - Smaller byte means smaller signed mantissa and hence smaller number when the exponents are equal.
+- One mantissa ending before another:
+ - This will result in the shorter being treated as smaller (since the trailing byte is `00`).
+ - Since all mantissas have at least one byte, this can’t happen in the leading mantissa byte.
+ - Thus the other number’s bytes from here on are not negative, and at least one of them must be non-zero, which means
+ its mantissa is bigger and thus it encodes a bigger number.
+
+Examples:
+
+
+| value | mexp | mantissa | mantissa in bytes | encodes as |
+| ---------: | ----: | -------- | ----------------- | -------------------- |
+| 1.1 | 1 | 0.0110 | . 01 10 | C1·01·81 8A·00 |
+| 1 | 1 | 0.01 | . 01 | C1·01·81·00 |
+| 0.01 | 0 | 0.01 | . 01 | C0·81·00 |
+| 0 | | | | 80 |
+| -0.01 | 0 | -0.01 | . -01 | 40·81·00 |
+| -1 | -1 | -0.01 | . -01 | 3F·FF·7F·00 |
+| -1.1 | -1 | -0.0110 | . -02 90 | 3F·FF·7E DA·00 |
+| -98.9 | -1 | -0.9890 | . -99 10 | 3F·FF·1D 8A·00 |
+| -99 | -1 | -0.99 | . -99 | 3F·FF·1D·00 |
+| -99.9 | -1 | -0.9990 | .-100 10 | 3F·FF·1C 8A·00 |
+| -8.1e2000 | -1001 | -0.0810 | . -09 90 | 3E·FC 17·77 DA·00 |
+| -8.1e-2000 | 999 | -0.0810 | . -09 90 | 42·03 E7·77 DA·00 |
+| 8.1e-2000 | -999 | 0.0810 | . 08 10 | BE·FC 19·88 8A·00 |
+| 8.1e2000 | 1001 | 0.0810 | . 08 10 | C2·03 E9·88 8A·00 |
+
+(mexp stands for “modulated exponent”, i.e. exponent * sign)
+
+The values are prefix-free, because no exponent’s encoding can be a prefix of another, and the mantissas can never have
+a `00` byte at any place other than the last byte, and thus all (1)-(4) are satisfied.
+
+## Nulls and empty encodings
+
+Some types in Cassandra (e.g. numbers) admit null values that are represented as empty byte buffers. This is
+distinct from null byte buffers, which can also appear in some cases. Particularly, null values in clustering
+columns, when allowed by the type, are interpreted as empty byte buffers, encoded with the empty separator `0x3F`.
+Unspecified clustering columns (at the end of a clustering specification), possible with `COMPACT STORAGE` or secondary
+indexes, use the null separator `0x3E`.
+
+## Reversed types
+
+Reversing a type is straightforward: flip all bits of the encoded byte sequence. Since the source type encoding must
+satisfy (3) and (4), the flipped bits also do for the reversed comparator. (It is also true that if the source type
+satisfies (1)-(2), the reversed will satisfy these too.)
+
+In a sequence we also must correct the empty encoding for a reversed type (since it must be greater than all values).
+Instead of `0x3F` we use `0x41` as the separator byte. Null encodings are not modified, as nulls compare smaller even
+in reversed types.
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java
new file mode 100644
index 0000000..83bb828
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java
@@ -0,0 +1,857 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * A stream of bytes, used for byte-order-comparable representations of data, and utilities to convert various values
+ * to their byte-ordered translation.
+ * See ByteComparable.md for details about the encoding scheme.
+ */
+public interface ByteSource
+{
+ /** Consume the next byte, unsigned. Must be between 0 and 255, or END_OF_STREAM if there are no more bytes. */
+ int next();
+
+ /** Value returned if at the end of the stream. */
+ int END_OF_STREAM = -1;
+
+ ByteSource EMPTY = () -> END_OF_STREAM;
+
+ /**
+ * Escape value. Used, among other things, to mark the end of subcomponents (so that shorter compares before anything longer).
+ * Actual zeros in input need to be escaped if this is in use (see {@link AbstractEscaper}).
+ */
+ int ESCAPE = 0x00;
+
+ // Zeros are encoded as a sequence of ESCAPE, 0 or more of ESCAPED_0_CONT, ESCAPED_0_DONE so zeroed spaces only grow by 1 byte
+ int ESCAPED_0_CONT = 0xFE;
+ int ESCAPED_0_DONE = 0xFF;
+
+ // All separators must be within these bounds
+ int MIN_SEPARATOR = 0x10;
+ int MAX_SEPARATOR = 0xEF;
+
+ // Next component marker.
+ int NEXT_COMPONENT = 0x40;
+ // Marker used to present null values represented by empty buffers (e.g. by Int32Type)
+ int NEXT_COMPONENT_EMPTY = 0x3F;
+ int NEXT_COMPONENT_EMPTY_REVERSED = 0x41;
+ // Marker for null components in tuples, maps, sets and clustering keys.
+ int NEXT_COMPONENT_NULL = 0x3E;
+
+ // Section for next component markers which is not allowed for use
+ int MIN_NEXT_COMPONENT = 0x3C;
+ int MAX_NEXT_COMPONENT = 0x44;
+
+ // Default terminator byte in sequences. Smaller than NEXT_COMPONENT_NULL, but larger than LT_NEXT_COMPONENT to
+ // ensure lexicographic compares go in the correct direction
+ int TERMINATOR = 0x38;
+ // These are special endings, for exclusive/inclusive bounds (i.e. smaller than anything with more components,
+ // bigger than anything with more components)
+ int LT_NEXT_COMPONENT = 0x20;
+ int GT_NEXT_COMPONENT = 0x60;
+
+ // Unsupported, for artificial bounds
+ int LTLT_NEXT_COMPONENT = 0x1F; // LT_NEXT_COMPONENT - 1
+ int GTGT_NEXT_COMPONENT = 0x61; // GT_NEXT_COMPONENT + 1
+
+ // Special value for components that should be excluded from the normal min/max span. (static rows)
+ int EXCLUDED = 0x18;
+
+ /**
+ * Encodes byte-accessible data as a byte-comparable source that has 0s escaped and finishes in an escaped
+ * state.
+ * This provides a weakly-prefix-free byte-comparable version of the content to use in sequences.
+ * (See {@link AbstractEscaper} for a detailed explanation.)
+ */
+ static <V> ByteSource of(ValueAccessor<V> accessor, V data, Version version)
+ {
+ return new AccessorEscaper<>(accessor, data, version);
+ }
+
+ /**
+ * Encodes a byte buffer as a byte-comparable source that has 0s escaped and finishes in an escape.
+ * This provides a weakly-prefix-free byte-comparable version of the content to use in sequences.
+ * (See ByteSource.BufferEscaper/Multi for explanation.)
+ */
+ static ByteSource of(ByteBuffer buf, Version version)
+ {
+ return new BufferEscaper(buf, version);
+ }
+
+ /**
+ * Encodes a byte array as a byte-comparable source that has 0s escaped and finishes in an escape.
+ * This provides a prefix-free byte-comparable version of the content to use in sequences.
+ * (See ByteSource.BufferEscaper/Multi for explanation.)
+ */
+ static ByteSource of(byte[] buf, Version version)
+ {
+ return new ArrayEscaper(buf, version);
+ }
+
+ /**
+ * Encodes a memory range as a byte-comparable source that has 0s escaped and finishes in an escape.
+ * This provides a weakly-prefix-free byte-comparable version of the content to use in sequences.
+ * (See ByteSource.BufferEscaper/Multi for explanation.)
+ */
+ static ByteSource ofMemory(long address, int length, ByteComparable.Version version)
+ {
+ return new MemoryEscaper(address, length, version);
+ }
+
+ /**
+ * Combines a chain of sources, turning their weak-prefix-free byte-comparable representation into the combination's
+ * prefix-free byte-comparable representation, with the included terminator character.
+ * For correctness, the terminator must be within MIN-MAX_SEPARATOR and outside the range reserved for
+ * NEXT_COMPONENT markers.
+ * Typically TERMINATOR, or LT/GT_NEXT_COMPONENT if used for partially specified bounds.
+ */
+ static ByteSource withTerminator(int terminator, ByteSource... srcs)
+ {
+ assert terminator >= MIN_SEPARATOR && terminator <= MAX_SEPARATOR;
+ assert terminator < MIN_NEXT_COMPONENT || terminator > MAX_NEXT_COMPONENT;
+ return new Multi(srcs, terminator);
+ }
+
+ /**
+ * As above, but permits any separator. The legacy format wasn't using weak prefix freedom and has some
+ * non-reversible transformations.
+ */
+ static ByteSource withTerminatorLegacy(int terminator, ByteSource... srcs)
+ {
+ return new Multi(srcs, terminator);
+ }
+
+ static ByteSource withTerminatorMaybeLegacy(Version version, int legacyTerminator, ByteSource... srcs)
+ {
+ return version == Version.LEGACY ? withTerminatorLegacy(legacyTerminator, srcs)
+ : withTerminator(TERMINATOR, srcs);
+ }
+
+ static ByteSource of(String s, Version version)
+ {
+ return new ArrayEscaper(s.getBytes(StandardCharsets.UTF_8), version);
+ }
+
+ static ByteSource of(long value)
+ {
+ return new Number(value ^ (1L<<63), 8);
+ }
+
+ static ByteSource of(int value)
+ {
+ return new Number(value ^ (1L<<31), 4);
+ }
+
+ /**
+ * Produce a source for a signed fixed-length number, also translating empty to null.
+ * The first byte has its sign bit inverted, and the rest are passed unchanged.
+ * Presumes that the length of the buffer is always either 0 or constant for the type, which permits decoding and
+ * ensures the representation is prefix-free.
+ */
+ static <V> ByteSource optionalSignedFixedLengthNumber(ValueAccessor<V> accessor, V data)
+ {
+ return !accessor.isEmpty(data) ? signedFixedLengthNumber(accessor, data) : null;
+ }
+
+ /**
+ * Produce a source for a signed fixed-length number.
+ * The first byte has its sign bit inverted, and the rest are passed unchanged.
+ * Presumes that the length of the buffer is always constant for the type.
+ */
+ static <V> ByteSource signedFixedLengthNumber(ValueAccessor<V> accessor, V data)
+ {
+ return new SignedFixedLengthNumber<>(accessor, data);
+ }
+
+ /**
+ * Produce a source for a signed fixed-length floating-point number, also translating empty to null.
+ * If sign bit is on, returns negated bytes. If not, add the sign bit value.
+ * (Sign of IEEE floats is the highest bit, the rest can be compared in magnitude by byte comparison.)
+ * Presumes that the length of the buffer is always either 0 or constant for the type, which permits decoding and
+ * ensures the representation is prefix-free.
+ */
+ static <V> ByteSource optionalSignedFixedLengthFloat(ValueAccessor<V> accessor, V data)
+ {
+ return !accessor.isEmpty(data) ? signedFixedLengthFloat(accessor, data) : null;
+ }
+
+ /**
+ * Produce a source for a signed fixed-length floating-point number.
+ * If sign bit is on, returns negated bytes. If not, add the sign bit value.
+ * (Sign of IEEE floats is the highest bit, the rest can be compared in magnitude by byte comparison.)
+ * Presumes that the length of the buffer is always constant for the type.
+ */
+ static <V> ByteSource signedFixedLengthFloat(ValueAccessor<V> accessor, V data)
+ {
+ return new SignedFixedLengthFloat<>(accessor, data);
+ }
+
+ /**
+ * Produce a source for a signed integer, stored using variable length encoding.
+ * The representation uses between 1 and 9 bytes, is prefix-free and compares
+ * correctly.
+ */
+ static ByteSource variableLengthInteger(long value)
+ {
+ return new VariableLengthInteger(value);
+ }
+
+ /**
+ * Returns a separator for two byte sources, i.e. something that is definitely > prevMax, and <= currMin, assuming
+ * prevMax < currMin.
+ * This returns the shortest prefix of currMin that is greater than prevMax.
+ */
+ public static ByteSource separatorPrefix(ByteSource prevMax, ByteSource currMin)
+ {
+ return new Separator(prevMax, currMin, true);
+ }
+
+ /**
+ * Returns a separator for two byte sources, i.e. something that is definitely > prevMax, and <= currMin, assuming
+ * prevMax < currMin.
+ * This is a source of length 1 longer than the common prefix of the two sources, with last byte one higher than the
+ * prevMax source.
+ */
+ public static ByteSource separatorGt(ByteSource prevMax, ByteSource currMin)
+ {
+ return new Separator(prevMax, currMin, false);
+ }
+
+ public static ByteSource oneByte(int i)
+ {
+ assert i >= 0 && i <= 0xFF : "Argument must be a valid unsigned byte.";
+ return new ByteSource()
+ {
+ boolean consumed = false;
+
+ @Override
+ public int next()
+ {
+ if (consumed)
+ return END_OF_STREAM;
+ consumed = true;
+ return i;
+ }
+ };
+ }
+
+ public static ByteSource cut(ByteSource src, int cutoff)
+ {
+ return new ByteSource()
+ {
+ int pos = 0;
+
+ @Override
+ public int next()
+ {
+ return pos++ < cutoff ? src.next() : END_OF_STREAM;
+ }
+ };
+ }
+
+ /**
+ * Wrap a ByteSource in a length-fixing facade.
+ *
+ * If the length of {@code src} is less than {@code cutoff}, then pad it on the right with {@code padding} until
+ * the overall length equals {@code cutoff}. If the length of {@code src} is greater than {@code cutoff}, then
+ * truncate {@code src} to that size. Effectively a noop if {@code src} happens to have length {@code cutoff}.
+ *
+ * @param src the input source to wrap
+ * @param cutoff the size of the source returned
+ * @param padding a padding byte (an int subject to a 0xFF mask)
+ */
+ public static ByteSource cutOrRightPad(ByteSource src, int cutoff, int padding)
+ {
+ return new ByteSource()
+ {
+ int pos = 0;
+
+ @Override
+ public int next()
+ {
+ if (pos++ >= cutoff)
+ {
+ return END_OF_STREAM;
+ }
+ int next = src.next();
+ return next == END_OF_STREAM ? padding : next;
+ }
+ };
+ }
+
+
+ /**
+ * Variable-length encoding. Escapes 0s as ESCAPE + zero or more ESCAPED_0_CONT + ESCAPED_0_DONE.
+ * If the source ends in 0, we use ESCAPED_0_CONT to make sure that the encoding remains smaller than that source
+ * with a further 0 at the end.
+ * Finishes in an escaped state (either with ESCAPE or ESCAPED_0_CONT), which in {@link Multi} is followed by
+ * a component separator between 0x10 and 0xFE.
+ *
+ * E.g. "A\0\0B" translates to 4100FEFF4200
+ * "A\0B\0" 4100FF4200FE (+00 for {@link Version#LEGACY})
+ * "A\0" 4100FE (+00 for {@link Version#LEGACY})
+ * "AB" 414200
+ *
+ * If in a single byte source, the bytes could be simply passed unchanged, but this would not allow us to
+ * combine components. This translation preserves order, and since the encoding for 0 is higher than the separator
+ * also makes sure shorter components are treated as smaller.
+ *
+ * The encoding is not prefix-free, since e.g. the encoding of "A" (4100) is a prefix of the encoding of "A\0"
+ * (4100FE), but the byte following the prefix is guaranteed to be FE or FF, which makes the encoding weakly
+ * prefix-free. Additionally, any such prefix sequence will compare smaller than the value to which it is a prefix,
+ * because any permitted separator byte will be smaller than the byte following the prefix.
+ */
+ abstract static class AbstractEscaper implements ByteSource
+ {
+ private final Version version;
+ private int bufpos;
+ private boolean escaped;
+
+ AbstractEscaper(int position, Version version)
+ {
+ this.bufpos = position;
+ this.version = version;
+ }
+
+ @Override
+ public final int next()
+ {
+ if (bufpos >= limit())
+ {
+ if (bufpos > limit())
+ return END_OF_STREAM;
+
+ ++bufpos;
+ if (escaped)
+ {
+ escaped = false;
+ if (version == Version.LEGACY)
+ --bufpos; // place an ESCAPE at the end of sequence ending in ESCAPE
+ return ESCAPED_0_CONT;
+ }
+ return ESCAPE;
+ }
+
+ int index = bufpos++;
+ int b = get(index) & 0xFF;
+ if (!escaped)
+ {
+ if (b == ESCAPE)
+ escaped = true;
+ return b;
+ }
+ else
+ {
+ if (b == ESCAPE)
+ return ESCAPED_0_CONT;
+ --bufpos;
+ escaped = false;
+ return ESCAPED_0_DONE;
+ }
+ }
+
+ protected abstract byte get(int index);
+
+ protected abstract int limit();
+ }
+
+ static class AccessorEscaper<V> extends AbstractEscaper
+ {
+ private final V data;
+ private final ValueAccessor<V> accessor;
+
+ private AccessorEscaper(ValueAccessor<V> accessor, V data, Version version)
+ {
+ super(0, version);
+ this.accessor = accessor;
+ this.data = data;
+ }
+
+ protected int limit()
+ {
+ return accessor.size(data);
+ }
+
+ protected byte get(int index)
+ {
+ return accessor.getByte(data, index);
+ }
+ }
+
+ static class BufferEscaper extends AbstractEscaper
+ {
+ private final ByteBuffer buf;
+
+ private BufferEscaper(ByteBuffer buf, Version version)
+ {
+ super(buf.position(), version);
+ this.buf = buf;
+ }
+
+ protected int limit()
+ {
+ return buf.limit();
+ }
+
+ protected byte get(int index)
+ {
+ return buf.get(index);
+ }
+ }
+
+ static class ArrayEscaper extends AbstractEscaper
+ {
+ private final byte[] buf;
+
+ private ArrayEscaper(byte[] buf, Version version)
+ {
+ super(0, version);
+ this.buf = buf;
+ }
+
+ @Override
+ protected byte get(int index)
+ {
+ return buf[index];
+ }
+
+ @Override
+ protected int limit()
+ {
+ return buf.length;
+ }
+ }
+
+ static class MemoryEscaper extends AbstractEscaper
+ {
+ private final long address;
+ private final int length;
+
+ MemoryEscaper(long address, int length, ByteComparable.Version version)
+ {
+ super(0, version);
+ this.address = address;
+ this.length = length;
+ }
+
+ protected byte get(int index)
+ {
+ return MemoryUtil.getByte(address + index);
+ }
+
+ protected int limit()
+ {
+ return length;
+ }
+ }
+
+ /**
+ * Fixed length signed number encoding. Inverts first bit (so that neg < pos), then just posts all bytes from the
+ * buffer. Assumes buffer is of correct length.
+ */
+ static class SignedFixedLengthNumber<V> implements ByteSource
+ {
+ private final ValueAccessor<V> accessor;
+ private final V data;
+ private int bufpos;
+
+ public SignedFixedLengthNumber(ValueAccessor<V> accessor, V data)
+ {
+ this.accessor = accessor;
+ this.data = data;
+ this.bufpos = 0;
+ }
+
+ @Override
+ public int next()
+ {
+ if (bufpos >= accessor.size(data))
+ return END_OF_STREAM;
+ int v = accessor.getByte(data, bufpos) & 0xFF;
+ if (bufpos == 0)
+ v ^= 0x80;
+ ++bufpos;
+ return v;
+ }
+ }
+
+ /**
+ * Variable-length encoding for unsigned integers.
+ * The encoding is similar to UTF-8 encoding.
+ * Numbers between 0 and 127 are encoded in one byte, using 0 in the most significant bit.
+ * Larger values have 1s in as many of the most significant bits as the number of additional bytes
+ * in the representation, followed by a 0. This ensures that longer numbers compare larger than shorter
+ * ones. Since we never use a longer representation than necessary, this implies numbers compare correctly.
+ * As the number of bytes is specified in the bits of the first, no value is a prefix of another.
+ */
+ static class VariableLengthUnsignedInteger implements ByteSource
+ {
+ private final long value;
+ private int pos = -1;
+
+ public VariableLengthUnsignedInteger(long value)
+ {
+ this.value = value;
+ }
+
+ @Override
+ public int next()
+ {
+ if (pos == -1)
+ {
+ int bitsMinusOne = 63 - (Long.numberOfLeadingZeros(value | 1)); // 0 to 63 (the | 1 is to make sure 0 maps to 0 (1 bit))
+ int bytesMinusOne = bitsMinusOne / 7;
+ int mask = -256 >> bytesMinusOne; // sequence of bytesMinusOne 1s in the most-significant bits
+ pos = bytesMinusOne * 8;
+ return (int) ((value >>> pos) | mask) & 0xFF;
+ }
+ pos -= 8;
+ if (pos < 0)
+ return END_OF_STREAM;
+ return (int) (value >>> pos) & 0xFF;
+ }
+ }
+
+ /**
+ * Variable-length encoding for signed integers.
+ * The encoding is based on the unsigned encoding above, where the first bit stored is the inverted sign,
+ * followed by as many matching bits as there are additional bytes in the encoding, followed by the two's
+ * complement of the number.
+ * Because of the inverted sign bit, negative numbers compare smaller than positives, and because the length
+ * bits match the sign, longer positive numbers compare greater and longer negative ones compare smaller.
+ *
+ * Examples:
+ * 0 encodes as 80
+ * 1 encodes as 81
+ * -1 encodes as 7F
+ * 63 encodes as BF
+ * 64 encodes as C040
+ * -64 encodes as 40
+ * -65 encodes as 3FBF
+ * 2^20-1 encodes as EFFFFF
+ * 2^20 encodes as F0100000
+ * -2^20 encodes as 100000
+ * 2^64-1 encodes as FFFFFFFFFFFFFFFFFF
+ * -2^64 encodes as 000000000000000000
+ *
+ * As the number of bytes is specified in bits 2-9, no value is a prefix of another.
+ */
+ static class VariableLengthInteger implements ByteSource
+ {
+ private final long value;
+ private int pos;
+
+ public VariableLengthInteger(long value)
+ {
+ long negativeMask = value >> 63; // -1 for negative, 0 for positive
+ value ^= negativeMask;
+
+ int bits = 64 - Long.numberOfLeadingZeros(value | 1); // 1 to 63 (can't be 64 because we flip negative numbers)
+ int bytes = bits / 7 + 1; // 0-6 bits 1 byte 7-13 2 bytes etc to 56-63 9 bytes
+ if (bytes >= 9)
+ {
+ value |= 0x8000000000000000L; // 8th bit, which doesn't fit the first byte
+ pos = negativeMask < 0 ? 256 : -1; // out of 0-64 range integer such that & 0xFF is 0x00 for negative and 0xFF for positive
+ }
+ else
+ {
+ long mask = (-0x100 >> bytes) & 0xFF; // one in sign bit and as many more as there are extra bytes
+ pos = bytes * 8;
+ value = value | (mask << (pos - 8));
+ }
+
+ value ^= negativeMask;
+ this.value = value;
+ }
+
+ @Override
+ public int next()
+ {
+ if (pos <= 0 || pos > 64)
+ {
+ if (pos == 0)
+ return END_OF_STREAM;
+ else
+ {
+ // 8-byte value, returning first byte
+ int result = pos & 0xFF; // 0x00 for negative numbers, 0xFF for positive
+ pos = 64;
+ return result;
+ }
+ }
+ pos -= 8;
+ return (int) (value >>> pos) & 0xFF;
+ }
+ }
+
+ static class Number implements ByteSource
+ {
+ private final long value;
+ private int pos;
+
+ public Number(long value, int length)
+ {
+ this.value = value;
+ this.pos = length;
+ }
+
+ @Override
+ public int next()
+ {
+ if (pos == 0)
+ return END_OF_STREAM;
+ return (int) ((value >> (--pos * 8)) & 0xFF);
+ }
+ }
+
+ /**
+ * Fixed length signed floating point number encoding. First bit is sign. If positive, add sign bit value to make
+ * greater than all negatives. If not, invert all content to make negatives with bigger magnitude smaller.
+ */
+ static class SignedFixedLengthFloat<V> implements ByteSource
+ {
+ private final ValueAccessor<V> accessor;
+ private final V data;
+ private int bufpos;
+ private boolean invert;
+
+ public SignedFixedLengthFloat(ValueAccessor<V> accessor, V data)
+ {
+ this.accessor = accessor;
+ this.data = data;
+ this.bufpos = 0;
+ }
+
+ @Override
+ public int next()
+ {
+ if (bufpos >= accessor.size(data))
+ return END_OF_STREAM;
+ int v = accessor.getByte(data, bufpos) & 0xFF;
+ if (bufpos == 0)
+ {
+ invert = v >= 0x80;
+ v |= 0x80;
+ }
+ if (invert)
+ v = v ^ 0xFF;
+ ++bufpos;
+ return v;
+ }
+ }
+
+ /**
+ * Combination of multiple byte sources. Adds {@link NEXT_COMPONENT} before sources, or {@link NEXT_COMPONENT_NULL} if next is null.
+ */
+ static class Multi implements ByteSource
+ {
+ private final ByteSource[] srcs;
+ private int srcnum = -1;
+ private final int sequenceTerminator;
+
+ Multi(ByteSource[] srcs, int sequenceTerminator)
+ {
+ this.srcs = srcs;
+ this.sequenceTerminator = sequenceTerminator;
+ }
+
+ @Override
+ public int next()
+ {
+ if (srcnum == srcs.length)
+ return END_OF_STREAM;
+
+ int b = END_OF_STREAM;
+ if (srcnum >= 0 && srcs[srcnum] != null)
+ b = srcs[srcnum].next();
+ if (b > END_OF_STREAM)
+ return b;
+
+ ++srcnum;
+ if (srcnum == srcs.length)
+ return sequenceTerminator;
+ if (srcs[srcnum] == null)
+ return NEXT_COMPONENT_NULL;
+ return NEXT_COMPONENT;
+ }
+ }
+
+ /**
+ * Construct the shortest common prefix of prevMax and currMin that separates those two byte streams.
+ * If {@code useCurr == true} the last byte of the returned stream comes from {@code currMin} and is the first
+ * byte which is greater than byte on the corresponding position of {@code prevMax}.
+ * Otherwise, the last byte of the returned stream comes from {@code prevMax} and is incremented by one, still
+ * guaranteeing that it is <= than the byte on the corresponding position of {@code currMin}.
+ */
+ static class Separator implements ByteSource
+ {
+ private final ByteSource prev;
+ private final ByteSource curr;
+ private boolean done = false;
+ private final boolean useCurr;
+
+ Separator(ByteSource prevMax, ByteSource currMin, boolean useCurr)
+ {
+ this.prev = prevMax;
+ this.curr = currMin;
+ this.useCurr = useCurr;
+ }
+
+ @Override
+ public int next()
+ {
+ if (done)
+ return END_OF_STREAM;
+ int p = prev.next();
+ int c = curr.next();
+ assert p <= c : prev + " not less than " + curr;
+ if (p == c)
+ return c;
+ done = true;
+ return useCurr ? c : p + 1;
+ }
+ }
+
+ static <V> ByteSource optionalFixedLength(ValueAccessor<V> accessor, V data)
+ {
+ return !accessor.isEmpty(data) ? fixedLength(accessor, data) : null;
+ }
+
+ /**
+ * A byte source of the given bytes without any encoding.
+ * The resulting source is only guaranteed to give correct comparison results and be prefix-free if the
+ * underlying type has a fixed length.
+ * In tests, this method is also used to generate non-escaped test cases.
+ */
+ public static <V> ByteSource fixedLength(ValueAccessor<V> accessor, V data)
+ {
+ return new ByteSource()
+ {
+ int pos = -1;
+
+ @Override
+ public int next()
+ {
+ return ++pos < accessor.size(data) ? accessor.getByte(data, pos) & 0xFF : END_OF_STREAM;
+ }
+ };
+ }
+
+ /**
+ * A byte source of the given bytes without any encoding.
+ * The resulting source is only guaranteed to give correct comparison results and be prefix-free if the
+ * underlying type has a fixed length.
+ * In tests, this method is also used to generate non-escaped test cases.
+ */
+ public static ByteSource fixedLength(ByteBuffer b)
+ {
+ return new ByteSource()
+ {
+ int pos = b.position() - 1;
+
+ @Override
+ public int next()
+ {
+ return ++pos < b.limit() ? b.get(pos) & 0xFF : END_OF_STREAM;
+ }
+ };
+ }
+
+ /**
+ * A byte source of the given bytes without any encoding.
+ * If used in a sequence, the resulting source is only guaranteed to give correct comparison results if the
+ * underlying type has a fixed length.
+ * In tests, this method is also used to generate non-escaped test cases.
+ */
+ public static ByteSource fixedLength(byte[] b)
+ {
+ return fixedLength(b, 0, b.length);
+ }
+
+ public static ByteSource fixedLength(byte[] b, int offset, int length)
+ {
+ checkArgument(offset >= 0 && offset <= b.length);
+ checkArgument(length >= 0 && offset + length <= b.length);
+
+ return new ByteSource()
+ {
+ int pos = offset - 1;
+
+ @Override
+ public int next()
+ {
+ return ++pos < offset + length ? b[pos] & 0xFF : END_OF_STREAM;
+ }
+ };
+ }
+
+ public class Peekable implements ByteSource
+ {
+ private static final int NONE = Integer.MIN_VALUE;
+
+ private final ByteSource wrapped;
+ private int peeked = NONE;
+
+ public Peekable(ByteSource wrapped)
+ {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public int next()
+ {
+ if (peeked != NONE)
+ {
+ int val = peeked;
+ peeked = NONE;
+ return val;
+ }
+ else
+ return wrapped.next();
+ }
+
+ public int peek()
+ {
+ if (peeked == NONE)
+ peeked = wrapped.next();
+ return peeked;
+ }
+ }
+
+ public static Peekable peekable(ByteSource p)
+ {
+ // When given a null source, we're better off not wrapping it and just returning null. This way existing
+ // code that doesn't know about ByteSource.Peekable, but handles correctly null ByteSources won't be thrown
+ // off by a non-null instance that semantically should have been null.
+ if (p == null)
+ return null;
+ return (p instanceof Peekable)
+ ? (Peekable) p
+ : new Peekable(p);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java
new file mode 100644
index 0000000..16b6679
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSourceInverse.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.marshal.ValueAccessor;
+
+/**
+ * Contains inverse transformation utilities for {@link ByteSource}s.
+ *
+ * See ByteComparable.md for details about the encoding scheme.
+ */
+public final class ByteSourceInverse
+{
+ private static final int INITIAL_BUFFER_CAPACITY = 32;
+ private static final int BYTE_ALL_BITS = 0xFF;
+ private static final int BYTE_NO_BITS = 0x00;
+ private static final int BYTE_SIGN_BIT = 1 << 7;
+ private static final int SHORT_SIGN_BIT = 1 << 15;
+ private static final int INT_SIGN_BIT = 1 << 31;
+ private static final long LONG_SIGN_BIT = 1L << 63;
+
+ /**
+ * Consume the given number of bytes and produce a long from them, effectively treating the bytes as a big-endian
+ * unsigned encoding of the number.
+ */
+ public static long getUnsignedFixedLengthAsLong(ByteSource byteSource, int length)
+ {
+ Preconditions.checkNotNull(byteSource);
+ Preconditions.checkArgument(length >= 1 && length <= 8, "Between 1 and 8 bytes can be read at a time");
+
+ long result = 0;
+ for (int i = 0; i < length; ++i)
+ result = (result << 8) | getAndCheckByte(byteSource, i, length); // note: this must use the unsigned byte value
+
+ return result;
+ }
+
+ /**
+ * Produce the bytes for an encoded signed fixed-length number.
+ * The first byte has its sign bit inverted, and the rest are passed unchanged.
+ */
+ public static <V> V getSignedFixedLength(ValueAccessor<V> accessor, ByteSource byteSource, int length)
+ {
+ Preconditions.checkNotNull(byteSource);
+ Preconditions.checkArgument(length >= 1, "At least 1 byte should be read");
+
+ V result = accessor.allocate(length);
+ // The first byte needs to have its sign flipped
+ accessor.putByte(result, 0, (byte) (getAndCheckByte(byteSource, 0, length) ^ BYTE_SIGN_BIT));
+ // and the rest can be retrieved unchanged.
+ for (int i = 1; i < length; ++i)
+ accessor.putByte(result, i, (byte) getAndCheckByte(byteSource, i, length));
+ return result;
+ }
+
+ /**
+ * Produce the bytes for an encoded signed fixed-length number, also translating null to empty buffer.
+ * The first byte has its sign bit inverted, and the rest are passed unchanged.
+ */
+ public static <V> V getOptionalSignedFixedLength(ValueAccessor<V> accessor, ByteSource byteSource, int length)
+ {
+ return byteSource == null ? accessor.empty() : getSignedFixedLength(accessor, byteSource, length);
+ }
+
+ /**
+ * Produce the bytes for an encoded signed fixed-length floating-point number.
+ * If sign bit is on, returns negated bytes. If not, clears the sign bit and passes the rest of the bytes unchanged.
+ */
+ public static <V> V getSignedFixedLengthFloat(ValueAccessor<V> accessor, ByteSource byteSource, int length)
+ {
+ Preconditions.checkNotNull(byteSource);
+ Preconditions.checkArgument(length >= 1, "At least 1 byte should be read");
+
+ V result = accessor.allocate(length);
+
+ int xor;
+ int first = getAndCheckByte(byteSource, 0, length);
+ if (first < 0x80)
+ {
+ // Negative number. Invert all bits.
+ xor = BYTE_ALL_BITS;
+ first ^= xor;
+ }
+ else
+ {
+ // Positive number. Invert only the sign bit.
+ xor = BYTE_NO_BITS;
+ first ^= BYTE_SIGN_BIT;
+ }
+ accessor.putByte(result, 0, (byte) first);
+
+ // xor is now applied to the rest of the bytes to flip their bits if necessary.
+ for (int i = 1; i < length; ++i)
+ accessor.putByte(result, i, (byte) (getAndCheckByte(byteSource, i, length) ^ xor));
+
+ return result;
+ }
+
+ /**
+ * Produce the bytes for an encoded signed fixed-length floating-point number, also translating null to an empty
+ * buffer.
+ * If sign bit is on, returns negated bytes. If not, clears the sign bit and passes the rest of the bytes unchanged.
+ */
+ public static <V> V getOptionalSignedFixedLengthFloat(ValueAccessor<V> accessor, ByteSource byteSource, int length)
+ {
+ return byteSource == null ? accessor.empty() : getSignedFixedLengthFloat(accessor, byteSource, length);
+ }
+
+ /**
+ * Consume the next length bytes from the source unchanged.
+ */
+ public static <V> V getFixedLength(ValueAccessor<V> accessor, ByteSource byteSource, int length)
+ {
+ Preconditions.checkNotNull(byteSource);
+ Preconditions.checkArgument(length >= 1, "At least 1 byte should be read");
+
+ V result = accessor.allocate(length);
+ for (int i = 0; i < length; ++i)
+ accessor.putByte(result, i, (byte) getAndCheckByte(byteSource, i, length));
+ return result;
+ }
+
+ /**
+ * Consume the next length bytes from the source unchanged, also translating null to an empty buffer.
+ */
+ public static <V> V getOptionalFixedLength(ValueAccessor<V> accessor, ByteSource byteSource, int length)
+ {
+ return byteSource == null ? accessor.empty() : getFixedLength(accessor, byteSource, length);
+ }
+
+ /**
+ * Consume the next {@code int} from the current position of the given {@link ByteSource}. The source position is
+ * modified accordingly (moved 4 bytes forward).
+ * <p>
+ * The source is not strictly required to represent just the encoding of an {@code int} value, so theoretically
+ * this API could be used for reading data in 4-byte strides. Nevertheless its usage is fairly limited because:
+ * <ol>
+ * <li>...it presupposes signed fixed-length encoding for the encoding of the original value</li>
+ * <li>...it decodes the data returned on each stride as an {@code int} (i.e. it inverts its leading bit)</li>
+ * <li>...it doesn't provide any meaningful guarantees (with regard to throwing) in case there are not enough
+ * bytes to read, in case a special escape value was not interpreted as such, etc.</li>
+ * </ol>
+ * </p>
+ *
+ * @param byteSource A non-null byte source, containing at least 4 bytes.
+ */
+ public static int getSignedInt(ByteSource byteSource)
+ {
+ return (int) getUnsignedFixedLengthAsLong(byteSource, 4) ^ INT_SIGN_BIT;
+ }
+
+ /**
+ * Consume the next {@code long} from the current position of the given {@link ByteSource}. The source position is
+ * modified accordingly (moved 8 bytes forward).
+ * <p>
+ * The source is not strictly required to represent just the encoding of a {@code long} value, so theoretically
+ * this API could be used for reading data in 8-byte strides. Nevertheless its usage is fairly limited because:
+ * <ol>
+ * <li>...it presupposes signed fixed-length encoding for the encoding of the original value</li>
+ * <li>...it decodes the data returned on each stride as a {@code long} (i.e. it inverts its leading bit)</li>
+ * <li>...it doesn't provide any meaningful guarantees (with regard to throwing) in case there are not enough
+ * bytes to read, in case a special escape value was not interpreted as such, etc.</li>
+ * </ol>
+ * </p>
+ *
+ * @param byteSource A non-null byte source, containing at least 8 bytes.
+ */
+ public static long getSignedLong(ByteSource byteSource)
+ {
+ return getUnsignedFixedLengthAsLong(byteSource, 8) ^ LONG_SIGN_BIT;
+ }
+
+ /**
+ * Converts the given {@link ByteSource} to a {@code byte}.
+ *
+ * @param byteSource A non-null byte source, containing at least 1 byte.
+ */
+ public static byte getSignedByte(ByteSource byteSource)
+ {
+ return (byte) (getAndCheckByte(Preconditions.checkNotNull(byteSource), 0, 1) ^ BYTE_SIGN_BIT);
+ }
+
+ /**
+ * Converts the given {@link ByteSource} to a {@code short}. All terms and conditions valid for
+ * {@link #getSignedInt(ByteSource)} and {@link #getSignedLong(ByteSource)} translate to this as well.
+ *
+ * @param byteSource A non-null byte source, containing at least 2 bytes.
+ *
+ * @see #getSignedInt(ByteSource)
+ * @see #getSignedLong(ByteSource)
+ */
+ public static short getSignedShort(ByteSource byteSource)
+ {
+ return (short) (getUnsignedFixedLengthAsLong(byteSource, 2) ^ SHORT_SIGN_BIT);
+ }
+
+ /**
+ * Decode a variable-length signed integer.
+ */
+ public static long getVariableLengthInteger(ByteSource byteSource)
+ {
+ int signAndMask = getAndCheckByte(byteSource);
+
+ long sum = 0;
+ int bytes;
+ // For every bit after the sign that matches the sign, read one more byte.
+ for (bytes = 0; bytes < 7 && sameByteSign(signAndMask << (bytes + 1), signAndMask); ++bytes)
+ sum = (sum << 8) | getAndCheckByte(byteSource);
+
+ // The eighth length bit is stored in the second byte.
+ if (bytes == 7 && sameByteSign((int) (sum >> 48), signAndMask))
+ return ((sum << 8) | getAndCheckByte(byteSource)) ^ LONG_SIGN_BIT; // 9-byte encoding, use bytes 2-9 with inverted sign
+ else
+ {
+ sum |= (((long) signAndMask) << bytes * 8); // add the rest of the bits
+ long signMask = -0x40L << bytes * 7; // mask of the bits that should be replaced by the sign
+ long sign = (byte) (signAndMask ^ 0x80) >> 7; // -1 if negative (0 leading bit), 0 otherwise
+ return sum & ~signMask | sign & signMask;
+ }
+ }
+
+ /**
+ * Decode a variable-length unsigned integer, passing all bytes read through XOR with the given xorWith parameter.
+ *
+ * Used in BigInteger encoding to read number length, where negative numbers have their length negated
+ * (i.e. xorWith = 0xFF) to ensure correct ordering.
+ */
+ public static long getVariableLengthUnsignedIntegerXoring(ByteSource byteSource, int xorWith)
+ {
+ int signAndMask = getAndCheckByte(byteSource) ^ xorWith;
+
+ long sum = 0;
+ int bytes;
+ // Read an extra byte while the next most significant bit is 1.
+ for (bytes = 0; bytes <= 7 && ((signAndMask << bytes) & 0x80) != 0; ++bytes)
+ sum = (sum << 8) | getAndCheckByte(byteSource) ^ xorWith;
+
+ // Strip the length bits from the leading byte.
+ signAndMask &= ~(-256 >> bytes);
+ return sum | (((long) signAndMask) << bytes * 8); // Add the rest of the bits of the leading byte.
+ }
+
+ /** Returns true if the two parameters treated as bytes have the same sign. */
+ private static boolean sameByteSign(int a, int b)
+ {
+ return ((a ^ b) & 0x80) == 0;
+ }
+
+
+ private static int getAndCheckByte(ByteSource byteSource)
+ {
+ return getAndCheckByte(byteSource, -1, -1);
+ }
+
+ private static int getAndCheckByte(ByteSource byteSource, int pos, int length)
+ {
+ int data = byteSource.next();
+ if (data == ByteSource.END_OF_STREAM)
+ throw new IllegalArgumentException(
+ length > 0 ? String.format("Unexpected end of stream reached after %d bytes (expected >= %d)", pos, length)
+ : "Unexpected end of stream");
+ assert data >= BYTE_NO_BITS && data <= BYTE_ALL_BITS
+ : "A ByteSource must produce unsigned bytes and end in END_OF_STREAM";
+ return data;
+ }
+
+ /**
+ * Reads a single variable-length byte sequence (blob, string, ...) encoded according to the scheme described
+ * in ByteComparable.md, decoding it back to its original, unescaped form.
+ *
+ * @param byteSource The source of the variable-length bytes sequence.
+ * @return A byte array containing the original, unescaped bytes of the given source. Unescaped here means
+ * not including any of the escape sequences of the encoding scheme used for variable-length byte sequences.
+ */
+ public static byte[] getUnescapedBytes(ByteSource.Peekable byteSource)
+ {
+ return byteSource == null ? null : readBytes(unescape(byteSource));
+ }
+
+ /**
+ * As above, but converts the result to a ByteSource.
+ */
+ public static ByteSource unescape(ByteSource.Peekable byteSource)
+ {
+ return new ByteSource() {
+ boolean escaped = false;
+
+ @Override
+ public int next()
+ {
+ if (!escaped)
+ {
+ int data = byteSource.next(); // we consume this byte no matter what it is
+ if (data > ByteSource.ESCAPE)
+ return data; // most used path leads here
+
+ assert data != ByteSource.END_OF_STREAM : "Invalid escaped byte sequence";
+ escaped = true;
+ }
+
+ int next = byteSource.peek();
+ switch (next)
+ {
+ case END_OF_STREAM:
+ // The end of a byte-comparable outside of a multi-component sequence. No matter what we have
+ // seen or peeked before, we should stop now.
+ byteSource.next();
+ return END_OF_STREAM;
+ case ESCAPED_0_DONE:
+ // The end of 1 or more consecutive 0x00 value bytes.
+ escaped = false;
+ byteSource.next();
+ return ESCAPE;
+ case ESCAPED_0_CONT:
+ // Escaped sequence continues
+ byteSource.next();
+ return ESCAPE;
+ default:
+ // An ESCAPE or ESCAPED_0_CONT won't be followed by either another ESCAPED_0_CONT, an
+ // ESCAPED_0_DONE, or an END_OF_STREAM only when the byte-comparable is part of a multi-component
+ // sequence and we have reached the end of the encoded byte-comparable. In this case, the byte
+ // we have just peeked is the separator or terminator byte between or at the end of components
+ // (which by contact must be 0x10 - 0xFE, which cannot conflict with our special bytes).
+ assert next >= ByteSource.MIN_SEPARATOR && next <= ByteSource.MAX_SEPARATOR : next;
+ // Unlike above, we don't consume this byte (the sequence decoding needs it).
+ return END_OF_STREAM;
+ }
+ }
+ };
+ }
+
+ /**
+ * Reads the bytes of the given source into a byte array. Doesn't do any transformation on the bytes, just reads
+ * them until it reads an {@link ByteSource#END_OF_STREAM} byte, after which it returns an array of all the read
+ * bytes, <strong>excluding the {@link ByteSource#END_OF_STREAM}</strong>.
+ * <p>
+ * This method sizes a tentative internal buffer array at {@code initialBufferCapacity}. However, if
+ * {@code byteSource} exceeds this size, the buffer array is recreated with doubled capacity as many times as
+ * necessary. If, after {@code byteSource} is fully exhausted, the number of bytes read from it does not exactly
+ * match the current size of the tentative buffer array, then it is copied into another array sized to fit the
+ * number of bytes read; otherwise, it is returned without that final copy step.
+ *
+ * @param byteSource The source which bytes we're interested in.
+ * @param initialBufferCapacity The initial size of the internal buffer.
+ * @return A byte array containing exactly all the read bytes. In case of a {@code null} source, the returned byte
+ * array will be empty.
+ */
+ public static byte[] readBytes(ByteSource byteSource, final int initialBufferCapacity)
+ {
+ Preconditions.checkNotNull(byteSource);
+
+ int readBytes = 0;
+ byte[] buf = new byte[initialBufferCapacity];
+ int data;
+ while ((data = byteSource.next()) != ByteSource.END_OF_STREAM)
+ {
+ buf = ensureCapacity(buf, readBytes);
+ buf[readBytes++] = (byte) data;
+ }
+
+ if (readBytes != buf.length)
+ {
+ buf = Arrays.copyOf(buf, readBytes);
+ }
+ return buf;
+ }
+
+ /**
+ * Reads the bytes of the given source into a byte array. Doesn't do any transformation on the bytes, just reads
+ * them until it reads an {@link ByteSource#END_OF_STREAM} byte, after which it returns an array of all the read
+ * bytes, <strong>excluding the {@link ByteSource#END_OF_STREAM}</strong>.
+ * <p>
+ * This is equivalent to {@link #readBytes(ByteSource, int)} where the second actual parameter is
+ * {@linkplain #INITIAL_BUFFER_CAPACITY} ({@value INITIAL_BUFFER_CAPACITY}).
+ *
+ * @param byteSource The source which bytes we're interested in.
+ * @return A byte array containing exactly all the read bytes. In case of a {@code null} source, the returned byte
+ * array will be empty.
+ */
+ public static byte[] readBytes(ByteSource byteSource)
+ {
+ return readBytes(byteSource, INITIAL_BUFFER_CAPACITY);
+ }
+
+ /**
+ * Ensures the given buffer has capacity for taking data with the given length - if it doesn't, it returns a copy
+ * of the buffer, but with double the capacity.
+ */
+ private static byte[] ensureCapacity(byte[] buf, int dataLengthInBytes)
+ {
+ if (dataLengthInBytes == buf.length)
+ // We won't gain much with guarding against overflow. We'll overflow when dataLengthInBytes >= 1 << 30,
+ // and if we do guard, we'll be able to extend the capacity to Integer.MAX_VALUE (which is 1 << 31 - 1).
+ // Controlling the exception that will be thrown shouldn't matter that much, and in practice, we almost
+ // surely won't be reading gigabytes of ByteSource data at once.
+ return Arrays.copyOf(buf, dataLengthInBytes * 2);
+ else
+ return buf;
+ }
+
+ /**
+ * Converts the given {@link ByteSource} to a UTF-8 {@link String}.
+ *
+ * @param byteSource The source we're interested in.
+ * @return A UTF-8 string corresponding to the given source.
+ */
+ public static String getString(ByteSource.Peekable byteSource)
+ {
+ if (byteSource == null)
+ return null;
+
+ byte[] data = getUnescapedBytes(byteSource);
+
+ return new String(data, StandardCharsets.UTF_8);
+ }
+
+ /*
+ * Multi-component sequence utilities.
+ */
+
+ /**
+ * A utility for consuming components from a peekable multi-component sequence.
+ * It uses the component separators, so the given sequence needs to have its last component fully consumed, in
+ * order for the next consumable byte to be a separator. Identifying the end of the component that will then be
+ * consumed is the responsibility of the consumer (the user of this method).
+ * @param source A peekable multi-component sequence, which next byte is a component separator.
+ * @return the given multi-component sequence if its next component is not null, or {@code null} if it is.
+ */
+ public static ByteSource.Peekable nextComponentSource(ByteSource.Peekable source)
+ {
+ return nextComponentSource(source, source.next());
+ }
+
+ /**
+ * A utility for consuming components from a peekable multi-component sequence, very similar to
+ * {@link #nextComponentSource(ByteSource.Peekable)} - the difference being that here the separator can be passed
+ * in case it had to be consumed beforehand.
+ */
+ public static ByteSource.Peekable nextComponentSource(ByteSource.Peekable source, int separator)
+ {
+ return nextComponentNull(separator)
+ ? null
+ : source;
+ }
+
+ public static boolean nextComponentNull(int separator)
+ {
+ return separator == ByteSource.NEXT_COMPONENT_NULL || separator == ByteSource.NEXT_COMPONENT_EMPTY
+ || separator == ByteSource.NEXT_COMPONENT_EMPTY_REVERSED;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java b/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java
index 83cd7d3..0020e1a 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java
@@ -494,11 +494,11 @@
public String toString()
{
String description = description();
- String state = state();
+ String state = stateInfo();
return description == null ? state : (state + ' ' + description);
}
- private String state()
+ private String stateInfo()
{
Object result = this.result;
if (isSuccess(result))
diff --git a/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java b/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java
new file mode 100644
index 0000000..ecf0d45
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/concurrent/LightweightRecycler.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+
+interface LightweightRecyclerPoolHolder<T>
+{
+ ArrayDeque<T> get();
+}
+
+/**
+ * A simple thread local object reuse facility with limited capacity and no attempt at rebalancing pooling between
+ * threads. This is meant to be put in place where churn is high, but single object allocation and footprint are not
+ * so high to justify a more sophisticated approach.
+ *
+ * @param <T>
+ * @see ThreadLocals#createLightweightRecycler(int)
+ */
+public interface LightweightRecycler<T> extends LightweightRecyclerPoolHolder<T>
+{
+ /**
+ * @return a reusable instance, or null if none is available
+ */
+ default T reuse()
+ {
+ return get().pollFirst();
+ }
+
+ /**
+ * @return a reusable instance, or allocate one via the provided supplier
+ */
+ default T reuseOrAllocate(Supplier<T> supplier)
+ {
+ final T reuse = reuse();
+ return reuse != null ? reuse : supplier.get();
+ }
+
+ /**
+ * @param t to be recycled, if t is a collection it will be cleared before recycling, but not cleared if not
+ * recycled
+ * @return true if t was recycled, false otherwise
+ */
+ default boolean tryRecycle(T t)
+ {
+ Objects.requireNonNull(t);
+
+ final ArrayDeque<T> pool = get();
+ if (pool.size() < capacity())
+ {
+ if (t instanceof Collection)
+ ((Collection<?>) t).clear();
+ pool.offerFirst(t);
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ /**
+ * @return current count of available instances for reuse
+ */
+ default int available()
+ {
+ return get().size();
+ }
+
+
+ /**
+ * @return maximum capacity of the recycler
+ */
+ int capacity();
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Ref.java b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
index 90650cf..1e27dc7 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Ref.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
@@ -55,6 +55,7 @@
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.UNSAFE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DEBUG_REF_COUNT;
import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
import static org.apache.cassandra.utils.Throwables.maybeFail;
import static org.apache.cassandra.utils.Throwables.merge;
@@ -93,7 +94,7 @@
public final class Ref<T> implements RefCounted<T>
{
static final Logger logger = LoggerFactory.getLogger(Ref.class);
- public static final boolean DEBUG_ENABLED = System.getProperty("cassandra.debugrefcount", "false").equalsIgnoreCase("true");
+ public static final boolean DEBUG_ENABLED = TEST_DEBUG_REF_COUNT.getBoolean();
static OnLeak ON_LEAK;
@Shared(scope = SIMULATION)
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Refs.java b/src/java/org/apache/cassandra/utils/concurrent/Refs.java
index e5d9c37..fb6067e 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Refs.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Refs.java
@@ -20,8 +20,13 @@
*/
package org.apache.cassandra.utils.concurrent;
-import java.util.*;
-
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import javax.annotation.Nullable;
import com.google.common.base.Function;
@@ -204,7 +209,7 @@
/**
* Acquire a reference to all of the provided objects, or none
*/
- public static <T extends RefCounted<T>> Refs<T> tryRef(Iterable<T> reference)
+ public static <T extends RefCounted<T>> Refs<T> tryRef(Iterable<? extends T> reference)
{
HashMap<T, Ref<T>> refs = new HashMap<>();
for (T rc : reference)
@@ -220,7 +225,7 @@
return new Refs<T>(refs);
}
- public static <T extends RefCounted<T>> Refs<T> ref(Iterable<T> reference)
+ public static <T extends RefCounted<T>> Refs<T> ref(Iterable<? extends T> reference)
{
Refs<T> refs = tryRef(reference);
if (refs != null)
diff --git a/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java b/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java
index d643d1d..243e94a 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java
@@ -34,4 +34,12 @@
public Throwable close(Throwable accumulate);
public void addTo(Ref.IdentityCollection identities);
-}
+
+ static <T extends SharedCloseable> T sharedCopyOrNull(T sharedCloseable)
+ {
+ if (sharedCloseable != null)
+ return (T) sharedCloseable.sharedCopy();
+
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/concurrent/ThreadLocals.java b/src/java/org/apache/cassandra/utils/concurrent/ThreadLocals.java
new file mode 100644
index 0000000..378ba45
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/concurrent/ThreadLocals.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.ArrayDeque;
+
+import io.netty.util.concurrent.FastThreadLocal;
+
+public final class ThreadLocals
+{
+ private ThreadLocals()
+ {
+ }
+
+ public static <T> LightweightRecycler<T> createLightweightRecycler(int limit)
+ {
+ return new FastThreadLocalLightweightRecycler<>(limit);
+ }
+
+ /**
+ * A {@link LightweightRecycler} which is backed by a {@link FastThreadLocal}.
+ */
+ private static final class FastThreadLocalLightweightRecycler<T> extends FastThreadLocal<ArrayDeque<T>> implements LightweightRecycler<T>
+ {
+ private final int capacity;
+
+ public FastThreadLocalLightweightRecycler(int capacity)
+ {
+ super();
+ this.capacity = capacity;
+ }
+
+ protected ArrayDeque<T> initialValue()
+ {
+ return new ArrayDeque<>(capacity);
+ }
+
+ /**
+ * @return maximum capacity of the recycler
+ */
+ public int capacity()
+ {
+ return capacity;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java b/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
index eda9153..e710d44 100644
--- a/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
+++ b/src/java/org/apache/cassandra/utils/logging/LogbackLoggingSupport.java
@@ -20,8 +20,11 @@
import java.lang.management.ManagementFactory;
import java.security.AccessControlException;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
import javax.management.JMX;
import javax.management.ObjectName;
@@ -57,6 +60,8 @@
@Override
public void onStartup()
{
+ checkOnlyOneVirtualTableAppender();
+
// The default logback configuration in conf/logback.xml allows reloading the
// configuration when the configuration file has changed (every 60 seconds by default).
// This requires logback to use file I/O APIs. But file I/O is not allowed from UDFs.
@@ -132,6 +137,46 @@
return logLevelMaps;
}
+ @Override
+ public Optional<Appender<?>> getAppender(Class<?> appenderClass, String name)
+ {
+ LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+ for (Logger logBackLogger : lc.getLoggerList())
+ {
+ for (Iterator<Appender<ILoggingEvent>> iterator = logBackLogger.iteratorForAppenders(); iterator.hasNext();)
+ {
+ Appender<ILoggingEvent> appender = iterator.next();
+ if (appender.getClass() == appenderClass && appender.getName().equals(name))
+ return Optional.of(appender);
+ }
+ }
+
+ return Optional.empty();
+ }
+
+ private void checkOnlyOneVirtualTableAppender()
+ {
+ int count = 0;
+ LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+ List<String> virtualAppenderNames = new ArrayList<>();
+ for (Logger logBackLogger : lc.getLoggerList())
+ {
+ for (Iterator<Appender<ILoggingEvent>> iterator = logBackLogger.iteratorForAppenders(); iterator.hasNext();)
+ {
+ Appender<?> appender = iterator.next();
+ if (appender instanceof VirtualTableAppender)
+ {
+ virtualAppenderNames.add(appender.getName());
+ count += 1;
+ }
+ }
+ }
+
+ if (count > 1)
+ throw new IllegalStateException(String.format("There are multiple appenders of class %s of names %s. There is only one appender of such class allowed.",
+ VirtualTableAppender.class.getName(), String.join(",", virtualAppenderNames)));
+ }
+
private boolean hasAppenders(Logger logBackLogger)
{
Iterator<Appender<ILoggingEvent>> it = logBackLogger.iteratorForAppenders();
diff --git a/src/java/org/apache/cassandra/utils/logging/LoggingSupport.java b/src/java/org/apache/cassandra/utils/logging/LoggingSupport.java
index 8ea83be..35e1197 100644
--- a/src/java/org/apache/cassandra/utils/logging/LoggingSupport.java
+++ b/src/java/org/apache/cassandra/utils/logging/LoggingSupport.java
@@ -19,6 +19,9 @@
package org.apache.cassandra.utils.logging;
import java.util.Map;
+import java.util.Optional;
+
+import ch.qos.logback.core.Appender;
/**
* Common abstraction of functionality which can be implemented for different logging backend implementations (slf4j bindings).
@@ -49,4 +52,9 @@
* @return a map of logger names and their associated log level as string representations.
*/
Map<String, String> getLoggingLevels();
+
+ default Optional<Appender<?>> getAppender(Class<?> appenderClass, String appenderName)
+ {
+ return Optional.empty();
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/logging/VirtualTableAppender.java b/src/java/org/apache/cassandra/utils/logging/VirtualTableAppender.java
new file mode 100644
index 0000000..2820b29
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/logging/VirtualTableAppender.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.logging;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import ch.qos.logback.classic.spi.LoggingEvent;
+import ch.qos.logback.core.AppenderBase;
+import org.apache.cassandra.audit.FileAuditLogger;
+import org.apache.cassandra.db.virtual.LogMessagesTable;
+import org.apache.cassandra.db.virtual.VirtualKeyspace;
+import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
+import org.apache.cassandra.db.virtual.VirtualTable;
+
+import static org.apache.cassandra.db.virtual.LogMessagesTable.LOGS_VIRTUAL_TABLE_DEFAULT_ROWS;
+import static org.apache.cassandra.db.virtual.LogMessagesTable.TABLE_NAME;
+import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_VIEWS;
+
+/**
+ * Appends Cassandra logs to virtual table system_views.system_logs
+ */
+public final class VirtualTableAppender extends AppenderBase<LoggingEvent>
+{
+ public static final String APPENDER_NAME = "CQLLOG";
+
+ private static final Set<String> forbiddenLoggers = ImmutableSet.of(FileAuditLogger.class.getName());
+
+ private LogMessagesTable logs;
+
+ // for holding messages until virtual registry contains logs virtual table
+ // as it takes some time during startup of a node to initialise virtual tables but messages are
+ // logged already
+ private final List<LoggingEvent> messageBuffer = new LinkedList<>();
+
+ @Override
+ protected void append(LoggingEvent eventObject)
+ {
+ if (!forbiddenLoggers.contains(eventObject.getLoggerName()))
+ {
+ if (logs == null)
+ {
+ logs = getVirtualTable();
+ if (logs == null)
+ addToBuffer(eventObject);
+ else
+ logs.add(eventObject);
+ }
+ else
+ logs.add(eventObject);
+ }
+ }
+
+ @Override
+ public void stop()
+ {
+ messageBuffer.clear();
+ super.stop();
+ }
+
+ /**
+ * Flushes all logs which were appended before virtual table was registered.
+ *
+ * @see org.apache.cassandra.service.CassandraDaemon#setupVirtualKeyspaces
+ */
+ public void flushBuffer()
+ {
+ Optional.ofNullable(getVirtualTable()).ifPresent(vtable -> {
+ messageBuffer.forEach(vtable::add);
+ messageBuffer.clear();
+ });
+ }
+
+ private LogMessagesTable getVirtualTable()
+ {
+ VirtualKeyspace keyspace = VirtualKeyspaceRegistry.instance.getKeyspaceNullable(VIRTUAL_VIEWS);
+
+ if (keyspace == null)
+ return null;
+
+ Optional<VirtualTable> logsTable = keyspace.tables()
+ .stream()
+ .filter(vt -> vt.name().equals(TABLE_NAME))
+ .findFirst();
+
+ if (!logsTable.isPresent())
+ return null;
+
+ VirtualTable vt = logsTable.get();
+
+ if (!(vt instanceof LogMessagesTable))
+ throw new IllegalStateException(String.format("Virtual table %s.%s is not backed by an instance of %s but by %s",
+ VIRTUAL_VIEWS,
+ TABLE_NAME,
+ LogMessagesTable.class.getName(),
+ vt.getClass().getName()));
+
+ return (LogMessagesTable) vt;
+ }
+
+ private void addToBuffer(LoggingEvent eventObject)
+ {
+ // we restrict how many logging events we can put into buffer,
+ // so we are not growing without any bound when things go south
+ if (messageBuffer.size() < LOGS_VIRTUAL_TABLE_DEFAULT_ROWS)
+ messageBuffer.add(eventObject);
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java b/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java
index d66c02c..34b9eaa 100644
--- a/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java
+++ b/src/java/org/apache/cassandra/utils/memory/EnsureOnHeap.java
@@ -32,6 +32,8 @@
public abstract class EnsureOnHeap extends Transformation
{
+ public static final EnsureOnHeap NOOP = new NoOp();
+
public abstract DecoratedKey applyToPartitionKey(DecoratedKey key);
public abstract UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition);
public abstract SearchIterator<Clustering<?>, Row> applyToPartition(SearchIterator<Clustering<?>, Row> partition);
diff --git a/src/java/org/apache/cassandra/utils/memory/HeapPool.java b/src/java/org/apache/cassandra/utils/memory/HeapPool.java
index 532d11f..659715d 100644
--- a/src/java/org/apache/cassandra/utils/memory/HeapPool.java
+++ b/src/java/org/apache/cassandra/utils/memory/HeapPool.java
@@ -30,8 +30,6 @@
public class HeapPool extends MemtablePool
{
- private static final EnsureOnHeap ENSURE_NOOP = new EnsureOnHeap.NoOp();
-
public HeapPool(long maxOnHeapMemory, float cleanupThreshold, MemtableCleaner cleaner)
{
super(maxOnHeapMemory, 0, cleanupThreshold, cleaner);
@@ -59,7 +57,7 @@
public EnsureOnHeap ensureOnHeap()
{
- return ENSURE_NOOP;
+ return EnsureOnHeap.NOOP;
}
public Cloner cloner(OpOrder.Group opGroup)
@@ -124,7 +122,7 @@
@Override
public EnsureOnHeap ensureOnHeap()
{
- return ENSURE_NOOP;
+ return EnsureOnHeap.NOOP;
}
public Cloner cloner(OpOrder.Group opGroup)
diff --git a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
index 8543e6f..c2455bb 100644
--- a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
+++ b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
@@ -73,6 +73,21 @@
public static final int MAX_SIZE = 10;
+ /**
+ * Throw when attempting to decode a vint and the output type
+ * doesn't have enough space to fit the value that was decoded
+ */
+ public static class VIntOutOfRangeException extends RuntimeException
+ {
+ public final long value;
+
+ private VIntOutOfRangeException(long value)
+ {
+ super(value + " is out of range for a 32-bit integer");
+ this.value = value;
+ }
+ }
+
public static long readUnsignedVInt(DataInput input) throws IOException
{
int firstByte = input.readByte();
@@ -101,14 +116,24 @@
}
/**
+ * Read up to a 32-bit integer back, using the unsigned (no zigzag) encoding.
+ *
* Note this method is the same as {@link #readUnsignedVInt(DataInput)},
* except that we do *not* block if there are not enough bytes in the buffer
* to reconstruct the value.
*
- * WARNING: this method is only safe for vints we know to be representable by a positive long value.
- *
- * @return -1 if there are not enough bytes in the input to read the value; else, the vint unsigned value.
+ * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer
*/
+ public static int getUnsignedVInt32(ByteBuffer input, int readerIndex)
+ {
+ return checkedCast(getUnsignedVInt(input, readerIndex));
+ }
+
+ public static int getVInt32(ByteBuffer input, int readerIndex)
+ {
+ return checkedCast(decodeZigZag64(getUnsignedVInt(input, readerIndex)));
+ }
+
public static long getUnsignedVInt(ByteBuffer input, int readerIndex)
{
return getUnsignedVInt(input, readerIndex, input.limit());
@@ -165,6 +190,32 @@
return decodeZigZag64(readUnsignedVInt(input));
}
+ /**
+ * Read up to a signed 32-bit integer back.
+ *
+ * Assumes the vint was written using {@link #writeVInt32(int, DataOutputPlus)} or similar
+ * that zigzag encodes the integer.
+ *
+ * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer
+ */
+ public static int readVInt32(DataInput input) throws IOException
+ {
+ return checkedCast(decodeZigZag64(readUnsignedVInt(input)));
+ }
+
+ /**
+ * Read up to a 32-bit integer.
+ *
+ * This method assumes the original integer was written using {@link #writeUnsignedVInt32(int, DataOutputPlus)}
+ * or similar that doesn't zigzag encodes the vint.
+ *
+ * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer
+ */
+ public static int readUnsignedVInt32(DataInput input) throws IOException
+ {
+ return checkedCast(readUnsignedVInt(input));
+ }
+
// & this with the first byte to give the value part for a given extraBytesToRead encoded in the byte
public static int firstByteValueMask(int extraBytesToRead)
{
@@ -186,6 +237,12 @@
return Integer.numberOfLeadingZeros(~firstByte) - 24;
}
+ @Deprecated
+ public static void writeUnsignedVInt(int value, DataOutputPlus output) throws IOException
+ {
+ throw new UnsupportedOperationException("Use writeUnsignedVInt32/readUnsignedVInt32");
+ }
+
@Inline
public static void writeUnsignedVInt(long value, DataOutputPlus output) throws IOException
{
@@ -200,7 +257,7 @@
int extraBytes = size - 1;
long mask = (long)VIntCoding.encodeExtraBytesToRead(extraBytes) << 56;
long register = (value << shift) | mask;
- output.writeBytes(register, size);
+ output.writeMostSignificantBytes(register, size);
}
else if (size == 9)
{
@@ -213,6 +270,17 @@
}
}
+ public static void writeUnsignedVInt32(int value, DataOutputPlus output) throws IOException
+ {
+ writeUnsignedVInt((long)value, output);
+ }
+
+ @Deprecated
+ public static void writeUnsignedVInt(int value, ByteBuffer output) throws IOException
+ {
+ throw new UnsupportedOperationException("Use writeUnsignedVInt32/getUnsignedVInt32");
+ }
+
@Inline
public static void writeUnsignedVInt(long value, ByteBuffer output)
{
@@ -251,17 +319,47 @@
}
@Inline
+ public static void writeUnsignedVInt32(int value, ByteBuffer output)
+ {
+ writeUnsignedVInt((long)value, output);
+ }
+
+ @Deprecated
+ public static void writeVInt(int value, DataOutputPlus output) throws IOException
+ {
+ throw new UnsupportedOperationException("Use writeVInt32/readVInt32");
+ }
+
+ @Inline
public static void writeVInt(long value, DataOutputPlus output) throws IOException
{
writeUnsignedVInt(encodeZigZag64(value), output);
}
@Inline
- public static void writeVInt(long value, ByteBuffer output) throws IOException
+ public static void writeVInt32(int value, DataOutputPlus output) throws IOException
+ {
+ writeVInt((long)value, output);
+ }
+
+ @Deprecated
+ public static void writeVInt(int value, ByteBuffer output)
+ {
+ throw new UnsupportedOperationException("Use writeVInt32/getVInt32");
+ }
+
+ @Inline
+ public static void writeVInt(long value, ByteBuffer output)
{
writeUnsignedVInt(encodeZigZag64(value), output);
}
+ @Inline
+ public static void writeVInt32(int value, ByteBuffer output)
+ {
+ writeVInt((long)value, output);
+ }
+
/**
* @return a TEMPORARY THREAD LOCAL BUFFER containing the encoded bytes of the value
* This byte[] must be discarded by the caller immediately, and synchronously
@@ -326,4 +424,12 @@
// the formula below is hand-picked to match the original 9 - ((magnitude - 1) / 7)
return (639 - magnitude * 9) >> 6;
}
+
+ public static int checkedCast(long value)
+ {
+ int result = (int)value;
+ if ((long)result != value)
+ throw new VIntOutOfRangeException(value);
+ return result;
+ }
}
diff --git a/src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory b/src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory
new file mode 100644
index 0000000..79037c5
--- /dev/null
+++ b/src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory
@@ -0,0 +1,2 @@
+org.apache.cassandra.io.sstable.format.big.BigFormat$BigFormatFactory
+org.apache.cassandra.io.sstable.format.bti.BtiFormat$BtiFormatFactory
diff --git a/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java b/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java
new file mode 100644
index 0000000..2e5d202
--- /dev/null
+++ b/test/anttasks/org/apache/cassandra/anttasks/JdkProperties.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.anttasks;
+
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.Task;
+
+public class JdkProperties extends Task
+{
+
+ public void execute()
+ {
+ Project project = getProject();
+ project.setNewProperty("java.version." + project.getProperty("ant.java.version").replace("1.", ""), "true");
+ project.setNewProperty("use-jdk" + project.getProperty("ant.java.version").replace("1.", ""), "true");
+ }
+}
diff --git a/test/burn/org/apache/cassandra/net/MessageGenerator.java b/test/burn/org/apache/cassandra/net/MessageGenerator.java
index 43ea16e..80e647c 100644
--- a/test/burn/org/apache/cassandra/net/MessageGenerator.java
+++ b/test/burn/org/apache/cassandra/net/MessageGenerator.java
@@ -149,7 +149,7 @@
{
int length = messagingVersion < VERSION_40
? in.readInt()
- : (int) in.readUnsignedVInt();
+ : in.readUnsignedVInt32();
long id = in.readLong();
if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN)
id = Long.reverseBytes(id);
@@ -162,7 +162,7 @@
if (messagingVersion < VERSION_40)
out.writeInt(payload.length);
else
- out.writeUnsignedVInt(payload.length);
+ out.writeUnsignedVInt32(payload.length);
}
static long serializedSize(byte[] payload, int messagingVersion)
diff --git a/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java b/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java
index a050245..e604697 100644
--- a/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java
+++ b/test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java
@@ -30,7 +30,7 @@
import java.util.stream.Collectors;
import com.google.common.util.concurrent.RateLimiter;
-import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
index 01b4493..9aa13f9 100644
--- a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
+++ b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java
@@ -52,6 +52,7 @@
import static com.google.common.collect.Iterables.transform;
import static java.util.Comparator.naturalOrder;
import static java.util.Comparator.reverseOrder;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BTREE_FAN_FACTOR;
import static org.apache.cassandra.utils.btree.BTree.iterable;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.junit.Assert.assertEquals;
@@ -190,7 +191,7 @@
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
- update.put(i, new Integer(i));
+ update.put(i, Integer.valueOf(i));
CountingFunction function = new CountingFunction((x) -> x);
Object[] original = selection.testAsSet.tree();
@@ -210,7 +211,7 @@
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
- update.put(i, new Integer(i));
+ update.put(i, Integer.valueOf(i));
CountingFunction function = new CountingFunction((x) -> update.getOrDefault(x, x));
Object[] original = selection.testAsSet.tree();
@@ -230,7 +231,7 @@
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
- update.put(i, new Integer(i));
+ update.put(i, Integer.valueOf(i));
CountingFunction function = new CountingFunction(update::get);
Object[] original = selection.testAsSet.tree();
@@ -249,7 +250,7 @@
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
- update.put(i, new Integer(i));
+ update.put(i, Integer.valueOf(i));
CountingFunction function = new CountingFunction((x) -> update.containsKey(x) ? null : x);
Object[] original = selection.testAsSet.tree();
@@ -1135,7 +1136,7 @@
for (String arg : args)
{
if (arg.startsWith("fan="))
- System.setProperty("cassandra.btree.fanfactor", arg.substring(4));
+ BTREE_FAN_FACTOR.setString(arg.substring(4));
else if (arg.startsWith("min="))
minTreeSize = Integer.parseInt(arg.substring(4));
else if (arg.startsWith("max="))
diff --git a/test/conf/cassandra-murmur.yaml b/test/conf/cassandra-murmur.yaml
index c0c2ae7..385ed46 100644
--- a/test/conf/cassandra-murmur.yaml
+++ b/test/conf/cassandra-murmur.yaml
@@ -39,6 +39,6 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
sasi_indexes_enabled: true
materialized_views_enabled: true
diff --git a/test/conf/cassandra-old.yaml b/test/conf/cassandra-old.yaml
index 86983ac..10981c2 100644
--- a/test/conf/cassandra-old.yaml
+++ b/test/conf/cassandra-old.yaml
@@ -43,7 +43,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size_in_mb: 16
enable_user_defined_functions: true
-enable_scripted_user_defined_functions: true
+enable_scripted_user_defined_functions: false
prepared_statements_cache_size_mb:
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
diff --git a/test/conf/cassandra-pem-jks-sslcontextfactory.yaml b/test/conf/cassandra-pem-jks-sslcontextfactory.yaml
index 1f10e6c..ad9d998 100644
--- a/test/conf/cassandra-pem-jks-sslcontextfactory.yaml
+++ b/test/conf/cassandra-pem-jks-sslcontextfactory.yaml
@@ -140,7 +140,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
prepared_statements_cache_size: 1MiB
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
diff --git a/test/conf/cassandra-pem-sslcontextfactory-invalidconfiguration.yaml b/test/conf/cassandra-pem-sslcontextfactory-invalidconfiguration.yaml
index 8c7d910..15aaae0 100644
--- a/test/conf/cassandra-pem-sslcontextfactory-invalidconfiguration.yaml
+++ b/test/conf/cassandra-pem-sslcontextfactory-invalidconfiguration.yaml
@@ -137,7 +137,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
prepared_statements_cache_size: 1MiB
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
diff --git a/test/conf/cassandra-pem-sslcontextfactory.yaml b/test/conf/cassandra-pem-sslcontextfactory.yaml
index 26d0f1f..36daecd 100644
--- a/test/conf/cassandra-pem-sslcontextfactory.yaml
+++ b/test/conf/cassandra-pem-sslcontextfactory.yaml
@@ -141,7 +141,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
prepared_statements_cache_size: 1MiB
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
diff --git a/test/conf/cassandra-seeds.yaml b/test/conf/cassandra-seeds.yaml
index 1c38f8e..2a206f1 100644
--- a/test/conf/cassandra-seeds.yaml
+++ b/test/conf/cassandra-seeds.yaml
@@ -40,4 +40,4 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
diff --git a/test/conf/cassandra-sslcontextfactory-invalidconfiguration.yaml b/test/conf/cassandra-sslcontextfactory-invalidconfiguration.yaml
index d3970cb..6e6b21f 100644
--- a/test/conf/cassandra-sslcontextfactory-invalidconfiguration.yaml
+++ b/test/conf/cassandra-sslcontextfactory-invalidconfiguration.yaml
@@ -72,7 +72,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
prepared_statements_cache_size: 1MiB
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
diff --git a/test/conf/cassandra-sslcontextfactory.yaml b/test/conf/cassandra-sslcontextfactory.yaml
index fde4bfd..add1666 100644
--- a/test/conf/cassandra-sslcontextfactory.yaml
+++ b/test/conf/cassandra-sslcontextfactory.yaml
@@ -75,7 +75,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
prepared_statements_cache_size: 1MiB
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml
index 89c5685..6e3059a 100644
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@ -43,7 +43,7 @@
row_cache_class_name: org.apache.cassandra.cache.OHCProvider
row_cache_size: 16MiB
user_defined_functions_enabled: true
-scripted_user_defined_functions_enabled: true
+scripted_user_defined_functions_enabled: false
prepared_statements_cache_size: 1MiB
corrupted_tombstone_strategy: exception
stream_entire_sstables: true
@@ -56,6 +56,9 @@
allow_nodetool_archive_command: true
auto_hints_cleanup_enabled: true
+heap_dump_path: build/test
+dump_heap_on_uncaught_exception: true
+
read_thresholds_enabled: true
coordinator_read_size_warn_threshold: 1024KiB
coordinator_read_size_fail_threshold: 4096KiB
@@ -67,13 +70,15 @@
memtable:
configurations:
skiplist:
- inherits: default
class_name: SkipListMemtable
+ trie:
+ class_name: TrieMemtable
+ parameters:
+ shards: 4
skiplist_sharded:
class_name: ShardedSkipListMemtable
parameters:
serialize_writes: false
- shards: 4
skiplist_sharded_locking:
inherits: skiplist_sharded
parameters:
@@ -81,7 +86,6 @@
skiplist_remapped:
inherits: skiplist
test_fullname:
- inherits: default
class_name: org.apache.cassandra.db.memtable.TestMemtable
test_shortname:
class_name: TestMemtable
@@ -105,3 +109,7 @@
class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryMethod
test_invalid_factory_field:
class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryField
+ test_memtable_metrics:
+ class_name: TrieMemtable
+# Note: keep the memtable configuration at the end of the file, so that the default mapping can be changed without
+# duplicating the whole section above.
diff --git a/test/conf/cassandra_ssl_test.truststore b/test/conf/cassandra_ssl_test.truststore
index 49cf332..ab01af3 100644
--- a/test/conf/cassandra_ssl_test.truststore
+++ b/test/conf/cassandra_ssl_test.truststore
Binary files differ
diff --git a/test/conf/cassandra_ssl_test_endpoint_verify.keystore b/test/conf/cassandra_ssl_test_endpoint_verify.keystore
new file mode 100644
index 0000000..951385b
--- /dev/null
+++ b/test/conf/cassandra_ssl_test_endpoint_verify.keystore
Binary files differ
diff --git a/test/conf/cassandra_ssl_test_outbound.keystore b/test/conf/cassandra_ssl_test_outbound.keystore
new file mode 100644
index 0000000..7dbf466
--- /dev/null
+++ b/test/conf/cassandra_ssl_test_outbound.keystore
Binary files differ
diff --git a/test/conf/logback-dtest_with_vtable_appender.xml b/test/conf/logback-dtest_with_vtable_appender.xml
new file mode 100644
index 0000000..c9fd108
--- /dev/null
+++ b/test/conf/logback-dtest_with_vtable_appender.xml
@@ -0,0 +1,66 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration debug="false" scan="true" scanPeriod="60 seconds">
+ <define name="cluster_id" class="org.apache.cassandra.distributed.impl.ClusterIDDefiner" />
+ <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
+
+ <!-- Shutdown hook ensures that async appender flushes -->
+ <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+
+ <appender name="INSTANCEFILE" class="ch.qos.logback.core.FileAppender">
+ <file>./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log</file>
+ <encoder>
+ <pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %msg%n</pattern>
+ </encoder>
+ <immediateFlush>true</immediateFlush>
+ </appender>
+
+ <appender name="INSTANCESTDERR" target="System.err" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>WARN</level>
+ </filter>
+ </appender>
+
+ <appender name="INSTANCESTDOUT" target="System.out" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>DEBUG</level>
+ </filter>
+ </appender>
+
+ <logger name="org.apache.hadoop" level="WARN"/>
+
+ <appender name="CQLLOG" class="org.apache.cassandra.utils.logging.VirtualTableAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+
+ <root level="DEBUG">
+ <appender-ref ref="INSTANCEFILE" /> <!-- use blocking to avoid race conditions with appending and searching -->
+ <appender-ref ref="INSTANCESTDERR" />
+ <appender-ref ref="INSTANCESTDOUT" />
+ <appender-ref ref="CQLLOG" />
+ </root>
+</configuration>
diff --git a/test/conf/logback-dtest_with_vtable_appender_invalid.xml b/test/conf/logback-dtest_with_vtable_appender_invalid.xml
new file mode 100644
index 0000000..1b30c14
--- /dev/null
+++ b/test/conf/logback-dtest_with_vtable_appender_invalid.xml
@@ -0,0 +1,73 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration debug="false" scan="true" scanPeriod="60 seconds">
+ <define name="cluster_id" class="org.apache.cassandra.distributed.impl.ClusterIDDefiner" />
+ <define name="instance_id" class="org.apache.cassandra.distributed.impl.InstanceIDDefiner" />
+
+ <!-- Shutdown hook ensures that async appender flushes -->
+ <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+
+ <appender name="INSTANCEFILE" class="ch.qos.logback.core.FileAppender">
+ <file>./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log</file>
+ <encoder>
+ <pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %msg%n</pattern>
+ </encoder>
+ <immediateFlush>true</immediateFlush>
+ </appender>
+
+ <appender name="INSTANCESTDERR" target="System.err" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>WARN</level>
+ </filter>
+ </appender>
+
+ <appender name="INSTANCESTDOUT" target="System.out" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%-5level [%thread] ${instance_id} %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>DEBUG</level>
+ </filter>
+ </appender>
+
+ <logger name="org.apache.hadoop" level="WARN"/>
+
+ <appender name="CQLLOG" class="org.apache.cassandra.utils.logging.VirtualTableAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+
+ <appender name="CQLLOG2" class="org.apache.cassandra.utils.logging.VirtualTableAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+
+ <root level="DEBUG">
+ <appender-ref ref="INSTANCEFILE" /> <!-- use blocking to avoid race conditions with appending and searching -->
+ <appender-ref ref="INSTANCESTDERR" />
+ <appender-ref ref="INSTANCESTDOUT" />
+ <appender-ref ref="CQLLOG" />
+ <appender-ref ref="CQLLOG2" /> <!-- invalid, we can not have multiple appenders of VirtualTableAppender class -->
+ </root>
+</configuration>
diff --git a/test/conf/trie_memtable.yaml b/test/conf/trie_memtable.yaml
new file mode 100644
index 0000000..c43ca80
--- /dev/null
+++ b/test/conf/trie_memtable.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Note: this attaches at the end of cassandra.yaml, where the memtable configuration setting must be.
+ default:
+ inherits: trie
+
+# Change the default SSTable format to BTI.
+# Note: This can also be achieved by passing -Dcassandra.sstable.format.default=bti
+sstable:
+ selected_format: bti
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-CompressionInfo.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..db075a8
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Data.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Data.db
new file mode 100644
index 0000000..f909d5e
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Digest.crc32 b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Digest.crc32
new file mode 100644
index 0000000..9b7b9f8
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Digest.crc32
@@ -0,0 +1 @@
+3501696673
\ No newline at end of file
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Filter.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Partitions.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Partitions.db
new file mode 100644
index 0000000..daf1b01
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Rows.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Rows.db
new file mode 100644
index 0000000..46defad
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Rows.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Statistics.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Statistics.db
new file mode 100644
index 0000000..e38018c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-TOC.txt b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-TOC.txt
new file mode 100644
index 0000000..298910c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-CompressionInfo.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..76fa6ee
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Data.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Data.db
new file mode 100644
index 0000000..6f1a05a
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Digest.crc32 b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Digest.crc32
new file mode 100644
index 0000000..a661246
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1748826086
\ No newline at end of file
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Filter.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Partitions.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Partitions.db
new file mode 100644
index 0000000..daf1b01
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Rows.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Rows.db
new file mode 100644
index 0000000..31347f9
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Rows.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Statistics.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Statistics.db
new file mode 100644
index 0000000..b822174
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-TOC.txt b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-TOC.txt
new file mode 100644
index 0000000..298910c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-CompressionInfo.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..ef68317
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Data.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Data.db
new file mode 100644
index 0000000..edba924
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Digest.crc32 b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Digest.crc32
new file mode 100644
index 0000000..f1d25be
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Digest.crc32
@@ -0,0 +1 @@
+4104327237
\ No newline at end of file
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Filter.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Partitions.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Partitions.db
new file mode 100644
index 0000000..e20b4e2
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/serialization/2.0/db.WriteResponse.bin b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Rows.db
similarity index 100%
rename from test/data/serialization/2.0/db.WriteResponse.bin
rename to test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Rows.db
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Statistics.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Statistics.db
new file mode 100644
index 0000000..59007b8
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-TOC.txt b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-TOC.txt
new file mode 100644
index 0000000..298910c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-CompressionInfo.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-CompressionInfo.db
new file mode 100644
index 0000000..1db9aa0
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Data.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Data.db
new file mode 100644
index 0000000..89b36f1
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Digest.crc32 b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Digest.crc32
new file mode 100644
index 0000000..d17ace7
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Digest.crc32
@@ -0,0 +1 @@
+1102192488
\ No newline at end of file
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Filter.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Partitions.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Partitions.db
new file mode 100644
index 0000000..773d3c8
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Partitions.db
Binary files differ
diff --git a/test/data/serialization/2.0/db.WriteResponse.bin b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Rows.db
similarity index 100%
copy from test/data/serialization/2.0/db.WriteResponse.bin
copy to test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Rows.db
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Statistics.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Statistics.db
new file mode 100644
index 0000000..b282579
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-TOC.txt b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-TOC.txt
new file mode 100644
index 0000000..298910c
--- /dev/null
+++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Statistics.db
+Digest.crc32
+TOC.txt
+CompressionInfo.db
+Filter.db
+Partitions.db
+Rows.db
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-CompressionInfo.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..2abf0a8
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Data.db
new file mode 100644
index 0000000..8db5dad
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32
new file mode 100644
index 0000000..a3554fa
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32
@@ -0,0 +1 @@
+516351458
\ No newline at end of file
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Index.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Index.db
new file mode 100644
index 0000000..8b23bee
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Statistics.db
new file mode 100644
index 0000000..ef1355c
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-TOC.txt b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-TOC.txt
new file mode 100644
index 0000000..6ea912e
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Data.db
+Statistics.db
+Summary.db
+Filter.db
+Digest.crc32
+Index.db
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-CompressionInfo.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..d0ab8d2
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Data.db
new file mode 100644
index 0000000..c212d3d
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32
new file mode 100644
index 0000000..3e0bf3e
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32
@@ -0,0 +1 @@
+3677152410
\ No newline at end of file
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Index.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Index.db
new file mode 100644
index 0000000..a8fdb3c
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Statistics.db
new file mode 100644
index 0000000..4a3ab07
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-TOC.txt b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-TOC.txt
new file mode 100644
index 0000000..6ea912e
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Data.db
+Statistics.db
+Summary.db
+Filter.db
+Digest.crc32
+Index.db
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-CompressionInfo.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..ef68317
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Data.db
new file mode 100644
index 0000000..fe53589
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Digest.crc32
new file mode 100644
index 0000000..67f6298
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Digest.crc32
@@ -0,0 +1 @@
+1155625239
\ No newline at end of file
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Index.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Index.db
new file mode 100644
index 0000000..b3094bf
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Statistics.db
new file mode 100644
index 0000000..f9940d0
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-TOC.txt b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-TOC.txt
new file mode 100644
index 0000000..6ea912e
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Data.db
+Statistics.db
+Summary.db
+Filter.db
+Digest.crc32
+Index.db
+CompressionInfo.db
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-CompressionInfo.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..1db9aa0
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-CompressionInfo.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Data.db
new file mode 100644
index 0000000..1431e3e
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Data.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32
new file mode 100644
index 0000000..1864117
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32
@@ -0,0 +1 @@
+4045937701
\ No newline at end of file
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Filter.db
new file mode 100644
index 0000000..8868e5c
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Filter.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Index.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Index.db
new file mode 100644
index 0000000..59e65ca
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Index.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Statistics.db
new file mode 100644
index 0000000..ead79f6
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Statistics.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Summary.db
Binary files differ
diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-TOC.txt b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-TOC.txt
new file mode 100644
index 0000000..6ea912e
--- /dev/null
+++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Data.db
+Statistics.db
+Summary.db
+Filter.db
+Digest.crc32
+Index.db
+CompressionInfo.db
diff --git a/test/data/serialization/0.7/db.RangeSliceCommand.bin b/test/data/serialization/0.7/db.RangeSliceCommand.bin
deleted file mode 100644
index d6a0b05..0000000
--- a/test/data/serialization/0.7/db.RangeSliceCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/db.Row.bin b/test/data/serialization/0.7/db.Row.bin
deleted file mode 100644
index 27de8bb..0000000
--- a/test/data/serialization/0.7/db.Row.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/db.RowMutation.bin b/test/data/serialization/0.7/db.RowMutation.bin
deleted file mode 100644
index 5465220..0000000
--- a/test/data/serialization/0.7/db.RowMutation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/db.SliceByNamesReadCommand.bin b/test/data/serialization/0.7/db.SliceByNamesReadCommand.bin
deleted file mode 100644
index aa892d5..0000000
--- a/test/data/serialization/0.7/db.SliceByNamesReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/db.SliceFromReadCommand.bin b/test/data/serialization/0.7/db.SliceFromReadCommand.bin
deleted file mode 100644
index 7160d24..0000000
--- a/test/data/serialization/0.7/db.SliceFromReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/db.Truncation.bin b/test/data/serialization/0.7/db.Truncation.bin
deleted file mode 100644
index ea67995..0000000
--- a/test/data/serialization/0.7/db.Truncation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/db.WriteResponse.bin b/test/data/serialization/0.7/db.WriteResponse.bin
deleted file mode 100644
index 9076238..0000000
--- a/test/data/serialization/0.7/db.WriteResponse.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/gms.EndpointState.bin b/test/data/serialization/0.7/gms.EndpointState.bin
deleted file mode 100644
index 054bced..0000000
--- a/test/data/serialization/0.7/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/gms.Gossip.bin b/test/data/serialization/0.7/gms.Gossip.bin
deleted file mode 100644
index 20b27a8..0000000
--- a/test/data/serialization/0.7/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/service.TreeRequest.bin b/test/data/serialization/0.7/service.TreeRequest.bin
deleted file mode 100644
index 5ad7e2a..0000000
--- a/test/data/serialization/0.7/service.TreeRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/service.TreeResponse.bin b/test/data/serialization/0.7/service.TreeResponse.bin
deleted file mode 100644
index b737785..0000000
--- a/test/data/serialization/0.7/service.TreeResponse.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/streaming.PendingFile.bin b/test/data/serialization/0.7/streaming.PendingFile.bin
deleted file mode 100644
index 6c0d61a..0000000
--- a/test/data/serialization/0.7/streaming.PendingFile.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/streaming.StreamHeader.bin b/test/data/serialization/0.7/streaming.StreamHeader.bin
deleted file mode 100644
index ad21a68..0000000
--- a/test/data/serialization/0.7/streaming.StreamHeader.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/streaming.StreamReply.bin b/test/data/serialization/0.7/streaming.StreamReply.bin
deleted file mode 100644
index 4b74058..0000000
--- a/test/data/serialization/0.7/streaming.StreamReply.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/streaming.StreamRequestMessage.bin b/test/data/serialization/0.7/streaming.StreamRequestMessage.bin
deleted file mode 100644
index 73f8d0e..0000000
--- a/test/data/serialization/0.7/streaming.StreamRequestMessage.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/utils.BloomFilter.bin b/test/data/serialization/0.7/utils.BloomFilter.bin
deleted file mode 100644
index 3ad7fba..0000000
--- a/test/data/serialization/0.7/utils.BloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/utils.EstimatedHistogram.bin b/test/data/serialization/0.7/utils.EstimatedHistogram.bin
deleted file mode 100644
index bedd39b..0000000
--- a/test/data/serialization/0.7/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/0.7/utils.LegacyBloomFilter.bin b/test/data/serialization/0.7/utils.LegacyBloomFilter.bin
deleted file mode 100644
index faef1b8..0000000
--- a/test/data/serialization/0.7/utils.LegacyBloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.RangeSliceCommand.bin b/test/data/serialization/1.0/db.RangeSliceCommand.bin
deleted file mode 100644
index ae24915..0000000
--- a/test/data/serialization/1.0/db.RangeSliceCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.Row.bin b/test/data/serialization/1.0/db.Row.bin
deleted file mode 100644
index 1850f41..0000000
--- a/test/data/serialization/1.0/db.Row.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.RowMutation.bin b/test/data/serialization/1.0/db.RowMutation.bin
deleted file mode 100644
index 0992cb5..0000000
--- a/test/data/serialization/1.0/db.RowMutation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.SliceByNamesReadCommand.bin b/test/data/serialization/1.0/db.SliceByNamesReadCommand.bin
deleted file mode 100644
index aa892d5..0000000
--- a/test/data/serialization/1.0/db.SliceByNamesReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.SliceFromReadCommand.bin b/test/data/serialization/1.0/db.SliceFromReadCommand.bin
deleted file mode 100644
index 7160d24..0000000
--- a/test/data/serialization/1.0/db.SliceFromReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.Truncation.bin b/test/data/serialization/1.0/db.Truncation.bin
deleted file mode 100644
index ea67995..0000000
--- a/test/data/serialization/1.0/db.Truncation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.WriteResponse.bin b/test/data/serialization/1.0/db.WriteResponse.bin
deleted file mode 100644
index 9076238..0000000
--- a/test/data/serialization/1.0/db.WriteResponse.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/db.migration.Keyspace1.bin b/test/data/serialization/1.0/db.migration.Keyspace1.bin
deleted file mode 100644
index e239c5e..0000000
--- a/test/data/serialization/1.0/db.migration.Keyspace1.bin
+++ /dev/null
@@ -1 +0,0 @@
-·l/Dl7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiTWlncmF0aW9uIiwibmFtZXNwYWNlIjoib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWlncmF0aW9uLmF2cm8iLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRfdmVyc2lvbiIsInR5cGUiOnsidHlwZSI6ImZpeGVkIiwibmFtZSI6IlVVSUQiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvIiwic2l6ZSI6MTZ9fSx7Im5hbWUiOiJuZXdfdmVyc2lvbiIsInR5cGUiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvLlVVSUQifSx7Im5hbWUiOiJyb3dfbXV0YXRpb24iLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJjbGFzc25hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoibWlncmF0aW9uIiwidHlwZSI6W3sidHlwZSI6InJlY29yZCIsIm5hbWUiOiJBZGRDb2x1bW5GYW1pbHkiLCJmaWVsZHMiOlt7Im5hbWUiOiJjZiIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiRHJvcENvbHVtbkZhbWlseSIsImZpZWxkcyI6W3sibmFtZSI6ImtzbmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJjZm5hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiQWRkS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrcyIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJLc0RlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOiJDZkRlZiJ9fSx7Im5hbWUiOiJkdXJhYmxlX3dyaXRlcyIsInR5cGUiOiJib29sZWFuIiwiZGVmYXVsdCI6dHJ1ZX1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLktzRGVmIl19fV19LHsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJEcm9wS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrc25hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRLcyIsInR5cGUiOiJLc0RlZiJ9LHsibmFtZSI6Im5ld0tzIiwidHlwZSI6IktzRGVmIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlQ29sdW1uRmFtaWx5IiwiZmllbGRzIjpbeyJuYW1lIjoibWV0YWRhdGEiLCJ0eXBlIjoiQ2ZEZWYifV19XX1dfahL81B3WRHhAAD+jr7q2fyoTBpgd1kR4QAA/o6+6tn8us8CAAZzeXN0ZW0AJGE4NGMxYTYwLTc3NTktMTFlMS0wMDAwLWZlOGViZWVhZDlmYwAAAAEAAAADAQAAAAOAAAAAgAAAAAAAAAAAAAAKAAtBdnJvL1NjaGVtYQAAAAE2T6gPCgAACu97InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiS3NEZWYiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uYXZybyIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX19LHsibmFtZSI6ImR1cmFibGVfd3JpdGVzIiwidHlwZSI6ImJvb2xlYW4iLCJkZWZhdWx0Ijp0cnVlfV0sImFsaWFzZXMiOlsib3JnLmFwYWNoZS5jYXNzYW5kcmEuY29uZmlnLmF2cm8uS3NEZWYiXX0ADUtleUNhY2hlU3BhY2UAAAABNk+oDwoAAARwGktleUNhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIGGktleUNhY2hlU3BhY2USU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAA8D8AmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AphAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAaS2V5Q2FjaGVTcGFjZRJTdGFuZGFyZDIAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAADwPwCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCkEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpLZXlDYWNoZVNwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAOA/AJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKIQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlMQAAAAE2T6gPCgAAJuASS2V5c3BhY2UxVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACNBJLZXlzcGFjZTEaU3VwZXJEaXJlY3RHQwAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQAAAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwD0DwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEQSmRiY1V0ZjgAEFN0YW5kYXJkAFBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlVURjhUeXBlAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVVRGOFR5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A+A8AAhBmb3J0eXR3b1ZvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkludGVnZXJUeXBlAgAQQ29sdW1uNDIAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxEkpkYmNBc2NpaQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AP4PAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMQxTdXBlcjEAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AN4PAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMSBWYWx1ZXNXaXRoUXVvdGVzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAABqCEEAmpmZmZmZuT8BAIC8aQJQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5VVEY4VHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDcDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEMU3VwZXIyAApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDgDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEMU3VwZXIzAApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDiDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEMU3VwZXI0AApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVVRGOFR5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDkDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEMU3VwZXI1AApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A5g8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxDFN1cGVyNgAKU3VwZXIAXm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTGV4aWNhbFVVSURUeXBlAFBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlVURjhUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A6A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxGlN1cGVyQ291bnRlcjEAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAmJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkNvdW50ZXJDb2x1bW5UeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PAgAAAAAAAPA/APIPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRBKZGJjTG9uZwAQU3RhbmRhcmQAUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUCAAAAAAAAAAAAAAAAAAAAAABqCEEAmpmZmZmZuT8BAIC8aQJQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwD6DwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTESU3RhbmRhcmQ0ABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A1g8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxEEluZGV4ZWQyABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A7A8AAhJiaXJ0aGRhdGVQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQIAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEWSmRiY0ludGVnZXIAEFN0YW5kYXJkAFZvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkludGVnZXJUeXBlAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuSW50ZWdlclR5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A9g8AAgIqUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVVRGOFR5cGUCABBDb2x1bW40MgAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTESU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A1A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxEEluZGV4ZWQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A6g8AAhJiaXJ0aGRhdGVQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQAAACQ2MjY5NzI3NDY4NjQ2MTc0NjUAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxElN0YW5kYXJkMgAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/ANIPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDQDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEwU3RhbmRhcmREeW5hbWljQ29tcG9zaXRlABBTdGFuZGFyZACkAm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuRHluYW1pY0NvbXBvc2l0ZVR5cGUodD0+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUsYj0+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUpAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AIIQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMSJTdGFuZGFyZENvbXBvc2l0ZQAQU3RhbmRhcmQA4gJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkNvbXBvc2l0ZVR5cGUob3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUsb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUsb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5JbnRlZ2VyVHlwZSkCAAAAAAAAAAAAAAAAAAAAAABqCEEAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AgBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxEkpkYmNCeXRlcwAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/APwPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMSBTdGFuZGFyZEludGVnZXIxABBTdGFuZGFyZABWb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5JbnRlZ2VyVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDuDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEQQ291bnRlcjEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAmJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkNvdW50ZXJDb2x1bW5UeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PAgAAAAAAAPA/APAPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRpTdGFuZGFyZExvbmcyABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A2g8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxGlN0YW5kYXJkTG9uZzEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDYDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABAAlLZXlzcGFjZTIAAAABNk+oDwoAAAepEktleXNwYWNlMlZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjEAAgoSS2V5c3BhY2UyEEluZGV4ZWQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AjBAAAhJiaXJ0aGRhdGVQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQAAACQ2MjY5NzI3NDY4NjQ2MTc0NjUAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UyElN0YW5kYXJkMwAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AIYQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMhJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCEEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTIMU3VwZXIzAApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AiBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UyDFN1cGVyNAAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFhvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlRpbWVVVUlEVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AIoQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlMwAAAAE2T6gPCgAAA1ASS2V5c3BhY2UzVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICNQACBBJLZXlzcGFjZTMQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCQEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTMSU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AjhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2U0AAAAATZPqA8KAAAHhhJLZXlzcGFjZTRWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIzAAIKEktleXNwYWNlNBJTdGFuZGFyZDMAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCUEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQSU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AkhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U0DFN1cGVyMwAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJYQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNAxTdXBlcjQAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBYb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCYEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQMU3VwZXI1AApTdXBlcgBYb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AmhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2U1AAAAATZPqA8KAAADDRJLZXlzcGFjZTVWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIyAAIEEktleXNwYWNlNRJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCcEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTUQQ291bnRlcjEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAmJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkNvdW50ZXJDb2x1bW5UeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJ4QAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlNgAAAAE2T6gPCgAAAfYSS2V5c3BhY2U2Vm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACAhJLZXlzcGFjZTYQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCgEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABABBOb0NvbW1pdGxvZ1NwYWNlAAAAATZPqA8KAAABuiBOb0NvbW1pdGxvZ1NwYWNlVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACAiBOb0NvbW1pdGxvZ1NwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKwQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAAADVJvd0NhY2hlU3BhY2UAAAABNk+oDwoAAAMWGlJvd0NhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIEGlJvd0NhY2hlU3BhY2UQQ2FjaGVkQ0YAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAABZQAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCqEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpSb3dDYWNoZVNwYWNlHENGV2l0aG91dENhY2hlABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AqBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAVpvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uQWRkS2V5c3BhY2UEEktleXNwYWNlMVZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjEAAjQSS2V5c3BhY2UxGlN1cGVyRGlyZWN0R0MAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAAAJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A9A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxEEpkYmNVdGY4ABBTdGFuZGFyZABQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5VVEY4VHlwZQIAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAlBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlVURjhUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/APgPAAIQZm9ydHl0d29Wb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5JbnRlZ2VyVHlwZQIAEENvbHVtbjQyAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRJKZGJjQXNjaWkAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkFzY2lpVHlwZQIAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkFzY2lpVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwD+DwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEMU3VwZXIxAApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDeDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEgVmFsdWVzV2l0aFF1b3RlcwAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVVRGOFR5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A3A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxDFN1cGVyMgAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A4A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxDFN1cGVyMwAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A4g8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxDFN1cGVyNAAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlVURjhUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A5A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxDFN1cGVyNQAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AOYPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMQxTdXBlcjYAClN1cGVyAF5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxleGljYWxVVUlEVHlwZQBQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5VVEY4VHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AOgPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRpTdXBlckNvdW50ZXIxAApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAABqCEEAmpmZmZmZuT8BAIC8aQJib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Db3VudGVyQ29sdW1uVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwIAAAAAAADwPwDyDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEQSmRiY0xvbmcAEFN0YW5kYXJkAFBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A+g8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxElN0YW5kYXJkNAAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/ANYPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRBJbmRleGVkMgAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkFzY2lpVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AOwPAAISYmlydGhkYXRlUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUCACQ2MjY5NzI3NDY4NjQ2MTc0NjUAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxFkpkYmNJbnRlZ2VyABBTdGFuZGFyZABWb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5JbnRlZ2VyVHlwZQIAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAlZvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkludGVnZXJUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/APYPAAICKlBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlVURjhUeXBlAgAQQ29sdW1uNDIAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxElN0YW5kYXJkMwAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/ANQPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRBJbmRleGVkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkFzY2lpVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AOoPAAISYmlydGhkYXRlUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUAAAAkNjI2OTcyNzQ2ODY0NjE3NDY1AAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRJTdGFuZGFyZDIAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwDSDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTESU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A0A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxMFN0YW5kYXJkRHluYW1pY0NvbXBvc2l0ZQAQU3RhbmRhcmQApAJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkR5bmFtaWNDb21wb3NpdGVUeXBlKHQ9Pm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVGltZVVVSURUeXBlLGI9Pm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlKQIAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCCEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEiU3RhbmRhcmRDb21wb3NpdGUAEFN0YW5kYXJkAOICb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Db21wb3NpdGVUeXBlKG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlLG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVGltZVVVSURUeXBlLG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuSW50ZWdlclR5cGUpAgAAAAAAAAAAAAAAAAAAAAAAaghBAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AIAQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRJKZGJjQnl0ZXMAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAGoIQQCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwD8DwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEgU3RhbmRhcmRJbnRlZ2VyMQAQU3RhbmRhcmQAVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuSW50ZWdlclR5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A7g8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UxEENvdW50ZXIxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAABqCEEAmpmZmZmZuT8BAIC8aQJib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Db3VudGVyQ29sdW1uVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwIAAAAAAADwPwDwDwAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTEaU3RhbmRhcmRMb25nMgAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/ANoPAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMRpTdGFuZGFyZExvbmcxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8A2A8AAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQ==
\ No newline at end of file
diff --git a/test/data/serialization/1.0/db.migration.Keyspace2.bin b/test/data/serialization/1.0/db.migration.Keyspace2.bin
deleted file mode 100644
index 0128443..0000000
--- a/test/data/serialization/1.0/db.migration.Keyspace2.bin
+++ /dev/null
@@ -1 +0,0 @@
-YØ/Dl7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiTWlncmF0aW9uIiwibmFtZXNwYWNlIjoib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWlncmF0aW9uLmF2cm8iLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRfdmVyc2lvbiIsInR5cGUiOnsidHlwZSI6ImZpeGVkIiwibmFtZSI6IlVVSUQiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvIiwic2l6ZSI6MTZ9fSx7Im5hbWUiOiJuZXdfdmVyc2lvbiIsInR5cGUiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvLlVVSUQifSx7Im5hbWUiOiJyb3dfbXV0YXRpb24iLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJjbGFzc25hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoibWlncmF0aW9uIiwidHlwZSI6W3sidHlwZSI6InJlY29yZCIsIm5hbWUiOiJBZGRDb2x1bW5GYW1pbHkiLCJmaWVsZHMiOlt7Im5hbWUiOiJjZiIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiRHJvcENvbHVtbkZhbWlseSIsImZpZWxkcyI6W3sibmFtZSI6ImtzbmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJjZm5hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiQWRkS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrcyIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJLc0RlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOiJDZkRlZiJ9fSx7Im5hbWUiOiJkdXJhYmxlX3dyaXRlcyIsInR5cGUiOiJib29sZWFuIiwiZGVmYXVsdCI6dHJ1ZX1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLktzRGVmIl19fV19LHsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJEcm9wS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrc25hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRLcyIsInR5cGUiOiJLc0RlZiJ9LHsibmFtZSI6Im5ld0tzIiwidHlwZSI6IktzRGVmIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlQ29sdW1uRmFtaWx5IiwiZmllbGRzIjpbeyJuYW1lIjoibWV0YWRhdGEiLCJ0eXBlIjoiQ2ZEZWYifV19XX1dfahxB4B3WRHhAAD+jr7q2fyocQeBd1kR4QAA/o6+6tn8yrMBAAZzeXN0ZW0AJGE4NzEwNzgxLTc3NTktMTFlMS0wMDAwLWZlOGViZWVhZDlmYwAAAAEAAAADAQAAAAOAAAAAgAAAAAAAAAAAAAAJAAtBdnJvL1NjaGVtYQAAAAE2T6gP+QAACu97InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiS3NEZWYiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uYXZybyIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX19LHsibmFtZSI6ImR1cmFibGVfd3JpdGVzIiwidHlwZSI6ImJvb2xlYW4iLCJkZWZhdWx0Ijp0cnVlfV0sImFsaWFzZXMiOlsib3JnLmFwYWNoZS5jYXNzYW5kcmEuY29uZmlnLmF2cm8uS3NEZWYiXX0ADUtleUNhY2hlU3BhY2UAAAABNk+oD/kAAARwGktleUNhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIGGktleUNhY2hlU3BhY2USU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAA8D8AmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AphAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAaS2V5Q2FjaGVTcGFjZRJTdGFuZGFyZDIAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAADwPwCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCkEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpLZXlDYWNoZVNwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAOA/AJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKIQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlMgAAAAE2T6gP+QAAB6kSS2V5c3BhY2UyVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACChJLZXlzcGFjZTIQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCMEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTISU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AhhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UyElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AIQQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMgxTdXBlcjMAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCIEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTIMU3VwZXI0AApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAWG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVGltZVVVSURUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AihAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2UzAAAAATZPqA/5AAADUBJLZXlzcGFjZTNWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgI1AAIEEktleXNwYWNlMxBJbmRleGVkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkFzY2lpVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJAQAAISYmlydGhkYXRlUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUAAAAkNjI2OTcyNzQ2ODY0NjE3NDY1AAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMxJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCOEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABAAlLZXlzcGFjZTQAAAABNk+oD/kAAAeGEktleXNwYWNlNFZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjMAAgoSS2V5c3BhY2U0ElN0YW5kYXJkMwAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJQQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNBJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCSEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQMU3VwZXIzAApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AlhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U0DFN1cGVyNAAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFhvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlRpbWVVVUlEVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJgQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNAxTdXBlcjUAClN1cGVyAFhvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLlRpbWVVVUlEVHlwZQBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCaEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABAAlLZXlzcGFjZTUAAAABNk+oD/kAAAMNEktleXNwYWNlNVZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjIAAgQSS2V5c3BhY2U1ElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJwQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNRBDb3VudGVyMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCYm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQ291bnRlckNvbHVtblR5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AnhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2U2AAAAATZPqA/5AAAB9hJLZXlzcGFjZTZWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAICEktleXNwYWNlNhBJbmRleGVkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkFzY2lpVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKAQAAISYmlydGhkYXRlUG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuTG9uZ1R5cGUAAAAkNjI2OTcyNzQ2ODY0NjE3NDY1AAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEAEE5vQ29tbWl0bG9nU3BhY2UAAAABNk+oD/kAAAG6IE5vQ29tbWl0bG9nU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAICIE5vQ29tbWl0bG9nU3BhY2USU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8ArBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAAANUm93Q2FjaGVTcGFjZQAAAAE2T6gP+QAAAxYaUm93Q2FjaGVTcGFjZVZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjEAAgQaUm93Q2FjaGVTcGFjZRBDYWNoZWRDRgAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAFlAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKoQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAGlJvd0NhY2hlU3BhY2UcQ0ZXaXRob3V0Q2FjaGUAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCoEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABWm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1pZ3JhdGlvbi5BZGRLZXlzcGFjZQQSS2V5c3BhY2UyVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACChJLZXlzcGFjZTIQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCMEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTISU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AhhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UyElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AIQQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlMgxTdXBlcjMAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCIEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTIMU3VwZXI0AApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAWG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVGltZVVVSURUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AihAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQ==
\ No newline at end of file
diff --git a/test/data/serialization/1.0/db.migration.Keyspace3.bin b/test/data/serialization/1.0/db.migration.Keyspace3.bin
deleted file mode 100644
index 37a092e..0000000
--- a/test/data/serialization/1.0/db.migration.Keyspace3.bin
+++ /dev/null
@@ -1 +0,0 @@
-I´/Dl7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiTWlncmF0aW9uIiwibmFtZXNwYWNlIjoib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWlncmF0aW9uLmF2cm8iLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRfdmVyc2lvbiIsInR5cGUiOnsidHlwZSI6ImZpeGVkIiwibmFtZSI6IlVVSUQiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvIiwic2l6ZSI6MTZ9fSx7Im5hbWUiOiJuZXdfdmVyc2lvbiIsInR5cGUiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvLlVVSUQifSx7Im5hbWUiOiJyb3dfbXV0YXRpb24iLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJjbGFzc25hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoibWlncmF0aW9uIiwidHlwZSI6W3sidHlwZSI6InJlY29yZCIsIm5hbWUiOiJBZGRDb2x1bW5GYW1pbHkiLCJmaWVsZHMiOlt7Im5hbWUiOiJjZiIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiRHJvcENvbHVtbkZhbWlseSIsImZpZWxkcyI6W3sibmFtZSI6ImtzbmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJjZm5hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiQWRkS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrcyIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJLc0RlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOiJDZkRlZiJ9fSx7Im5hbWUiOiJkdXJhYmxlX3dyaXRlcyIsInR5cGUiOiJib29sZWFuIiwiZGVmYXVsdCI6dHJ1ZX1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLktzRGVmIl19fV19LHsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJEcm9wS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrc25hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRLcyIsInR5cGUiOiJLc0RlZiJ9LHsibmFtZSI6Im5ld0tzIiwidHlwZSI6IktzRGVmIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlQ29sdW1uRmFtaWx5IiwiZmllbGRzIjpbeyJuYW1lIjoibWV0YWRhdGEiLCJ0eXBlIjoiQ2ZEZWYifV19XX1dfahy3EB3WRHhAAD+jr7q2fyoctxBd1kR4QAA/o6+6tn8yJQBAAZzeXN0ZW0AJGE4NzJkYzQxLTc3NTktMTFlMS0wMDAwLWZlOGViZWVhZDlmYwAAAAEAAAADAQAAAAOAAAAAgAAAAAAAAAAAAAAIAAtBdnJvL1NjaGVtYQAAAAE2T6gQBAAACu97InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiS3NEZWYiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uYXZybyIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX19LHsibmFtZSI6ImR1cmFibGVfd3JpdGVzIiwidHlwZSI6ImJvb2xlYW4iLCJkZWZhdWx0Ijp0cnVlfV0sImFsaWFzZXMiOlsib3JnLmFwYWNoZS5jYXNzYW5kcmEuY29uZmlnLmF2cm8uS3NEZWYiXX0ADUtleUNhY2hlU3BhY2UAAAABNk+oEAQAAARwGktleUNhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIGGktleUNhY2hlU3BhY2USU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAA8D8AmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AphAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAaS2V5Q2FjaGVTcGFjZRJTdGFuZGFyZDIAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAADwPwCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCkEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpLZXlDYWNoZVNwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAOA/AJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKIQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlMwAAAAE2T6gQBAAAA1ASS2V5c3BhY2UzVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICNQACBBJLZXlzcGFjZTMQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCQEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTMSU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AjhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2U0AAAAATZPqBAEAAAHhhJLZXlzcGFjZTRWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIzAAIKEktleXNwYWNlNBJTdGFuZGFyZDMAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCUEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQSU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AkhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U0DFN1cGVyMwAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJYQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNAxTdXBlcjQAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBYb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCYEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQMU3VwZXI1AApTdXBlcgBYb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AmhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2U1AAAAATZPqBAEAAADDRJLZXlzcGFjZTVWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIyAAIEEktleXNwYWNlNRJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCcEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTUQQ291bnRlcjEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAmJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkNvdW50ZXJDb2x1bW5UeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJ4QAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlNgAAAAE2T6gQBAAAAfYSS2V5c3BhY2U2Vm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACAhJLZXlzcGFjZTYQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCgEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABABBOb0NvbW1pdGxvZ1NwYWNlAAAAATZPqBAEAAABuiBOb0NvbW1pdGxvZ1NwYWNlVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACAiBOb0NvbW1pdGxvZ1NwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKwQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAAADVJvd0NhY2hlU3BhY2UAAAABNk+oEAQAAAMWGlJvd0NhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIEGlJvd0NhY2hlU3BhY2UQQ2FjaGVkQ0YAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAABZQAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCqEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpSb3dDYWNoZVNwYWNlHENGV2l0aG91dENhY2hlABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AqBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAVpvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uQWRkS2V5c3BhY2UEEktleXNwYWNlM1ZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjUAAgQSS2V5c3BhY2UzEEluZGV4ZWQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AkBAAAhJiaXJ0aGRhdGVQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQAAACQ2MjY5NzI3NDY4NjQ2MTc0NjUAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2UzElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AI4QAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAE=
\ No newline at end of file
diff --git a/test/data/serialization/1.0/db.migration.Keyspace4.bin b/test/data/serialization/1.0/db.migration.Keyspace4.bin
deleted file mode 100644
index 2e05da4..0000000
--- a/test/data/serialization/1.0/db.migration.Keyspace4.bin
+++ /dev/null
@@ -1 +0,0 @@
-JÈ/Dl7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiTWlncmF0aW9uIiwibmFtZXNwYWNlIjoib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWlncmF0aW9uLmF2cm8iLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRfdmVyc2lvbiIsInR5cGUiOnsidHlwZSI6ImZpeGVkIiwibmFtZSI6IlVVSUQiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvIiwic2l6ZSI6MTZ9fSx7Im5hbWUiOiJuZXdfdmVyc2lvbiIsInR5cGUiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvLlVVSUQifSx7Im5hbWUiOiJyb3dfbXV0YXRpb24iLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJjbGFzc25hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoibWlncmF0aW9uIiwidHlwZSI6W3sidHlwZSI6InJlY29yZCIsIm5hbWUiOiJBZGRDb2x1bW5GYW1pbHkiLCJmaWVsZHMiOlt7Im5hbWUiOiJjZiIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiRHJvcENvbHVtbkZhbWlseSIsImZpZWxkcyI6W3sibmFtZSI6ImtzbmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJjZm5hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiQWRkS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrcyIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJLc0RlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOiJDZkRlZiJ9fSx7Im5hbWUiOiJkdXJhYmxlX3dyaXRlcyIsInR5cGUiOiJib29sZWFuIiwiZGVmYXVsdCI6dHJ1ZX1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLktzRGVmIl19fV19LHsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJEcm9wS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrc25hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRLcyIsInR5cGUiOiJLc0RlZiJ9LHsibmFtZSI6Im5ld0tzIiwidHlwZSI6IktzRGVmIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlQ29sdW1uRmFtaWx5IiwiZmllbGRzIjpbeyJuYW1lIjoibWV0YWRhdGEiLCJ0eXBlIjoiQ2ZEZWYifV19XX1dfah0ifB3WRHhAAD+jr7q2fyodInxd1kR4QAA/o6+6tn8+IYBAAZzeXN0ZW0AJGE4NzQ4OWYxLTc3NTktMTFlMS0wMDAwLWZlOGViZWVhZDlmYwAAAAEAAAADAQAAAAOAAAAAgAAAAAAAAAAAAAAHAAtBdnJvL1NjaGVtYQAAAAE2T6gQDwAACu97InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiS3NEZWYiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uYXZybyIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX19LHsibmFtZSI6ImR1cmFibGVfd3JpdGVzIiwidHlwZSI6ImJvb2xlYW4iLCJkZWZhdWx0Ijp0cnVlfV0sImFsaWFzZXMiOlsib3JnLmFwYWNoZS5jYXNzYW5kcmEuY29uZmlnLmF2cm8uS3NEZWYiXX0ADUtleUNhY2hlU3BhY2UAAAABNk+oEA8AAARwGktleUNhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIGGktleUNhY2hlU3BhY2USU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAA8D8AmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AphAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAaS2V5Q2FjaGVTcGFjZRJTdGFuZGFyZDIAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAADwPwCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCkEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpLZXlDYWNoZVNwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAOA/AJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKIQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlNAAAAAE2T6gQDwAAB4YSS2V5c3BhY2U0Vm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMwACChJLZXlzcGFjZTQSU3RhbmRhcmQzABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AlBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U0ElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJIQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNAxTdXBlcjMAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCWEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQMU3VwZXI0AApTdXBlcgBSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUAWG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVGltZVVVSURUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AmBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U0DFN1cGVyNQAKU3VwZXIAWG9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuVGltZVVVSURUeXBlAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJoQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlNQAAAAE2T6gQDwAAAw0SS2V5c3BhY2U1Vm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMgACBBJLZXlzcGFjZTUSU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AnBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U1EENvdW50ZXIxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Db3VudGVyQ29sdW1uVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCeEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABAAlLZXlzcGFjZTYAAAABNk+oEA8AAAH2EktleXNwYWNlNlZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjEAAgISS2V5c3BhY2U2EEluZGV4ZWQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQXNjaWlUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AoBAAAhJiaXJ0aGRhdGVQb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Mb25nVHlwZQAAACQ2MjY5NzI3NDY4NjQ2MTc0NjUAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAQTm9Db21taXRsb2dTcGFjZQAAAAE2T6gQDwAAAbogTm9Db21taXRsb2dTcGFjZVZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjEAAgIgTm9Db21taXRsb2dTcGFjZRJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCsEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAAAAA1Sb3dDYWNoZVNwYWNlAAAAATZPqBAPAAADFhpSb3dDYWNoZVNwYWNlVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACBBpSb3dDYWNoZVNwYWNlEENhY2hlZENGABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAWUAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AqhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAaUm93Q2FjaGVTcGFjZRxDRldpdGhvdXRDYWNoZQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKgQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAFab3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWlncmF0aW9uLkFkZEtleXNwYWNlBBJLZXlzcGFjZTRWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIzAAIKEktleXNwYWNlNBJTdGFuZGFyZDMAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCUEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQSU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AkhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAASS2V5c3BhY2U0DFN1cGVyMwAKU3VwZXIAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJYQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNAxTdXBlcjQAClN1cGVyAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQBYb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCYEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTQMU3VwZXI1AApTdXBlcgBYb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5UaW1lVVVJRFR5cGUAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AmhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQ==
\ No newline at end of file
diff --git a/test/data/serialization/1.0/db.migration.Keyspace5.bin b/test/data/serialization/1.0/db.migration.Keyspace5.bin
deleted file mode 100644
index ec0704a..0000000
--- a/test/data/serialization/1.0/db.migration.Keyspace5.bin
+++ /dev/null
@@ -1 +0,0 @@
-:¨/Dl7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiTWlncmF0aW9uIiwibmFtZXNwYWNlIjoib3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWlncmF0aW9uLmF2cm8iLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRfdmVyc2lvbiIsInR5cGUiOnsidHlwZSI6ImZpeGVkIiwibmFtZSI6IlVVSUQiLCJuYW1lc3BhY2UiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvIiwic2l6ZSI6MTZ9fSx7Im5hbWUiOiJuZXdfdmVyc2lvbiIsInR5cGUiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS51dGlscy5hdnJvLlVVSUQifSx7Im5hbWUiOiJyb3dfbXV0YXRpb24iLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJjbGFzc25hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoibWlncmF0aW9uIiwidHlwZSI6W3sidHlwZSI6InJlY29yZCIsIm5hbWUiOiJBZGRDb2x1bW5GYW1pbHkiLCJmaWVsZHMiOlt7Im5hbWUiOiJjZiIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJDZkRlZiIsImZpZWxkcyI6W3sibmFtZSI6ImtleXNwYWNlIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiY29sdW1uX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoiY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InN1YmNvbXBhcmF0b3JfdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21tZW50IiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6ImtleV9jYWNoZV9zaXplIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlYWRfcmVwYWlyX2NoYW5jZSIsInR5cGUiOlsiZG91YmxlIiwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGVfb25fd3JpdGUiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOmZhbHNlfSx7Im5hbWUiOiJnY19ncmFjZV9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImRlZmF1bHRfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJrZXlfdmFsaWRhdGlvbl9jbGFzcyIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtaW5fY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWF4X2NvbXBhY3Rpb25fdGhyZXNob2xkIiwidHlwZSI6WyJudWxsIiwiaW50Il0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InJvd19jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjB9LHsibmFtZSI6ImtleV9jYWNoZV9zYXZlX3BlcmlvZF9pbl9zZWNvbmRzIiwidHlwZSI6WyJpbnQiLCJudWxsIl0sImRlZmF1bHQiOjM2MDB9LHsibmFtZSI6InJvd19jYWNoZV9rZXlzX3RvX3NhdmUiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoibWVyZ2Vfc2hhcmRzX2NoYW5jZSIsInR5cGUiOlsibnVsbCIsImRvdWJsZSJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJpZCIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjb2x1bW5fbWV0YWRhdGEiLCJ0eXBlIjpbeyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNvbHVtbkRlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoiYnl0ZXMifSx7Im5hbWUiOiJ2YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6InN0cmluZyJ9LHsibmFtZSI6ImluZGV4X3R5cGUiLCJ0eXBlIjpbeyJ0eXBlIjoiZW51bSIsIm5hbWUiOiJJbmRleFR5cGUiLCJzeW1ib2xzIjpbIktFWVMiLCJDVVNUT00iXSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5JbmRleFR5cGUiXX0sIm51bGwiXX0seyJuYW1lIjoiaW5kZXhfbmFtZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Db2x1bW5EZWYiXX19LCJudWxsIl19LHsibmFtZSI6InJvd19jYWNoZV9wcm92aWRlciIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdLCJkZWZhdWx0Ijoib3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuQ29uY3VycmVudExpbmtlZEhhc2hDYWNoZVByb3ZpZGVyIn0seyJuYW1lIjoia2V5X2FsaWFzIiwidHlwZSI6WyJudWxsIiwiYnl0ZXMiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcGFjdGlvbl9zdHJhdGVneSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiY29tcHJlc3Npb25fb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJibG9vbV9maWx0ZXJfZnBfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5DZkRlZiJdfX1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiRHJvcENvbHVtbkZhbWlseSIsImZpZWxkcyI6W3sibmFtZSI6ImtzbmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJjZm5hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiQWRkS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrcyIsInR5cGUiOnsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJLc0RlZiIsImZpZWxkcyI6W3sibmFtZSI6Im5hbWUiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoic3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlt7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifSwibnVsbCJdfSx7Im5hbWUiOiJyZXBsaWNhdGlvbl9mYWN0b3IiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiY2ZfZGVmcyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiaXRlbXMiOiJDZkRlZiJ9fSx7Im5hbWUiOiJkdXJhYmxlX3dyaXRlcyIsInR5cGUiOiJib29sZWFuIiwiZGVmYXVsdCI6dHJ1ZX1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLktzRGVmIl19fV19LHsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJEcm9wS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJrc25hbWUiLCJ0eXBlIjoic3RyaW5nIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlS2V5c3BhY2UiLCJmaWVsZHMiOlt7Im5hbWUiOiJvbGRLcyIsInR5cGUiOiJLc0RlZiJ9LHsibmFtZSI6Im5ld0tzIiwidHlwZSI6IktzRGVmIn1dfSx7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiVXBkYXRlQ29sdW1uRmFtaWx5IiwiZmllbGRzIjpbeyJuYW1lIjoibWV0YWRhdGEiLCJ0eXBlIjoiQ2ZEZWYifV19XX1dfah1wnB3WRHhAAD+jr7q2fyodemAd1kR4QAA/o6+6tn8vGgABnN5c3RlbQAkYTg3NWU5ODAtNzc1OS0xMWUxLTAwMDAtZmU4ZWJlZWFkOWZjAAAAAQAAAAMBAAAAA4AAAACAAAAAAAAAAAAAAAYAC0F2cm8vU2NoZW1hAAAAATZPqBAYAAAK73sidHlwZSI6InJlY29yZCIsIm5hbWUiOiJLc0RlZiIsIm5hbWVzcGFjZSI6Im9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1pZ3JhdGlvbi5hdnJvIiwiZmllbGRzIjpbeyJuYW1lIjoibmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJzdHJhdGVneV9jbGFzcyIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJzdHJhdGVneV9vcHRpb25zIiwidHlwZSI6W3sidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9LCJudWxsIl19LHsibmFtZSI6InJlcGxpY2F0aW9uX2ZhY3RvciIsInR5cGUiOlsiaW50IiwibnVsbCJdfSx7Im5hbWUiOiJjZl9kZWZzIiwidHlwZSI6eyJ0eXBlIjoiYXJyYXkiLCJpdGVtcyI6eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IkNmRGVmIiwiZmllbGRzIjpbeyJuYW1lIjoia2V5c3BhY2UiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoibmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJjb2x1bW5fdHlwZSIsInR5cGUiOlsic3RyaW5nIiwibnVsbCJdfSx7Im5hbWUiOiJjb21wYXJhdG9yX3R5cGUiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoic3ViY29tcGFyYXRvcl90eXBlIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6ImNvbW1lbnQiLCJ0eXBlIjpbInN0cmluZyIsIm51bGwiXX0seyJuYW1lIjoicm93X2NhY2hlX3NpemUiLCJ0eXBlIjpbImRvdWJsZSIsIm51bGwiXX0seyJuYW1lIjoia2V5X2NhY2hlX3NpemUiLCJ0eXBlIjpbImRvdWJsZSIsIm51bGwiXX0seyJuYW1lIjoicmVhZF9yZXBhaXJfY2hhbmNlIiwidHlwZSI6WyJkb3VibGUiLCJudWxsIl19LHsibmFtZSI6InJlcGxpY2F0ZV9vbl93cml0ZSIsInR5cGUiOiJib29sZWFuIiwiZGVmYXVsdCI6ZmFsc2V9LHsibmFtZSI6ImdjX2dyYWNlX3NlY29uZHMiLCJ0eXBlIjpbImludCIsIm51bGwiXX0seyJuYW1lIjoiZGVmYXVsdF92YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6WyJudWxsIiwic3RyaW5nIl0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6ImtleV92YWxpZGF0aW9uX2NsYXNzIiwidHlwZSI6WyJudWxsIiwic3RyaW5nIl0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6Im1pbl9jb21wYWN0aW9uX3RocmVzaG9sZCIsInR5cGUiOlsibnVsbCIsImludCJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtYXhfY29tcGFjdGlvbl90aHJlc2hvbGQiLCJ0eXBlIjpbIm51bGwiLCJpbnQiXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoicm93X2NhY2hlX3NhdmVfcGVyaW9kX2luX3NlY29uZHMiLCJ0eXBlIjpbImludCIsIm51bGwiXSwiZGVmYXVsdCI6MH0seyJuYW1lIjoia2V5X2NhY2hlX3NhdmVfcGVyaW9kX2luX3NlY29uZHMiLCJ0eXBlIjpbImludCIsIm51bGwiXSwiZGVmYXVsdCI6MzYwMH0seyJuYW1lIjoicm93X2NhY2hlX2tleXNfdG9fc2F2ZSIsInR5cGUiOlsibnVsbCIsImludCJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtZXJnZV9zaGFyZHNfY2hhbmNlIiwidHlwZSI6WyJudWxsIiwiZG91YmxlIl0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6ImlkIiwidHlwZSI6WyJpbnQiLCJudWxsIl19LHsibmFtZSI6ImNvbHVtbl9tZXRhZGF0YSIsInR5cGUiOlt7InR5cGUiOiJhcnJheSIsIml0ZW1zIjp7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiQ29sdW1uRGVmIiwiZmllbGRzIjpbeyJuYW1lIjoibmFtZSIsInR5cGUiOiJieXRlcyJ9LHsibmFtZSI6InZhbGlkYXRpb25fY2xhc3MiLCJ0eXBlIjoic3RyaW5nIn0seyJuYW1lIjoiaW5kZXhfdHlwZSIsInR5cGUiOlt7InR5cGUiOiJlbnVtIiwibmFtZSI6IkluZGV4VHlwZSIsInN5bWJvbHMiOlsiS0VZUyIsIkNVU1RPTSJdLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLkluZGV4VHlwZSJdfSwibnVsbCJdfSx7Im5hbWUiOiJpbmRleF9uYW1lIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl19LHsibmFtZSI6ImluZGV4X29wdGlvbnMiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6Im1hcCIsInZhbHVlcyI6InN0cmluZyJ9XSwiZGVmYXVsdCI6bnVsbH1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLkNvbHVtbkRlZiJdfX0sIm51bGwiXX0seyJuYW1lIjoicm93X2NhY2hlX3Byb3ZpZGVyIiwidHlwZSI6WyJzdHJpbmciLCJudWxsIl0sImRlZmF1bHQiOiJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5Db25jdXJyZW50TGlua2VkSGFzaENhY2hlUHJvdmlkZXIifSx7Im5hbWUiOiJrZXlfYWxpYXMiLCJ0eXBlIjpbIm51bGwiLCJieXRlcyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wYWN0aW9uX3N0cmF0ZWd5IiwidHlwZSI6WyJudWxsIiwic3RyaW5nIl0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6ImNvbXBhY3Rpb25fc3RyYXRlZ3lfb3B0aW9ucyIsInR5cGUiOlsibnVsbCIseyJ0eXBlIjoibWFwIiwidmFsdWVzIjoic3RyaW5nIn1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJjb21wcmVzc2lvbl9vcHRpb25zIiwidHlwZSI6WyJudWxsIix7InR5cGUiOiJtYXAiLCJ2YWx1ZXMiOiJzdHJpbmcifV0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6ImJsb29tX2ZpbHRlcl9mcF9jaGFuY2UiLCJ0eXBlIjpbIm51bGwiLCJkb3VibGUiXSwiZGVmYXVsdCI6bnVsbH1dLCJhbGlhc2VzIjpbIm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNvbmZpZy5hdnJvLkNmRGVmIl19fX0seyJuYW1lIjoiZHVyYWJsZV93cml0ZXMiLCJ0eXBlIjoiYm9vbGVhbiIsImRlZmF1bHQiOnRydWV9XSwiYWxpYXNlcyI6WyJvcmcuYXBhY2hlLmNhc3NhbmRyYS5jb25maWcuYXZyby5Lc0RlZiJdfQANS2V5Q2FjaGVTcGFjZQAAAAE2T6gQGAAABHAaS2V5Q2FjaGVTcGFjZVZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjEAAgYaS2V5Q2FjaGVTcGFjZRJTdGFuZGFyZDMAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAADwPwCamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCmEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpLZXlDYWNoZVNwYWNlElN0YW5kYXJkMgAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAPA/AJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKQQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAGktleUNhY2hlU3BhY2USU3RhbmRhcmQxABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAA4D8AmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AohAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQAJS2V5c3BhY2U1AAAAATZPqBAYAAADDRJLZXlzcGFjZTVWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIyAAIEEktleXNwYWNlNRJTdGFuZGFyZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCcEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABJLZXlzcGFjZTUQQ291bnRlcjEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAmJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkNvdW50ZXJDb2x1bW5UeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJ4QAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAEACUtleXNwYWNlNgAAAAE2T6gQGAAAAfYSS2V5c3BhY2U2Vm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACAhJLZXlzcGFjZTYQSW5kZXhlZDEAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAAAAAAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5Bc2NpaVR5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCgEAACEmJpcnRoZGF0ZVBvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkxvbmdUeXBlAAAAJDYyNjk3Mjc0Njg2NDYxNzQ2NQAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAAAABABBOb0NvbW1pdGxvZ1NwYWNlAAAAATZPqBAYAAABuiBOb0NvbW1pdGxvZ1NwYWNlVm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmxvY2F0b3IuU2ltcGxlU3RyYXRlZ3kAAiRyZXBsaWNhdGlvbl9mYWN0b3ICMQACAiBOb0NvbW1pdGxvZ1NwYWNlElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AKwQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAAAAADVJvd0NhY2hlU3BhY2UAAAABNk+oEBgAAAMWGlJvd0NhY2hlU3BhY2VWb3JnLmFwYWNoZS5jYXNzYW5kcmEubG9jYXRvci5TaW1wbGVTdHJhdGVneQACJHJlcGxpY2F0aW9uX2ZhY3RvcgIxAAIEGlJvd0NhY2hlU3BhY2UQQ2FjaGVkQ0YAEFN0YW5kYXJkAFJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIAAAAAAAAAAABZQAAAAAAAAAAAAACamZmZmZm5PwEAgLxpAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCCAJAAAAAAAL+////DwKamZmZmZm5PwCqEAAAAGZvcmcuYXBhY2hlLmNhc3NhbmRyYS5jYWNoZS5TZXJpYWxpemluZ0NhY2hlUHJvdmlkZXIAAn5vcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5jb21wYWN0aW9uLlNpemVUaWVyZWRDb21wYWN0aW9uU3RyYXRlZ3kCAAIAABpSb3dDYWNoZVNwYWNlHENGV2l0aG91dENhY2hlABBTdGFuZGFyZABSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAmpmZmZmZuT8BAIC8aQJSb3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIubWFyc2hhbC5CeXRlc1R5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AqBAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAVpvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5taWdyYXRpb24uQWRkS2V5c3BhY2UEEktleXNwYWNlNVZvcmcuYXBhY2hlLmNhc3NhbmRyYS5sb2NhdG9yLlNpbXBsZVN0cmF0ZWd5AAIkcmVwbGljYXRpb25fZmFjdG9yAjIAAgQSS2V5c3BhY2U1ElN0YW5kYXJkMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAlJvcmcuYXBhY2hlLmNhc3NhbmRyYS5kYi5tYXJzaGFsLkJ5dGVzVHlwZQIIAkAAAAAAAv7///8PApqZmZmZmbk/AJwQAAAAZm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmNhY2hlLlNlcmlhbGl6aW5nQ2FjaGVQcm92aWRlcgACfm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLmNvbXBhY3Rpb24uU2l6ZVRpZXJlZENvbXBhY3Rpb25TdHJhdGVneQIAAgAAEktleXNwYWNlNRBDb3VudGVyMQAQU3RhbmRhcmQAUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAgAAAAAAAAAAAAAAAAAAAAAAAAAAAJqZmZmZmbk/AQCAvGkCYm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQ291bnRlckNvbHVtblR5cGUCUm9yZy5hcGFjaGUuY2Fzc2FuZHJhLmRiLm1hcnNoYWwuQnl0ZXNUeXBlAggCQAAAAAAC/v///w8CmpmZmZmZuT8AnhAAAABmb3JnLmFwYWNoZS5jYXNzYW5kcmEuY2FjaGUuU2VyaWFsaXppbmdDYWNoZVByb3ZpZGVyAAJ+b3JnLmFwYWNoZS5jYXNzYW5kcmEuZGIuY29tcGFjdGlvbi5TaXplVGllcmVkQ29tcGFjdGlvblN0cmF0ZWd5AgACAAAAAQ==
\ No newline at end of file
diff --git a/test/data/serialization/1.0/gms.EndpointState.bin b/test/data/serialization/1.0/gms.EndpointState.bin
deleted file mode 100644
index 2383c5a..0000000
--- a/test/data/serialization/1.0/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/gms.Gossip.bin b/test/data/serialization/1.0/gms.Gossip.bin
deleted file mode 100644
index 20b27a8..0000000
--- a/test/data/serialization/1.0/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/service.TreeRequest.bin b/test/data/serialization/1.0/service.TreeRequest.bin
deleted file mode 100644
index b12a1b8..0000000
--- a/test/data/serialization/1.0/service.TreeRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/service.TreeResponse.bin b/test/data/serialization/1.0/service.TreeResponse.bin
deleted file mode 100644
index 4d94e2d..0000000
--- a/test/data/serialization/1.0/service.TreeResponse.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/streaming.PendingFile.bin b/test/data/serialization/1.0/streaming.PendingFile.bin
deleted file mode 100644
index ada0844..0000000
--- a/test/data/serialization/1.0/streaming.PendingFile.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/streaming.StreamHeader.bin b/test/data/serialization/1.0/streaming.StreamHeader.bin
deleted file mode 100644
index 90143b8..0000000
--- a/test/data/serialization/1.0/streaming.StreamHeader.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/streaming.StreamReply.bin b/test/data/serialization/1.0/streaming.StreamReply.bin
deleted file mode 100644
index 4b74058..0000000
--- a/test/data/serialization/1.0/streaming.StreamReply.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/streaming.StreamRequestMessage.bin b/test/data/serialization/1.0/streaming.StreamRequestMessage.bin
deleted file mode 100644
index 7d19ad9..0000000
--- a/test/data/serialization/1.0/streaming.StreamRequestMessage.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/utils.BloomFilter.bin b/test/data/serialization/1.0/utils.BloomFilter.bin
deleted file mode 100644
index f403170..0000000
--- a/test/data/serialization/1.0/utils.BloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/utils.EstimatedHistogram.bin b/test/data/serialization/1.0/utils.EstimatedHistogram.bin
deleted file mode 100644
index bedd39b..0000000
--- a/test/data/serialization/1.0/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.0/utils.LegacyBloomFilter.bin b/test/data/serialization/1.0/utils.LegacyBloomFilter.bin
deleted file mode 100644
index faef1b8..0000000
--- a/test/data/serialization/1.0/utils.LegacyBloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.RangeSliceCommand.bin b/test/data/serialization/1.2/db.RangeSliceCommand.bin
deleted file mode 100644
index 273d738..0000000
--- a/test/data/serialization/1.2/db.RangeSliceCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.Row.bin b/test/data/serialization/1.2/db.Row.bin
deleted file mode 100644
index ed5ad9d..0000000
--- a/test/data/serialization/1.2/db.Row.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.RowMutation.bin b/test/data/serialization/1.2/db.RowMutation.bin
deleted file mode 100644
index e62daed..0000000
--- a/test/data/serialization/1.2/db.RowMutation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.SliceByNamesReadCommand.bin b/test/data/serialization/1.2/db.SliceByNamesReadCommand.bin
deleted file mode 100644
index 9b9956d..0000000
--- a/test/data/serialization/1.2/db.SliceByNamesReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.SliceFromReadCommand.bin b/test/data/serialization/1.2/db.SliceFromReadCommand.bin
deleted file mode 100644
index 3e68cd3..0000000
--- a/test/data/serialization/1.2/db.SliceFromReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.Truncation.bin b/test/data/serialization/1.2/db.Truncation.bin
deleted file mode 100644
index ea67995..0000000
--- a/test/data/serialization/1.2/db.Truncation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/db.WriteResponse.bin b/test/data/serialization/1.2/db.WriteResponse.bin
deleted file mode 100644
index 9076238..0000000
--- a/test/data/serialization/1.2/db.WriteResponse.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/gms.EndpointState.bin b/test/data/serialization/1.2/gms.EndpointState.bin
deleted file mode 100644
index cd85279..0000000
--- a/test/data/serialization/1.2/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/gms.Gossip.bin b/test/data/serialization/1.2/gms.Gossip.bin
deleted file mode 100644
index af5ac57..0000000
--- a/test/data/serialization/1.2/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/service.TreeRequest.bin b/test/data/serialization/1.2/service.TreeRequest.bin
deleted file mode 100644
index b12a1b8..0000000
--- a/test/data/serialization/1.2/service.TreeRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/service.TreeResponse.bin b/test/data/serialization/1.2/service.TreeResponse.bin
deleted file mode 100644
index 90f3c6f..0000000
--- a/test/data/serialization/1.2/service.TreeResponse.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/streaming.PendingFile.bin b/test/data/serialization/1.2/streaming.PendingFile.bin
deleted file mode 100644
index efc8f77..0000000
--- a/test/data/serialization/1.2/streaming.PendingFile.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/streaming.StreamHeader.bin b/test/data/serialization/1.2/streaming.StreamHeader.bin
deleted file mode 100644
index ac5b7ac..0000000
--- a/test/data/serialization/1.2/streaming.StreamHeader.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/streaming.StreamReply.bin b/test/data/serialization/1.2/streaming.StreamReply.bin
deleted file mode 100644
index 6933316..0000000
--- a/test/data/serialization/1.2/streaming.StreamReply.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/streaming.StreamRequestMessage.bin b/test/data/serialization/1.2/streaming.StreamRequestMessage.bin
deleted file mode 100644
index fd53579..0000000
--- a/test/data/serialization/1.2/streaming.StreamRequestMessage.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/utils.BloomFilter.bin b/test/data/serialization/1.2/utils.BloomFilter.bin
deleted file mode 100644
index 6b4e7de..0000000
--- a/test/data/serialization/1.2/utils.BloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/utils.EstimatedHistogram.bin b/test/data/serialization/1.2/utils.EstimatedHistogram.bin
deleted file mode 100644
index bedd39b..0000000
--- a/test/data/serialization/1.2/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/1.2/utils.LegacyBloomFilter.bin b/test/data/serialization/1.2/utils.LegacyBloomFilter.bin
deleted file mode 100644
index faef1b8..0000000
--- a/test/data/serialization/1.2/utils.LegacyBloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/db.RangeSliceCommand.bin b/test/data/serialization/2.0/db.RangeSliceCommand.bin
deleted file mode 100644
index 099e429..0000000
--- a/test/data/serialization/2.0/db.RangeSliceCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/db.SliceByNamesReadCommand.bin b/test/data/serialization/2.0/db.SliceByNamesReadCommand.bin
deleted file mode 100644
index e9c33a2..0000000
--- a/test/data/serialization/2.0/db.SliceByNamesReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/db.SliceFromReadCommand.bin b/test/data/serialization/2.0/db.SliceFromReadCommand.bin
deleted file mode 100644
index 1beede3..0000000
--- a/test/data/serialization/2.0/db.SliceFromReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/db.Truncation.bin b/test/data/serialization/2.0/db.Truncation.bin
deleted file mode 100644
index ea67995..0000000
--- a/test/data/serialization/2.0/db.Truncation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/gms.EndpointState.bin b/test/data/serialization/2.0/gms.EndpointState.bin
deleted file mode 100644
index cd89893..0000000
--- a/test/data/serialization/2.0/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/gms.Gossip.bin b/test/data/serialization/2.0/gms.Gossip.bin
deleted file mode 100644
index af5ac57..0000000
--- a/test/data/serialization/2.0/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/service.SyncComplete.bin b/test/data/serialization/2.0/service.SyncComplete.bin
deleted file mode 100644
index 66c72e1..0000000
--- a/test/data/serialization/2.0/service.SyncComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/service.SyncRequest.bin b/test/data/serialization/2.0/service.SyncRequest.bin
deleted file mode 100644
index 8918405..0000000
--- a/test/data/serialization/2.0/service.SyncRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/service.ValidationComplete.bin b/test/data/serialization/2.0/service.ValidationComplete.bin
deleted file mode 100644
index 0c8d7be..0000000
--- a/test/data/serialization/2.0/service.ValidationComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/service.ValidationRequest.bin b/test/data/serialization/2.0/service.ValidationRequest.bin
deleted file mode 100644
index 4ec4c47..0000000
--- a/test/data/serialization/2.0/service.ValidationRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/streaming.PendingFile.bin b/test/data/serialization/2.0/streaming.PendingFile.bin
deleted file mode 100644
index efc8f77..0000000
--- a/test/data/serialization/2.0/streaming.PendingFile.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/streaming.StreamHeader.bin b/test/data/serialization/2.0/streaming.StreamHeader.bin
deleted file mode 100644
index f7e5edc..0000000
--- a/test/data/serialization/2.0/streaming.StreamHeader.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/streaming.StreamReply.bin b/test/data/serialization/2.0/streaming.StreamReply.bin
deleted file mode 100644
index 0094ecc..0000000
--- a/test/data/serialization/2.0/streaming.StreamReply.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/streaming.StreamRequestMessage.bin b/test/data/serialization/2.0/streaming.StreamRequestMessage.bin
deleted file mode 100644
index 71aaf78..0000000
--- a/test/data/serialization/2.0/streaming.StreamRequestMessage.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/utils.BloomFilter.bin b/test/data/serialization/2.0/utils.BloomFilter.bin
deleted file mode 100644
index 12f72f5..0000000
--- a/test/data/serialization/2.0/utils.BloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/utils.EstimatedHistogram.bin b/test/data/serialization/2.0/utils.EstimatedHistogram.bin
deleted file mode 100644
index bedd39b..0000000
--- a/test/data/serialization/2.0/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.0/utils.LegacyBloomFilter.bin b/test/data/serialization/2.0/utils.LegacyBloomFilter.bin
deleted file mode 100644
index faef1b8..0000000
--- a/test/data/serialization/2.0/utils.LegacyBloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/db.RangeSliceCommand.bin b/test/data/serialization/2.1/db.RangeSliceCommand.bin
deleted file mode 100644
index f852df0..0000000
--- a/test/data/serialization/2.1/db.RangeSliceCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/db.SliceByNamesReadCommand.bin b/test/data/serialization/2.1/db.SliceByNamesReadCommand.bin
deleted file mode 100644
index e9c33a2..0000000
--- a/test/data/serialization/2.1/db.SliceByNamesReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/db.SliceFromReadCommand.bin b/test/data/serialization/2.1/db.SliceFromReadCommand.bin
deleted file mode 100644
index 1beede3..0000000
--- a/test/data/serialization/2.1/db.SliceFromReadCommand.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/db.Truncation.bin b/test/data/serialization/2.1/db.Truncation.bin
deleted file mode 100644
index ea67995..0000000
--- a/test/data/serialization/2.1/db.Truncation.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/db.WriteResponse.bin b/test/data/serialization/2.1/db.WriteResponse.bin
deleted file mode 100644
index e69de29..0000000
--- a/test/data/serialization/2.1/db.WriteResponse.bin
+++ /dev/null
diff --git a/test/data/serialization/2.1/gms.EndpointState.bin b/test/data/serialization/2.1/gms.EndpointState.bin
deleted file mode 100644
index f87fc77..0000000
--- a/test/data/serialization/2.1/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/gms.Gossip.bin b/test/data/serialization/2.1/gms.Gossip.bin
deleted file mode 100644
index af5ac57..0000000
--- a/test/data/serialization/2.1/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/service.SyncComplete.bin b/test/data/serialization/2.1/service.SyncComplete.bin
deleted file mode 100644
index 533abe2..0000000
--- a/test/data/serialization/2.1/service.SyncComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/service.SyncRequest.bin b/test/data/serialization/2.1/service.SyncRequest.bin
deleted file mode 100644
index 2bb8bf9..0000000
--- a/test/data/serialization/2.1/service.SyncRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/service.ValidationComplete.bin b/test/data/serialization/2.1/service.ValidationComplete.bin
deleted file mode 100644
index 6eff48f..0000000
--- a/test/data/serialization/2.1/service.ValidationComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/service.ValidationRequest.bin b/test/data/serialization/2.1/service.ValidationRequest.bin
deleted file mode 100644
index e774d05..0000000
--- a/test/data/serialization/2.1/service.ValidationRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/utils.BloomFilter.bin b/test/data/serialization/2.1/utils.BloomFilter.bin
deleted file mode 100644
index 357042a..0000000
--- a/test/data/serialization/2.1/utils.BloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/utils.BloomFilter1000.bin b/test/data/serialization/2.1/utils.BloomFilter1000.bin
deleted file mode 100644
index 619679c..0000000
--- a/test/data/serialization/2.1/utils.BloomFilter1000.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/2.1/utils.EstimatedHistogram.bin b/test/data/serialization/2.1/utils.EstimatedHistogram.bin
deleted file mode 100644
index bedd39b..0000000
--- a/test/data/serialization/2.1/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/gms.EndpointState.bin b/test/data/serialization/3.0/gms.EndpointState.bin
deleted file mode 100644
index a230ae1..0000000
--- a/test/data/serialization/3.0/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/gms.Gossip.bin b/test/data/serialization/3.0/gms.Gossip.bin
deleted file mode 100644
index af5ac57..0000000
--- a/test/data/serialization/3.0/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/service.SyncComplete.bin b/test/data/serialization/3.0/service.SyncComplete.bin
deleted file mode 100644
index 73ea4b4..0000000
--- a/test/data/serialization/3.0/service.SyncComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/service.SyncRequest.bin b/test/data/serialization/3.0/service.SyncRequest.bin
deleted file mode 100644
index 7e09777..0000000
--- a/test/data/serialization/3.0/service.SyncRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/service.ValidationComplete.bin b/test/data/serialization/3.0/service.ValidationComplete.bin
deleted file mode 100644
index b8f0fb9..0000000
--- a/test/data/serialization/3.0/service.ValidationComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/service.ValidationRequest.bin b/test/data/serialization/3.0/service.ValidationRequest.bin
deleted file mode 100644
index a00763b..0000000
--- a/test/data/serialization/3.0/service.ValidationRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/utils.BloomFilter.bin b/test/data/serialization/3.0/utils.BloomFilter.bin
deleted file mode 100644
index e8bfb4f..0000000
--- a/test/data/serialization/3.0/utils.BloomFilter.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/3.0/utils.EstimatedHistogram.bin b/test/data/serialization/3.0/utils.EstimatedHistogram.bin
deleted file mode 100644
index bedd39b..0000000
--- a/test/data/serialization/3.0/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.0/gms.EndpointState.bin b/test/data/serialization/4.0/gms.EndpointState.bin
index 17fc088..083dbb7 100644
--- a/test/data/serialization/4.0/gms.EndpointState.bin
+++ b/test/data/serialization/4.0/gms.EndpointState.bin
Binary files differ
diff --git a/test/data/serialization/4.0/gms.Gossip.bin b/test/data/serialization/4.0/gms.Gossip.bin
index 2fbd5d4..7a4fb56 100644
--- a/test/data/serialization/4.0/gms.Gossip.bin
+++ b/test/data/serialization/4.0/gms.Gossip.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.SyncComplete.bin b/test/data/serialization/4.0/service.SyncComplete.bin
index 4e8caa6..39a6243 100644
--- a/test/data/serialization/4.0/service.SyncComplete.bin
+++ b/test/data/serialization/4.0/service.SyncComplete.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.SyncRequest.bin b/test/data/serialization/4.0/service.SyncRequest.bin
index 17bb014..f853b20 100644
--- a/test/data/serialization/4.0/service.SyncRequest.bin
+++ b/test/data/serialization/4.0/service.SyncRequest.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.ValidationComplete.bin b/test/data/serialization/4.0/service.ValidationComplete.bin
index 7402c9e..bb4de43 100644
--- a/test/data/serialization/4.0/service.ValidationComplete.bin
+++ b/test/data/serialization/4.0/service.ValidationComplete.bin
Binary files differ
diff --git a/test/data/serialization/4.0/service.ValidationRequest.bin b/test/data/serialization/4.0/service.ValidationRequest.bin
index fa4a913..04c492a 100644
--- a/test/data/serialization/4.0/service.ValidationRequest.bin
+++ b/test/data/serialization/4.0/service.ValidationRequest.bin
Binary files differ
diff --git a/test/data/serialization/4.1/gms.EndpointState.bin b/test/data/serialization/4.1/gms.EndpointState.bin
deleted file mode 100644
index 083dbb7..0000000
--- a/test/data/serialization/4.1/gms.EndpointState.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.1/gms.Gossip.bin b/test/data/serialization/4.1/gms.Gossip.bin
deleted file mode 100644
index 7a4fb56..0000000
--- a/test/data/serialization/4.1/gms.Gossip.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.1/service.SyncComplete.bin b/test/data/serialization/4.1/service.SyncComplete.bin
deleted file mode 100644
index 39a6243..0000000
--- a/test/data/serialization/4.1/service.SyncComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.1/service.SyncRequest.bin b/test/data/serialization/4.1/service.SyncRequest.bin
deleted file mode 100644
index f853b20..0000000
--- a/test/data/serialization/4.1/service.SyncRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.1/service.ValidationComplete.bin b/test/data/serialization/4.1/service.ValidationComplete.bin
deleted file mode 100644
index bb4de43..0000000
--- a/test/data/serialization/4.1/service.ValidationComplete.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.1/service.ValidationRequest.bin b/test/data/serialization/4.1/service.ValidationRequest.bin
deleted file mode 100644
index 04c492a..0000000
--- a/test/data/serialization/4.1/service.ValidationRequest.bin
+++ /dev/null
Binary files differ
diff --git a/test/data/serialization/4.1/utils.EstimatedHistogram.bin b/test/data/serialization/4.1/utils.EstimatedHistogram.bin
deleted file mode 100644
index e878eda..0000000
--- a/test/data/serialization/4.1/utils.EstimatedHistogram.bin
+++ /dev/null
Binary files differ
diff --git a/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java b/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java
index 5b23b86..7a46935 100644
--- a/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java
+++ b/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java
@@ -32,6 +32,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.api.IInstance;
@@ -467,25 +468,24 @@
return netVersion;
}
- public static void withProperty(String prop, boolean value, Runnable r)
+ public static void withProperty(CassandraRelevantProperties prop, boolean value, Runnable r)
{
withProperty(prop, Boolean.toString(value), r);
}
- public static void withProperty(String prop, String value, Runnable r)
+ public static void withProperty(CassandraRelevantProperties prop, String value, Runnable r)
{
- String before = System.getProperty(prop);
+ String prev = prop.setString(value);
try
{
- System.setProperty(prop, value);
r.run();
}
finally
{
- if (before == null)
- System.clearProperty(prop);
+ if (prev == null)
+ prop.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
else
- System.setProperty(prop, before);
+ prop.setString(prev);
}
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/api/Row.java b/test/distributed/org/apache/cassandra/distributed/api/Row.java
index 33272ed..9d08cb2 100644
--- a/test/distributed/org/apache/cassandra/distributed/api/Row.java
+++ b/test/distributed/org/apache/cassandra/distributed/api/Row.java
@@ -62,7 +62,7 @@
this.nameIndex = nameIndex;
}
- void setResults(Object[] results)
+ public void setResults(Object[] results)
{
this.results = results;
}
diff --git a/test/distributed/org/apache/cassandra/distributed/fuzz/HarryHelper.java b/test/distributed/org/apache/cassandra/distributed/fuzz/HarryHelper.java
index 2997fa6..66178f2 100644
--- a/test/distributed/org/apache/cassandra/distributed/fuzz/HarryHelper.java
+++ b/test/distributed/org/apache/cassandra/distributed/fuzz/HarryHelper.java
@@ -26,19 +26,28 @@
import harry.model.clock.OffsetClock;
import harry.model.sut.PrintlnSut;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_ALLOW_SIMPLE_STRATEGY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_MINIMUM_REPLICATION_FACTOR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOG4J2_DISABLE_JMX;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOG4J2_DISABLE_JMX_LEGACY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOG4J_SHUTDOWN_HOOK_ENABLED;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RELOCATED_SHADED_IO_NETTY_TRANSPORT_NONATIVE;
+
public class HarryHelper
{
public static void init()
{
- System.setProperty("log4j2.disableJmx", "true"); // setting both ways as changes between versions
- System.setProperty("log4j2.disable.jmx", "true");
- System.setProperty("log4j.shutdownHookEnabled", "false");
- System.setProperty("cassandra.allow_simplestrategy", "true"); // makes easier to share OSS tests without RF limits
- System.setProperty("cassandra.minimum_replication_factor", "0"); // makes easier to share OSS tests without RF limits
-
- System.setProperty("cassandra.disable_tcactive_openssl", "true");
- System.setProperty("relocated.shaded.io.netty.transport.noNative", "true");
- System.setProperty("org.apache.cassandra.disable_mbean_registration", "true");
+ // setting both ways as changes between versions
+ LOG4J2_DISABLE_JMX.setBoolean(true);
+ LOG4J2_DISABLE_JMX_LEGACY.setBoolean(true);
+ LOG4J_SHUTDOWN_HOOK_ENABLED.setBoolean(false);
+ CASSANDRA_ALLOW_SIMPLE_STRATEGY.setBoolean(true); // makes easier to share OSS tests without RF limits
+ CASSANDRA_MINIMUM_REPLICATION_FACTOR.setInt(0); // makes easier to share OSS tests without RF limits
+ DISABLE_TCACTIVE_OPENSSL.setBoolean(true);
+ RELOCATED_SHADED_IO_NETTY_TRANSPORT_NONATIVE.setBoolean(true);
+ ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true);
InJvmSut.init();
QueryingNoOpChecker.init();
diff --git a/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java b/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
index 627d360..c7d0fbb 100644
--- a/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
+++ b/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
@@ -306,7 +306,8 @@
new AbstractMarker.Raw(values.size() - 1)));
}
- StatementRestrictions restrictions = new StatementRestrictions(StatementType.DELETE,
+ StatementRestrictions restrictions = new StatementRestrictions(null,
+ StatementType.DELETE,
metadata,
builder.build(),
new VariableSpecifications(variableNames),
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
index 7d33372..78b14d2 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
@@ -187,7 +187,7 @@
{
// those properties may be set for unit-test optimizations; those should not be used when running dtests
- CassandraRelevantProperties.FLUSH_LOCAL_SCHEMA_CHANGES.reset();
+ CassandraRelevantProperties.TEST_FLUSH_LOCAL_SCHEMA_CHANGES.reset();
CassandraRelevantProperties.NON_GRACEFUL_SHUTDOWN.reset();
}
@@ -650,6 +650,16 @@
return stream().map(IInstance::coordinator);
}
+ public List<I> get(int... nodes)
+ {
+ if (nodes == null || nodes.length == 0)
+ throw new IllegalArgumentException("No nodes provided");
+ List<I> list = new ArrayList<>(nodes.length);
+ for (int i : nodes)
+ list.add(get(i));
+ return list;
+ }
+
/**
* WARNING: we index from 1 here, for consistency with inet address!
*/
@@ -1033,6 +1043,8 @@
@Override
public void close()
{
+ logger.info("Closing cluster {}", this.clusterId);
+ FBUtilities.closeQuietly(instanceInitializer);
FBUtilities.waitOnFutures(instances.stream()
.filter(i -> !i.isShutdown())
.map(IInstance::shutdown)
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index d4cb1cb..435de65 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -25,6 +25,7 @@
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
import java.security.Permission;
import java.util.ArrayList;
import java.util.Collections;
@@ -96,8 +97,8 @@
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
-import org.apache.cassandra.io.sstable.IndexSummaryManager;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -146,11 +147,17 @@
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.logging.LoggingSupportFactory;
import org.apache.cassandra.utils.memory.BufferPools;
import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_RANGE_MOVEMENT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_SIMULTANEOUS_MOVES_ALLOW;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_SUITENAME;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_TESTTAG;
import static org.apache.cassandra.distributed.api.Feature.BLANK_GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
@@ -216,14 +223,14 @@
{
// the path used is defined by test/conf/logback-dtest.xml and looks like the following
// ./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log
- String tag = System.getProperty("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED");
- String suite = System.getProperty("suitename", "suitename_IS_UNDEFINED");
+ String tag = TEST_CASSANDRA_TESTTAG.getString();
+ String suite = TEST_CASSANDRA_SUITENAME.getString();
String clusterId = ClusterIDDefiner.getId();
String instanceId = InstanceIDDefiner.getInstanceId();
- File f = new File(String.format("build/test/logs/%s/%s/%s/%s/system.log", tag, suite, clusterId, instanceId));
+ File f = new File(FileSystems.getDefault(), String.format("build/test/logs/%s/%s/%s/%s/system.log", tag, suite, clusterId, instanceId));
// when creating a cluster globally in a test class we get the logs without the suite, try finding those logs:
if (!f.exists())
- f = new File(String.format("build/test/logs/%s/%s/%s/system.log", tag, clusterId, instanceId));
+ f = new File(FileSystems.getDefault(), String.format("build/test/logs/%s/%s/%s/system.log", tag, clusterId, instanceId));
if (!f.exists())
throw new AssertionError("Unable to locate system.log under " + new File("build/test/logs").absolutePath() + "; make sure ICluster.setup() is called or extend TestBaseImpl and do not define a static beforeClass function with @BeforeClass");
return new FileLogAction(f);
@@ -441,7 +448,7 @@
batch.id.serialize(out);
out.writeLong(batch.creationTime);
- out.writeUnsignedVInt(batch.getEncodedMutations().size());
+ out.writeUnsignedVInt32(batch.getEncodedMutations().size());
for (ByteBuffer mutation : batch.getEncodedMutations())
{
@@ -508,7 +515,14 @@
}
else
{
- header.verb.stage.executor().execute(ExecutorLocals.create(state), () -> MessagingService.instance().inboundSink.accept(messageIn));
+ ExecutorPlus executor = header.verb.stage.executor();
+ if (executor.isShutdown())
+ {
+ MessagingService.instance().metrics.recordDroppedMessage(messageIn, messageIn.elapsedSinceCreated(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
+ inInstancelogger.warn("Dropping message {} due to stage {} being shutdown", messageIn, header.verb.stage);
+ return;
+ }
+ executor.execute(ExecutorLocals.create(state), () -> MessagingService.instance().inboundSink.accept(messageIn));
}
};
}
@@ -578,9 +592,9 @@
if (config.has(GOSSIP))
{
// TODO: hacky
- System.setProperty("cassandra.ring_delay_ms", "15000");
- System.setProperty("cassandra.consistent.rangemovement", "false");
- System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true");
+ RING_DELAY.setLong(15000);
+ CONSISTENT_RANGE_MOVEMENT.setBoolean(false);
+ CONSISTENT_SIMULTANEOUS_MOVES_ALLOW.setBoolean(true);
}
mkdirs();
@@ -590,9 +604,12 @@
DistributedTestSnitch.assign(config.networkTopology());
DatabaseDescriptor.daemonInitialization();
+ LoggingSupportFactory.getLoggingSupport().onStartup();
+
FileUtils.setFSErrorHandler(new DefaultFSErrorHandler());
DatabaseDescriptor.createAllDirectories();
CassandraDaemon.getInstanceForTesting().migrateSystemDataIfNeeded();
+ CassandraDaemon.logSystemInfo(inInstancelogger);
CommitLog.instance.start();
CassandraDaemon.getInstanceForTesting().runStartupChecks();
@@ -619,6 +636,9 @@
// Start up virtual table support
CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces();
+ // clean up debris in data directories
+ CassandraDaemon.getInstanceForTesting().scrubDataDirectories();
+
Keyspace.setInitialized();
// Replay any CommitLogSegments found on disk
@@ -663,6 +683,7 @@
propagateMessagingVersions(cluster); // fake messaging needs to know messaging version for filters
}
internodeMessagingStarted = true;
+
JVMStabilityInspector.replaceKiller(new InstanceKiller(Instance.this::shutdown));
// TODO: this is more than just gossip
@@ -931,9 +952,9 @@
public NodeToolResult nodetoolResult(boolean withNotifications, String... commandAndArgs)
{
return sync(() -> {
- try (CapturingOutput output = new CapturingOutput())
+ try (CapturingOutput output = new CapturingOutput();
+ DTestNodeTool nodetool = new DTestNodeTool(withNotifications, output.delegate))
{
- DTestNodeTool nodetool = new DTestNodeTool(withNotifications, output.delegate);
// install security manager to get informed about the exit-code
System.setSecurityManager(new SecurityManager()
{
@@ -1017,15 +1038,18 @@
}
}
- public static class DTestNodeTool extends NodeTool {
+ public static class DTestNodeTool extends NodeTool implements AutoCloseable
+ {
private final StorageServiceMBean storageProxy;
private final CollectingNotificationListener notifications = new CollectingNotificationListener();
-
+ private final InternalNodeProbe internalNodeProbe;
private Throwable latestError;
- public DTestNodeTool(boolean withNotifications, Output output) {
+ public DTestNodeTool(boolean withNotifications, Output output)
+ {
super(new InternalNodeProbeFactory(withNotifications), output);
- storageProxy = new InternalNodeProbe(withNotifications).getStorageService();
+ internalNodeProbe = new InternalNodeProbe(withNotifications);
+ storageProxy = internalNodeProbe.getStorageService();
storageProxy.addNotificationListener(notifications, null, null);
}
@@ -1071,6 +1095,12 @@
super.err(e);
latestError = e;
}
+
+ @Override
+ public void close()
+ {
+ internalNodeProbe.close();
+ }
}
private static final class CollectingNotificationListener implements NotificationListener
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java b/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java
index 06fdabb..9c347e0 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/TracingUtil.java
@@ -27,7 +27,6 @@
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.utils.TimeUUID;
-
/**
* Utilities for accessing the system_traces table from in-JVM dtests
*/
@@ -104,14 +103,4 @@
}
return traces;
}
-
- // Set up the wait for tracing time system property, returning the previous value.
- // Handles being called again to reset with the original value, replacing the null
- // with the default value.
- public static String setWaitForTracingEventTimeoutSecs(String timeoutInSeconds)
- {
- return System.setProperty("cassandra.wait_for_tracing_events_timeout_secs",
- timeoutInSeconds == null ? "0" : timeoutInSeconds);
-
- }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
index 097206a..381bb09 100644
--- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
+++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
@@ -21,7 +21,6 @@
import java.lang.management.ManagementFactory;
import java.util.Iterator;
import java.util.Map;
-import javax.management.ListenerNotFoundException;
import com.google.common.collect.Multimap;
@@ -45,14 +44,13 @@
import org.apache.cassandra.service.GCInspector;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.StorageServiceMBean;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.tools.NodeProbe;
-import org.mockito.Mockito;
public class InternalNodeProbe extends NodeProbe
{
private final boolean withNotifications;
+ private boolean previousSkipNotificationListeners = false;
public InternalNodeProbe(boolean withNotifications)
{
@@ -66,26 +64,10 @@
mbeanServerConn = null;
jmxc = null;
- if (withNotifications)
- {
- ssProxy = StorageService.instance;
- }
- else
- {
- // replace the notification apis with a no-op method
- StorageServiceMBean mock = Mockito.spy(StorageService.instance);
- Mockito.doNothing().when(mock).addNotificationListener(Mockito.any(), Mockito.any(), Mockito.any());
- try
- {
- Mockito.doNothing().when(mock).removeNotificationListener(Mockito.any(), Mockito.any(), Mockito.any());
- Mockito.doNothing().when(mock).removeNotificationListener(Mockito.any());
- }
- catch (ListenerNotFoundException e)
- {
- throw new AssertionError(e);
- }
- ssProxy = mock;
- }
+ previousSkipNotificationListeners = StorageService.instance.skipNotificationListeners;
+ StorageService.instance.skipNotificationListeners = !withNotifications;
+
+ ssProxy = StorageService.instance;
msProxy = MessagingService.instance();
streamProxy = StreamManager.instance;
compactionProxy = CompactionManager.instance;
@@ -105,7 +87,7 @@
@Override
public void close()
{
- // nothing to close. no-op
+ StorageService.instance.skipNotificationListeners = previousSkipNotificationListeners;
}
@Override
diff --git a/test/distributed/org/apache/cassandra/distributed/shared/Byteman.java b/test/distributed/org/apache/cassandra/distributed/shared/Byteman.java
index b4dd10c..c5c76e3 100644
--- a/test/distributed/org/apache/cassandra/distributed/shared/Byteman.java
+++ b/test/distributed/org/apache/cassandra/distributed/shared/Byteman.java
@@ -47,11 +47,13 @@
import org.jboss.byteman.agent.Transformer;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_BYTEMAN_TRANSFORMATIONS_DEBUG;
+
public final class Byteman
{
private static final Logger logger = LoggerFactory.getLogger(Byteman.class);
- private static final boolean DEBUG_TRANSFORMATIONS = Boolean.getBoolean("cassandra.test.byteman.transformations.debug");
+ private static final boolean DEBUG_TRANSFORMATIONS = TEST_BYTEMAN_TRANSFORMATIONS_DEBUG.getBoolean();
private static final Method METHOD;
private static final URL BYTEMAN;
diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
index dc280f3..c313821 100644
--- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
+++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
@@ -40,6 +40,10 @@
import java.util.stream.Collectors;
import com.google.common.util.concurrent.Futures;
+
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.io.util.File;
import org.junit.Assert;
@@ -59,6 +63,9 @@
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SCHEMA_DELAY_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BROADCAST_INTERVAL_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS_FIRST_BOOT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY;
import static org.assertj.core.api.Assertions.assertThat;
/**
@@ -251,13 +258,13 @@
return start(inst, properties -> {
// lower this so the replacement waits less time
- properties.setProperty("cassandra.broadcast_interval_ms", Long.toString(TimeUnit.SECONDS.toMillis(30)));
+ properties.set(BROADCAST_INTERVAL_MS, Long.toString(TimeUnit.SECONDS.toMillis(30)));
// default is 30s, lowering as it should be faster
- properties.setProperty("cassandra.ring_delay_ms", Long.toString(TimeUnit.SECONDS.toMillis(10)));
+ properties.set(RING_DELAY, Long.toString(TimeUnit.SECONDS.toMillis(10)));
properties.set(BOOTSTRAP_SCHEMA_DELAY_MS, TimeUnit.SECONDS.toMillis(10));
// state which node to replace
- properties.setProperty("cassandra.replace_address_first_boot", toReplace.config().broadcastAddress().getAddress().getHostAddress());
+ properties.set(REPLACE_ADDRESS_FIRST_BOOT, toReplace.config().broadcastAddress().getAddress().getHostAddress());
fn.accept(inst, properties);
});
@@ -554,6 +561,63 @@
});
}
+ public static void awaitGossipSchemaMatch(ICluster<? extends IInstance> cluster)
+ {
+ cluster.forEach(ClusterUtils::awaitGossipSchemaMatch);
+ }
+
+ public static void awaitGossipSchemaMatch(IInstance instance)
+ {
+ if (!instance.config().has(Feature.GOSSIP))
+ {
+ // when gosisp isn't enabled, don't bother waiting on gossip to settle...
+ return;
+ }
+ awaitGossip(instance, "Schema IDs did not match", all -> {
+ String current = null;
+ for (Map.Entry<String, Map<String, String>> e : all.entrySet())
+ {
+ Map<String, String> state = e.getValue();
+ // has the instance joined?
+ String status = state.get(ApplicationState.STATUS_WITH_PORT.name());
+ if (status == null)
+ status = state.get(ApplicationState.STATUS.name());
+ if (status == null || !status.contains(VersionedValue.STATUS_NORMAL))
+ continue; // ignore instances not joined yet
+ String schema = state.get("SCHEMA");
+ if (schema == null)
+ throw new AssertionError("Unable to find schema for " + e.getKey() + "; status was " + status);
+ schema = schema.split(":")[1];
+
+ if (current == null)
+ {
+ current = schema;
+ }
+ else if (!current.equals(schema))
+ {
+ return false;
+ }
+ }
+ return true;
+ });
+ }
+
+ public static void awaitGossipStateMatch(ICluster<? extends IInstance> cluster, IInstance expectedInGossip, ApplicationState key)
+ {
+ Set<String> matches = null;
+ for (int i = 0; i < 100; i++)
+ {
+ matches = cluster.stream().map(ClusterUtils::gossipInfo)
+ .map(gi -> Objects.requireNonNull(gi.get(getBroadcastAddressString(expectedInGossip))))
+ .map(m -> m.get(key.name()))
+ .collect(Collectors.toSet());
+ if (matches.isEmpty() || matches.size() == 1)
+ return;
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+ throw new AssertionError("Expected ApplicationState." + key + " to match, but saw " + matches);
+ }
+
/**
* Get the gossip information from the node. Currently only address, generation, and heartbeat are returned
*
diff --git a/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java b/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java
index 88987c2..d17d3e6 100644
--- a/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java
+++ b/test/distributed/org/apache/cassandra/distributed/shared/WithProperties.java
@@ -23,92 +23,73 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.function.Supplier;
import com.google.common.base.Joiner;
import org.apache.cassandra.config.CassandraRelevantProperties;
+import static org.apache.cassandra.config.CassandraRelevantProperties.convertToString;
+
public final class WithProperties implements AutoCloseable
{
- private final List<Property> properties = new ArrayList<>();
+ private final List<Runnable> rollback = new ArrayList<>();
public WithProperties()
{
}
- public WithProperties(String... kvs)
- {
- with(kvs);
- }
-
- public void with(String... kvs)
+ public WithProperties with(String... kvs)
{
assert kvs.length % 2 == 0 : "Input must have an even amount of inputs but given " + kvs.length;
for (int i = 0; i <= kvs.length - 2; i = i + 2)
- {
with(kvs[i], kvs[i + 1]);
- }
+ return this;
}
- public void setProperty(String key, String value)
+ public WithProperties set(CassandraRelevantProperties prop, String value)
{
- with(key, value);
+ return set(prop, () -> prop.setString(value));
}
- public void set(CassandraRelevantProperties prop, String value)
+ public WithProperties set(CassandraRelevantProperties prop, String... values)
{
- with(prop.getKey(), value);
+ return set(prop, Arrays.asList(values));
}
- public void set(CassandraRelevantProperties prop, String... values)
+ public WithProperties set(CassandraRelevantProperties prop, Collection<String> values)
{
- set(prop, Arrays.asList(values));
+ return set(prop, Joiner.on(",").join(values));
}
- public void set(CassandraRelevantProperties prop, Collection<String> values)
+ public WithProperties set(CassandraRelevantProperties prop, boolean value)
{
- set(prop, Joiner.on(",").join(values));
+ return set(prop, () -> prop.setBoolean(value));
}
- public void set(CassandraRelevantProperties prop, boolean value)
+ public WithProperties set(CassandraRelevantProperties prop, long value)
{
- set(prop, Boolean.toString(value));
+ return set(prop, () -> prop.setLong(value));
}
- public void set(CassandraRelevantProperties prop, long value)
+ private void with(String key, String value)
{
- set(prop, Long.toString(value));
+ String previous = System.setProperty(key, value); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
+ rollback.add(previous == null ? () -> System.clearProperty(key) : () -> System.setProperty(key, previous)); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
}
- public void with(String key, String value)
+ private WithProperties set(CassandraRelevantProperties prop, Supplier<Object> prev)
{
- String previous = System.setProperty(key, value);
- properties.add(new Property(key, previous));
+ String previous = convertToString(prev.get());
+ rollback.add(previous == null ? prop::clearValue : () -> prop.setString(previous));
+ return this;
}
-
@Override
public void close()
{
- Collections.reverse(properties);
- properties.forEach(s -> {
- if (s.value == null)
- System.getProperties().remove(s.key);
- else
- System.setProperty(s.key, s.value);
- });
- properties.clear();
- }
-
- private static final class Property
- {
- private final String key;
- private final String value;
-
- private Property(String key, String value)
- {
- this.key = key;
- this.value = value;
- }
+ Collections.reverse(rollback);
+ rollback.forEach(Runnable::run);
+ rollback.clear();
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java
new file mode 100644
index 0000000..7a94905
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.distributed.Cluster.build;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.awaitility.Awaitility.await;
+
+public class AllowAutoSnapshotTest extends TestBaseImpl
+{
+ @Test
+ public void testAllowAutoSnapshotOnAutoSnapshotEnabled() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true"));
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table;"));
+
+ checkSnapshots(c, true, "test_table", "dropped");
+
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH allow_auto_snapshot = false"));
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table2;"));
+
+ checkSnapshots(c, false, "test_table2", "dropped");
+ }
+ }
+
+ @Test
+ public void testAllowAutoSnapshotOnAutoSnapshotDisabled() throws Exception
+ {
+ try (Cluster c = getCluster(false))
+ {
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true"));
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table;"));
+
+ checkSnapshots(c, false, "test_table", "dropped");
+
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH allow_auto_snapshot = false"));
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table2;"));
+
+ checkSnapshots(c, false, "test_table2", "dropped");
+ }
+ }
+
+ @Test
+ public void testDisableAndEnableAllowAutoSnapshot() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true"));
+
+ c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH allow_auto_snapshot = false"));
+ c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH allow_auto_snapshot = true"));
+
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table;"));
+
+ checkSnapshots(c, true, "test_table", "dropped");
+ }
+ }
+
+ @Test
+ public void testTruncateAllowAutoSnapshot() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ // allow_auto_snapshot = true
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true"));
+
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1);"), ALL);
+
+ c.schemaChange(withKeyspace("TRUNCATE TABLE %s.test_table;"));
+
+ checkSnapshots(c, true, "test_table", "truncated");
+
+ // allow_auto_snapshot = false
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH allow_auto_snapshot = false"));
+
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table2 (a, b) VALUES ('a', 1);"), ALL);
+
+ c.schemaChange(withKeyspace("TRUNCATE TABLE %s.test_table2;"));
+
+ checkSnapshots(c, false, "test_table2", "truncated");
+ }
+ }
+
+ @Test
+ public void testMaterializedViewAllowAutoSnapshot() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ // materialized view allow_auto_snapshot = false
+ c.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c1 int, c2 int, v1 int, v2 int, PRIMARY KEY (k, c1, c2))"));
+ c.schemaChange(withKeyspace("CREATE MATERIALIZED VIEW %s.mv1 AS SELECT * FROM t WHERE k IS NOT NULL AND c1 IS NOT NULL AND c2 IS NOT NULL PRIMARY KEY (c1, k, c2) WITH allow_auto_snapshot = false"));
+
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.t (k, c1, c2, v1, v2) VALUES (1, 2, 3, 4, 5);"), ALL);
+
+ c.schemaChange(withKeyspace("DROP MATERIALIZED VIEW %s.mv1;"));
+
+ checkSnapshots(c, false, "mv1", "dropped");
+
+ // materialized view allow_auto_snapshot = true
+ c.schemaChange(withKeyspace("CREATE MATERIALIZED VIEW %s.mv1 AS SELECT * FROM t WHERE k IS NOT NULL AND c1 IS NOT NULL AND c2 IS NOT NULL PRIMARY KEY (c1, k, c2) WITH allow_auto_snapshot = true"));
+
+ c.schemaChange(withKeyspace("DROP MATERIALIZED VIEW %s.mv1;"));
+ checkSnapshots(c, true, "mv1", "dropped");
+ }
+ }
+
+ private Cluster getCluster(boolean autoSnapshotEnabled) throws IOException
+ {
+ return init(build(2).withConfig(c -> c.with(GOSSIP)
+ .set("auto_snapshot", autoSnapshotEnabled)
+ .set("materialized_views_enabled", true)).start());
+ }
+
+ private void checkSnapshots(Cluster cluster, boolean shouldContain, String table, String snapshotPrefix)
+ {
+ for (int i = 1; i < cluster.size() + 1; i++)
+ {
+ final int node = i; // has to be effectively final for the usage in "until" method
+ await().until(() -> cluster.get(node).appliesOnInstance((IIsolatedExecutor.SerializableTriFunction<Boolean, String, String, Boolean>) (shouldContainSnapshot, tableName, prefix) -> {
+ Stream<String> stream = StorageService.instance.getSnapshotDetails(Collections.emptyMap()).keySet().stream();
+ Predicate<String> predicate = tag -> tag.startsWith(prefix + '-') && tag.endsWith('-' + tableName);
+ return shouldContainSnapshot ? stream.anyMatch(predicate) : stream.noneMatch(predicate);
+ }).apply(shouldContain, table, snapshotPrefix));
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/AlterTest.java b/test/distributed/org/apache/cassandra/distributed/test/AlterTest.java
index 2061c291..bc7159b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/AlterTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/AlterTest.java
@@ -38,10 +38,10 @@
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.ClusterUtils;
-import org.apache.cassandra.distributed.util.QueryResultUtil;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Throwables;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
@@ -99,7 +99,7 @@
{
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance gossippingOnlyMember = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", Boolean.toString(false), () -> gossippingOnlyMember.startup(cluster));
+ withProperty(JOIN_RING, false, () -> gossippingOnlyMember.startup(cluster));
int attempts = 0;
// it takes some time the underlying structure is populated otherwise the test is flaky
@@ -158,12 +158,8 @@
cluster.schemaChange("ALTER TABLE " + KEYSPACE + ".tbl WITH memtable = 'testconfig'", false, node1);
// the above should succeed, the configuration is acceptable to node1
- final String schema1 = QueryResultUtil.expand(node1.executeInternalWithResult("SELECT * FROM system_schema.tables WHERE keyspace_name=?", KEYSPACE));
- final String schema2 = QueryResultUtil.expand(node2.executeInternalWithResult("SELECT * FROM system_schema.tables WHERE keyspace_name=?", KEYSPACE));
- logger.info("node1 schema: \n{}", schema1);
- logger.info("node2 schema: \n{}", schema2);
- Assert.assertEquals(schema1, schema2);
- List<String> errorInLog = node2.logs().grep(mark, "ERROR.*Invalid memtable configuration.*").getResult();
+ ClusterUtils.awaitGossipSchemaMatch(cluster);
+ List<String> errorInLog = node2.logs().grep(mark,"ERROR.*Invalid memtable configuration.*").getResult();
Assert.assertTrue(errorInLog.size() > 0);
logger.info(Lists.listToString(errorInLog));
@@ -175,9 +171,7 @@
"testconfig", ImmutableMap.of(
"class_name", "NotExistingMemtable")))));
node3.startup(cluster);
- final String schema3 = QueryResultUtil.expand(node3.executeInternalWithResult("SELECT * FROM system_schema.tables WHERE keyspace_name=?", KEYSPACE));
- logger.info("node3 schema: \n{}", schema3);
- Assert.assertEquals(schema1, schema3);
+ ClusterUtils.awaitGossipSchemaMatch(cluster);
errorInLog = node3.logs().grep("ERROR.*Invalid memtable configuration.*").getResult();
Assert.assertTrue(errorInLog.size() > 0);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java b/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
index 3f50c30..627b695 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
@@ -24,7 +24,9 @@
import java.util.Map;
import java.util.concurrent.TimeoutException;
+import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
@@ -36,13 +38,33 @@
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.Byteman;
import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.utils.Shared;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_WRITE_SURVEY;
+
/**
* Replaces python dtest bootstrap_test.py::TestBootstrap::test_bootstrap_binary_disabled
*/
public class BootstrapBinaryDisabledTest extends TestBaseImpl
{
+ static WithProperties properties;
+
+ @BeforeClass
+ public static void beforeClass() throws Throwable
+ {
+ TestBaseImpl.beforeClass();
+ properties = new WithProperties().set(RESET_BOOTSTRAP_PROGRESS, false);
+ }
+
+ @AfterClass
+ public static void afterClass()
+ {
+ properties.close();
+ }
+
@Test
public void test() throws IOException, TimeoutException
{
@@ -92,9 +114,9 @@
config.forEach(nodeConfig::set);
//TODO can we make this more isolated?
- System.setProperty("cassandra.ring_delay_ms", "5000");
+ RING_DELAY.setLong(5000);
if (isWriteSurvey)
- System.setProperty("cassandra.write_survey", "true");
+ TEST_WRITE_SURVEY.setBoolean(true);
RewriteEnabled.enable();
cluster.bootstrap(nodeConfig).startup();
@@ -108,6 +130,7 @@
.failure()
.errorContains("Cannot join the ring until bootstrap completes");
+ node.nodetoolResult("bootstrap", "resume").asserts().failure();
RewriteEnabled.disable();
node.nodetoolResult("bootstrap", "resume").asserts().success();
if (isWriteSurvey)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java b/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java
index aafbc45..60e8db0 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java
@@ -35,6 +35,7 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_USE_SELF_EXECUTION;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ;
@@ -45,7 +46,7 @@
@BeforeClass
public static void beforeClass() throws Throwable
{
- System.setProperty("cassandra.paxos.use_self_execution", "false");
+ PAXOS_USE_SELF_EXECUTION.setBoolean(false);
TestBaseImpl.beforeClass();
Consumer<IInstanceConfig> conf = config -> config
.set("paxos_variant", "v2")
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASTest.java b/test/distributed/org/apache/cassandra/distributed/test/CASTest.java
index 8e2b8ac..dc79256 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/CASTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/CASTest.java
@@ -39,6 +39,7 @@
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_USE_SELF_EXECUTION;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ANY;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM;
@@ -76,7 +77,7 @@
@BeforeClass
public static void beforeClass() throws Throwable
{
- System.setProperty("cassandra.paxos.use_self_execution", "false");
+ PAXOS_USE_SELF_EXECUTION.setBoolean(false);
TestBaseImpl.beforeClass();
Consumer<IInstanceConfig> conf = config -> config
.set("paxos_variant", "v2")
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ClusterStorageUsageTest.java b/test/distributed/org/apache/cassandra/distributed/test/ClusterStorageUsageTest.java
new file mode 100644
index 0000000..9c379d1
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/ClusterStorageUsageTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * This class verifies the behavior of our global disk usage metrics across different cluster and replication
+ * configurations. In addition, it verifies that they are properly exposed via the public metric registry.
+ *
+ * Disk usage metrics are characterized by how they handle compression and replication:
+ *
+ * "compressed" -> indicates raw disk usage
+ * "uncompressed" -> indicates uncompressed file size (which is equivalent to "compressed" with no compression enabled)
+ * "replicated" -> includes disk usage for data outside the node's primary range
+ * "unreplicated" -> indicates disk usage scaled down by replication factor across the entire cluster
+ */
+public class ClusterStorageUsageTest extends TestBaseImpl
+{
+ private static final DefaultNameFactory FACTORY = new DefaultNameFactory("Storage");
+ private static final int MUTATIONS = 1000;
+
+ @Test
+ public void testNoReplication() throws Throwable
+ {
+ // With a replication factor of 1 for our only user keyspace, system keyspaces using local replication, and
+ // empty distributed system tables, replicated and unreplicated versions of our compressed and uncompressed
+ // metrics should be equivalent.
+
+ try (Cluster cluster = init(builder().withNodes(2).start(), 1))
+ {
+ populateUserKeyspace(cluster);
+ verifyLoadMetricsWithoutReplication(cluster.get(1));
+ verifyLoadMetricsWithoutReplication(cluster.get(2));
+ }
+ }
+
+ private void verifyLoadMetricsWithoutReplication(IInvokableInstance node)
+ {
+ long compressedLoad = getLoad(node);
+ long uncompressedLoad = getUncompressedLoad(node);
+ assertThat(compressedLoad).isEqualTo(getUnreplicatedLoad(node));
+ assertThat(uncompressedLoad).isEqualTo(getUnreplicatedUncompressedLoad(node));
+ assertThat(uncompressedLoad).isGreaterThan(compressedLoad);
+ }
+
+ @Test
+ public void testSimpleReplication() throws Throwable
+ {
+ // With a replication factor of 2 for our only user keyspace, disk space used by that keyspace should
+ // be scaled down by a factor of 2, while contributions from system keyspaces are unaffected.
+
+ try (Cluster cluster = init(builder().withNodes(3).start(), 2))
+ {
+ populateUserKeyspace(cluster);
+
+ verifyLoadMetricsWithReplication(cluster.get(1));
+ verifyLoadMetricsWithReplication(cluster.get(2));
+ verifyLoadMetricsWithReplication(cluster.get(3));
+ }
+ }
+
+ @Test
+ public void testMultiDatacenterReplication() throws Throwable
+ {
+ // With a replication factor of 1 for our only user keyspace in two DCs, disk space used by that keyspace should
+ // be scaled down by a factor of 2, while contributions from system keyspaces are unaffected.
+
+ try (Cluster cluster = builder().withDC("DC1", 2).withDC("DC2", 2).start())
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1': 1, 'DC2': 1};");
+ populateUserKeyspace(cluster);
+
+ verifyLoadMetricsWithReplication(cluster.get(1));
+ verifyLoadMetricsWithReplication(cluster.get(2));
+ verifyLoadMetricsWithReplication(cluster.get(3));
+ verifyLoadMetricsWithReplication(cluster.get(4));
+ }
+ }
+
+ private void populateUserKeyspace(Cluster cluster)
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, v text, PRIMARY KEY (pk));"));
+
+ for (int i = 0; i < MUTATIONS; i++) {
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (pk, v) VALUES (?,?)"), ConsistencyLevel.ALL, i, "compressable");
+ }
+
+ cluster.forEach((i) -> i.flush(KEYSPACE));
+ }
+
+ private void verifyLoadMetricsWithReplication(IInvokableInstance node)
+ {
+ long unreplicatedLoad = getUnreplicatedLoad(node);
+ long expectedUnreplicatedLoad = computeUnreplicatedMetric(node, table -> table.metric.liveDiskSpaceUsed.getCount());
+ assertThat(expectedUnreplicatedLoad).isEqualTo(unreplicatedLoad);
+ assertThat(getLoad(node)).isGreaterThan(unreplicatedLoad);
+
+ long unreplicatedUncompressedLoad = getUnreplicatedUncompressedLoad(node);
+ long expectedUnreplicatedUncompressedLoad = computeUnreplicatedMetric(node, table -> table.metric.uncompressedLiveDiskSpaceUsed.getCount());
+ assertThat(expectedUnreplicatedUncompressedLoad).isEqualTo(unreplicatedUncompressedLoad);
+ assertThat(getUncompressedLoad(node)).isGreaterThan(unreplicatedUncompressedLoad);
+ }
+
+ private long getLoad(IInvokableInstance node)
+ {
+ return node.metrics().getCounter(FACTORY.createMetricName("Load").getMetricName());
+ }
+
+ private long getUncompressedLoad(IInvokableInstance node1)
+ {
+ return node1.metrics().getCounter(FACTORY.createMetricName("UncompressedLoad").getMetricName());
+ }
+
+ private long getUnreplicatedLoad(IInvokableInstance node)
+ {
+ return (Long) node.metrics().getGauge(FACTORY.createMetricName("UnreplicatedLoad").getMetricName());
+ }
+
+ private long getUnreplicatedUncompressedLoad(IInvokableInstance node)
+ {
+ return (Long) node.metrics().getGauge(FACTORY.createMetricName("UnreplicatedUncompressedLoad").getMetricName());
+ }
+
+ private long computeUnreplicatedMetric(IInvokableInstance node, SerializableFunction<ColumnFamilyStore, Long> metric)
+ {
+ return node.callOnInstance(() ->
+ {
+ long sum = 0;
+
+ for (Keyspace keyspace : Keyspace.all())
+ for (ColumnFamilyStore table : keyspace.getColumnFamilyStores())
+ sum += metric.apply(table) / keyspace.getReplicationStrategy().getReplicationFactor().fullReplicas;
+
+ return sum;
+ });
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ColumnMaskTest.java b/test/distributed/org/apache/cassandra/distributed/test/ColumnMaskTest.java
new file mode 100644
index 0000000..3dd197c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/ColumnMaskTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.function.Consumer;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+import com.datastax.driver.core.policies.LoadBalancingPolicy;
+import org.apache.cassandra.auth.CassandraRoleManager;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.impl.RowUtil;
+import org.apache.cassandra.distributed.util.SingleHostLoadBalancingPolicy;
+
+import static com.datastax.driver.core.Cluster.Builder;
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.auth.CassandraRoleManager.DEFAULT_SUPERUSER_NAME;
+import static org.apache.cassandra.auth.CassandraRoleManager.DEFAULT_SUPERUSER_PASSWORD;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+import static org.awaitility.Awaitility.await;
+
+/**
+ * Tests for dynamic data masking.
+ */
+public class ColumnMaskTest extends TestBaseImpl
+{
+ private static final String SELECT = withKeyspace("SELECT * FROM %s.t");
+ private static final String USERNAME = "ddm_user";
+ private static final String PASSWORD = "ddm_password";
+
+ /**
+ * Tests that column masks are propagated to all nodes in the cluster.
+ */
+ @Test
+ public void testMaskPropagation() throws Throwable
+ {
+ try (Cluster cluster = createClusterWithAuhentication(3))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int PRIMARY KEY, v text MASKED WITH DEFAULT) WITH read_repair='NONE'"));
+ cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.t(k, v) VALUES (1, 'secret1')"));
+ cluster.get(2).executeInternal(withKeyspace("INSERT INTO %s.t(k, v) VALUES (2, 'secret2')"));
+ cluster.get(3).executeInternal(withKeyspace("INSERT INTO %s.t(k, v) VALUES (3, 'secret3')"));
+
+ assertRowsInAllCoordinators(cluster, row(1, "****"), row(2, "****"), row(3, "****"));
+
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER v DROP MASKED"));
+ assertRowsInAllCoordinators(cluster, row(1, "secret1"), row(2, "secret2"), row(3, "secret3"));
+
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER v MASKED WITH mask_inner(null, 1)"));
+ assertRowsInAllCoordinators(cluster, row(1, "******1"), row(2, "******2"), row(3, "******3"));
+
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER v MASKED WITH mask_inner(3, null)"));
+ assertRowsInAllCoordinators(cluster, row(1, "sec****"), row(2, "sec****"), row(3, "sec****"));
+ }
+ }
+
+ /**
+ * Tests that column masks are properly loaded at startup.
+ */
+ @Test
+ public void testMaskLoading() throws Throwable
+ {
+ try (Cluster cluster = createClusterWithAuhentication(1))
+ {
+ IInvokableInstance node = cluster.get(1);
+
+ cluster.schemaChange(withKeyspace("CREATE FUNCTION %s.custom_mask(column text, replacement text) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return replacement;'"));
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (" +
+ "a text MASKED WITH DEFAULT, " +
+ "b text MASKED WITH mask_replace('redacted'), " +
+ "c text MASKED WITH mask_inner(null, 1), " +
+ "d text MASKED WITH mask_inner(3, null), " +
+ "e text MASKED WITH %<s.custom_mask('obscured'), " +
+ "PRIMARY KEY (a, b))"));
+ String insert = withKeyspace("INSERT INTO %s.t(a, b, c, d, e) VALUES (?, ?, ?, ?, ?)");
+ node.executeInternal(insert, "secret1", "secret1", "secret1", "secret1", "secret1");
+ node.executeInternal(insert, "secret2", "secret2", "secret2", "secret2", "secret2");
+ assertRowsWithRestart(node,
+ row("****", "redacted", "******1", "sec****", "obscured"),
+ row("****", "redacted", "******2", "sec****", "obscured"));
+
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER a DROP MASKED"));
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER b MASKED WITH mask_null()"));
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER c MASKED WITH mask_inner(null, null, '#')"));
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER d MASKED WITH mask_inner(3, 1, '#')"));
+ cluster.schemaChange(withKeyspace("ALTER TABLE %s.t ALTER e MASKED WITH %<s.custom_mask('censored')"));
+ assertRowsWithRestart(node,
+ row("secret1", null, "#######", "sec###1", "censored"),
+ row("secret2", null, "#######", "sec###2", "censored"));
+ }
+ }
+
+ private static Cluster createClusterWithAuhentication(int nodeCount) throws IOException
+ {
+ Cluster cluster = init(Cluster.build()
+ .withNodes(nodeCount)
+ .withConfig(conf -> conf.with(GOSSIP, NATIVE_PROTOCOL)
+ .set("dynamic_data_masking_enabled", "true")
+ .set("user_defined_functions_enabled", "true")
+ .set("authenticator", "PasswordAuthenticator")
+ .set("authorizer", "CassandraAuthorizer"))
+ .start());
+
+ // create a user without UNMASK permission
+ withAuthenticatedSession(cluster.get(1), DEFAULT_SUPERUSER_NAME, DEFAULT_SUPERUSER_PASSWORD, session -> {
+ session.execute(format("CREATE USER IF NOT EXISTS %s WITH PASSWORD '%s'", USERNAME, PASSWORD));
+ session.execute(format("GRANT ALL ON KEYSPACE %s TO %s", KEYSPACE, USERNAME));
+ session.execute(format("REVOKE UNMASK ON KEYSPACE %s FROM %s", KEYSPACE, USERNAME));
+ });
+
+ return cluster;
+ }
+
+ private static void assertRowsInAllCoordinators(Cluster cluster, Object[]... expectedRows)
+ {
+ for (int i = 1; i < cluster.size(); i++)
+ {
+ assertRowsWithAuthentication(cluster.get(i), expectedRows);
+ }
+ }
+
+ private static void assertRowsWithRestart(IInvokableInstance node, Object[]... expectedRows) throws Throwable
+ {
+ // test querying with in-memory column definitions
+ assertRowsWithAuthentication(node, expectedRows);
+
+ // restart the nodes to reload the column definitions from disk
+ node.shutdown().get();
+ node.startup();
+
+ // test querying with the column definitions loaded from disk
+ assertRowsWithAuthentication(node, expectedRows);
+ }
+
+ private static void assertRowsWithAuthentication(IInvokableInstance node, Object[]... expectedRows)
+ {
+ withAuthenticatedSession(node, USERNAME, PASSWORD, session -> {
+ Statement statement = new SimpleStatement(SELECT).setConsistencyLevel(ConsistencyLevel.ALL);
+ ResultSet resultSet = session.execute(statement);
+ assertRows(RowUtil.toObjects(resultSet), expectedRows);
+ });
+ }
+
+ private static void withAuthenticatedSession(IInvokableInstance instance, String username, String password, Consumer<Session> consumer)
+ {
+ // wait for existing roles
+ await().pollDelay(1, SECONDS)
+ .pollInterval(1, SECONDS)
+ .atMost(1, MINUTES)
+ .until(() -> instance.callOnInstance(CassandraRoleManager::hasExistingRoles));
+
+ // use a load balancing policy that ensures that we actually connect to the desired node
+ InetAddress address = instance.broadcastAddress().getAddress();
+ LoadBalancingPolicy lbc = new SingleHostLoadBalancingPolicy(address);
+
+ Builder builder = com.datastax.driver.core.Cluster.builder()
+ .addContactPoints(address)
+ .withLoadBalancingPolicy(lbc)
+ .withCredentials(username, password);
+
+ try (com.datastax.driver.core.Cluster cluster = builder.build();
+ Session session = cluster.connect())
+ {
+ consumer.accept(session);
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java
new file mode 100644
index 0000000..099f87d
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileStoreUtils;
+import org.apache.cassandra.io.util.PathUtils;
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.ActiveCompactions;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.junit.Assert.fail;
+
+public class CompactionDiskSpaceTest extends TestBaseImpl
+{
+ @Test
+ public void testNoSpaceLeft() throws IOException
+ {
+ try(Cluster cluster = init(Cluster.build(1)
+ .withConfig(config -> config.set("min_free_space_per_drive_in_mb", "0"))
+ .withDataDirCount(3)
+ .withInstanceInitializer(BB::install)
+ .start()))
+ {
+ cluster.schemaChange("create table "+KEYSPACE+".tbl (id int primary key, x int) with compaction = {'class':'SizeTieredCompactionStrategy'}");
+ cluster.coordinator(1).execute("insert into "+KEYSPACE+".tbl (id, x) values (1,1)", ConsistencyLevel.ALL);
+ cluster.get(1).flush(KEYSPACE);
+ cluster.setUncaughtExceptionsFilter((t) -> t.getMessage() != null && t.getMessage().contains("Not enough space for compaction") && t.getMessage().contains(KEYSPACE+".tbl"));
+ cluster.get(1).runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+ BB.estimatedRemaining.set(2000);
+ BB.freeSpace.set(2000);
+ BB.sstableDir = cfs.getLiveSSTables().iterator().next().descriptor.directory;
+ try
+ {
+ cfs.forceMajorCompaction();
+ fail("This should fail, we have 2000b free space, 2000b estimated remaining compactions and the new compaction > 0");
+ }
+ catch (Exception e)
+ {
+ // ignored
+ }
+ // and available space again:
+ BB.estimatedRemaining.set(1000);
+ cfs.forceMajorCompaction();
+
+ BB.estimatedRemaining.set(2000);
+ BB.freeSpace.set(10000);
+ cfs.forceMajorCompaction();
+
+ // make sure we fail if other dir on the same file store runs out of disk
+ BB.freeSpace.set(0);
+ for (Directories.DataDirectory newDir : cfs.getDirectories().getWriteableLocations())
+ {
+ File newSSTableDir = cfs.getDirectories().getLocationForDisk(newDir);
+ if (!BB.sstableDir.equals(newSSTableDir))
+ {
+ BB.sstableDir = cfs.getDirectories().getLocationForDisk(newDir);
+ break;
+ }
+ }
+ try
+ {
+ cfs.forceMajorCompaction();
+ fail("this should fail, data dirs share filestore");
+ }
+ catch (Exception e)
+ {
+ //ignored
+ }
+ });
+ }
+ }
+
+ public static class BB
+ {
+ static final AtomicLong estimatedRemaining = new AtomicLong();
+ static final AtomicLong freeSpace = new AtomicLong();
+ static File sstableDir;
+ public static void install(ClassLoader cl, Integer node)
+ {
+ new ByteBuddy().rebase(ActiveCompactions.class)
+ .method(named("estimatedRemainingWriteBytes"))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+
+ new ByteBuddy().rebase(FileStoreUtils.class)
+ .method(named("tryGetSpace"))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static Map<File, Long> estimatedRemainingWriteBytes()
+ {
+ if (sstableDir != null)
+ return ImmutableMap.of(sstableDir, estimatedRemaining.get());
+ return Collections.emptyMap();
+ }
+
+ public static long tryGetSpace(FileStore fileStore, PathUtils.IOToLongFunction<FileStore> function)
+ {
+ try
+ {
+ if (sstableDir != null && Files.getFileStore(sstableDir.toPath()).equals(fileStore))
+ return freeSpace.get();
+ }
+ catch (IOException e)
+ {
+ // ignore
+ }
+ return Long.MAX_VALUE;
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/CompactionOverlappingSSTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/CompactionOverlappingSSTableTest.java
index 6a65c91..44caf85 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/CompactionOverlappingSSTableTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/CompactionOverlappingSSTableTest.java
@@ -18,8 +18,10 @@
package org.apache.cassandra.distributed.test;
+import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -96,13 +98,13 @@
public static void install(ClassLoader cl, Integer i)
{
new ByteBuddy().rebase(Directories.class)
- .method(named("hasAvailableDiskSpace").and(takesArguments(2)))
+ .method(named("hasDiskSpaceForCompactionsAndStreams").and(takesArguments(2)))
.intercept(MethodDelegation.to(BB.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
}
- public static boolean hasAvailableDiskSpace(long ignore1, long ignore2)
+ public static boolean hasDiskSpaceForCompactionsAndStreams(Map<File,Long> ignore1, Map<File,Long> ignore2)
{
if (enabled.get())
{
diff --git a/test/distributed/org/apache/cassandra/distributed/test/DataResurrectionCheckTest.java b/test/distributed/org/apache/cassandra/distributed/test/DataResurrectionCheckTest.java
index 1b297da..f48c872 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/DataResurrectionCheckTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/DataResurrectionCheckTest.java
@@ -25,11 +25,11 @@
import org.junit.Test;
-import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.StartupChecksOptions;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.DataResurrectionCheck;
@@ -40,6 +40,7 @@
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD;
import static org.apache.cassandra.config.StartupChecksOptions.ENABLED_PROPERTY;
import static org.apache.cassandra.distributed.Cluster.build;
import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
@@ -59,11 +60,9 @@
@Test
public void testDataResurrectionCheck() throws Exception
{
- try
+ // set it to 1 hour so check will be not updated after it is written, for test purposes
+ try (WithProperties properties = new WithProperties().set(CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD, 3600000))
{
- // set it to 1 hour so check will be not updated after it is written, for test purposes
- System.setProperty(CassandraRelevantProperties.CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD.getKey(), "3600000");
-
// start the node with the check enabled, it will just pass fine as there are not any user tables yet
// and system tables are young enough
try (Cluster cluster = build().withNodes(1)
@@ -129,10 +128,6 @@
EXCLUDED_KEYSPACES_CONFIG_PROPERTY, "ks3"));
}
}
- finally
- {
- System.clearProperty(CassandraRelevantProperties.CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD.getKey());
- }
}
private Throwable executeChecksOnInstance(IInvokableInstance instance, final String... config)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java
new file mode 100644
index 0000000..2de8f54
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import com.google.common.util.concurrent.Futures;
+import org.junit.Test;
+
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.snapshot.SnapshotManifest;
+import org.apache.cassandra.utils.Pair;
+
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class EphemeralSnapshotTest extends TestBaseImpl
+{
+ private static final String snapshotName = "snapshotname";
+ private static final String snapshotName2 = "second-snapshot";
+ private static final String tableName = "city";
+
+ @Test
+ public void testStartupRemovesEphemeralSnapshotOnEphemeralFlagInManifest() throws Exception
+ {
+ try (Cluster c = init(builder().withNodes(1)
+ .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
+ .start()))
+ {
+ Pair<String, String[]> initialisationData = initialise(c);
+
+ rewriteManifestToEphemeral(initialisationData.left, initialisationData.right);
+
+ verify(c.get(1));
+ }
+ }
+
+ // TODO this test might be deleted once we get rid of ephemeral marker file for good in 4.3
+ @Test
+ public void testStartupRemovesEphemeralSnapshotOnMarkerFile() throws Exception
+ {
+ try (Cluster c = init(builder().withNodes(1)
+ .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
+ .start()))
+ {
+ Pair<String, String[]> initialisationData = initialise(c);
+
+ String tableId = initialisationData.left;
+ String[] dataDirs = initialisationData.right;
+
+ // place ephemeral marker file into snapshot directory pretending it was created as ephemeral
+ Path ephemeralMarkerFile = Paths.get(dataDirs[0])
+ .resolve(KEYSPACE)
+ .resolve(format("%s-%s", tableName, tableId))
+ .resolve("snapshots")
+ .resolve(snapshotName)
+ .resolve("ephemeral.snapshot");
+
+ Files.createFile(ephemeralMarkerFile);
+
+ verify(c.get(1));
+ }
+ }
+
+ @Test
+ public void testEphemeralSnapshotIsNotClearableFromNodetool() throws Exception
+ {
+ try (Cluster c = init(builder().withNodes(1)
+ .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
+ .start()))
+ {
+ IInvokableInstance instance = c.get(1);
+
+ Pair<String, String[]> initialisationData = initialise(c);
+ rewriteManifestToEphemeral(initialisationData.left, initialisationData.right);
+
+ assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
+ instance.nodetoolResult("clearsnapshot", "-t", snapshotName).asserts().success();
+ // ephemeral snapshot was not removed as it can not be (from nodetool / user operation)
+ assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
+
+ assertFalse(instance.logs().grep("Skipping deletion of ephemeral snapshot 'snapshotname' in keyspace distributed_test_keyspace. " +
+ "Ephemeral snapshots are not removable by a user.").getResult().isEmpty());
+ }
+ }
+
+ @Test
+ public void testClearingAllSnapshotsFromNodetoolWillKeepEphemeralSnaphotsIntact() throws Exception
+ {
+ try (Cluster c = init(builder().withNodes(1)
+ .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
+ .start()))
+ {
+ IInvokableInstance instance = c.get(1);
+
+ Pair<String, String[]> initialisationData = initialise(c);
+
+ rewriteManifestToEphemeral(initialisationData.left, initialisationData.right);
+
+ instance.nodetoolResult("clearsnapshot", "--all").asserts().success();
+ assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
+ assertFalse(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName2));
+ }
+ }
+
+ private Pair<String, String[]> initialise(Cluster c)
+ {
+ c.schemaChange(withKeyspace("CREATE TABLE IF NOT EXISTS %s." + tableName + " (cityid int PRIMARY KEY, name text)"));
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s." + tableName + "(cityid, name) VALUES (1, 'Canberra');"), ONE);
+ IInvokableInstance instance = c.get(1);
+
+ instance.flush(KEYSPACE);
+
+ assertEquals(0, instance.nodetool("snapshot", "-kt", withKeyspace("%s." + tableName), "-t", snapshotName));
+ waitForSnapshot(instance, snapshotName);
+
+ // take one more snapshot, this one is not ephemeral,
+ // starting Cassandra will clear ephemerals, but it will not affect non-ephemeral snapshots
+ assertEquals(0, instance.nodetool("snapshot", "-kt", withKeyspace("%s." + tableName), "-t", snapshotName2));
+ waitForSnapshot(instance, snapshotName2);
+
+ String tableId = instance.callOnInstance((IIsolatedExecutor.SerializableCallable<String>) () -> {
+ return Keyspace.open(KEYSPACE).getMetadata().tables.get(tableName).get().id.asUUID().toString().replaceAll("-", "");
+ });
+
+ String[] dataDirs = (String[]) instance.config().get("data_file_directories");
+
+ return Pair.create(tableId, dataDirs);
+ }
+
+
+ private void verify(IInvokableInstance instance)
+ {
+ // by default, we do not see ephemerals
+ assertFalse(instance.nodetoolResult("listsnapshots").getStdout().contains(snapshotName));
+
+ // we see them via -e flag
+ assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
+
+ Futures.getUnchecked(instance.shutdown());
+
+ // startup should remove ephemeral snapshot
+ instance.startup();
+
+ assertFalse(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
+ assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName2));
+ }
+
+ private void waitForSnapshot(IInvokableInstance instance, String snapshotName)
+ {
+ await().timeout(20, SECONDS)
+ .pollInterval(1, SECONDS)
+ .until(() -> instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
+ }
+
+ private void rewriteManifestToEphemeral(String tableId, String[] dataDirs) throws Exception
+ {
+ // rewrite manifest, pretend that it is ephemeral
+ Path manifestPath = findManifest(dataDirs, tableId);
+ SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(new File(manifestPath));
+ SnapshotManifest manifestWithEphemeralFlag = new SnapshotManifest(manifest.files, null, manifest.createdAt, true);
+ manifestWithEphemeralFlag.serializeToJsonFile(new File(manifestPath));
+ }
+
+ private Path findManifest(String[] dataDirs, String tableId)
+ {
+ for (String dataDir : dataDirs)
+ {
+ Path manifest = Paths.get(dataDir)
+ .resolve(KEYSPACE)
+ .resolve(format("%s-%s", tableName, tableId))
+ .resolve("snapshots")
+ .resolve(snapshotName)
+ .resolve("manifest.json");
+
+ if (Files.exists(manifest))
+ {
+ return manifest;
+ }
+ }
+
+ throw new IllegalStateException("Unable to find manifest!");
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
index eb1cdff..405279a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
@@ -64,9 +64,9 @@
import org.apache.cassandra.distributed.impl.InstanceKiller;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.ForwardingSSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.RepairParallelism;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingResponseDoesNotLogTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingResponseDoesNotLogTest.java
index 1071938..4032cc7 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailingResponseDoesNotLogTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingResponseDoesNotLogTest.java
@@ -40,6 +40,7 @@
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.LogAction;
import org.apache.cassandra.distributed.api.LogResult;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.service.ClientState;
@@ -49,6 +50,8 @@
import org.apache.cassandra.utils.MD5Digest;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CUSTOM_QUERY_HANDLER_CLASS;
+
/**
* This class is rather impelemntation specific. It is possible that changes made will cause this tests to fail,
* so updating to the latest logic is fine.
@@ -68,13 +71,15 @@
@Test
public void dispatcherErrorDoesNotLock() throws IOException
{
- System.setProperty("cassandra.custom_query_handler_class", AlwaysRejectErrorQueryHandler.class.getName());
- try (Cluster cluster = Cluster.build(1)
+ try (WithProperties properties = new WithProperties().set(CUSTOM_QUERY_HANDLER_CLASS, AlwaysRejectErrorQueryHandler.class.getName());
+ Cluster cluster = Cluster.build(1)
.withConfig(c -> c.with(Feature.NATIVE_PROTOCOL, Feature.GOSSIP)
- .set("client_error_reporting_exclusions", ImmutableMap.of("subnets", Collections.singletonList("127.0.0.1")))
- )
- .start())
+ .set("client_error_reporting_exclusions", ImmutableMap.of("subnets", Collections.singletonList("127.0.0.1"))))
+ .start();
+
+ )
{
+
try (SimpleClient client = SimpleClient.builder("127.0.0.1", 9042).build().connect(false))
{
client.execute("SELECT * FROM system.peers", ConsistencyLevel.ONE);
@@ -92,10 +97,6 @@
matches = logs.grep("Unexpected exception during request");
Assertions.assertThat(matches.getResult()).isEmpty();
}
- finally
- {
- System.clearProperty("cassandra.custom_query_handler_class");
- }
}
public static class AlwaysRejectErrorQueryHandler implements QueryHandler
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingTruncationTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingTruncationTest.java
index dd419db..9ce2bf3 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailingTruncationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingTruncationTest.java
@@ -30,13 +30,12 @@
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_BBFAILHELPER_ENABLED;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class FailingTruncationTest extends TestBaseImpl
{
- private static final String BB_FAIL_HELPER_PROP = "test.bbfailhelper.enabled";
-
@Test
public void testFailingTruncation() throws IOException
{
@@ -45,7 +44,7 @@
.start()))
{
cluster.setUncaughtExceptionsFilter(t -> "truncateBlocking".equals(t.getMessage()));
- System.setProperty(BB_FAIL_HELPER_PROP, "true");
+ TEST_BBFAILHELPER_ENABLED.setBoolean(true);
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
try
{
@@ -76,7 +75,7 @@
public static void truncateBlocking()
{
- if (Boolean.getBoolean(BB_FAIL_HELPER_PROP))
+ if (TEST_BBFAILHELPER_ENABLED.getBoolean())
throw new RuntimeException("truncateBlocking");
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java b/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java
index 3b54398..3314c2a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FrozenUDTTest.java
@@ -65,14 +65,14 @@
{
cluster.schemaChange("create type " + KEYSPACE + ".a (foo text)");
cluster.schemaChange("create table " + KEYSPACE + ".x (id int, ck frozen<a>, i int, primary key (id, ck))");
- cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (1, system.fromjson('{\"foo\":\"\"}'), 1)", ConsistencyLevel.ALL);
- cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (1, system.fromjson('{\"foo\":\"a\"}'), 2)", ConsistencyLevel.ALL);
+ cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (1, system.from_json('{\"foo\":\"\"}'), 1)", ConsistencyLevel.ALL);
+ cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (1, system.from_json('{\"foo\":\"a\"}'), 2)", ConsistencyLevel.ALL);
cluster.forEach(i -> i.flush(KEYSPACE));
Runnable check = () -> {
- assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.fromjson('{\"foo\":\"\"}')", ConsistencyLevel.ALL),
+ assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.from_json('{\"foo\":\"\"}')", ConsistencyLevel.ALL),
row(1));
- assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.fromjson('{\"foo\":\"a\"}')", ConsistencyLevel.ALL),
+ assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.from_json('{\"foo\":\"a\"}')", ConsistencyLevel.ALL),
row(2));
};
@@ -80,10 +80,10 @@
cluster.schemaChange("alter type " + KEYSPACE + ".a add bar text");
check.run();
- assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.fromjson('{\"foo\":\"\",\"bar\":\"\"}')", ConsistencyLevel.ALL));
- cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (1, system.fromjson('{\"foo\":\"\",\"bar\":\"\"}'), 3)", ConsistencyLevel.ALL);
+ assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.from_json('{\"foo\":\"\",\"bar\":\"\"}')", ConsistencyLevel.ALL));
+ cluster.coordinator(1).execute("insert into " + KEYSPACE + ".x (id, ck, i) VALUES (1, system.from_json('{\"foo\":\"\",\"bar\":\"\"}'), 3)", ConsistencyLevel.ALL);
check.run();
- assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.fromjson('{\"foo\":\"\",\"bar\":\"\"}')", ConsistencyLevel.ALL),
+ assertRows(cluster.coordinator(1).execute("select i from " + KEYSPACE + ".x WHERE id = 1 and ck = system.from_json('{\"foo\":\"\",\"bar\":\"\"}')", ConsistencyLevel.ALL),
row(3));
}
}
@@ -143,11 +143,11 @@
private String json(int i)
{
- return String.format("system.fromjson('{\"foo\":\"%d\"}')", i);
+ return String.format("system.from_json('{\"foo\":\"%d\"}')", i);
}
private String json(int i, int j)
{
- return String.format("system.fromjson('{\"foo\":\"%d\", \"bar\":\"%d\"}')", i, j);
+ return String.format("system.from_json('{\"foo\":\"%d\", \"bar\":\"%d\"}')", i, j);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
index c5f2a07..3905411 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
@@ -51,6 +51,7 @@
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@@ -299,7 +300,7 @@
{
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance gossippingOnlyMember = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", Boolean.toString(false), () -> gossippingOnlyMember.startup(cluster));
+ withProperty(JOIN_RING, false, () -> gossippingOnlyMember.startup(cluster));
assertTrue(gossippingOnlyMember.callOnInstance((IIsolatedExecutor.SerializableCallable<Boolean>)
() -> StorageService.instance.isGossipRunning()));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/HintedHandoffAddRemoveNodesTest.java b/test/distributed/org/apache/cassandra/distributed/test/HintedHandoffAddRemoveNodesTest.java
index 5cf1ab6..add6fdf 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/HintedHandoffAddRemoveNodesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/HintedHandoffAddRemoveNodesTest.java
@@ -23,18 +23,22 @@
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.action.GossipHelper;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.metrics.HintsServiceMetrics;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.service.StorageService;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.apache.cassandra.distributed.action.GossipHelper.decommission;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.TWO;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@@ -46,6 +50,39 @@
*/
public class HintedHandoffAddRemoveNodesTest extends TestBaseImpl
{
+ @SuppressWarnings("Convert2MethodRef")
+ @Test
+ public void shouldAvoidHintTransferOnDecommission() throws Exception
+ {
+ try (Cluster cluster = init(builder().withNodes(3)
+ .withConfig(config -> config.set("transfer_hints_on_decommission", false).with(GOSSIP))
+ .withoutVNodes()
+ .start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.decom_no_hints_test (key int PRIMARY KEY, value int)"));
+
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.decom_no_hints_test (key, value) VALUES (?, ?)"), ALL, 0, 0);
+ long hintsBeforeShutdown = countTotalHints(cluster.get(1));
+ assertThat(hintsBeforeShutdown).isEqualTo(0);
+ long hintsDelivered = countHintsDelivered(cluster.get(1));
+ assertThat(hintsDelivered).isEqualTo(0);
+
+ // Shutdown node 3 so hints can be written against it.
+ cluster.get(3).shutdown().get();
+
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.decom_no_hints_test (key, value) VALUES (?, ?)"), TWO, 0, 0);
+ Awaitility.await().until(() -> countTotalHints(cluster.get(1)) > 0);
+ long hintsAfterShutdown = countTotalHints(cluster.get(1));
+ assertThat(hintsAfterShutdown).isEqualTo(1);
+
+ cluster.get(1).nodetoolResult("decommission", "--force").asserts().success();
+ long hintsDeliveredByDecom = countHintsDelivered(cluster.get(1));
+ String mode = cluster.get(1).callOnInstance(() -> StorageService.instance.getOperationMode());
+ assertEquals(StorageService.Mode.DECOMMISSIONED.toString(), mode);
+ assertThat(hintsDeliveredByDecom).isEqualTo(0);
+ }
+ }
+
/**
* Replaces Python dtest {@code hintedhandoff_test.py:TestHintedHandoff.test_hintedhandoff_decom()}.
*/
@@ -130,6 +167,12 @@
return instance.callOnInstance(() -> StorageMetrics.totalHints.getCount());
}
+ @SuppressWarnings("Convert2MethodRef")
+ private long countHintsDelivered(IInvokableInstance instance)
+ {
+ return instance.callOnInstance(() -> HintsServiceMetrics.hintsSucceeded.getCount());
+ }
+
@SuppressWarnings("SameParameterValue")
private void populate(Cluster cluster, String table, int coordinator, int start, int count, ConsistencyLevel cl)
{
diff --git a/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java b/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java
index 474a8b7..16a97b0 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java
@@ -36,6 +36,7 @@
import org.apache.cassandra.tools.ToolRunner;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS;
import static org.apache.cassandra.distributed.shared.ClusterUtils.assertRingIs;
import static org.apache.cassandra.distributed.shared.ClusterUtils.getDirectories;
import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked;
@@ -64,11 +65,13 @@
getDirectories(nodeToReplace).forEach(FileUtils::deleteRecursive);
nodeToReplace.config().set("auto_bootstrap", auto_bootstrap);
+
// we need to override the host id because otherwise the node will not be considered as a new node
((InstanceConfig) nodeToReplace.config()).setHostId(UUID.randomUUID());
Assertions.assertThatThrownBy(() -> nodeToReplace.startup())
- .hasMessage("A node with address /127.0.0.3:7012 already exists, cancelling join. Use cassandra.replace_address if you want to replace this node.");
+ .hasMessage("A node with address /127.0.0.3:7012 already exists, cancelling join. Use " +
+ REPLACE_ADDRESS.getKey() + " if you want to replace this node.");
}
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java b/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java
index 62f0a73..27c26fb 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java
@@ -17,21 +17,38 @@
*/
package org.apache.cassandra.distributed.test;
+import java.io.Closeable;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
import java.util.HashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableMap;
import org.junit.Test;
+import org.apache.cassandra.auth.AllowAllInternodeAuthenticator;
+import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable;
import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.net.InboundMessageHandlers;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.OutboundConnections;
+import org.awaitility.Awaitility;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -39,6 +56,114 @@
public final class InternodeEncryptionEnforcementTest extends TestBaseImpl
{
+
+ @Test
+ public void testInboundConnectionsAreRejectedWhenAuthFails() throws IOException, TimeoutException
+ {
+ // RejectInboundConnections authenticator is configured only for instance 1 of the cluster
+ Cluster.Builder builder = createCluster(RejectInboundConnections.class);
+
+ final ExecutorService executorService = Executors.newSingleThreadExecutor();
+ try (Cluster cluster = builder.start(); Closeable es = executorService::shutdown)
+ {
+ executorService.submit(() -> openConnections(cluster));
+
+ /*
+ * Instance (1) should be able to make outbound connections to instance (2) but Instance (1) should not be
+ * accepting any inbound connections. we should wait for the authentication failure log on Instance (1)
+ */
+ SerializableRunnable runnable = () ->
+ {
+ // There should be no inbound handlers as authentication fails & we remove handlers.
+ assertEquals(0, MessagingService.instance().messageHandlers.values().size());
+
+ // Verify that the failure is due to authentication failure
+ final RejectInboundConnections authenticator = (RejectInboundConnections) DatabaseDescriptor.getInternodeAuthenticator();
+ assertTrue(authenticator.authenticationFailed);
+ };
+
+ // Wait for authentication to fail
+ cluster.get(1).logs().watchFor("Unable to authenticate peer");
+ cluster.get(1).runOnInstance(runnable);
+
+ }
+ }
+
+ @Test
+ public void testOutboundConnectionsAreRejectedWhenAuthFails() throws IOException, TimeoutException
+ {
+ Cluster.Builder builder = createCluster(RejectOutboundAuthenticator.class);
+
+ final ExecutorService executorService = Executors.newSingleThreadExecutor();
+ try (Cluster cluster = builder.start(); Closeable es = executorService::shutdown)
+ {
+ executorService.submit(() -> openConnections(cluster));
+
+ /*
+ * Instance (1) should not be able to make outbound connections to instance (2) but Instance (2) should be
+ * accepting outbound connections from Instance (1)
+ */
+ SerializableRunnable runnable = () ->
+ {
+ // There should be no outbound connections as authentication fails on Instance (1).
+ OutboundConnections outbound = getOnlyElement(MessagingService.instance().channelManagers.values());
+ assertTrue(!outbound.small.isConnected() && !outbound.large.isConnected() && !outbound.urgent.isConnected());
+
+ // Verify that the failure is due to authentication failure
+ final RejectOutboundAuthenticator authenticator = (RejectOutboundAuthenticator) DatabaseDescriptor.getInternodeAuthenticator();
+ assertTrue(authenticator.authenticationFailed);
+ };
+
+ // Wait for authentication to fail
+ cluster.get(1).logs().watchFor("Authentication failed");
+ cluster.get(1).runOnInstance(runnable);
+ }
+ }
+
+ @Test
+ public void testOutboundConnectionsAreInterruptedWhenAuthFails() throws IOException, TimeoutException
+ {
+ Cluster.Builder builder = createCluster(AllowFirstAndRejectOtherOutboundAuthenticator.class);
+ final ExecutorService executorService = Executors.newSingleThreadExecutor();
+
+ try (Cluster cluster = builder.start(); Closeable es = executorService::shutdown)
+ {
+ executorService.submit(() -> openConnections(cluster));
+
+ // Verify that authentication is failed and Interrupt is called on outbound connections.
+ cluster.get(1).logs().watchFor("Authentication failed to");
+ cluster.get(1).logs().watchFor("Interrupted outbound connections to");
+
+ /*
+ * Check if outbound connections are zero
+ */
+ SerializableRunnable runnable = () ->
+ {
+ // Verify that there is only one successful outbound connection
+ final AllowFirstAndRejectOtherOutboundAuthenticator authenticator = (AllowFirstAndRejectOtherOutboundAuthenticator) DatabaseDescriptor.getInternodeAuthenticator();
+ assertEquals(1, authenticator.successfulOutbound.get());
+ assertTrue(authenticator.failedOutbound.get() > 0);
+
+ // There should be no outbound connections as authentication fails.
+ OutboundConnections outbound = getOnlyElement(MessagingService.instance().channelManagers.values());
+ Awaitility.await().until(() -> !outbound.small.isConnected() && !outbound.large.isConnected() && !outbound.urgent.isConnected());
+ };
+ cluster.get(1).runOnInstance(runnable);
+ }
+ }
+
+ @Test
+ public void testConnectionsAreAcceptedWhenAuthSucceds() throws IOException
+ {
+ verifyAuthenticationSucceeds(AllowAllInternodeAuthenticator.class);
+ }
+
+ @Test
+ public void testAuthenticationWithCertificateAuthenticator() throws IOException
+ {
+ verifyAuthenticationSucceeds(CertificateVerifyAuthenticator.class);
+ }
+
@Test
public void testConnectionsAreRejectedWithInvalidConfig() throws Throwable
{
@@ -49,16 +174,18 @@
c.with(Feature.NETWORK);
c.with(Feature.NATIVE_PROTOCOL);
+ HashMap<String, Object> encryption = new HashMap<>();
+ encryption.put("optional", "false");
+ encryption.put("internode_encryption", "none");
if (c.num() == 1)
{
- HashMap<String, Object> encryption = new HashMap<>();
encryption.put("keystore", "test/conf/cassandra_ssl_test.keystore");
encryption.put("keystore_password", "cassandra");
encryption.put("truststore", "test/conf/cassandra_ssl_test.truststore");
encryption.put("truststore_password", "cassandra");
- encryption.put("internode_encryption", "dc");
- c.set("server_encryption_options", encryption);
+ encryption.put("internode_encryption", "all");
}
+ c.set("server_encryption_options", encryption);
})
.withNodeIdTopology(ImmutableMap.of(1, NetworkTopology.dcAndRack("dc1", "r1a"),
2, NetworkTopology.dcAndRack("dc2", "r2a")));
@@ -153,4 +280,164 @@
cluster.schemaChange("CREATE KEYSPACE test_connections_from_2 " +
"WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};", false, cluster.get(2));
}
+
+ private void verifyAuthenticationSucceeds(final Class authenticatorClass) throws IOException
+ {
+ Cluster.Builder builder = createCluster(authenticatorClass);
+ try (Cluster cluster = builder.start())
+ {
+ openConnections(cluster);
+
+ /*
+ * instance (1) should connect to instance (2) without any issues;
+ * instance (2) should connect to instance (1) without any issues.
+ */
+
+ SerializableRunnable runnable = () ->
+ {
+ // There should be inbound connections as authentication succeeds.
+ InboundMessageHandlers inbound = getOnlyElement(MessagingService.instance().messageHandlers.values());
+ assertTrue(inbound.count() > 0);
+
+ // There should be outbound connections as authentication succeeds.
+ OutboundConnections outbound = getOnlyElement(MessagingService.instance().channelManagers.values());
+ assertTrue(outbound.small.isConnected() || outbound.large.isConnected() || outbound.urgent.isConnected());
+ };
+
+ cluster.get(1).runOnInstance(runnable);
+ cluster.get(2).runOnInstance(runnable);
+ }
+ }
+
+ private Cluster.Builder createCluster(final Class authenticatorClass)
+ {
+ return builder()
+ .withNodes(2)
+ .withConfig(c ->
+ {
+ c.with(Feature.NETWORK);
+ c.with(Feature.NATIVE_PROTOCOL);
+
+ HashMap<String, Object> encryption = new HashMap<>();
+ encryption.put("keystore", "test/conf/cassandra_ssl_test.keystore");
+ encryption.put("keystore_password", "cassandra");
+ encryption.put("truststore", "test/conf/cassandra_ssl_test.truststore");
+ encryption.put("truststore_password", "cassandra");
+ encryption.put("internode_encryption", "all");
+ encryption.put("require_client_auth", "true");
+ c.set("server_encryption_options", encryption);
+ if (c.num() == 1)
+ {
+ c.set("internode_authenticator", authenticatorClass.getName());
+ }
+ else
+ {
+ c.set("internode_authenticator", AllowAllInternodeAuthenticator.class.getName());
+ }
+ })
+ .withNodeIdTopology(ImmutableMap.of(1, NetworkTopology.dcAndRack("dc1", "r1a"),
+ 2, NetworkTopology.dcAndRack("dc2", "r2a")));
+ }
+
+ // Authenticator that validates certificate authentication
+ public static class CertificateVerifyAuthenticator implements IInternodeAuthenticator
+ {
+ @Override
+ public boolean authenticate(InetAddress remoteAddress, int remotePort, Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ if (connectionType == InternodeConnectionDirection.OUTBOUND_PRECONNECT)
+ {
+ return true;
+ }
+ try
+ {
+ // Check if the presented certificates during internode authentication are the ones in the keystores
+ // configured in the cassandra.yaml configuration.
+ KeyStore keyStore = KeyStore.getInstance("JKS");
+ char[] keyStorePassword = "cassandra".toCharArray();
+ InputStream keyStoreData = new FileInputStream("test/conf/cassandra_ssl_test.keystore");
+ keyStore.load(keyStoreData, keyStorePassword);
+ return certificates != null && certificates.length != 0 && keyStore.getCertificate("cassandra_ssl_test").equals(certificates[0]);
+ }
+ catch (Exception e)
+ {
+ return false;
+ }
+ }
+
+ @Override
+ public void validateConfiguration() throws ConfigurationException
+ {
+
+ }
+ }
+
+ public static class RejectConnectionsAuthenticator implements IInternodeAuthenticator
+ {
+ boolean authenticationFailed = false;
+
+ @Override
+ public boolean authenticate(InetAddress remoteAddress, int remotePort, Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ authenticationFailed = true;
+ return false;
+ }
+
+ @Override
+ public void validateConfiguration() throws ConfigurationException
+ {
+
+ }
+ }
+
+ public static class RejectInboundConnections extends RejectConnectionsAuthenticator
+ {
+ @Override
+ public boolean authenticate(InetAddress remoteAddress, int remotePort, Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ if (connectionType == InternodeConnectionDirection.INBOUND)
+ {
+ return super.authenticate(remoteAddress, remotePort, certificates, connectionType);
+ }
+ return true;
+ }
+ }
+
+ public static class RejectOutboundAuthenticator extends RejectConnectionsAuthenticator
+ {
+ @Override
+ public boolean authenticate(InetAddress remoteAddress, int remotePort, Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ if (connectionType == InternodeConnectionDirection.OUTBOUND)
+ {
+ return super.authenticate(remoteAddress, remotePort, certificates, connectionType);
+ }
+ return true;
+ }
+ }
+
+ public static class AllowFirstAndRejectOtherOutboundAuthenticator extends RejectOutboundAuthenticator
+ {
+ AtomicInteger successfulOutbound = new AtomicInteger();
+ AtomicInteger failedOutbound = new AtomicInteger();
+
+ @Override
+ public boolean authenticate(InetAddress remoteAddress, int remotePort, Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ if (connectionType == InternodeConnectionDirection.OUTBOUND)
+ {
+ if (successfulOutbound.compareAndSet(0, 1))
+ {
+ return true;
+ }
+ else
+ {
+ failedOutbound.incrementAndGet();
+ authenticationFailed = true;
+ return false;
+ }
+ }
+ return true;
+ }
+ }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java b/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java
index 089d17b..f11ad16 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java
@@ -95,6 +95,16 @@
}
@Test
+ public void jvmArgumentLoggingTest() throws IOException
+ {
+ try (Cluster cluster = Cluster.build(1).start())
+ {
+ LogAction logs = cluster.get(1).logs();
+ Assertions.assertThat(logs.grep("JVM Arguments").getResult()).isNotEmpty();
+ }
+ }
+
+ @Test
public void nonSharedConfigClassTest() throws IOException
{
Map<String,Object> commitLogCompression = ImmutableMap.of("class_name", "org.apache.cassandra.io.compress.LZ4Compressor",
diff --git a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java
index 665d58c..b63179b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorThrowableTest.java
@@ -34,7 +34,6 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@ -48,10 +47,9 @@
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.ForwardingSSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
-import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Throwables;
@@ -231,22 +229,14 @@
throw throwFSError();
}
- @Override
- public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
- {
- if (shouldThrowCorrupted)
- throw throwCorrupted();
- throw throwFSError();
- }
-
private CorruptSSTableException throwCorrupted()
{
- throw new CorruptSSTableException(new IOException("failed to get position"), descriptor.baseFilename());
+ throw new CorruptSSTableException(new IOException("failed to get position"), descriptor.baseFile());
}
private FSError throwFSError()
{
- throw new FSReadError(new IOException("failed to get position"), descriptor.baseFilename());
+ throw new FSReadError(new IOException("failed to get position"), descriptor.baseFile());
}
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java b/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java
index a4dd8e6..7006795 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MessageForwardingTest.java
@@ -39,8 +39,10 @@
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.impl.IsolatedExecutor;
import org.apache.cassandra.distributed.impl.TracingUtil;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.utils.TimeUUID;
+import static org.apache.cassandra.config.CassandraRelevantProperties.WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
public class MessageForwardingTest extends TestBaseImpl
@@ -48,15 +50,16 @@
@Test
public void mutationsForwardedToAllReplicasTest()
{
- String originalTraceTimeout = TracingUtil.setWaitForTracingEventTimeoutSecs("1");
final int numInserts = 100;
Map<InetAddress, Integer> forwardFromCounts = new HashMap<>();
Map<InetAddress, Integer> commitCounts = new HashMap<>();
- try (Cluster cluster = (Cluster) init(builder()
+ try (WithProperties properties = new WithProperties().set(WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS, 1);
+ Cluster cluster = (Cluster) init(builder()
.withDC("dc0", 1)
.withDC("dc1", 3)
- .start()))
+ .start());
+ )
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck))");
@@ -115,9 +118,5 @@
{
Assert.fail("Threw exception: " + e);
}
- finally
- {
- TracingUtil.setWaitForTracingEventTimeoutSecs(originalTraceTimeout);
- }
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MigrationCoordinatorTest.java b/test/distributed/org/apache/cassandra/distributed/test/MigrationCoordinatorTest.java
index ca89b43..e75783d 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MigrationCoordinatorTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MigrationCoordinatorTest.java
@@ -30,8 +30,10 @@
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.schema.Schema;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_RANGE_MOVEMENT;
import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORED_SCHEMA_CHECK_ENDPOINTS;
import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORED_SCHEMA_CHECK_VERSIONS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@@ -41,11 +43,9 @@
@Before
public void setUp()
{
- System.clearProperty("cassandra.replace_address");
- System.clearProperty("cassandra.consistent.rangemovement");
-
- System.clearProperty(IGNORED_SCHEMA_CHECK_VERSIONS.getKey());
- System.clearProperty(IGNORED_SCHEMA_CHECK_VERSIONS.getKey());
+ REPLACE_ADDRESS.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+ CONSISTENT_RANGE_MOVEMENT.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+ IGNORED_SCHEMA_CHECK_VERSIONS.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
}
/**
* We shouldn't wait on versions only available from a node being replaced
@@ -67,7 +67,7 @@
IInstanceConfig config = cluster.newInstanceConfig();
config.set("auto_bootstrap", true);
- System.setProperty("cassandra.replace_address", replacementAddress.getHostAddress());
+ REPLACE_ADDRESS.setString(replacementAddress.getHostAddress());
cluster.bootstrap(config).startup();
}
}
@@ -89,7 +89,7 @@
IInstanceConfig config = cluster.newInstanceConfig();
config.set("auto_bootstrap", true);
IGNORED_SCHEMA_CHECK_ENDPOINTS.setString(ignoredEndpoint.getHostAddress());
- System.setProperty("cassandra.consistent.rangemovement", "false");
+ CONSISTENT_RANGE_MOVEMENT.setBoolean(false);
cluster.bootstrap(config).startup();
}
}
@@ -116,7 +116,7 @@
IInstanceConfig config = cluster.newInstanceConfig();
config.set("auto_bootstrap", true);
IGNORED_SCHEMA_CHECK_VERSIONS.setString(initialVersion.toString() + ',' + oldVersion);
- System.setProperty("cassandra.consistent.rangemovement", "false");
+ CONSISTENT_RANGE_MOVEMENT.setBoolean(false);
cluster.bootstrap(config).startup();
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java
index fc1bca3..cdbe1cd 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java
@@ -31,6 +31,7 @@
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@@ -42,7 +43,7 @@
static
{
- System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default
+ RING_DELAY.setLong(5000);
}
private void move(boolean forwards) throws Throwable
diff --git a/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java b/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java
index 0826954..304136b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java
@@ -37,6 +37,7 @@
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.TOCComponent;
import org.apache.cassandra.io.util.File;
public class MultipleDataDirectoryTest extends TestBaseImpl
@@ -151,14 +152,14 @@
// getting a new file index in order to move SSTable between directories.
second = cfs.newSSTableDescriptor(second.directory);
// now we just move all sstables from first to second
- for (Component component : SSTableReader.componentsFor(first))
+ for (Component component : TOCComponent.loadOrCreate(first))
{
- File file = new File(first.filenameFor(component));
+ File file = first.fileFor(component);
if (file.exists())
{
try
{
- Files.copy(file.toPath(), new File(second.filenameFor(component)).toPath());
+ Files.copy(file.toPath(), second.fileFor(component).toPath());
}
catch (IOException e)
{
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NativeMixedVersionTest.java b/test/distributed/org/apache/cassandra/distributed/test/NativeMixedVersionTest.java
index 26d0186..29605f0 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NativeMixedVersionTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NativeMixedVersionTest.java
@@ -30,8 +30,11 @@
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.IO_NETTY_EVENTLOOP_THREADS;
+
public class NativeMixedVersionTest extends TestBaseImpl
{
@Test
@@ -39,15 +42,17 @@
{
// make sure to limit the netty thread pool to size 1, this will make the test determanistic as all work
// will happen on the single thread.
- System.setProperty("io.netty.eventLoopThreads", "1");
- try (Cluster cluster = Cluster.build(1)
+ try (WithProperties properties = new WithProperties().set(IO_NETTY_EVENTLOOP_THREADS, 1);
+ Cluster cluster = Cluster.build(1)
.withConfig(c ->
c.with(Feature.values())
.set("read_thresholds_enabled", true)
.set("local_read_size_warn_threshold", "1KiB")
)
- .start())
+ .start();
+ )
{
+
init(cluster);
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck1 int, value blob, PRIMARY KEY (pk, ck1))"));
IInvokableInstance node = cluster.get(1);
@@ -78,9 +83,5 @@
List<String> result = node.logs().grep("Warnings present in message with version less than").getResult();
Assertions.assertThat(result).isEmpty();
}
- finally
- {
- System.clearProperty("io.netty.eventLoopThreads");
- }
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java b/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java
index f965572..11ba994 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java
@@ -34,6 +34,7 @@
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.impl.RowUtil;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
@@ -103,7 +104,7 @@
{
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance gossippingOnlyMember = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", Boolean.toString(false), () -> gossippingOnlyMember.startup(cluster));
+ withProperty(JOIN_RING, false, () -> gossippingOnlyMember.startup(cluster));
assertTrue(gossippingOnlyMember.callOnInstance((IIsolatedExecutor.SerializableCallable<Boolean>)
() -> StorageService.instance.isNativeTransportRunning()));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java
index c5a810c..5f2caaf 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java
@@ -18,18 +18,33 @@
package org.apache.cassandra.distributed.test;
+import java.io.FileInputStream;
+import java.io.InputStream;
import java.net.InetAddress;
+import java.security.KeyStore;
import java.util.Collections;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.TrustManagerFactory;
+
import com.google.common.collect.ImmutableMap;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import com.datastax.driver.core.SSLOptions;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
+import com.datastax.shaded.netty.handler.ssl.SslContext;
+import com.datastax.shaded.netty.handler.ssl.SslContextBuilder;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
public class NativeTransportEncryptionOptionsTest extends AbstractEncryptionOptionsImpl
{
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
@Test
public void nodeWillNotStartWithBadKeystore() throws Throwable
{
@@ -219,4 +234,90 @@
assertCannotStartDueToConfigurationException(cluster);
}
}
+
+ @Test
+ public void testEndpointVerificationDisabledIpNotInSAN() throws Throwable
+ {
+ // When required_endpoint_verification is set to false, client certificate Ip/hostname should be validated
+ // The certificate in cassandra_ssl_test_outbound.keystore does not have IP/hostname embeded, so when
+ // require_endpoint_verification is false, the connection should be established
+ testEndpointVerification(false, true);
+ }
+
+ @Test
+ public void testEndpointVerificationEnabledIpNotInSAN() throws Throwable
+ {
+ // When required_endpoint_verification is set to true, client certificate Ip/hostname should be validated
+ // The certificate in cassandra_ssl_test_outbound.keystore does not have IP/hostname emebeded, so when
+ // require_endpoint_verification is true, the connection should not be established
+ testEndpointVerification(true, false);
+ }
+
+ @Test
+ public void testEndpointVerificationEnabledWithIPInSan() throws Throwable
+ {
+ // When required_endpoint_verification is set to true, client certificate Ip/hostname should be validated
+ // The certificate in cassandra_ssl_test_outbound.keystore have IP/hostname emebeded, so when
+ // require_endpoint_verification is true, the connection should be established
+ testEndpointVerification(true, true);
+ }
+
+ private void testEndpointVerification(boolean requireEndpointVerification, boolean ipInSAN) throws Throwable
+ {
+ try (Cluster cluster = builder().withNodes(1).withConfig(c -> {
+ c.with(Feature.NATIVE_PROTOCOL);
+ c.set("client_encryption_options",
+ ImmutableMap.builder().putAll(validKeystore)
+ .put("enabled", true)
+ .put("require_client_auth", true)
+ .put("require_endpoint_verification", requireEndpointVerification)
+ .build());
+ }).start())
+ {
+ InetAddress address = cluster.get(1).config().broadcastAddress().getAddress();
+ SslContextBuilder sslContextBuilder = SslContextBuilder.forClient();
+ if (ipInSAN)
+ sslContextBuilder.keyManager(createKeyManagerFactory("test/conf/cassandra_ssl_test_endpoint_verify.keystore", "cassandra"));
+ else
+ sslContextBuilder.keyManager(createKeyManagerFactory("test/conf/cassandra_ssl_test_outbound.keystore", "cassandra"));
+
+ SslContext sslContext = sslContextBuilder.trustManager(createTrustManagerFactory("test/conf/cassandra_ssl_test.truststore", "cassandra"))
+ .build();
+ final SSLOptions sslOptions = socketChannel -> sslContext.newHandler(socketChannel.alloc());
+ com.datastax.driver.core.Cluster driverCluster = com.datastax.driver.core.Cluster.builder()
+ .addContactPoint(address.getHostAddress())
+ .withSSL(sslOptions)
+ .build();
+
+ if (!ipInSAN)
+ {
+ expectedException.expect(NoHostAvailableException.class);
+ }
+
+ driverCluster.connect();
+ }
+ }
+
+ private KeyManagerFactory createKeyManagerFactory(final String keyStorePath,
+ final String keyStorePassword) throws Exception
+ {
+ final InputStream stream = new FileInputStream(keyStorePath);
+ final KeyStore ks = KeyStore.getInstance("JKS");
+ ks.load(stream, keyStorePassword.toCharArray());
+ final KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ kmf.init(ks, keyStorePassword.toCharArray());
+ return kmf;
+ }
+
+ private TrustManagerFactory createTrustManagerFactory(final String trustStorePath,
+ final String trustStorePassword) throws Exception
+ {
+ final InputStream stream = new FileInputStream(trustStorePath);
+ final KeyStore ts = KeyStore.getInstance("JKS");
+ ts.load(stream, trustStorePassword.toCharArray());
+ final TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(ts);
+ return tmf;
+ }
+
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java b/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
index 9087f96..24a65e3 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/NodeToolTest.java
@@ -116,4 +116,35 @@
ringResult.asserts().stderrContains("is not permitted as this cache is disabled");
}
}
+
+ @Test
+ public void testInfoOutput() throws Throwable
+ {
+ try (ICluster<?> cluster = init(builder().withNodes(1).start()))
+ {
+ NodeToolResult ringResult = cluster.get(1).nodetoolResult("info");
+ ringResult.asserts().stdoutContains("ID");
+ ringResult.asserts().stdoutContains("Gossip active");
+ ringResult.asserts().stdoutContains("Native Transport active");
+ ringResult.asserts().stdoutContains("Load");
+ ringResult.asserts().stdoutContains("Uncompressed load");
+ ringResult.asserts().stdoutContains("Generation");
+ ringResult.asserts().stdoutContains("Uptime");
+ ringResult.asserts().stdoutContains("Heap Memory");
+ }
+ }
+
+ @Test
+ public void testVersionIncludesGitSHAWhenVerbose() throws Throwable
+ {
+ NODE.nodetoolResult("version")
+ .asserts()
+ .success()
+ .stdoutNotContains("GitSHA:");
+
+ NODE.nodetoolResult("version", "--verbose")
+ .asserts()
+ .success()
+ .stdoutContains("GitSHA:");
+ }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/OversizedMutationTest.java b/test/distributed/org/apache/cassandra/distributed/test/OversizedMutationTest.java
index 2b0d0a0..fcd600c 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/OversizedMutationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/OversizedMutationTest.java
@@ -36,7 +36,7 @@
{
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (key int PRIMARY KEY, val blob)"));
String payload = StringUtils.repeat('1', 1024 * 49);
- String query = "INSERT INTO %s.t (key, val) VALUES (1, textAsBlob('" + payload + "'))";
+ String query = "INSERT INTO %s.t (key, val) VALUES (1, text_as_blob('" + payload + "'))";
Assertions.assertThatThrownBy(() -> cluster.coordinator(1).execute(withKeyspace(query), ALL))
.hasMessageContaining("Rejected an oversized mutation (")
.hasMessageContaining("/49152) for keyspace: distributed_test_keyspace. Top keys are: t.1");
@@ -53,8 +53,8 @@
cluster.schemaChange(withKeyspace("CREATE TABLE ks1.t (key int PRIMARY KEY, val blob)"));
String payload = StringUtils.repeat('1', 1024 * 48);
String query = "BEGIN BATCH\n" +
- "INSERT INTO ks1.t (key, val) VALUES (1, textAsBlob('" + payload + "'))\n" +
- "INSERT INTO ks1.t (key, val) VALUES (2, textAsBlob('222'))\n" +
+ "INSERT INTO ks1.t (key, val) VALUES (1, text_as_blob('" + payload + "'))\n" +
+ "INSERT INTO ks1.t (key, val) VALUES (2, text_as_blob('222'))\n" +
"APPLY BATCH";
Assertions.assertThatThrownBy(() -> cluster.coordinator(1).execute(withKeyspace(query), ALL))
.hasMessageContaining("Rejected an oversized mutation (")
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PartitionDenylistTest.java b/test/distributed/org/apache/cassandra/distributed/test/PartitionDenylistTest.java
index 382981f..d8e6378 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PartitionDenylistTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PartitionDenylistTest.java
@@ -34,6 +34,9 @@
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_RANGE_MOVEMENT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_SIMULTANEOUS_MOVES_ALLOW;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@@ -58,9 +61,9 @@
public void checkStartupWithoutTriggeringUnavailable() throws IOException, InterruptedException, ExecutionException, TimeoutException
{
int nodeCount = 4;
- System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default
- System.setProperty("cassandra.consistent.rangemovement", "false");
- System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true");
+ RING_DELAY.setLong(5000); // down from 30s default
+ CONSISTENT_RANGE_MOVEMENT.setBoolean(false);
+ CONSISTENT_SIMULTANEOUS_MOVES_ALLOW.setBoolean(true);
try (Cluster cluster = Cluster.build(nodeCount)
.withConfig(config -> config
.with(NETWORK)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
index 574b84f..40050bf 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
@@ -27,6 +27,8 @@
import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
+
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.awaitility.Awaitility;
import org.junit.Assert;
import org.junit.Test;
@@ -63,6 +65,7 @@
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.exceptions.CasWriteTimeoutException;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -91,6 +94,8 @@
import org.apache.cassandra.utils.Pair;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTO_REPAIR_FREQUENCY_SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_PAXOS_AUTO_REPAIRS;
import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME;
import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
import static org.apache.cassandra.service.paxos.BallotGenerator.Global.staleBallot;
@@ -106,7 +111,7 @@
static
{
- CassandraRelevantProperties.PAXOS_EXECUTE_ON_SELF.setBoolean(false);
+ CassandraRelevantProperties.PAXOS_USE_SELF_EXECUTION.setBoolean(false);
DatabaseDescriptor.daemonInitialization();
}
@@ -239,7 +244,7 @@
)
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (k int primary key, v int)");
- cluster.get(3).shutdown().get();
+ ClusterUtils.stopUnchecked(cluster.get(3));
InetAddressAndPort node3 = InetAddressAndPort.getByAddress(cluster.get(3).broadcastAddress());
// make sure node1 knows node3 is down
@@ -330,21 +335,20 @@
@Test
public void paxosAutoRepair() throws Throwable
{
- System.setProperty("cassandra.auto_repair_frequency_seconds", "1");
- System.setProperty("cassandra.disable_paxos_auto_repairs", "true");
- try (Cluster cluster = init(Cluster.create(3, cfg -> cfg
+ try (WithProperties properties = new WithProperties().set(AUTO_REPAIR_FREQUENCY_SECONDS, 1).set(DISABLE_PAXOS_AUTO_REPAIRS, true);
+ Cluster cluster = init(Cluster.create(3, cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_repair_enabled", true)
- .set("truncate_request_timeout_in_ms", 1000L)))
- )
+ .set("truncate_request_timeout_in_ms", 1000L)));
+ )
{
cluster.forEach(i -> {
Assert.assertFalse(CassandraRelevantProperties.CLOCK_GLOBAL.isPresent());
- Assert.assertEquals("1", System.getProperty("cassandra.auto_repair_frequency_seconds"));
- Assert.assertEquals("true", System.getProperty("cassandra.disable_paxos_auto_repairs"));
+ Assert.assertEquals(1, CassandraRelevantProperties.AUTO_REPAIR_FREQUENCY_SECONDS.getInt());
+ Assert.assertTrue(CassandraRelevantProperties.DISABLE_PAXOS_AUTO_REPAIRS.getBoolean());
});
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
- cluster.get(3).shutdown().get();
+ ClusterUtils.stopUnchecked(cluster.get(3));
cluster.verbs(Verb.PAXOS_COMMIT_REQ).drop();
try
{
@@ -369,7 +373,7 @@
for (int i=0; i<20; i++)
{
if (!cluster.get(1).callsOnInstance(() -> PaxosState.uncommittedTracker().hasInflightAutoRepairs()).call()
- && !cluster.get(2).callsOnInstance(() -> PaxosState.uncommittedTracker().hasInflightAutoRepairs()).call())
+ && !cluster.get(2).callsOnInstance(() -> PaxosState.uncommittedTracker().hasInflightAutoRepairs()).call())
break;
logger.info("Waiting for auto repairs to finish...");
Thread.sleep(1000);
@@ -377,11 +381,6 @@
assertUncommitted(cluster.get(1), KEYSPACE, TABLE, 0);
assertUncommitted(cluster.get(2), KEYSPACE, TABLE, 0);
}
- finally
- {
- System.clearProperty("cassandra.auto_repair_frequency_seconds");
- System.clearProperty("cassandra.disable_paxos_auto_repairs");
- }
}
@Test
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
index bfd4e95..3251f84 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
@@ -96,7 +96,7 @@
static
{
- CassandraRelevantProperties.PAXOS_EXECUTE_ON_SELF.setBoolean(false);
+ CassandraRelevantProperties.PAXOS_USE_SELF_EXECUTION.setBoolean(false);
DatabaseDescriptor.daemonInitialization();
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java
new file mode 100644
index 0000000..e3679e9
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.awaitility.Awaitility.await;
+import static org.hamcrest.Matchers.emptyString;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class PreviewRepairSnapshotTest extends TestBaseImpl
+{
+ /**
+ * Makes sure we only snapshot sstables containing the mismatching token
+ * <p>
+ * 1. create 100 sstables per instance, compaction disabled, one token per sstable
+ * 2. make 3 tokens mismatching on node2, one token per sstable
+ * 3. run preview repair
+ * 4. make sure that only the sstables containing the token are in the snapshot
+ */
+ @Test
+ public void testSnapshotOfSStablesContainingMismatchingTokens() throws IOException
+ {
+ try (Cluster cluster = init(Cluster.build(2).withConfig(config ->
+ config.set("snapshot_on_repaired_data_mismatch", true)
+ .with(GOSSIP)
+ .with(NETWORK)).start()))
+ {
+ Set<Integer> tokensToMismatch = Sets.newHashSet(1, 50, 99);
+ cluster.schemaChange(withKeyspace("create table %s.tbl (id int primary key) with compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled':false }"));
+ // 1 token per sstable;
+ for (int i = 0; i < 100; i++)
+ {
+ cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (id) values (?)"), ConsistencyLevel.ALL, i);
+ cluster.stream().forEach(instance -> instance.flush(KEYSPACE));
+ }
+ cluster.stream().forEach(instance -> instance.flush(KEYSPACE));
+ for (int i = 1; i <= 2; i++)
+ markRepaired(cluster, i);
+
+ cluster.get(1)
+ .nodetoolResult("repair", "-vd", "-pr", KEYSPACE, "tbl")
+ .asserts()
+ .success()
+ .stdoutContains("Repaired data is in sync");
+
+ Set<Token> mismatchingTokens = new HashSet<>();
+ for (Integer token : tokensToMismatch)
+ {
+ cluster.get(2).executeInternal(withKeyspace("insert into %s.tbl (id) values (?)"), token);
+ cluster.get(2).flush(KEYSPACE);
+ Object[][] res = cluster.get(2).executeInternal(withKeyspace("select token(id) from %s.tbl where id = ?"), token);
+ mismatchingTokens.add(new Murmur3Partitioner.LongToken((long) res[0][0]));
+ }
+
+ markRepaired(cluster, 2);
+
+ cluster.get(1)
+ .nodetoolResult("repair", "-vd", KEYSPACE, "tbl")
+ .asserts()
+ .success()
+ .stdoutContains("Repaired data is inconsistent");
+
+ cluster.get(1).runOnInstance(checkSnapshot(mismatchingTokens, 3));
+ // node2 got the duplicate mismatch-tokens above, so it should exist in exactly 6 sstables
+ cluster.get(2).runOnInstance(checkSnapshot(mismatchingTokens, 6));
+ }
+ }
+
+ private IIsolatedExecutor.SerializableRunnable checkSnapshot(Set<Token> mismatchingTokens, int expectedSnapshotSize)
+ {
+ return () -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+
+ String snapshotTag = await().atMost(1, MINUTES)
+ .pollInterval(100, MILLISECONDS)
+ .until(() -> {
+ for (String tag : cfs.listSnapshots().keySet())
+ {
+ // we create the snapshot schema file last, so when this exists we know the snapshot is complete;
+ if (cfs.getDirectories().getSnapshotSchemaFile(tag).exists())
+ return tag;
+ }
+
+ return "";
+ }, not(emptyString()));
+
+ Set<SSTableReader> inSnapshot = new HashSet<>();
+
+ try (Refs<SSTableReader> sstables = cfs.getSnapshotSSTableReaders(snapshotTag))
+ {
+ inSnapshot.addAll(sstables);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ assertEquals(expectedSnapshotSize, inSnapshot.size());
+
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ Bounds<Token> sstableBounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken());
+ boolean shouldBeInSnapshot = false;
+ for (Token mismatchingToken : mismatchingTokens)
+ {
+ if (sstableBounds.contains(mismatchingToken))
+ {
+ assertFalse(shouldBeInSnapshot);
+ shouldBeInSnapshot = true;
+ }
+ }
+ assertEquals(shouldBeInSnapshot, inSnapshot.contains(sstable));
+ }
+ };
+ }
+
+ private void markRepaired(Cluster cluster, int instance)
+ {
+ cluster.get(instance).runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ try
+ {
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor,
+ System.currentTimeMillis(),
+ null,
+ false);
+ sstable.reloadSSTableMetadata();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
index 90e29f2..a0b643f 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
@@ -73,9 +73,7 @@
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.distributed.api.IMessageFilters.Matcher;
import static org.apache.cassandra.distributed.impl.Instance.deserializeMessage;
-import static org.apache.cassandra.distributed.test.PreviewRepairTest.DelayFirstRepairTypeMessageFilter.finalizePropose;
import static org.apache.cassandra.distributed.test.PreviewRepairTest.DelayFirstRepairTypeMessageFilter.validationRequest;
-import static org.apache.cassandra.net.Verb.FINALIZE_PROPOSE_MSG;
import static org.apache.cassandra.net.Verb.VALIDATION_REQ;
import static org.apache.cassandra.service.StorageService.instance;
import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
@@ -187,10 +185,14 @@
previewRepairStarted.await();
// this needs to finish before the preview repair is unpaused on node2
cluster.get(1).callOnInstance(repair(options(false, false)));
+ RepairResult irResult = cluster.get(1).callOnInstance(repair(options(false, false)));
continuePreviewRepair.signalAll();
RepairResult rs = rsFuture.get();
- assertFalse(rs.success); // preview repair should have failed
+ assertFalse(rs.success); // preview repair was started before IR, but has lower priority, so its task will get cancelled
assertFalse(rs.wasInconsistent); // and no mismatches should have been reported
+
+ assertTrue(irResult.success); // IR was started after preview repair, but has a higher priority, so it'll be allowed to finish
+ assertFalse(irResult.wasInconsistent);
}
finally
{
@@ -226,34 +228,21 @@
.messagesMatching(validationRequest(previewRepairStarted, continuePreviewRepair))
.drop();
- Condition irRepairStarted = newOneTimeCondition();
- Condition continueIrRepair = newOneTimeCondition();
- // this blocks the IR from committing, so we can reenable the preview
- cluster.filters()
- .outbound()
- .verbs(FINALIZE_PROPOSE_MSG.id)
- .from(1).to(2)
- .messagesMatching(finalizePropose(irRepairStarted, continueIrRepair))
- .drop();
-
Future<RepairResult> previewResult = cluster.get(1).asyncCallsOnInstance(repair(options(true, false))).call();
previewRepairStarted.await();
- // trigger IR and wait till its ready to commit
+ // trigger IR and wait till it's ready to commit
Future<RepairResult> irResult = cluster.get(1).asyncCallsOnInstance(repair(options(false, false))).call();
- irRepairStarted.await();
+ RepairResult ir = irResult.get();
+ assertTrue(ir.success); // IR was submitted after preview repair has acquired sstables, but has higher priority
+ assertFalse(ir.wasInconsistent); // not preview, so we don't care about preview notification
// unblock preview repair and wait for it to complete
continuePreviewRepair.signalAll();
RepairResult rs = previewResult.get();
- assertFalse(rs.success); // preview repair should have failed
+ assertFalse(rs.success); // preview repair was started earlier than IR session; but has smaller priority
assertFalse(rs.wasInconsistent); // and no mismatches should have been reported
-
- continueIrRepair.signalAll();
- RepairResult ir = irResult.get();
- assertTrue(ir.success);
- assertFalse(ir.wasInconsistent); // not preview, so we don't care about preview notification
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ProfileLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/ProfileLoadTest.java
new file mode 100644
index 0000000..a83ea7c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/ProfileLoadTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+public class ProfileLoadTest extends TestBaseImpl
+{
+ @Test
+ public void testScheduledSamplingTaskLogs() throws IOException
+ {
+ try (Cluster cluster = init(Cluster.build(1).start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck));"));
+
+ // start the scheduled profileload task that samples for 1 second and every second.
+ cluster.get(1).nodetoolResult("profileload", "1000", "-i", "1000").asserts().success();
+
+ Random rnd = new Random();
+ // 800 * 2ms = 1.6 seconds. It logs every second. So it logs at least once.
+ for (int i = 0; i < 800; i++)
+ {
+ cluster.coordinator(1)
+ .execute(withKeyspace("INSERT INTO %s.tbl (pk, ck, v) VALUES (?,?,?)"),
+ ConsistencyLevel.QUORUM, rnd.nextInt(), rnd.nextInt(), i);
+ Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
+ }
+ // --list should display all active tasks.
+ String expectedOutput = String.format("KEYSPACE TABLE%n" + "%8s %5s", "*", "*");
+ cluster.get(1).nodetoolResult("profileload", "--list")
+ .asserts()
+ .success()
+ .stdoutContains(expectedOutput);
+
+ // loop assert the log contains the frequency readout; give this 15 seconds which should be plenty of time
+ // even on very badly underprovisioned environments
+ int timeout = 15;
+ boolean testPassed = false;
+ while (timeout-- > 0)
+ {
+ List<String> freqHeadings = cluster.get(1)
+ .logs()
+ .grep("Frequency of (reads|writes|cas contentions) by partition")
+ .getResult();
+ if (freqHeadings.size() > 3)
+ {
+ testPassed = true;
+ break;
+ }
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+ Assert.assertTrue("The scheduled task should at least run and log once", testPassed);
+
+ List<String> startSamplingLogs = cluster.get(1)
+ .logs()
+ .grep("Starting to sample tables")
+ .getResult();
+ Assert.assertTrue("It should start sampling at least once", startSamplingLogs.size() > 0);
+
+ // stop the scheduled sampling
+ cluster.get(1).nodetoolResult("profileload", "--stop").asserts().success();
+
+ // wait for the last schedule to be stopped. --list should list nothing after stopping
+ assertListEmpty(cluster.get(1));
+
+ // schedule on the specific table
+ cluster.get(1).nodetoolResult("profileload", KEYSPACE, "tbl", "1000", "-i", "1000").asserts().success();
+ expectedOutput = String.format("%" + KEYSPACE.length() + "s %5s%n" +
+ "%s %5s",
+ "KEYSPACE", "TABLE",
+ KEYSPACE, "tbl");
+ cluster.get(1).nodetoolResult("profileload", "--list")
+ .asserts()
+ .success()
+ .stdoutContains(expectedOutput);
+ // stop all should stop the task scheduled with the specific table
+ cluster.get(1).nodetoolResult("profileload", "--stop")
+ .asserts().success();
+ assertListEmpty(cluster.get(1));
+ }
+ }
+
+ @Test
+ public void testPreventDuplicatedSchedule() throws IOException
+ {
+ try (Cluster cluster = init(Cluster.build(1).start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck));"));
+
+ // New sampling; we are good
+ cluster.get(1).nodetoolResult("profileload", KEYSPACE, "tbl", "1000", "-i", "1000")
+ .asserts()
+ .success()
+ .stdoutNotContains("Unable to schedule sampling for keyspace");
+
+ // Duplicated sampling (against the same table) but different interval. Nodetool should reject
+ cluster.get(1).nodetoolResult("profileload", KEYSPACE, "tbl", "1000", "-i", "1000")
+ .asserts()
+ .success()
+ .stdoutContains("Unable to schedule sampling for keyspace");
+
+ // The "sampling all" request creates overlaps, so it should be rejected too
+ cluster.get(1).nodetoolResult("profileload", "1000", "-i", "1000")
+ .asserts()
+ .success()
+ .stdoutContains("Unable to schedule sampling for keyspace");
+
+ cluster.get(1).nodetoolResult("profileload", KEYSPACE, "tbl", "--stop").asserts().success();
+ assertListEmpty(cluster.get(1));
+
+ cluster.get(1).nodetoolResult("profileload", "nonexistks", "nonexisttbl", "--stop")
+ .asserts()
+ .success()
+ .stdoutContains("Unable to stop the non-existent scheduled sampling");
+ }
+ }
+
+ private void assertListEmpty(IInvokableInstance instance)
+ {
+ Uninterruptibles.sleepUninterruptibly(1500, TimeUnit.MILLISECONDS);
+ Assert.assertEquals("--list should list nothing",
+ "KEYSPACE TABLE\n",
+ instance.nodetoolResult("profileload", "--list").getStdout());
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
new file mode 100644
index 0000000..09e56e0
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Row;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class QueriesTableTest extends TestBaseImpl
+{
+ public static final int ITERATIONS = 256;
+
+ @Test
+ public void shouldExposeReadsAndWrites() throws Throwable
+ {
+ try (Cluster cluster = init(Cluster.build(1).start()))
+ {
+ ExecutorService executor = Executors.newFixedThreadPool(16);
+
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (k int primary key, v int)");
+
+ AtomicInteger reads = new AtomicInteger(0);
+ AtomicInteger writes = new AtomicInteger(0);
+ AtomicInteger paxos = new AtomicInteger(0);
+
+ for (int i = 0; i < ITERATIONS; i++)
+ {
+ int k = i;
+ executor.execute(() -> cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (k, v) VALUES (" + k + ", 0)", ConsistencyLevel.ALL));
+ executor.execute(() -> cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 10 WHERE k = " + (k - 1) + " IF v = 0", ConsistencyLevel.ALL));
+ executor.execute(() -> cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = " + (k - 1), ConsistencyLevel.ALL));
+
+ executor.execute(() ->
+ {
+ SimpleQueryResult result = cluster.get(1).executeInternalWithResult("SELECT * FROM system_views.queries");
+
+ while (result.hasNext())
+ {
+ Row row = result.next();
+ String threadId = row.get("thread_id").toString();
+ String task = row.get("task").toString();
+
+ if (threadId.contains("Read") && task.contains("SELECT"))
+ reads.incrementAndGet();
+ else if (threadId.contains("Mutation") && task.contains("Mutation"))
+ writes.incrementAndGet();
+ else if (threadId.contains("Mutation") && task.contains("Paxos"))
+ paxos.incrementAndGet();
+ }
+ });
+ }
+
+ executor.shutdown();
+ assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
+
+ // We should see at least one read, write, and conditional update in the "queries" table.
+ assertThat(reads.get()).isGreaterThan(0).isLessThanOrEqualTo(ITERATIONS);
+ assertThat(writes.get()).isGreaterThan(0).isLessThanOrEqualTo(ITERATIONS);
+ assertThat(paxos.get()).isGreaterThan(0).isLessThanOrEqualTo(ITERATIONS);
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
index 9a70e89..83c2806 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
@@ -34,7 +34,6 @@
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Mutation;
@@ -59,6 +58,7 @@
import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT;
import static org.apache.cassandra.db.Keyspace.open;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
@@ -253,7 +253,7 @@
assertRows(cluster.get(2).executeInternal(query));
// alter RF
- System.setProperty(Config.PROPERTY_PREFIX + "allow_alter_rf_during_range_movement", "true");
+ ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT.setBoolean(true);
cluster.schemaChange(withKeyspace("ALTER KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': 2}"));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
index f1266fa..0fc2554 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorBase.java
@@ -34,6 +34,8 @@
import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairParallelism;
import org.apache.cassandra.distributed.test.DistributedRepairUtils.RepairType;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS;
+
public class RepairCoordinatorBase extends TestBaseImpl
{
protected static Cluster CLUSTER;
@@ -67,7 +69,7 @@
// This only works because the way CI works
// In CI a new JVM is spun up for each test file, so this doesn't have to worry about another test file
// getting this set first
- System.setProperty("cassandra.nodetool.jmx_notification_poll_interval_seconds", "1");
+ NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS.setLong(1);
}
@BeforeClass
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
index f6da888..b702855 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
@@ -20,10 +20,8 @@
import java.io.IOException;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
@@ -31,14 +29,6 @@
import java.util.stream.Stream;
import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.concurrent.SEPExecutor;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.locator.AbstractReplicationStrategy;
-import org.apache.cassandra.locator.EndpointsForToken;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.ReplicaLayout;
-import org.apache.cassandra.locator.ReplicaUtils;
-import org.apache.cassandra.utils.Throwables;
import org.junit.Assert;
import org.junit.Test;
@@ -46,21 +36,32 @@
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.concurrent.SEPExecutor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.format.StatsComponent;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.ReplicaLayout;
+import org.apache.cassandra.locator.ReplicaUtils;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageProxy.LocalReadRunnable;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
+import org.apache.cassandra.utils.Throwables;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
@@ -516,10 +517,7 @@
{
SSTableReader sstable = sstables.next();
Descriptor descriptor = sstable.descriptor;
- Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer()
- .deserialize(descriptor, EnumSet.of(MetadataType.STATS));
-
- StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
+ StatsMetadata stats = StatsComponent.load(descriptor).statsMetadata();
Assert.assertEquals("repaired at is set for sstable: " + descriptor,
stats.repairedAt,
ActiveRepairService.UNREPAIRED_SSTABLE);
@@ -568,10 +566,7 @@
{
SSTableReader sstable = sstables.next();
Descriptor descriptor = sstable.descriptor;
- Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer()
- .deserialize(descriptor, EnumSet.of(MetadataType.STATS));
-
- StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
+ StatsMetadata stats = StatsComponent.load(descriptor).statsMetadata();
Assert.assertTrue("repaired at is not set for sstable: " + descriptor, stats.repairedAt > 0);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
index dd37bd3..db2244b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
@@ -36,7 +36,6 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.compaction.DateTieredCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
@@ -151,7 +150,6 @@
public final void testCompactionStrategiesWithMixedSSTables() throws Exception
{
testCompactionStrategiesWithMixedSSTables(SizeTieredCompactionStrategy.class,
- DateTieredCompactionStrategy.class,
TimeWindowCompactionStrategy.class,
LeveledCompactionStrategy.class);
}
@@ -432,8 +430,16 @@
private static void assertSSTablesCount(Set<Descriptor> descs, String tableName, int expectedSeqGenIds, int expectedUUIDGenIds)
{
- List<String> seqSSTables = descs.stream().filter(desc -> desc.id instanceof SequenceBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList());
- List<String> uuidSSTables = descs.stream().filter(desc -> desc.id instanceof UUIDBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList());
+ List<String> seqSSTables = descs.stream()
+ .filter(desc -> desc.id instanceof SequenceBasedSSTableId)
+ .map(descriptor -> descriptor.baseFile().toString())
+ .sorted()
+ .collect(Collectors.toList());
+ List<String> uuidSSTables = descs.stream()
+ .filter(desc -> desc.id instanceof UUIDBasedSSTableId)
+ .map(descriptor -> descriptor.baseFile().toString())
+ .sorted()
+ .collect(Collectors.toList());
assertThat(seqSSTables).describedAs("SSTables of %s with sequence based id", tableName).hasSize(expectedSeqGenIds);
assertThat(uuidSSTables).describedAs("SSTables of %s with UUID based id", tableName).hasSize(expectedUUIDGenIds);
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java
new file mode 100644
index 0000000..9dbc0e4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.index.internal.CassandraIndex;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+public class SecondaryIndexCompactionTest extends TestBaseImpl
+{
+ @Test
+ public void test2iCompaction() throws IOException
+ {
+ try (Cluster cluster = init(Cluster.build(1).start()))
+ {
+ cluster.schemaChange(withKeyspace("create table %s.tbl (id int, ck int, something int, else int, primary key (id, ck));"));
+ cluster.schemaChange(withKeyspace("create index tbl_idx on %s.tbl (ck)"));
+
+ for (int i = 0; i < 10; i++)
+ cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (id, ck, something, else) values (?, ?, ?, ?)"), ConsistencyLevel.ALL, i, i, i, i);
+
+ cluster.get(1).runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+ CassandraIndex i = (CassandraIndex) cfs.indexManager.getIndexByName("tbl_idx");
+ i.getIndexCfs().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+ Set<SSTableReader> idxSSTables = i.getIndexCfs().getLiveSSTables();
+ // emulate ongoing index compaction:
+ CompactionInfo.Holder h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables);
+ CompactionManager.instance.active.beginCompaction(h);
+ CompactionManager.instance.active.estimatedRemainingWriteBytes();
+ CompactionManager.instance.active.finishCompaction(h);
+ });
+ }
+ }
+
+ static class MockHolder extends CompactionInfo.Holder
+ {
+ private final Set<SSTableReader> sstables;
+ private final TableMetadata metadata;
+
+ public MockHolder(TableMetadata metadata, Set<SSTableReader> sstables)
+ {
+ this.metadata = metadata;
+ this.sstables = sstables;
+ }
+ @Override
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, nextTimeUUID(), sstables);
+ }
+
+ @Override
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java
new file mode 100644
index 0000000..5d72660
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.cassandra.io.util.File;
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.ActiveCompactions;
+import org.apache.cassandra.db.compaction.CompactionStrategyManager;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.streaming.StreamManager;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+public class StreamsDiskSpaceTest extends TestBaseImpl
+{
+ @Test
+ public void testAbortStreams() throws Exception
+ {
+ try(Cluster cluster = init(Cluster.build(2)
+ .withConfig(config -> config.set("hinted_handoff_enabled", false)
+ .with(GOSSIP)
+ .with(NETWORK))
+ .withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, StreamManager.class, "getTotalRemainingOngoingBytes"))
+ .start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
+ for (int i = 0; i < 10000; i++)
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (id, t) values (?,?)", i, i);
+ cluster.get(1).flush(KEYSPACE);
+ cluster.get(2).runOnInstance(() -> BB.ongoing.set(Long.MAX_VALUE / 2));
+ cluster.get(1).nodetoolResult("repair", "-full").asserts().failure();
+ cluster.get(2).nodetoolResult("repair", "-full").asserts().failure();
+ cluster.get(2).runOnInstance(() -> BB.ongoing.set(0));
+ cluster.get(1).nodetoolResult("repair", "-full").asserts().success();
+ }
+ }
+
+ @Test
+ public void testAbortStreamsWhenOngoingCompactionsLeaveInsufficientSpace() throws Exception
+ {
+ try(Cluster cluster = init(Cluster.build(2)
+ .withConfig(config -> config.set("hinted_handoff_enabled", false)
+ .with(GOSSIP)
+ .with(NETWORK))
+ .withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteBytes"))
+ .start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
+ for (int i = 0; i < 10000; i++)
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (id, t) values (?,?)", i, i);
+ cluster.get(1).flush(KEYSPACE);
+ cluster.get(2).runOnInstance(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+ BB.datadir = cfs.getDirectories().getLocationForDisk(cfs.getDirectories().getWriteableLocation(0));
+ });
+ cluster.get(2).runOnInstance(() -> BB.ongoing.set(Long.MAX_VALUE / 2));
+
+ cluster.get(1).nodetoolResult("repair", "-full").asserts().failure();
+ cluster.get(2).nodetoolResult("repair", "-full").asserts().failure();
+ cluster.get(2).runOnInstance(() -> BB.ongoing.set(0));
+ cluster.get(1).nodetoolResult("repair", "-full").asserts().success();
+ }
+ }
+
+ @Test
+ public void testAbortStreamsWhenTooManyPendingCompactions() throws Exception
+ {
+ try(Cluster cluster = init(Cluster.build(2)
+ .withConfig(config -> config.set("hinted_handoff_enabled", false)
+ .set("reject_repair_compaction_threshold", 1024)
+ .with(GOSSIP)
+ .with(NETWORK))
+ .withInstanceInitializer(BB::installCSMGetEstimatedRemainingTasks)
+ .start()))
+ {
+ cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
+ for (int i = 0; i < 10000; i++)
+ cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (id, t) values (?,?)", i, i);
+ cluster.get(1).flush(KEYSPACE);
+ cluster.get(2).runOnInstance(() -> {
+ BB.ongoing.set(DatabaseDescriptor.getRepairPendingCompactionRejectThreshold() + 1);
+ });
+
+ cluster.get(1).nodetoolResult("repair", "-full").asserts().failure();
+ cluster.get(2).nodetoolResult("repair", "-full").asserts().failure();
+ cluster.get(2).runOnInstance(() -> BB.ongoing.set(0));
+ cluster.get(1).nodetoolResult("repair", "-full").asserts().success();
+ }
+ }
+
+ public static class BB
+ {
+ public static AtomicLong ongoing = new AtomicLong();
+ public static File datadir;
+ private static void doInstall(ClassLoader cl, int id, Class<?> clazz, String method)
+ {
+ if (id != 2)
+ return;
+ new ByteBuddy().rebase(clazz)
+ .method(named(method))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static long getTotalRemainingOngoingBytes()
+ {
+ return ongoing.get();
+ }
+
+ public static Map<File, Long> estimatedRemainingWriteBytes()
+ {
+ Map<File, Long> ret = new HashMap<>();
+ if (datadir != null)
+ ret.put(datadir, ongoing.get());
+ return ret;
+ }
+
+ public static int getEstimatedRemainingTasks(int additionalSSTables, long additionalBytes, boolean isIncremental)
+ {
+ return (int) ongoing.get();
+ }
+
+ private static void installCSMGetEstimatedRemainingTasks(ClassLoader cl, int nodeNumber)
+ {
+ if (nodeNumber == 2)
+ {
+ new ByteBuddy().redefine(CompactionStrategyManager.class)
+ .method(named("getEstimatedRemainingTasks").and(takesArguments(3)))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java b/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java
new file mode 100644
index 0000000..8e2f866
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+
+import static org.apache.cassandra.Util.getBackups;
+import static org.apache.cassandra.distributed.Cluster.build;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.test.ExecUtil.rethrow;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TableLevelIncrementalBackupsTest extends TestBaseImpl
+{
+ @Test
+ public void testIncrementalBackupEnabledCreateTable() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH incremental_backups = true"));
+ disableCompaction(c, KEYSPACE, "test_table");
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 1, true, KEYSPACE, "test_table");
+
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 2, true, KEYSPACE, "test_table");
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table"));
+
+
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH incremental_backups = false"));
+ disableCompaction(c, KEYSPACE, "test_table2");
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table2 (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 0, false, KEYSPACE, "test_table2");
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table2"));
+ }
+ }
+
+ @Test
+ public void testIncrementalBackupEnabledAlterTable() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH incremental_backups = false"));
+ disableCompaction(c, KEYSPACE, "test_table");
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 0, false, KEYSPACE, "test_table");
+
+ c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH incremental_backups = true"));
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 1, true, KEYSPACE, "test_table");
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table"));
+
+
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH incremental_backups = true"));
+ disableCompaction(c, KEYSPACE, "test_table2");
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table2 (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 1, true, KEYSPACE, "test_table2");
+
+ c.schemaChange(withKeyspace("ALTER TABLE %s.test_table2 WITH incremental_backups = false"));
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table2 (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 1, false, KEYSPACE, "test_table2");
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table2"));
+ }
+ }
+
+ @Test
+ public void testIncrementalBackupWhenCreateTableByDefault() throws Exception
+ {
+ try (Cluster c = getCluster(true))
+ {
+ //incremental_backups is set to true by default
+ c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int)"));
+ disableCompaction(c, KEYSPACE, "test_table");
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 1, true, KEYSPACE, "test_table");
+
+ c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH incremental_backups = false"));
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 1, false, KEYSPACE, "test_table");
+
+ c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH incremental_backups = true"));
+ c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1)"), ALL);
+ flush(c, KEYSPACE);
+ assertBackupSSTablesCount(c, 2, true, KEYSPACE, "test_table");
+
+ c.schemaChange(withKeyspace("DROP TABLE %s.test_table"));
+ }
+ }
+
+ private Cluster getCluster(boolean incrementalBackups) throws IOException
+ {
+ return init(build(2).withDataDirCount(1).withConfig(c -> c.with(Feature.GOSSIP)
+ .set("incremental_backups", incrementalBackups)).start());
+ }
+
+ private void flush(Cluster cluster, String keyspace)
+ {
+ for (int i = 1; i < cluster.size() + 1; i++)
+ cluster.get(i).flush(keyspace);
+ }
+
+ private void disableCompaction(Cluster cluster, String keyspace, String table)
+ {
+ for (int i = 1; i < cluster.size() + 1; i++)
+ cluster.get(i).nodetool("disableautocompaction", keyspace, table);
+ }
+
+ private static void assertBackupSSTablesCount(Cluster cluster, int expectedSeqGenIds, boolean enable, String ks, String... tableNames)
+ {
+ for (int i = 1; i < cluster.size() + 1; i++)
+ {
+ cluster.get(i).runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertTableMetaIncrementalBackupEnable(ks, tableName, enable))));
+ cluster.get(i).runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(getBackups(ks, tableName), tableName, expectedSeqGenIds))));
+ }
+ }
+
+ private static void assertSSTablesCount(Set<Descriptor> descs, String tableName, int expectedSeqGenIds)
+ {
+ List<String> seqSSTables = descs.stream()
+ .filter(desc -> desc.id instanceof SequenceBasedSSTableId)
+ .map(descriptor -> descriptor.baseFile().toString())
+ .sorted()
+ .collect(Collectors.toList());
+ assertThat(seqSSTables).describedAs("SSTables of %s with sequence based id", tableName).hasSize(expectedSeqGenIds);
+ }
+
+ private static void assertTableMetaIncrementalBackupEnable(String ks, String tableName, boolean enable)
+ {
+ ColumnFamilyStore columnFamilyStore = ColumnFamilyStore.getIfExists(ks, tableName);
+ if (enable)
+ assertTrue(columnFamilyStore.isTableIncrementalBackupsEnabled());
+ else
+ assertFalse(columnFamilyStore.isTableIncrementalBackupsEnabled());
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
index e97a081..35c5904 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
@@ -27,6 +27,8 @@
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import com.google.common.collect.ImmutableSet;
@@ -42,6 +44,7 @@
import org.apache.cassandra.distributed.shared.DistributedTestBase;
import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SCHEMA_DELAY_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
public class TestBaseImpl extends DistributedTestBase
@@ -119,8 +122,8 @@
IInstanceConfig config = cluster.newInstanceConfig();
config.set("auto_bootstrap", true);
IInvokableInstance newInstance = cluster.bootstrap(config);
- withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000),
- () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster)));
+ withProperty(BOOTSTRAP_SCHEMA_DELAY_MS, Integer.toString(90 * 1000),
+ () -> withProperty(JOIN_RING, false, () -> newInstance.startup(cluster)));
newInstance.nodetoolResult("join").asserts().success();
}
@@ -190,9 +193,20 @@
{
// These keyspaces are under replicated by default, so must be updated when doing a multi-node cluster;
// else bootstrap will fail with 'Unable to find sufficient sources for streaming range <range> in keyspace <name>'
+ Map<String, Long> dcCounts = cluster.stream()
+ .map(i -> i.config().localDatacenter())
+ .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
+ String replica = "{'class': 'NetworkTopologyStrategy'";
+ for (Map.Entry<String, Long> e : dcCounts.entrySet())
+ {
+ String dc = e.getKey();
+ int rf = Math.min(e.getValue().intValue(), 3);
+ replica += ", '" + dc + "': " + rf;
+ }
+ replica += "}";
for (String ks : Arrays.asList("system_auth", "system_traces"))
{
- cluster.schemaChange("ALTER KEYSPACE " + ks + " WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': " + Math.min(cluster.size(), 3) + "}");
+ cluster.schemaChange("ALTER KEYSPACE " + ks + " WITH REPLICATION = " + replica);
}
// in real live repair is needed in this case, but in the test case it doesn't matter if the tables loose
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TimeoutAbortTest.java b/test/distributed/org/apache/cassandra/distributed/test/TimeoutAbortTest.java
new file mode 100644
index 0000000..f496530
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/TimeoutAbortTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_READ_ITERATION_DELAY_MS;
+import static org.junit.Assert.assertFalse;
+import static org.psjava.util.AssertStatus.assertTrue;
+
+public class TimeoutAbortTest extends TestBaseImpl
+{
+ @Test
+ public void timeoutTest() throws IOException, InterruptedException
+ {
+ TEST_READ_ITERATION_DELAY_MS.setInt(5000);
+ try (Cluster cluster = init(Cluster.build(1).start()))
+ {
+ cluster.schemaChange(withKeyspace("create table %s.tbl (id int, ck1 int, ck2 int, d int, primary key (id, ck1, ck2))"));
+ cluster.coordinator(1).execute(withKeyspace("delete from %s.tbl using timestamp 5 where id = 1 and ck1 = 77 "), ConsistencyLevel.ALL);
+ cluster.get(1).flush(KEYSPACE);
+ Thread.sleep(1000);
+ for (int i = 0; i < 100; i++)
+ cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (id, ck1, ck2, d) values (1,77,?,1) using timestamp 10"), ConsistencyLevel.ALL, i);
+ cluster.get(1).flush(KEYSPACE);
+ boolean caughtException = false;
+ try
+ {
+ cluster.coordinator(1).execute(withKeyspace("select * from %s.tbl where id=1 and ck1 = 77"), ConsistencyLevel.ALL);
+ }
+ catch (Exception e)
+ {
+ assertTrue(e.getClass().getName().contains("ReadTimeoutException"));
+ caughtException = true;
+ }
+ assertTrue(caughtException);
+ List<String> errors = cluster.get(1).logs().grepForErrors().getResult();
+ assertFalse(errors.toString(), errors.stream().anyMatch(s -> s.contains("open RT bound")));
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/UpdateSystemAuthAfterDCExpansionTest.java b/test/distributed/org/apache/cassandra/distributed/test/UpdateSystemAuthAfterDCExpansionTest.java
index 8765067..5313cee 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/UpdateSystemAuthAfterDCExpansionTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/UpdateSystemAuthAfterDCExpansionTest.java
@@ -22,6 +22,7 @@
import java.util.UUID;
import com.google.common.collect.ImmutableList;
+
import org.apache.cassandra.utils.concurrent.Condition;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -44,6 +45,7 @@
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.auth.AuthKeyspace.ROLES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
@@ -113,7 +115,7 @@
{
// reduce the time from 10s to prevent "Cannot process role related query as the role manager isn't yet setup."
// exception from CassandraRoleManager
- System.setProperty("cassandra.superuser_setup_delay_ms", "0");
+ SUPERUSER_SETUP_DELAY_MS.setLong(0);
TestBaseImpl.beforeClass();
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java b/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java
index ee89a81..2bdcaf1 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java
@@ -19,22 +19,209 @@
package org.apache.cassandra.distributed.test;
import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.junit.Assert;
import org.junit.Test;
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.ActiveCompactions;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.LogAction;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.CountDownLatch;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
public class UpgradeSSTablesTest extends TestBaseImpl
{
@Test
+ public void upgradeSSTablesInterruptsOngoingCompaction() throws Throwable
+ {
+ try (ICluster<IInvokableInstance> cluster = init(builder().withNodes(1).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck));");
+ cluster.get(1).acceptsOnInstance((String ks) -> {
+ ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
+ cfs.disableAutoCompaction();
+ CompactionManager.instance.setMaximumCompactorThreads(1);
+ CompactionManager.instance.setCoreCompactorThreads(1);
+ }).accept(KEYSPACE);
+
+ String blob = "blob";
+ for (int i = 0; i < 6; i++)
+ blob += blob;
+
+ for (int cnt = 0; cnt < 5; cnt++)
+ {
+ for (int i = 0; i < 100; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?,?,?)",
+ ConsistencyLevel.QUORUM, (cnt * 1000) + i, i, blob);
+ }
+ cluster.get(1).nodetool("flush", KEYSPACE, "tbl");
+ }
+
+ LogAction logAction = cluster.get(1).logs();
+ logAction.mark();
+ Future<?> future = cluster.get(1).asyncAcceptsOnInstance((String ks) -> {
+ ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
+ CompactionManager.instance.submitMaximal(cfs, FBUtilities.nowInSeconds(), false, OperationType.COMPACTION);
+ }).apply(KEYSPACE);
+ Assert.assertEquals(0, cluster.get(1).nodetool("upgradesstables", "-a", KEYSPACE, "tbl"));
+ future.get();
+ Assert.assertFalse(logAction.grep("Compaction interrupted").getResult().isEmpty());
+ }
+ }
+
+ @Test
+ public void compactionDoesNotCancelUpgradeSSTables() throws Throwable
+ {
+ try (ICluster<IInvokableInstance> cluster = init(builder().withNodes(1).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck));");
+ cluster.get(1).acceptsOnInstance((String ks) -> {
+ ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
+ cfs.disableAutoCompaction();
+ CompactionManager.instance.setMaximumCompactorThreads(1);
+ CompactionManager.instance.setCoreCompactorThreads(1);
+ }).accept(KEYSPACE);
+
+ String blob = "blob";
+ for (int i = 0; i < 6; i++)
+ blob += blob;
+
+ for (int cnt = 0; cnt < 5; cnt++)
+ {
+ for (int i = 0; i < 100; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?,?,?)",
+ ConsistencyLevel.QUORUM, (cnt * 1000) + i, i, blob);
+ }
+ cluster.get(1).nodetool("flush", KEYSPACE, "tbl");
+ }
+
+ LogAction logAction = cluster.get(1).logs();
+ logAction.mark();
+ Assert.assertEquals(0, cluster.get(1).nodetool("upgradesstables", "-a", KEYSPACE, "tbl"));
+ Assert.assertFalse(logAction.watchFor("Compacting").getResult().isEmpty());
+
+ cluster.get(1).acceptsOnInstance((String ks) -> {
+ ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
+ FBUtilities.allOf(CompactionManager.instance.submitMaximal(cfs, FBUtilities.nowInSeconds(), false, OperationType.COMPACTION))
+ .awaitUninterruptibly(1, TimeUnit.MINUTES);
+
+ }).accept(KEYSPACE);
+ Assert.assertTrue(logAction.grep("Compaction interrupted").getResult().isEmpty());
+ Assert.assertFalse(logAction.grep("Finished Upgrade sstables").getResult().isEmpty());
+ Assert.assertFalse(logAction.grep("Compacted (.*) 5 sstables to").getResult().isEmpty());
+ }
+ }
+
+ @Test
+ public void cleanupDoesNotInterruptUpgradeSSTables() throws Throwable
+ {
+ try (ICluster<IInvokableInstance> cluster = init(builder().withNodes(1).withInstanceInitializer(BB::install).start()))
+ {
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck));");
+
+ cluster.get(1).acceptsOnInstance((String ks) -> {
+ ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
+ cfs.disableAutoCompaction();
+ }).accept(KEYSPACE);
+
+ String blob = "blob";
+ for (int i = 0; i < 6; i++)
+ blob += blob;
+
+ for (int i = 0; i < 10000; i++)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?,?,?)",
+ ConsistencyLevel.QUORUM, i, i, blob);
+ }
+
+ cluster.get(1).nodetool("flush", KEYSPACE, "tbl");
+
+ LogAction logAction = cluster.get(1).logs();
+ logAction.mark();
+
+ // Start upgradingsstables - use BB to pause once inside ActiveCompactions.beginCompaction
+ Thread upgradeThread = new Thread(() -> {
+ cluster.get(1).nodetool("upgradesstables", "-a", KEYSPACE, "tbl");
+ });
+ upgradeThread.start();
+ Assert.assertTrue(cluster.get(1).callOnInstance(() -> BB.starting.awaitUninterruptibly(1, TimeUnit.MINUTES)));
+
+ // Start a scrub and make sure that it fails, log check later to make sure it was
+ // because it cannot cancel the active upgrade sstables
+ Assert.assertNotEquals(0, cluster.get(1).nodetool("scrub", KEYSPACE, "tbl"));
+
+ // Now resume the upgrade sstables so test can shut down
+ cluster.get(1).runOnInstance(() -> {
+ BB.start.decrement();
+ });
+ upgradeThread.join();
+
+ Assert.assertFalse(logAction.grep("Unable to cancel in-progress compactions, since they're running with higher or same priority: Upgrade sstables").getResult().isEmpty());
+ Assert.assertFalse(logAction.grep("Starting Scrub for ").getResult().isEmpty());
+ Assert.assertFalse(logAction.grep("Finished Upgrade sstables for distributed_test_keyspace.tbl successfully").getResult().isEmpty());
+ }
+ }
+
+ @Test
+ public void truncateWhileUpgrading() throws Throwable
+ {
+ try (ICluster<IInvokableInstance> cluster = init(builder().withNodes(1).start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v text, PRIMARY KEY (pk, ck)) "));
+ cluster.get(1).acceptsOnInstance((String ks) -> {
+ ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore("tbl");
+ cfs.disableAutoCompaction();
+ CompactionManager.instance.setMaximumCompactorThreads(1);
+ CompactionManager.instance.setCoreCompactorThreads(1);
+ }).accept(KEYSPACE);
+
+ String blob = "blob";
+ for (int i = 0; i < 10; i++)
+ blob += blob;
+
+ for (int i = 0; i < 500; i++)
+ {
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (pk, ck, v) VALUES (?,?,?)"),
+ ConsistencyLevel.QUORUM, i, i, blob);
+ if (i > 0 && i % 100 == 0)
+ cluster.get(1).nodetool("flush", KEYSPACE, "tbl");
+ }
+
+ LogAction logAction = cluster.get(1).logs();
+ logAction.mark();
+
+ Future<?> upgrade = CompletableFuture.runAsync(() -> {
+ cluster.get(1).nodetool("upgradesstables", "-a", KEYSPACE, "tbl");
+ });
+
+ cluster.schemaChange(withKeyspace("TRUNCATE %s.tbl"));
+ upgrade.get();
+ Assert.assertFalse(logAction.grep("Compaction interrupted").getResult().isEmpty());
+ }
+ }
+
+ @Test
public void rewriteSSTablesTest() throws Throwable
{
try (ICluster<IInvokableInstance> cluster = builder().withNodes(1).withDataDirCount(1).start())
@@ -74,7 +261,7 @@
cfs.disableAutoCompaction();
for (SSTableReader tbl : cfs.getLiveSSTables())
{
- maxTs = Math.max(maxTs, tbl.getCreationTimeFor(Component.DATA));
+ maxTs = Math.max(maxTs, tbl.getDataCreationTime());
}
return maxTs;
}).apply(KEYSPACE);
@@ -96,7 +283,7 @@
assert liveSSTables.size() == 2 : String.format("Expected 2 sstables, but got " + liveSSTables.size());
for (SSTableReader tbl : liveSSTables)
{
- if (tbl.getCreationTimeFor(Component.DATA) <= maxTs)
+ if (tbl.getDataCreationTime() <= maxTs)
count++;
else
skipped++;
@@ -115,4 +302,38 @@
}
}
}
+
+ public static class BB
+ {
+ // Will be initialized in the context of the instance class loader
+ static CountDownLatch starting = newCountDownLatch(1);
+ static CountDownLatch start = newCountDownLatch(1);
+
+ public static void install(ClassLoader classLoader, Integer num)
+ {
+ new ByteBuddy().rebase(ActiveCompactions.class)
+ .method(named("beginCompaction"))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ @SuppressWarnings("unused")
+ public static void beginCompaction(CompactionInfo.Holder ci, @SuperCall Callable<Void> zuperCall)
+ {
+ try
+ {
+ zuperCall.call();
+ if (ci.getCompactionInfo().getTaskType() == OperationType.UPGRADE_SSTABLES)
+ {
+ starting.decrement();
+ Assert.assertTrue(start.awaitUninterruptibly(1, TimeUnit.MINUTES));
+ }
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/VirtualTableLogsTest.java b/test/distributed/org/apache/cassandra/distributed/test/VirtualTableLogsTest.java
new file mode 100644
index 0000000..1fc2958
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/VirtualTableLogsTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import ch.qos.logback.classic.Level;
+import org.apache.cassandra.db.virtual.LogMessagesTable;
+import org.apache.cassandra.db.virtual.LogMessagesTable.LogMessage;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.utils.logging.VirtualTableAppender;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOGBACK_CONFIGURATION_FILE;
+import static org.apache.cassandra.db.virtual.LogMessagesTable.LEVEL_COLUMN_NAME;
+import static org.apache.cassandra.db.virtual.LogMessagesTable.LOGGER_COLUMN_NAME;
+import static org.apache.cassandra.db.virtual.LogMessagesTable.MESSAGE_COLUMN_NAME;
+import static org.apache.cassandra.db.virtual.LogMessagesTable.ORDER_IN_MILLISECOND_COLUMN_NAME;
+import static org.apache.cassandra.db.virtual.LogMessagesTable.TIMESTAMP_COLUMN_NAME;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class VirtualTableLogsTest extends TestBaseImpl
+{
+ @Test
+ public void testVTableOutput() throws Throwable
+ {
+ try (WithProperties properties = new WithProperties().set(LOGBACK_CONFIGURATION_FILE, "test/conf/logback-dtest_with_vtable_appender.xml");
+ Cluster cluster = Cluster.build(1)
+ .withConfig(c -> c.with(Feature.values()))
+ .start();
+ )
+ {
+ List<TestingLogMessage> rows = getRows(cluster);
+ assertFalse(rows.isEmpty());
+
+ rows.forEach(message -> assertTrue(Level.toLevel(message.level).isGreaterOrEqual(Level.INFO)));
+ }
+ }
+
+ @Test
+ public void testMultipleAppendersFailToStartNode() throws Throwable
+ {
+ LOGBACK_CONFIGURATION_FILE.setString("test/conf/logback-dtest_with_vtable_appender_invalid.xml");
+ try (WithProperties properties = new WithProperties().set(LOGBACK_CONFIGURATION_FILE, "test/conf/logback-dtest_with_vtable_appender_invalid.xml");
+ Cluster ignored = Cluster.build(1)
+ .withConfig(c -> c.with(Feature.values()))
+ .start();
+ )
+ {
+ fail("Node should not start as there is supposed to be invalid logback configuration file.");
+ }
+ catch (IllegalStateException ex)
+ {
+ assertEquals(format("There are multiple appenders of class %s " +
+ "of names CQLLOG,CQLLOG2. There is only one appender of such class allowed.",
+ VirtualTableAppender.class.getName()),
+ ex.getMessage());
+ }
+ }
+
+ private List<TestingLogMessage> getRows(Cluster cluster)
+ {
+ SimpleQueryResult simpleQueryResult = cluster.coordinator(1).executeWithResult(query("select * from %s"), ONE);
+ List<TestingLogMessage> rows = new ArrayList<>();
+ simpleQueryResult.forEachRemaining(row -> {
+ long timestamp = row.getTimestamp(TIMESTAMP_COLUMN_NAME).getTime();
+ String logger = row.getString(LOGGER_COLUMN_NAME);
+ String level = row.getString(LEVEL_COLUMN_NAME);
+ String message = row.getString(MESSAGE_COLUMN_NAME);
+ int order = row.getInteger(ORDER_IN_MILLISECOND_COLUMN_NAME);
+ TestingLogMessage logMessage = new TestingLogMessage(timestamp, logger, level, message, order);
+ rows.add(logMessage);
+ });
+ return rows;
+ }
+
+ private String query(String template)
+ {
+ return format(template, getTableName());
+ }
+
+ private String getTableName()
+ {
+ return format("%s.%s", SchemaConstants.VIRTUAL_VIEWS, LogMessagesTable.TABLE_NAME);
+ }
+
+ private static class TestingLogMessage extends LogMessage
+ {
+ private int order;
+
+ public TestingLogMessage(long timestamp, String logger, String level, String message, int order)
+ {
+ super(timestamp, logger, level, message);
+ this.order = order;
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java b/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java
new file mode 100644
index 0000000..499cf07
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.cdc;
+
+import java.util.function.Consumer;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertTrue;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public class ToggleCDCOnRepairEnabledTest extends TestBaseImpl
+{
+ @Test
+ public void testCDCOnRepairIsEnabled() throws Exception
+ {
+ testCDCOnRepairEnabled(true, cluster -> {
+ cluster.get(2).runOnInstance(() -> {
+ boolean containCDCInLog = CommitLog.instance.segmentManager
+ .getActiveSegments()
+ .stream()
+ .anyMatch(s -> s.getCDCState() == CommitLogSegment.CDCState.CONTAINS);
+ assertTrue("Mutation should be added to commit log when cdc_on_repair_enabled is true",
+ containCDCInLog);
+ });
+ });
+ }
+
+ @Test
+ public void testCDCOnRepairIsDisabled() throws Exception
+ {
+ testCDCOnRepairEnabled(false, cluster -> {
+ cluster.get(2).runOnInstance(() -> {
+ boolean containCDCInLog = CommitLog.instance.segmentManager
+ .getActiveSegments()
+ .stream()
+ .allMatch(s -> s.getCDCState() != CommitLogSegment.CDCState.CONTAINS);
+ assertTrue("No mutation should be added to commit log when cdc_on_repair_enabled is false",
+ containCDCInLog);
+ });
+ });
+ }
+
+ // test helper to repair data between nodes when cdc_on_repair_enabled is on or off.
+ private void testCDCOnRepairEnabled(boolean enabled, Consumer<Cluster> assertion) throws Exception
+ {
+ try (Cluster cluster = init(Cluster.build(2)
+ .withConfig(c -> c.set("cdc_enabled", true)
+ .set("cdc_on_repair_enabled", enabled)
+ .with(Feature.NETWORK)
+ .with(Feature.GOSSIP))
+ .start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k INT PRIMARY KEY, v INT) WITH cdc=true"));
+
+ // Data only in node1
+ cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"));
+ Object[][] result = cluster.get(1).executeInternal(withKeyspace("SELECT * FROM %s.tbl WHERE k = 1"));
+ assertRows(result, row(1, 1));
+ result = cluster.get(2).executeInternal(withKeyspace("SELECT * FROM %s.tbl WHERE k = 1"));
+ assertRows(result);
+
+ // repair
+ cluster.get(1).flush(KEYSPACE);
+ cluster.get(2).nodetool("repair", KEYSPACE, "tbl");
+
+ // verify node2 now have data
+ result = cluster.get(2).executeInternal(withKeyspace("SELECT * FROM %s.tbl WHERE k = 1"));
+ assertRows(result, row(1, 1));
+
+ assertion.accept(cluster);
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailPartitionSizeTest.java b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailPartitionSizeTest.java
new file mode 100644
index 0000000..b2ee8d7
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailPartitionSizeTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.guardrails;
+
+import java.io.IOException;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+
+import static java.nio.ByteBuffer.allocate;
+
+/**
+ * Tests the guardrail for the size of partition size, {@link Guardrails#partitionSize}.
+ * <p>
+ * This test only includes the activation of the guardrail during sstable writes, focusing on the emmitted log messages.
+ * The tests for config, client warnings and diagnostic events are in
+ * {@link org.apache.cassandra.db.guardrails.GuardrailPartitionSizeTest}.
+ */
+public class GuardrailPartitionSizeTest extends GuardrailTester
+{
+ private static final int WARN_THRESHOLD = 1024 * 1024; // bytes (1 MiB)
+ private static final int FAIL_THRESHOLD = WARN_THRESHOLD * 2; // bytes (2 MiB)
+
+ private static final int NUM_NODES = 1;
+ private static final int NUM_CLUSTERINGS = 5;
+
+ private static Cluster cluster;
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ cluster = init(Cluster.build(NUM_NODES)
+ .withConfig(c -> c.set("partition_size_warn_threshold", WARN_THRESHOLD + "B")
+ .set("partition_size_fail_threshold", FAIL_THRESHOLD + "B")
+ .set("compaction_large_partition_warning_threshold", "999GiB")
+ .set("memtable_heap_space", "512MiB")) // avoids flushes
+ .start());
+ cluster.disableAutoCompaction(KEYSPACE);
+ }
+
+ @AfterClass
+ public static void teardownCluster()
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ @Override
+ protected Cluster getCluster()
+ {
+ return cluster;
+ }
+
+ @Test
+ public void testPartitionSize()
+ {
+ testPartitionSize(WARN_THRESHOLD, FAIL_THRESHOLD);
+ }
+
+ @Test
+ public void testPartitionSizeWithDynamicUpdate()
+ {
+ int warn = WARN_THRESHOLD * 2;
+ int fail = FAIL_THRESHOLD * 2;
+ cluster.get(1).runOnInstance(() -> Guardrails.instance.setPartitionSizeThreshold(warn + "B", fail + "B"));
+ testPartitionSize(warn, fail);
+ }
+
+ private void testPartitionSize(int warn, int fail)
+ {
+ schemaChange("CREATE TABLE %s (k int, c int, v blob, PRIMARY KEY (k, c))");
+
+ // empty table
+ assertNotWarnedOnFlush();
+ assertNotWarnedOnCompact();
+
+ // keep partition size lower than thresholds
+ execute("INSERT INTO %s (k, c, v) VALUES (1, 1, ?)", allocate(1));
+ assertNotWarnedOnFlush();
+ assertNotWarnedOnCompact();
+
+ // exceed warn threshold
+ for (int c = 0; c < NUM_CLUSTERINGS; c++)
+ {
+ execute("INSERT INTO %s (k, c, v) VALUES (1, ?, ?)", c, allocate(warn / NUM_CLUSTERINGS));
+ }
+ assertWarnedOnFlush(expectedMessages(1));
+ assertWarnedOnCompact(expectedMessages(1));
+
+ // exceed fail threshold
+ for (int c = 0; c < NUM_CLUSTERINGS * 10; c++)
+ {
+ execute("INSERT INTO %s (k, c, v) VALUES (1, ?, ?)", c, allocate(fail / NUM_CLUSTERINGS));
+ }
+ assertFailedOnFlush(expectedMessages(1));
+ assertFailedOnCompact(expectedMessages(1));
+
+ // remove most of the data to be under the threshold again
+ execute("DELETE FROM %s WHERE k = 1 AND c > 1");
+ assertNotWarnedOnFlush();
+ assertNotWarnedOnCompact();
+
+ // exceed warn threshold in multiple partitions
+ for (int c = 0; c < NUM_CLUSTERINGS; c++)
+ {
+ execute("INSERT INTO %s (k, c, v) VALUES (1, ?, ?)", c, allocate(warn / NUM_CLUSTERINGS));
+ execute("INSERT INTO %s (k, c, v) VALUES (2, ?, ?)", c, allocate(warn / NUM_CLUSTERINGS));
+ }
+ assertWarnedOnFlush(expectedMessages(1, 2));
+ assertWarnedOnCompact(expectedMessages(1, 2));
+
+ // exceed warn threshold in a new partition
+ for (int c = 0; c < NUM_CLUSTERINGS; c++)
+ {
+ execute("INSERT INTO %s (k, c, v) VALUES (3, ?, ?)", c, allocate(warn / NUM_CLUSTERINGS));
+ }
+ assertWarnedOnFlush(expectedMessages(3));
+ assertWarnedOnCompact(expectedMessages(1, 2, 3));
+ }
+
+ private void execute(String query, Object... args)
+ {
+ cluster.coordinator(1).execute(format(query), ConsistencyLevel.ALL, args);
+ }
+
+ private String[] expectedMessages(int... keys)
+ {
+ String[] messages = new String[keys.length];
+ for (int i = 0; i < keys.length; i++)
+ messages[i] = String.format("Guardrail partition_size violated: Partition %s:%d", qualifiedTableName, keys[i]);
+ return messages;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java
new file mode 100644
index 0000000..7a8e421
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/FailedBootstrapTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.hostreplacement;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import net.bytebuddy.implementation.bind.annotation.This;
+import org.apache.cassandra.auth.CassandraRoleManager;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
+import org.apache.cassandra.streaming.StreamException;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.assertj.core.api.Assertions;
+import org.awaitility.Awaitility;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS;
+import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart;
+import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked;
+import static org.apache.cassandra.distributed.test.hostreplacement.HostReplacementTest.setupCluster;
+
+public class FailedBootstrapTest extends TestBaseImpl
+{
+ private static final Logger logger = LoggerFactory.getLogger(FailedBootstrapTest.class);
+
+ private static final int NODE_TO_REMOVE = 2;
+
+ @Test
+ public void roleSetupDoesNotProduceUnavailables() throws IOException
+ {
+ Cluster.Builder builder = Cluster.build(3)
+ .withConfig(c -> c.with(Feature.values()))
+ .withInstanceInitializer(BB::install);
+ TokenSupplier even = TokenSupplier.evenlyDistributedTokens(3, builder.getTokenCount());
+ builder = builder.withTokenSupplier((TokenSupplier) node -> even.tokens(node == 4 ? NODE_TO_REMOVE : node));
+ try (Cluster cluster = builder.start())
+ {
+ List<IInvokableInstance> alive = Arrays.asList(cluster.get(1), cluster.get(3));
+ IInvokableInstance nodeToRemove = cluster.get(NODE_TO_REMOVE);
+
+ setupCluster(cluster);
+
+ stopUnchecked(nodeToRemove);
+
+ // should fail to join, but should start up!
+ IInvokableInstance added = replaceHostAndStart(cluster, nodeToRemove, p -> p.set(SUPERUSER_SETUP_DELAY_MS, "1"));
+ // log gossip for debugging
+ alive.forEach(i -> {
+ NodeToolResult result = i.nodetoolResult("gossipinfo");
+ result.asserts().success();
+ logger.info("gossipinfo for node{}\n{}", i.config().num(), result.getStdout());
+ });
+
+ // CassandraRoleManager attempted to do distributed reads while bootstrap was still going (it failed, so still in bootstrap mode)
+ // so need to validate that is no longer happening and we incrementing org.apache.cassandra.metrics.ClientRequestMetrics.unavailables
+ // sleep larger than multiple retry attempts...
+ Awaitility.await()
+ .atMost(1, TimeUnit.MINUTES)
+ .until(() -> added.callOnInstance(() -> BB.SETUP_SCHEDULE_COUNTER.get()) >= 42); // why 42? just need something large enough to make sure multiple attempts happened
+
+ // do we have any read metrics have unavailables?
+ added.runOnInstance(() -> {
+ Assertions.assertThat(ClientRequestsMetricsHolder.readMetrics.unavailables.getCount()).describedAs("read unavailables").isEqualTo(0);
+ Assertions.assertThat(ClientRequestsMetricsHolder.casReadMetrics.unavailables.getCount()).describedAs("CAS read unavailables").isEqualTo(0);
+ });
+ }
+ }
+
+ public static class BB
+ {
+ public static void install(ClassLoader classLoader, Integer num)
+ {
+ if (num != 4)
+ return;
+
+ new ByteBuddy().rebase(StreamResultFuture.class)
+ .method(named("maybeComplete"))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+
+ new ByteBuddy().rebase(CassandraRoleManager.class)
+ .method(named("scheduleSetupTask"))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static void maybeComplete(@This StreamResultFuture future) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException
+ {
+ Method method = future.getClass().getSuperclass().getSuperclass().getDeclaredMethod("tryFailure", Throwable.class);
+ method.setAccessible(true);
+ method.invoke(future, new StreamException(future.getCurrentState(), "Stream failed"));
+ }
+
+ private static final AtomicInteger SETUP_SCHEDULE_COUNTER = new AtomicInteger(0);
+ public static void scheduleSetupTask(final Callable<?> setupTask, @SuperCall Runnable fn)
+ {
+ SETUP_SCHEDULE_COUNTER.incrementAndGet();
+ fn.run();
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java
index 3de0bf5..8219d43 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/HostReplacementTest.java
@@ -35,6 +35,7 @@
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.AssertUtils;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.assertj.core.api.Assertions;
@@ -210,6 +211,8 @@
fixDistributedSchemas(cluster);
init(cluster);
+ ClusterUtils.awaitGossipSchemaMatch(cluster);
+
populate(cluster);
cluster.forEach(i -> i.flush(KEYSPACE));
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java
index ef0f1ec..b935e24 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java
@@ -42,7 +42,7 @@
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.utils.JMXServerUtils;
-import static org.apache.cassandra.config.CassandraRelevantProperties.IS_DISABLED_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
import static org.apache.cassandra.cql3.CQLTester.getAutomaticallyAllocatedPort;
public class JMXGetterCheckTest extends TestBaseImpl
@@ -68,7 +68,7 @@
jmxServer.start();
String url = "service:jmx:rmi:///jndi/rmi://" + jmxHost + ":" + jmxPort + "/jmxrmi";
- IS_DISABLED_MBEAN_REGISTRATION.setBoolean(false);
+ ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(false);
try (Cluster cluster = Cluster.build(1).withConfig(c -> c.with(Feature.values())).start())
{
List<Named> errors = new ArrayList<>();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/metric/TableMetricTest.java b/test/distributed/org/apache/cassandra/distributed/test/metric/TableMetricTest.java
index 08c9324..a8ddfe0 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/metric/TableMetricTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/metric/TableMetricTest.java
@@ -47,7 +47,7 @@
import org.apache.cassandra.tracing.TraceKeyspace;
import org.apache.cassandra.utils.MBeanWrapper;
-import static org.apache.cassandra.config.CassandraRelevantProperties.IS_DISABLED_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
import static org.apache.cassandra.config.CassandraRelevantProperties.MBEAN_REGISTRATION_CLASS;
public class TableMetricTest extends TestBaseImpl
@@ -55,7 +55,7 @@
static
{
MBEAN_REGISTRATION_CLASS.setString(MapMBeanWrapper.class.getName());
- IS_DISABLED_MBEAN_REGISTRATION.setBoolean(false);
+ ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(false);
}
private static volatile Map<String, Collection<String>> SYSTEM_TABLES = null;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/metrics/RequestTimeoutTest.java b/test/distributed/org/apache/cassandra/distributed/test/metrics/RequestTimeoutTest.java
new file mode 100644
index 0000000..2799fca
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/metrics/RequestTimeoutTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.metrics;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import net.bytebuddy.implementation.bind.annotation.SuperMethod;
+import net.bytebuddy.implementation.bind.annotation.This;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.utils.AssertionUtils;
+import org.apache.cassandra.exceptions.CasWriteTimeoutException;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.service.paxos.Paxos;
+import org.apache.cassandra.utils.concurrent.Awaitable;
+import org.apache.cassandra.utils.concurrent.Condition;
+import org.assertj.core.api.Assertions;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static org.apache.cassandra.utils.AssertionUtils.isThrowable;
+
+public class RequestTimeoutTest extends TestBaseImpl
+{
+ private static final AtomicInteger NEXT = new AtomicInteger(0);
+ public static final int COORDINATOR = 1;
+ private static Cluster CLUSTER;
+
+ @BeforeClass
+ public static void init() throws IOException
+ {
+ CLUSTER = Cluster.build(3)
+ .withConfig(c -> c.set("truncate_request_timeout", "10s"))
+ .withInstanceInitializer(BB::install)
+ .start();
+ init(CLUSTER);
+ CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY, v int)"));
+ }
+
+ @AfterClass
+ public static void cleanup()
+ {
+ if (CLUSTER != null)
+ CLUSTER.close();
+ }
+
+ @Before
+ public void before()
+ {
+ CLUSTER.get(COORDINATOR).runOnInstance(() -> MessagingService.instance().callbacks.unsafeClear());
+ CLUSTER.filters().reset();
+ BB.reset();
+ }
+
+ @Test
+ public void insert()
+ {
+ CLUSTER.filters().verbs(Verb.MUTATION_REQ.id).to(2).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("INSERT INTO %s.tbl (pk, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NEXT.getAndIncrement(), NEXT.getAndIncrement()))
+ .is(isThrowable(WriteTimeoutException.class));
+ BB.assertIsTimeoutTrue();
+ }
+
+ @Test
+ public void update()
+ {
+ CLUSTER.filters().verbs(Verb.MUTATION_REQ.id).to(2).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("UPDATE %s.tbl SET v=? WHERE pk=?"), ConsistencyLevel.ALL, NEXT.getAndIncrement(), NEXT.getAndIncrement()))
+ .is(isThrowable(WriteTimeoutException.class));
+ BB.assertIsTimeoutTrue();
+ }
+
+ @Test
+ public void batchInsert()
+ {
+ CLUSTER.filters().verbs(Verb.MUTATION_REQ.id).to(2).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(batch(withKeyspace("INSERT INTO %s.tbl (pk, v) VALUES (?, ?)")), ConsistencyLevel.ALL, NEXT.getAndIncrement(), NEXT.getAndIncrement()))
+ .is(isThrowable(WriteTimeoutException.class));
+ BB.assertIsTimeoutTrue();
+ }
+
+ @Test
+ public void rangeSelect()
+ {
+ CLUSTER.filters().verbs(Verb.RANGE_REQ.id).to(2).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("SELECT * FROM %s.tbl"), ConsistencyLevel.ALL))
+ .is(isThrowable(ReadTimeoutException.class));
+ BB.assertIsTimeoutTrue();
+ }
+
+ @Test
+ public void select()
+ {
+ CLUSTER.filters().verbs(Verb.READ_REQ.id).to(2).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("SELECT * FROM %s.tbl WHERE pk=?"), ConsistencyLevel.ALL, NEXT.getAndIncrement()))
+ .is(isThrowable(ReadTimeoutException.class));
+ BB.assertIsTimeoutTrue();
+ }
+
+ @Test
+ public void truncate()
+ {
+ CLUSTER.filters().verbs(Verb.TRUNCATE_REQ.id).to(2).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("TRUNCATE %s.tbl"), ConsistencyLevel.ALL))
+ .is(AssertionUtils.rootCauseIs(TimeoutException.class));
+ BB.assertIsTimeoutTrue();
+ }
+
+ // don't call BB.assertIsTimeoutTrue(); for CAS, as it has its own logic
+
+ @Test
+ public void casV2PrepareInsert()
+ {
+ withPaxos(Config.PaxosVariant.v2);
+
+ CLUSTER.filters().verbs(Verb.PAXOS2_PREPARE_REQ.id).to(2, 3).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("INSERT INTO %s.tbl (pk, v) VALUES (?, ?) IF NOT EXISTS"), ConsistencyLevel.ALL, NEXT.getAndIncrement(), NEXT.getAndIncrement()))
+ .is(isThrowable(CasWriteTimeoutException.class));
+ }
+
+ @Test
+ public void casV2PrepareSelect()
+ {
+ withPaxos(Config.PaxosVariant.v2);
+
+ CLUSTER.filters().verbs(Verb.PAXOS2_PREPARE_REQ.id).to(2, 3).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("SELECT * FROM %s.tbl WHERE pk=?"), ConsistencyLevel.SERIAL, NEXT.getAndIncrement()))
+ .is(isThrowable(ReadTimeoutException.class)); // why does write have its own type but not read?
+ }
+
+ @Test
+ public void casV2CommitInsert()
+ {
+ withPaxos(Config.PaxosVariant.v2);
+
+ CLUSTER.filters().verbs(Verb.PAXOS_COMMIT_REQ.id).to(2, 3).drop();
+ Assertions.assertThatThrownBy(() -> CLUSTER.coordinator(COORDINATOR).execute(withKeyspace("INSERT INTO %s.tbl (pk, v) VALUES (?, ?) IF NOT EXISTS"), ConsistencyLevel.ALL, NEXT.getAndIncrement(), NEXT.getAndIncrement()))
+ .is(isThrowable(CasWriteTimeoutException.class));
+ }
+
+ private static void withPaxos(Config.PaxosVariant variant)
+ {
+ CLUSTER.forEach(i -> i.runOnInstance(() -> Paxos.setPaxosVariant(variant)));
+ }
+
+ private static String batch(String cql)
+ {
+ return "BEGIN " + BatchStatement.Type.UNLOGGED.name() + " BATCH\n" + cql + "\nAPPLY BATCH";
+ }
+
+ public static class BB
+ {
+ public static void install(ClassLoader cl, int num)
+ {
+ if (num != COORDINATOR)
+ return;
+ new ByteBuddy().rebase(Condition.Async.class)
+ .method(named("await").and(takesArguments(2)))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+
+ new ByteBuddy().rebase(RequestCallback.class)
+ .method(named("isTimeout"))
+ .intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static boolean await(long time, TimeUnit units, @This Awaitable self, @SuperMethod Method method) throws InterruptedException, InvocationTargetException, IllegalAccessException
+ {
+ // make sure that the underline condition is met before returnning true
+ // this way its know that the timeouts triggered!
+ while (!((boolean) method.invoke(self, time, units)))
+ {
+ }
+ return true;
+ }
+
+ private static final AtomicInteger TIMEOUTS = new AtomicInteger(0);
+ public static boolean isTimeout(Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint, @SuperCall Callable<Boolean> fn) throws Exception
+ {
+ boolean timeout = fn.call();
+ if (timeout)
+ TIMEOUTS.incrementAndGet();
+ return timeout;
+ }
+
+ public static void assertIsTimeoutTrue()
+ {
+ int timeouts = CLUSTER.get(COORDINATOR).callOnInstance(() -> TIMEOUTS.getAndSet(0));
+ Assertions.assertThat(timeouts).isGreaterThan(0);
+ }
+
+ public static void reset()
+ {
+ CLUSTER.get(COORDINATOR).runOnInstance(() -> TIMEOUTS.set(0));
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
index 423e78b..f31e3a8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
@@ -19,16 +19,24 @@
package org.apache.cassandra.distributed.test.ring;
import java.lang.management.ManagementFactory;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.After;
import org.junit.Assert;
+import org.junit.AssumptionViolatedException;
import org.junit.Before;
import org.junit.Test;
-import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICluster;
@@ -36,9 +44,14 @@
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.schema.SchemaConstants;
import static java.util.Arrays.asList;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MIGRATION_DELAY;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
import static org.apache.cassandra.distributed.action.GossipHelper.bootstrap;
import static org.apache.cassandra.distributed.action.GossipHelper.pullSchemaFrom;
import static org.apache.cassandra.distributed.action.GossipHelper.statusToBootstrap;
@@ -50,6 +63,8 @@
{
private long savedMigrationDelay;
+ static WithProperties properties;
+
@Before
public void beforeTest()
{
@@ -59,18 +74,139 @@
// When we are running multiple test cases in the class, where each starts a node but in the same JVM, the
// up-time will be more or less relevant only for the first test. In order to enforce the startup-like behaviour
// for each test case, the MIGRATION_DELAY time is adjusted accordingly
- savedMigrationDelay = CassandraRelevantProperties.MIGRATION_DELAY.getLong();
- CassandraRelevantProperties.MIGRATION_DELAY.setLong(ManagementFactory.getRuntimeMXBean().getUptime() + savedMigrationDelay);
+ properties = new WithProperties().set(MIGRATION_DELAY, ManagementFactory.getRuntimeMXBean().getUptime() + savedMigrationDelay);
}
@After
public void afterTest()
{
- CassandraRelevantProperties.MIGRATION_DELAY.setLong(savedMigrationDelay);
+ properties.close();
}
@Test
- public void bootstrapTest() throws Throwable
+ public void bootstrapWithResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.setBoolean(false);
+ bootstrapTest();
+ }
+
+ @Test
+ public void bootstrapWithoutResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.setBoolean(true);
+ bootstrapTest();
+ }
+
+ /**
+ * Confirm that a normal, non-resumed bootstrap without the reset_bootstrap_progress param specified works without issue.
+ * @throws Throwable
+ */
+ @Test
+ public void bootstrapUnspecifiedResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+ bootstrapTest();
+ }
+
+ /**
+ * Confirm that, in the absence of the reset_bootstrap_progress param being set and in the face of a found prior
+ * partial bootstrap, we error out and don't complete our bootstrap.
+ *
+ * Test w/out vnodes only; logic is identical for both run env but the token alloc in this test doesn't work for
+ * vnode env and it's not worth the lift to update it to work in both env.
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void bootstrapUnspecifiedFailsOnResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
+
+ // Need our partitioner active for rangeToBytes conversion below
+ Config c = DatabaseDescriptor.loadConfig();
+ DatabaseDescriptor.daemonInitialization(() -> c);
+
+ int originalNodeCount = 2;
+ int expandedNodeCount = originalNodeCount + 1;
+
+ boolean sawException = false;
+ try (Cluster cluster = builder().withNodes(originalNodeCount)
+ .withoutVNodes()
+ .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount))
+ .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0"))
+ .withConfig(config -> config.with(NETWORK, GOSSIP))
+ .start())
+ {
+ populate(cluster, 0, 100);
+
+ IInstanceConfig config = cluster.newInstanceConfig();
+ IInvokableInstance newInstance = cluster.bootstrap(config);
+ withProperty(JOIN_RING, false, () -> newInstance.startup(cluster));
+
+ cluster.forEach(statusToBootstrap(newInstance));
+
+ List<Token> tokens = cluster.tokens();
+ assert tokens.size() >= 3;
+
+ /*
+ Our local tokens:
+ Tokens in cluster tokens: [-3074457345618258603, 3074457345618258601, 9223372036854775805]
+
+ From the bootstrap process:
+ fetchReplicas in our test keyspace:
+ [FetchReplica
+ {local=Full(/127.0.0.3:7012,(-3074457345618258603,3074457345618258601]),
+ remote=Full(/127.0.0.1:7012,(-3074457345618258603,3074457345618258601])},
+ FetchReplica
+ {local=Full(/127.0.0.3:7012,(9223372036854775805,-3074457345618258603]),
+ remote=Full(/127.0.0.1:7012,(9223372036854775805,-3074457345618258603])},
+ FetchReplica
+ {local=Full(/127.0.0.3:7012,(3074457345618258601,9223372036854775805]),
+ remote=Full(/127.0.0.1:7012,(3074457345618258601,9223372036854775805])}]
+ */
+
+ // Insert some bogus ranges in the keyspace to be bootstrapped to trigger the check on available ranges on bootstrap.
+ // Note: these have to precisely overlap with the token ranges hit during streaming or they won't trigger the
+ // availability logic on bootstrap to then except out; we can't just have _any_ range for a keyspace, but rather,
+ // must have a range that overlaps with what we're trying to stream.
+ Set<Range <Token>> fullSet = new HashSet<>();
+ fullSet.add(new Range<>(tokens.get(0), tokens.get(1)));
+ fullSet.add(new Range<>(tokens.get(1), tokens.get(2)));
+ fullSet.add(new Range<>(tokens.get(2), tokens.get(0)));
+
+ // Should be fine to trigger on full ranges only but add a partial for good measure.
+ Set<Range <Token>> partialSet = new HashSet<>();
+ partialSet.add(new Range<>(tokens.get(2), tokens.get(1)));
+
+ String cql = String.format("INSERT INTO %s.%s (keyspace_name, full_ranges, transient_ranges) VALUES (?, ?, ?)",
+ SchemaConstants.SYSTEM_KEYSPACE_NAME,
+ SystemKeyspace.AVAILABLE_RANGES_V2);
+
+ newInstance.executeInternal(cql,
+ KEYSPACE,
+ fullSet.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()),
+ partialSet.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()));
+
+ // We expect bootstrap to throw an exception on node3 w/the seen ranges we've inserted
+ cluster.run(asList(pullSchemaFrom(cluster.get(1)),
+ bootstrap()),
+ newInstance.config().num());
+ }
+ catch (AssumptionViolatedException ave)
+ {
+ // We get an AssumptionViolatedException if we're in a test job configured w/vnodes
+ throw ave;
+ }
+ catch (RuntimeException rte)
+ {
+ if (rte.getMessage().contains("Discovered existing bootstrap data"))
+ sawException = true;
+ }
+ Assert.assertTrue("Expected to see a RuntimeException w/'Discovered existing bootstrap data' in the error message; did not.",
+ sawException);
+ }
+
+ private void bootstrapTest() throws Throwable
{
int originalNodeCount = 2;
int expandedNodeCount = originalNodeCount + 1;
@@ -85,7 +221,7 @@
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance newInstance = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", false,
+ withProperty(JOIN_RING, false,
() -> newInstance.startup(cluster));
cluster.forEach(statusToBootstrap(newInstance));
@@ -115,14 +251,14 @@
{
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance newInstance = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", false,
+ withProperty(JOIN_RING, false,
() -> newInstance.startup(cluster));
cluster.forEach(statusToBootstrap(newInstance));
populate(cluster, 0, 100);
- Assert.assertEquals(100, newInstance.executeInternal("SELECT *FROM " + KEYSPACE + ".tbl").length);
+ Assert.assertEquals(100, newInstance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl").length);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/CleanupFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/CleanupFailureTest.java
index 9c8f71f..8ec3630 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ring/CleanupFailureTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ring/CleanupFailureTest.java
@@ -28,6 +28,7 @@
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.TestBaseImpl;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.junit.Assert.assertEquals;
import static org.apache.cassandra.distributed.action.GossipHelper.statusToBootstrap;
import static org.apache.cassandra.distributed.action.GossipHelper.statusToDecommission;
@@ -86,7 +87,7 @@
{
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance bootstrappingNode = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", false,
+ withProperty(JOIN_RING, false,
() -> bootstrappingNode.startup(cluster));
// Start decomission on bootstrappingNode
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/PendingWritesTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/PendingWritesTest.java
index 2e702b2..0e0cf76 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ring/PendingWritesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ring/PendingWritesTest.java
@@ -41,6 +41,7 @@
import org.apache.cassandra.service.PendingRangeCalculatorService;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.apache.cassandra.distributed.action.GossipHelper.bootstrap;
import static org.apache.cassandra.distributed.action.GossipHelper.disseminateGossipState;
import static org.apache.cassandra.distributed.action.GossipHelper.statusToBootstrap;
@@ -65,8 +66,7 @@
BootstrapTest.populate(cluster, 0, 100);
IInstanceConfig config = cluster.newInstanceConfig();
IInvokableInstance newInstance = cluster.bootstrap(config);
- withProperty("cassandra.join_ring", false,
- () -> newInstance.startup(cluster));
+ withProperty(JOIN_RING, false, () -> newInstance.startup(cluster));
cluster.forEach(statusToBootstrap(newInstance));
cluster.run(bootstrap(false, Duration.ofSeconds(60), Duration.ofSeconds(60)), newInstance.config().num());
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/AbstractStreamFailureLogs.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/AbstractStreamFailureLogs.java
new file mode 100644
index 0000000..28aa6b0
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/AbstractStreamFailureLogs.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.distributed.test.streaming;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.streaming.CassandraIncomingFile;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.LogResult;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter;
+import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.assertj.core.api.Assertions;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class AbstractStreamFailureLogs extends TestBaseImpl
+{
+ protected static final Logger logger = LoggerFactory.getLogger(AbstractStreamFailureLogs.class);
+
+ protected static final int FAILING_NODE = 2;
+
+ protected void streamTest(boolean zeroCopyStreaming, String reason, Integer failedNode) throws IOException
+ {
+ try (Cluster cluster = Cluster.build(2)
+ .withInstanceInitializer(BBStreamHelper::install)
+ .withConfig(c -> c.with(Feature.values())
+ .set("stream_entire_sstables", zeroCopyStreaming)
+ // when die, this will try to halt JVM, which is easier to validate in the test
+ // other levels require checking state of the subsystems
+ .set("disk_failure_policy", "die"))
+ .start())
+ {
+ init(cluster);
+
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY)"));
+
+ triggerStreaming(cluster);
+ // make sure disk failure policy is not triggered
+
+ IInvokableInstance failingNode = cluster.get(failedNode);
+
+ searchForLog(failingNode, reason);
+ }
+ }
+
+ protected void triggerStreaming(Cluster cluster)
+ {
+ IInvokableInstance node1 = cluster.get(1);
+ IInvokableInstance node2 = cluster.get(2);
+
+ // repair will do streaming IFF there is a mismatch; so cause one
+ for (int i = 0; i < 10; i++)
+ node1.executeInternal(withKeyspace("INSERT INTO %s.tbl (pk) VALUES (?)"), i); // timestamp won't match, causing a mismatch
+
+ // trigger streaming; expected to fail as streaming socket closed in the middle (currently this is an unrecoverable event)
+ //Blocks until the stream is complete
+ node2.nodetoolResult("repair", "-full", KEYSPACE, "tbl").asserts().failure();
+ }
+
+ protected void searchForLog(IInvokableInstance failingNode, String reason)
+ {
+ searchForLog(failingNode, true, reason);
+ }
+
+ protected boolean searchForLog(IInvokableInstance failingNode, boolean failIfNoMatch, String reason)
+ {
+ LogResult<List<String>> result = failingNode.logs().grepForErrors(-1, Pattern.compile("Stream failed:"));
+ // grepForErrors will include all ERROR logs even if they don't match the pattern; for this reason need to filter after the fact
+ List<String> matches = result.getResult();
+
+ matches = matches.stream().filter(s -> s.startsWith("WARN")).collect(Collectors.toList());
+ logger.info("Stream failed logs found: {}", String.join("\n", matches));
+ if (matches.isEmpty() && !failIfNoMatch)
+ return false;
+
+ Assertions.assertThat(matches)
+ .describedAs("node%d expected to find %s but could not", failingNode.config().num(), reason)
+ .hasSize(1);
+ String logLine = matches.get(0);
+ Assertions.assertThat(logLine).contains(reason);
+
+ Matcher match = Pattern.compile(".*\\[Stream #(.*)\\]").matcher(logLine);
+ if (!match.find()) throw new AssertionError("Unable to parse: " + logLine);
+ UUID planId = UUID.fromString(match.group(1));
+ SimpleQueryResult qr = failingNode.executeInternalWithResult("SELECT * FROM system_views.streaming WHERE id=?", planId);
+ Assertions.assertThat(qr.hasNext()).isTrue();
+ Assertions.assertThat(qr.next().getString("failure_cause")).contains(reason);
+ return true;
+ }
+
+ public static class BBStreamHelper
+ {
+ @SuppressWarnings("unused")
+ public static int writeDirectlyToChannel(ByteBuffer buf, @SuperCall Callable<Integer> zuper) throws Exception
+ {
+ if (isCaller(SSTableZeroCopyWriter.class.getName(), "write"))
+ throw new RuntimeException("TEST");
+ // different context; pass through
+ return zuper.call();
+ }
+
+ @SuppressWarnings("unused")
+ public static boolean append(UnfilteredRowIterator partition, @SuperCall Callable<Boolean> zuper) throws Exception
+ {
+ if (isCaller(CassandraIncomingFile.class.getName(), "read")) // handles compressed and non-compressed
+ throw new java.nio.channels.ClosedChannelException();
+ // different context; pass through
+ return zuper.call();
+ }
+
+ public static void install(ClassLoader classLoader, Integer num)
+ {
+ if (num != FAILING_NODE)
+ return;
+ new ByteBuddy().rebase(SequentialWriter.class)
+ .method(named("writeDirectlyToChannel").and(takesArguments(1)))
+ .intercept(MethodDelegation.to(BBStreamHelper.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+
+ new ByteBuddy().rebase(RangeAwareSSTableWriter.class)
+ .method(named("append").and(takesArguments(1)))
+ .intercept(MethodDelegation.to(BBStreamHelper.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+
+ }
+ }
+
+ protected static boolean isCaller(String klass, String method)
+ {
+ StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+ for (int i = 0; i < stack.length; i++)
+ {
+ StackTraceElement e = stack[i];
+ if (klass.equals(e.getClassName()) && method.equals(e.getMethodName()))
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/BoundExceptionTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/BoundExceptionTest.java
new file mode 100644
index 0000000..23c445c
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/BoundExceptionTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.distributed.test.streaming;
+
+import org.junit.Test;
+
+import org.apache.cassandra.streaming.StreamSession;
+
+import static org.junit.Assert.assertEquals;
+
+public class BoundExceptionTest
+{
+ private static final int LIMIT = 2;
+
+ @Test
+ public void testSingleException()
+ {
+ Throwable exceptionToTest = exception("test exception");
+ StringBuilder boundedStackTrace = StreamSession.boundStackTrace(exceptionToTest, LIMIT, new StringBuilder());
+
+ String expectedStackTrace = "java.lang.RuntimeException: test exception\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:1)\n";
+
+ assertEquals(expectedStackTrace,boundedStackTrace.toString());
+ }
+
+ @Test
+ public void testNestedException()
+ {
+ Throwable exceptionToTest = exception(exception("the disk /foo/var is bad", exception("Bad disk somewhere")));
+ StringBuilder boundedStackTrace = StreamSession.boundStackTrace(exceptionToTest, LIMIT, new StringBuilder());
+
+ String expectedStackTrace = "java.lang.RuntimeException: java.lang.RuntimeException: the disk /foo/var is bad\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:1)\n" +
+ "java.lang.RuntimeException: the disk /foo/var is bad\n" +
+ "java.lang.RuntimeException: Bad disk somewhere\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:1)\n";
+
+ assertEquals(expectedStackTrace, boundedStackTrace.toString());
+ }
+
+ @Test
+ public void testExceptionCycle()
+ {
+ Exception e1 = exception("Test exception 1");
+ Exception e2 = exception("Test exception 2");
+
+ e1.initCause(e2);
+ e2.initCause(e1);
+
+ StringBuilder boundedStackTrace = StreamSession.boundStackTrace(e1, LIMIT, new StringBuilder());
+ String expectedStackTrace = "java.lang.RuntimeException: Test exception 1\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:1)\n" +
+ "java.lang.RuntimeException: Test exception 2\n" +
+ "[CIRCULAR REFERENCE: java.lang.RuntimeException: Test exception 1]\n";
+
+ assertEquals(expectedStackTrace, boundedStackTrace.toString());
+ }
+
+ @Test
+ public void testEmptyStackTrace()
+ {
+ Throwable exceptionToTest = exception("there are words here", 0);
+
+ StringBuilder boundedStackTrace = StreamSession.boundStackTrace(exceptionToTest, LIMIT, new StringBuilder());
+ String expectedStackTrace = "java.lang.RuntimeException: there are words here\n";
+
+ assertEquals(expectedStackTrace,boundedStackTrace.toString());
+ }
+
+ @Test
+ public void testEmptyNestedStackTrace()
+ {
+ Throwable exceptionToTest = exception(exception("the disk /foo/var is bad", exception("Bad disk somewhere"), 0), 0);
+
+ StringBuilder boundedStackTrace = StreamSession.boundStackTrace(exceptionToTest, LIMIT, new StringBuilder());
+ String expectedStackTrace = "java.lang.RuntimeException: java.lang.RuntimeException: the disk /foo/var is bad\n" +
+ "java.lang.RuntimeException: the disk /foo/var is bad\n" +
+ "java.lang.RuntimeException: Bad disk somewhere\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:1)\n";
+
+ assertEquals(expectedStackTrace, boundedStackTrace.toString());
+ }
+
+ @Test
+ public void testLimitLargerThanStackTrace()
+ {
+ Throwable exceptionToTest = exception(exception("the disk /foo/var is bad", exception("Bad disk somewhere")), 1);
+
+ StringBuilder boundedStackTrace = StreamSession.boundStackTrace(exceptionToTest, LIMIT, new StringBuilder());
+ String expectedStackTrace = "java.lang.RuntimeException: java.lang.RuntimeException: the disk /foo/var is bad\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "java.lang.RuntimeException: the disk /foo/var is bad\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "java.lang.RuntimeException: Bad disk somewhere\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:0)\n" +
+ "\torg.apache.cassandra.distributed.test.streaming.BoundExceptionTest.method(BoundExceptionTest.java:1)\n";
+
+ assertEquals(expectedStackTrace, boundedStackTrace.toString());
+ }
+
+ private static StackTraceElement[] frames(int length)
+ {
+ StackTraceElement[] frames = new StackTraceElement[length];
+ for (int i = 0; i < length; i++)
+ frames[i] = new StackTraceElement(BoundExceptionTest.class.getCanonicalName(), "method", BoundExceptionTest.class.getSimpleName() + ".java", i);
+ return frames;
+ }
+
+ private static RuntimeException exception(String msg)
+ {
+ return exception(msg, null);
+ }
+
+ private static RuntimeException exception(String msg, int length)
+ {
+ return exception(msg, null, length);
+ }
+
+ private static RuntimeException exception(Throwable cause)
+ {
+ return exception(null, cause);
+ }
+
+ private static RuntimeException exception(Throwable cause, int length)
+ {
+ return exception(null, cause, length);
+ }
+
+ private static RuntimeException exception(String msg, Throwable cause)
+ {
+ return exception(msg, cause, LIMIT * 2);
+ }
+
+ private static RuntimeException exception(String msg, Throwable cause, int length)
+ {
+ RuntimeException e;
+ if (msg != null && cause != null) e = new RuntimeException(msg, cause);
+ else if (msg != null) e = new RuntimeException(msg);
+ else if (cause != null) e = new RuntimeException(cause);
+ else e = new RuntimeException();
+ e.setStackTrace(frames(length));
+ return e;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java
new file mode 100644
index 0000000..4319768
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.streaming;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.junit.Assert.assertTrue;
+
+public class LCSStreamingKeepLevelTest extends TestBaseImpl
+{
+ @Test
+ public void testDecom() throws IOException
+ {
+ try (Cluster cluster = builder().withNodes(4)
+ .withConfig(config -> config.with(NETWORK, GOSSIP, NATIVE_PROTOCOL))
+ .withoutVNodes()
+ .withDataDirCount(1)
+ .start())
+ {
+ populate(cluster);
+
+ cluster.get(4).nodetoolResult("decommission").asserts().success();
+
+ assertEmptyL0(cluster);
+ }
+ }
+
+ @Test
+ public void testMove() throws IOException
+ {
+ try (Cluster cluster = builder().withNodes(4)
+ .withConfig(config -> config.with(NETWORK, GOSSIP, NATIVE_PROTOCOL))
+ .withoutVNodes()
+ .withDataDirCount(1)
+ .start())
+ {
+ populate(cluster);
+
+ long tokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(3).getToken()).token;
+ long prevTokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(2).getToken()).token;
+ // move node 4 to the middle point between its current position and the previous node
+ long newToken = (tokenVal + prevTokenVal) / 2;
+ cluster.get(4).nodetoolResult("move", String.valueOf(newToken)).asserts().success();
+
+ assertEmptyL0(cluster);
+ }
+ }
+
+ private static void populate(Cluster cluster)
+ {
+ cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2}"));
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.decom_test (id int PRIMARY KEY, value int) with compaction = { 'class':'LeveledCompactionStrategy', 'enabled':'false' }"));
+
+ for (int i = 0; i < 500; i++)
+ {
+ cluster.coordinator(1).execute(withKeyspace("insert into %s.decom_test (id, value) VALUES (?, ?)"), ConsistencyLevel.ALL, i, i);
+ if (i % 100 == 0)
+ cluster.forEach((inst) -> inst.flush(KEYSPACE));
+ }
+ cluster.forEach((i) -> i.flush(KEYSPACE));
+ relevel(cluster);
+ }
+
+ private static void relevel(Cluster cluster)
+ {
+ for (IInvokableInstance i : cluster)
+ {
+ i.runOnInstance(() -> {
+ Set<SSTableReader> sstables = Keyspace.open(KEYSPACE).getColumnFamilyStore("decom_test").getLiveSSTables();
+ int lvl = 1;
+ for (SSTableReader sstable : sstables)
+ {
+ try
+ {
+ sstable.mutateLevelAndReload(lvl++);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ assertEmptyL0(cluster);
+ }
+
+ private static void assertEmptyL0(Cluster cluster)
+ {
+ for (IInvokableInstance i : cluster)
+ {
+ i.runOnInstance(() -> {
+ for (SSTableReader sstable : Keyspace.open(KEYSPACE).getColumnFamilyStore("decom_test").getLiveSSTables())
+ assertTrue(sstable.getSSTableLevel() > 0);
+ });
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java
index fc52ab6..f2772bc 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java
@@ -37,8 +37,8 @@
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
-import org.apache.cassandra.io.sstable.format.big.BigTableZeroCopyWriter;
+import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter;
+import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter;
import org.apache.cassandra.io.util.SequentialWriter;
import org.assertj.core.api.Assertions;
@@ -150,7 +150,7 @@
@SuppressWarnings("unused")
public static int writeDirectlyToChannel(ByteBuffer buf, @SuperCall Callable<Integer> zuper) throws Exception
{
- if (isCaller(BigTableZeroCopyWriter.class.getName(), "write"))
+ if (isCaller(SSTableZeroCopyWriter.class.getName(), "write"))
throw new java.nio.channels.ClosedChannelException();
// different context; pass through
return zuper.call();
@@ -193,4 +193,4 @@
.load(classLoader, ClassLoadingStrategy.Default.INJECTION);
}
}
-}
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureDueToSessionFailedTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureDueToSessionFailedTest.java
new file mode 100644
index 0000000..b04ea1e
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureDueToSessionFailedTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.streaming;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+public class StreamFailureLogsFailureDueToSessionFailedTest extends AbstractStreamFailureLogs
+{
+ @Test
+ public void failureDueToSessionFailed() throws IOException
+ {
+ streamTest(true,"Remote peer /127.0.0.2:7012 failed stream session", 1);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureDueToSessionTimeoutTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureDueToSessionTimeoutTest.java
new file mode 100644
index 0000000..b473597
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureDueToSessionTimeoutTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.streaming;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.streaming.CassandraIncomingFile;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter;
+import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.Shared;
+import org.awaitility.Awaitility;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class StreamFailureLogsFailureDueToSessionTimeoutTest extends AbstractStreamFailureLogs
+{
+ @Test
+ public void failureDueToSessionTimeout() throws IOException
+ {
+ try (Cluster cluster = Cluster.build(2)
+ .withInstanceInitializer(BBStreamTimeoutHelper::install)
+ .withConfig(c -> c.with(Feature.values())
+ // when die, this will try to halt JVM, which is easier to validate in the test
+ // other levels require checking state of the subsystems
+ .set("stream_transfer_task_timeout", "1ms"))
+ .start())
+ {
+
+ init(cluster);
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY)"));
+
+ ForkJoinPool.commonPool().execute(() -> triggerStreaming(cluster));
+ try
+ {
+ Awaitility.await("Did not see stream running or timed out")
+ .atMost(3, TimeUnit.MINUTES)
+ .until(() -> State.STREAM_IS_RUNNING.await(false) || searchForLog(cluster.get(1), false, "Session timed out"));
+ }
+ finally
+ {
+ State.UNBLOCK_STREAM.signal();
+ }
+ Awaitility.await("Unable to find 'Session timed out'")
+ .atMost(1, TimeUnit.MINUTES)
+ .until(() -> searchForLog(cluster.get(1), false, "Session timed out"));
+ }
+ }
+
+ @Shared
+ public static class State
+ {
+ public static final TestCondition STREAM_IS_RUNNING = new TestCondition();
+ public static final TestCondition UNBLOCK_STREAM = new TestCondition();
+ }
+
+ @Shared
+ public static class TestCondition
+ {
+ private volatile boolean signaled = false;
+
+ public void await()
+ {
+ await(true);
+ }
+
+ public boolean await(boolean throwOnTimeout)
+ {
+ long deadlineNanos = Clock.Global.nanoTime() + TimeUnit.MINUTES.toNanos(1);
+ while (!signaled)
+ {
+ long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - Clock.Global.nanoTime());
+ if (remainingMillis <= 0)
+ {
+ if (throwOnTimeout) throw new UncheckedTimeoutException("Condition not met within 1 minute");
+ return false;
+ }
+ // await may block signal from triggering notify, so make sure not to block for more than 500ms
+ remainingMillis = Math.min(remainingMillis, 500);
+ synchronized (this)
+ {
+ try
+ {
+ this.wait(remainingMillis);
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+ }
+ return true;
+ }
+
+ public void signal()
+ {
+ signaled = true;
+ synchronized (this)
+ {
+ this.notify();
+ }
+ }
+ }
+
+ public static class BBStreamTimeoutHelper
+ {
+ @SuppressWarnings("unused")
+ public static int writeDirectlyToChannel(ByteBuffer buf, @SuperCall Callable<Integer> zuper) throws Exception
+ {
+ if (isCaller(SSTableZeroCopyWriter.class.getName(), "write"))
+ {
+ State.STREAM_IS_RUNNING.signal();
+ State.UNBLOCK_STREAM.await();
+ }
+ // different context; pass through
+ return zuper.call();
+ }
+
+ @SuppressWarnings("unused")
+ public static boolean append(UnfilteredRowIterator partition, @SuperCall Callable<Boolean> zuper) throws Exception
+ {
+ if (isCaller(CassandraIncomingFile.class.getName(), "read")) // handles compressed and non-compressed
+ {
+ State.STREAM_IS_RUNNING.signal();
+ State.UNBLOCK_STREAM.await();
+ }
+ // different context; pass through
+ return zuper.call();
+ }
+
+ public static void install(ClassLoader classLoader, Integer num)
+ {
+ if (num != FAILING_NODE)
+ return;
+ new ByteBuddy().rebase(SequentialWriter.class)
+ .method(named("writeDirectlyToChannel").and(takesArguments(1)))
+ .intercept(MethodDelegation.to(BBStreamTimeoutHelper.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+
+ new ByteBuddy().rebase(RangeAwareSSTableWriter.class)
+ .method(named("append").and(takesArguments(1)))
+ .intercept(MethodDelegation.to(BBStreamTimeoutHelper.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureInTheMiddleWithEOFTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureInTheMiddleWithEOFTest.java
new file mode 100644
index 0000000..9b0f4f4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureInTheMiddleWithEOFTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.streaming;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+public class StreamFailureLogsFailureInTheMiddleWithEOFTest extends AbstractStreamFailureLogs
+{
+ @Test
+ public void failureInTheMiddleWithEOF() throws IOException
+ {
+ streamTest(false, "Session peer /127.0.0.1:7012 Failed because there was an java.nio.channels.ClosedChannelException with state=STREAMING", FAILING_NODE);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureInTheMiddleWithUnknownTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureInTheMiddleWithUnknownTest.java
new file mode 100644
index 0000000..9d2507b
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamFailureLogsFailureInTheMiddleWithUnknownTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.streaming;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+public class StreamFailureLogsFailureInTheMiddleWithUnknownTest extends AbstractStreamFailureLogs
+{
+ @Test
+ public void failureInTheMiddleWithUnknown() throws IOException
+ {
+ streamTest(true, "java.lang.RuntimeException: TEST", FAILING_NODE);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java b/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java
index 33e6cd6..a0f7f60 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java
@@ -26,6 +26,7 @@
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.KIBIBYTES;
import static org.assertj.core.api.Assertions.assertThat;
@@ -37,13 +38,16 @@
{
AbstractClientSizeWarning.setupClass();
+ //noinspection Convert2MethodRef
+ Assume.assumeTrue(CLUSTER.get(1).callOnInstance(() -> BigFormat.isSelected()));
+
CLUSTER.stream().forEach(i -> i.runOnInstance(() -> {
DatabaseDescriptor.setRowIndexReadSizeWarnThreshold(new DataStorageSpec.LongBytesBound(1, KIBIBYTES));
DatabaseDescriptor.setRowIndexReadSizeFailThreshold(new DataStorageSpec.LongBytesBound(2, KIBIBYTES));
// hack to force multiple index entries
DatabaseDescriptor.setColumnIndexCacheSize(1 << 20);
- DatabaseDescriptor.setColumnIndexSize(0);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(0);
}));
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidReadTimeoutsTest.java b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidReadTimeoutsTest.java
new file mode 100644
index 0000000..227f7f1
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidReadTimeoutsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.topology;
+
+public class DecommissionAvoidReadTimeoutsTest extends DecommissionAvoidTimeouts
+{
+ @Override
+ protected String getQuery(String table)
+ {
+ return "SELECT * FROM " + table + " WHERE pk=?";
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java
new file mode 100644
index 0000000..05206de
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.topology;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.IInstanceInitializer;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.distributed.util.Coordinators;
+import org.apache.cassandra.distributed.util.QueryResultUtil;
+import org.apache.cassandra.distributed.util.byterewrite.StatusChangeListener;
+import org.apache.cassandra.distributed.util.byterewrite.StatusChangeListener.Hooks;
+import org.apache.cassandra.distributed.util.byterewrite.Undead;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.locator.DynamicEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.ReplicaCollection;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.AssertionUtils;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+
+public abstract class DecommissionAvoidTimeouts extends TestBaseImpl
+{
+ public static final int DECOM_NODE = 6;
+
+ @Test
+ public void test() throws IOException
+ {
+ try (Cluster cluster = Cluster.build(8)
+ .withRacks(2, 4)
+ .withInstanceInitializer(new BB())
+ .withConfig(c -> c.with(Feature.GOSSIP)
+ .set("transfer_hints_on_decommission", false)
+ .set("severity_during_decommission", 10000D)
+ .set("dynamic_snitch_badness_threshold", 0))
+ .start())
+ {
+ // failure happens in PendingRangeCalculatorService.update, so the keyspace is being removed
+ cluster.setUncaughtExceptionsFilter((ignore, throwable) -> !"Unknown keyspace system_distributed".equals(throwable.getMessage()));
+
+ fixDistributedSchemas(cluster);
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': 3, 'datacenter2': 3}");
+ String table = KEYSPACE + ".tbl";
+ cluster.schemaChange("CREATE TABLE " + table + " (pk blob PRIMARY KEY)");
+
+ List<IInvokableInstance> dc1 = cluster.get(1, 2, 3, 4);
+ List<IInvokableInstance> dc2 = cluster.get(5, 6, 7, 8);
+ IInvokableInstance toDecom = dc2.get(1);
+ List<Murmur3Partitioner.LongToken> tokens = ClusterUtils.getLocalTokens(toDecom).stream().map(t -> new Murmur3Partitioner.LongToken(Long.parseLong(t))).collect(Collectors.toList());
+
+ for (Murmur3Partitioner.LongToken token : tokens)
+ {
+ ByteBuffer key = Murmur3Partitioner.LongToken.keyForToken(token);
+
+ toDecom.coordinator().execute("INSERT INTO " + table + "(pk) VALUES (?)", ConsistencyLevel.EACH_QUORUM, key);
+ }
+
+ CompletableFuture<Void> nodetool = CompletableFuture.runAsync(() -> toDecom.nodetoolResult("decommission").asserts().success());
+
+ Hooks statusHooks = StatusChangeListener.hooks(DECOM_NODE);
+ statusHooks.leaving.awaitAndEnter();
+ // make sure all nodes see the severity change
+ ClusterUtils.awaitGossipStateMatch(cluster, cluster.get(DECOM_NODE), ApplicationState.SEVERITY);
+ cluster.forEach(i -> i.runOnInstance(() -> ((DynamicEndpointSnitch) DatabaseDescriptor.getEndpointSnitch()).updateScores()));
+
+ statusHooks.leave.await();
+ cluster.filters().verbs(Verb.GOSSIP_DIGEST_SYN.id).drop();
+ statusHooks.leave.enter();
+
+ nodetool.join();
+
+ List<String> failures = new ArrayList<>();
+ String query = getQuery(table);
+ for (Murmur3Partitioner.LongToken token : tokens)
+ {
+ ByteBuffer key = Murmur3Partitioner.LongToken.keyForToken(token);
+
+ for (IInvokableInstance i : dc1)
+ {
+ for (ConsistencyLevel cl : levels())
+ {
+ try
+ {
+ Coordinators.withTracing(i.coordinator(), query, cl, key);
+ }
+ catch (Coordinators.WithTraceException e)
+ {
+ Throwable cause = e.getCause();
+ if (AssertionUtils.isInstanceof(WriteTimeoutException.class).matches(cause) || AssertionUtils.isInstanceof(ReadTimeoutException.class).matches(cause))
+ {
+ List<String> traceMesssages = Arrays.asList("Sending mutation to remote replica",
+ "reading data from",
+ "reading digest from");
+ SimpleQueryResult filtered = QueryResultUtil.query(e.trace)
+ .select("activity")
+ .filter(row -> traceMesssages.stream().anyMatch(row.getString("activity")::startsWith))
+ .build();
+ InetAddressAndPort decomeNode = BB.address((byte) DECOM_NODE);
+ while (filtered.hasNext())
+ {
+ String log = filtered.next().getString("activity");
+ if (log.contains(decomeNode.toString()))
+ failures.add("Failure with node" + i.config().num() + ", cl=" + cl + ";\n\t" + cause.getMessage() + ";\n\tTrace activity=" + log);
+ }
+ }
+ else
+ {
+ throw e;
+ }
+ }
+ }
+ }
+ }
+ if (!failures.isEmpty()) throw new AssertionError(String.join("\n", failures));
+
+ // since only one tests exists per file, shutdown without blocking so .close does not timeout
+ try
+ {
+ FBUtilities.waitOnFutures(cluster.stream().map(IInstance::shutdown).collect(Collectors.toList()),
+ 1, TimeUnit.MINUTES);
+ }
+ catch (Exception e)
+ {
+ // ignore
+ }
+ }
+ }
+
+ protected abstract String getQuery(String table);
+
+ private static Collection<ConsistencyLevel> levels()
+ {
+ return EnumSet.of(ConsistencyLevel.QUORUM, ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM,
+ ConsistencyLevel.ONE);
+ }
+
+ public static class BB implements IInstanceInitializer, AutoCloseable
+ {
+ @Override
+ public void initialise(ClassLoader cl, ThreadGroup group, int node, int generation)
+ {
+ Undead.install(cl);
+ new ByteBuddy().rebase(DynamicEndpointSnitch.class)
+ .method(named("sortedByProximity")).intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+
+ if (node != DECOM_NODE) return;
+ StatusChangeListener.install(cl, node, StatusChangeListener.Status.LEAVING, StatusChangeListener.Status.LEAVE);
+ }
+
+ @Override
+ public void close() throws Exception
+ {
+ Undead.close();
+ StatusChangeListener.close();
+ }
+
+ public static <C extends ReplicaCollection<? extends C>> C sortedByProximity(final InetAddressAndPort address, C replicas, @SuperCall Callable<C> real) throws Exception
+ {
+ C result = real.call();
+ if (result.size() > 1)
+ {
+ InetAddressAndPort decom = address((byte) DECOM_NODE);
+ if (result.endpoints().contains(decom))
+ {
+ if (DynamicEndpointSnitch.getSeverity(decom) != 0)
+ {
+ Replica last = result.get(result.size() - 1);
+ if (!last.endpoint().equals(decom))
+ throw new AssertionError("Expected endpoint " + decom + " to be the last replica, but found " + last.endpoint() + "; " + result);
+ }
+ }
+ }
+ return result;
+ }
+
+ private static InetAddressAndPort address(byte num)
+ {
+ try
+ {
+ return InetAddressAndPort.getByAddressOverrideDefaults(InetAddress.getByAddress(new byte[]{ 127, 0, 0, num }), 7012);
+ }
+ catch (UnknownHostException e)
+ {
+ throw new AssertionError(e);
+ }
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidWriteTimeoutsTest.java b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidWriteTimeoutsTest.java
new file mode 100644
index 0000000..cc2ebc2
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidWriteTimeoutsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.topology;
+
+public class DecommissionAvoidWriteTimeoutsTest extends DecommissionAvoidTimeouts
+{
+ @Override
+ protected String getQuery(String table)
+ {
+ return "INSERT INTO " + table + "(pk) VALUES (?)";
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/BatchUpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/BatchUpgradeTest.java
index db5e2e1..7ba12ff 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/BatchUpgradeTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/BatchUpgradeTest.java
@@ -33,7 +33,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(2)
- .upgradesFrom(v40).setup((cluster) -> {
+ .upgradesToCurrentFrom(v40).setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".users (" +
"userid uuid PRIMARY KEY," +
"firstname ascii," +
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageColumnDeleteTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageColumnDeleteTest.java
index 720a1b5..2a2ece6 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageColumnDeleteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageColumnDeleteTest.java
@@ -33,7 +33,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(2)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v40)
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH COMPACT STORAGE");
})
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageHiddenColumnTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageHiddenColumnTest.java
index 4e5236c..93f770c 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageHiddenColumnTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageHiddenColumnTest.java
@@ -33,7 +33,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(2)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v40)
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, PRIMARY KEY (pk, ck)) WITH COMPACT STORAGE");
})
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageImplicitNullInClusteringTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageImplicitNullInClusteringTest.java
index 9d4824a..52b792e 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageImplicitNullInClusteringTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageImplicitNullInClusteringTest.java
@@ -33,7 +33,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(2)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v40)
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck1 int, ck2 int, v int, PRIMARY KEY (pk, ck1, ck2)) WITH COMPACT STORAGE");
})
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingTest.java
index 307d6dd..1643190 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingTest.java
@@ -33,7 +33,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(2)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v40)
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH COMPACT STORAGE");
for (int i = 1; i < 10; i++)
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java
index 6dd3bd9..691ad4e 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java
@@ -61,7 +61,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(1)
- .singleUpgrade(initialVersion())
+ .singleUpgradeToCurrentFrom(initialVersion())
.withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
.setup(c -> {
c.schemaChange(withKeyspace("CREATE TABLE %s.t (pk text, ck text, v text, " +
@@ -88,7 +88,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(1)
- .singleUpgrade(initialVersion())
+ .singleUpgradeToCurrentFrom(initialVersion())
.withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
.setup(c -> {
c.schemaChange(withKeyspace("CREATE TABLE %s.t (pk text, ck1 text, ck2 text, v text, " +
@@ -114,7 +114,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(1)
- .singleUpgrade(initialVersion())
+ .singleUpgradeToCurrentFrom(initialVersion())
.withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
.setup(c -> {
c.schemaChange(withKeyspace("CREATE TABLE %s.t (pk text PRIMARY KEY, v1 text, v2 text) WITH COMPACT STORAGE"));
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java
deleted file mode 100644
index 518a514..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import com.vdurmont.semver4j.Semver;
-
-/**
- * {@link CompactStoragePagingWithProtocolTester} for v30 -> CURRENT upgrade path.
- */
-public class CompactStoragePagingWithProtocolV30Test extends CompactStoragePagingWithProtocolTester
-{
- @Override
- protected Semver initialVersion()
- {
- return v30;
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java
deleted file mode 100644
index 003c372..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import com.vdurmont.semver4j.Semver;
-
-/**
- * {@link CompactStoragePagingWithProtocolTester} for v3X -> CURRENT upgrade path.
- */
-public class CompactStoragePagingWithProtocolV3XTest extends CompactStoragePagingWithProtocolTester
-{
- @Override
- protected Semver initialVersion()
- {
- return v3X;
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV41Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV41Test.java
new file mode 100644
index 0000000..9dd7dc4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV41Test.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.upgrade;
+
+import com.vdurmont.semver4j.Semver;
+
+/**
+ * {@link CompactStoragePagingWithProtocolTester} for v41 -> CURRENT upgrade path.
+ */
+public class CompactStoragePagingWithProtocolV41Test extends CompactStoragePagingWithProtocolTester
+{
+ @Override
+ protected Semver initialVersion()
+ {
+ return v41;
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactionHistorySystemTableUpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactionHistorySystemTableUpgradeTest.java
new file mode 100644
index 0000000..73e92dd
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactionHistorySystemTableUpgradeTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.upgrade;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import com.vdurmont.semver4j.Semver;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.tools.ToolRunner;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.ArrayList;
+
+import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
+import static org.apache.cassandra.tools.ToolRunner.invokeNodetoolJvmDtest;
+import static org.apache.cassandra.tools.nodetool.CompactionHistoryTest.assertCompactionHistoryOutPut;
+
+@RunWith(Parameterized.class)
+public class CompactionHistorySystemTableUpgradeTest extends UpgradeTestBase
+{
+ @Parameter
+ public Semver version;
+
+ @Parameters()
+ public static ArrayList<Semver> versions()
+ {
+ return Lists.newArrayList(v30, v3X, v40, v41);
+ }
+
+ @Test
+ public void compactionHistorySystemTableTest() throws Throwable
+ {
+ new TestCase()
+ .nodes(1)
+ .nodesToUpgrade(1)
+ .upgradesToCurrentFrom(version)
+ .setup((cluster) -> {
+ //create table
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tb (" +
+ "pk text PRIMARY KEY," +
+ "c1 text," +
+ "c2 int," +
+ "c3 int)");
+ // disable auto compaction
+ cluster.stream().forEach(node -> node.nodetool("disableautocompaction"));
+ // generate sstables
+ for (int i = 0; i != 10; ++i)
+ {
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tb (pk, c1, c2, c3) VALUES ('pk" + i + "', 'c1" + i + "', " + i + ',' + i + ')', ConsistencyLevel.ALL);
+ cluster.stream().forEach(node -> node.flush(KEYSPACE));
+ }
+ // force compact
+ cluster.stream().forEach(node -> node.forceCompact(KEYSPACE, "tb"));
+ }).runAfterClusterUpgrade((cluster) -> {
+ // disable auto compaction at start up
+ cluster.stream().forEach(node -> node.nodetool("disableautocompaction"));
+ ToolRunner.ToolResult toolHistory = invokeNodetoolJvmDtest(cluster.get(1), "compactionhistory");
+ toolHistory.assertOnCleanExit();
+ // upgraded system.compaction_history data verify
+ assertCompactionHistoryOutPut(toolHistory, KEYSPACE, "tb", ImmutableMap.of());
+
+ // force compact
+ cluster.stream().forEach(node -> node.nodetool("compact"));
+ toolHistory = invokeNodetoolJvmDtest(cluster.get(1), "compactionhistory");
+ toolHistory.assertOnCleanExit();
+ assertCompactionHistoryOutPut(toolHistory, KEYSPACE, "tb", ImmutableMap.of(COMPACTION_TYPE_PROPERTY, OperationType.MAJOR_COMPACTION.type));
+ })
+ .run();
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageNullClusteringValuesTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageNullClusteringValuesTest.java
index 1657765..2e5578d 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageNullClusteringValuesTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageNullClusteringValuesTest.java
@@ -33,7 +33,7 @@
public void testNullClusteringValues() throws Throwable
{
new TestCase().nodes(1)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v30)
.withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL).set("enable_drop_compact_storage", true))
.setup(cluster -> {
String create = "CREATE TABLE %s.%s(k int, c1 int, c2 int, v int, PRIMARY KEY (k, c1, c2)) " +
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageTest.java
index c645085..9846264 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageTest.java
@@ -35,7 +35,7 @@
new TestCase()
.nodes(2)
.nodesToUpgrade(1, 2)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v30)
.withConfig(config -> config.with(GOSSIP, NETWORK).set("enable_drop_compact_storage", true))
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, PRIMARY KEY (pk, ck)) WITH COMPACT STORAGE");
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/GroupByTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/GroupByTest.java
index b09b882..b2971cd 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/GroupByTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/GroupByTest.java
@@ -35,7 +35,7 @@
// CASSANDRA-16582: group-by across mixed version cluster would fail with ArrayIndexOutOfBoundException
new UpgradeTestBase.TestCase()
.nodes(2)
- .upgradesFrom(v3X)
+ .upgradesToCurrentFrom(v3X)
.nodesToUpgrade(1)
.withConfig(config -> config.with(GOSSIP, NETWORK))
.setup(cluster -> {
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java
index 4e50eb1..1ca23ae 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java
@@ -74,6 +74,14 @@
testAvailability(true, initial, writeConsistencyLevel, readConsistencyLevel);
}
+ protected static void testAvailability(Semver initial,
+ ConsistencyLevel writeConsistencyLevel,
+ ConsistencyLevel readConsistencyLevel) throws Throwable
+ {
+ testAvailability(true, initial, writeConsistencyLevel, readConsistencyLevel);
+ testAvailability(false, initial, writeConsistencyLevel, readConsistencyLevel);
+ }
+
private static void testAvailability(boolean upgradedCoordinator,
Semver initial,
ConsistencyLevel writeConsistencyLevel,
@@ -82,7 +90,7 @@
new TestCase()
.nodes(NUM_NODES)
.nodesToUpgrade(upgradedCoordinator ? 1 : 2)
- .upgrades(initial, UpgradeTestBase.CURRENT)
+ .upgradesToCurrentFrom(initial)
.withConfig(config -> config.set("read_request_timeout_in_ms", SECONDS.toMillis(5))
.set("write_request_timeout_in_ms", SECONDS.toMillis(5)))
// use retry of 10ms so that each check is consistent
@@ -93,6 +101,7 @@
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k uuid, c int, v int, PRIMARY KEY (k, c)) WITH speculative_retry = '10ms'"));
cluster.setUncaughtExceptionsFilter(throwable -> throwable instanceof RejectedExecutionException);
})
+ .runBeforeClusterUpgrade(cluster -> cluster.filters().reset())
.runAfterNodeUpgrade((cluster, n) -> {
ICoordinator coordinator = cluster.coordinator(COORDINATOR);
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java
index 8ea94ea..59554d1 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java
@@ -25,7 +25,7 @@
*/
public class MixedModeAvailabilityV3XOneAllTest extends MixedModeAvailabilityTestBase
{
- public MixedModeAvailabilityV3XOneAllTest() throws Throwable
+ public MixedModeAvailabilityV3XOneAllTest()
{
super(v3X, ConsistencyLevel.ONE, ConsistencyLevel.ALL);
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeConsistencyTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeConsistencyTestBase.java
index f98fc8a..519625e 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeConsistencyTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeConsistencyTestBase.java
@@ -42,11 +42,6 @@
{
protected static void testConsistency(Semver initial) throws Throwable
{
- testConsistency(initial, UpgradeTestBase.CURRENT);
- }
-
- protected static void testConsistency(Semver initial, Semver upgrade) throws Throwable
- {
List<Tester> testers = new ArrayList<>();
testers.addAll(Tester.create(1, ALL));
testers.addAll(Tester.create(2, ALL, QUORUM));
@@ -55,7 +50,7 @@
new TestCase()
.nodes(3)
.nodesToUpgrade(1)
- .upgrades(initial, upgrade)
+ .upgradesToCurrentFrom(initial)
.withConfig(config -> config.set("read_request_timeout_in_ms", SECONDS.toMillis(30))
.set("write_request_timeout_in_ms", SECONDS.toMillis(30)))
.setup(cluster -> {
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeFrom3ReplicationTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeFrom3ReplicationTest.java
index 4902385..69d3dbe 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeFrom3ReplicationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeFrom3ReplicationTest.java
@@ -18,19 +18,65 @@
package org.apache.cassandra.distributed.upgrade;
+import java.util.ArrayList;
+import java.util.List;
+
import org.junit.Test;
-public class MixedModeFrom3ReplicationTest extends MixedModeReplicationTestBase
-{
- @Test
- public void testSimpleStrategy30to3X() throws Throwable
- {
- testSimpleStrategy(v30, v3X);
- }
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public class MixedModeFrom3ReplicationTest extends UpgradeTestBase
+{
@Test
public void testSimpleStrategy() throws Throwable
{
- testSimpleStrategy(v30);
+ String insert = "INSERT INTO test_simple.names (key, name) VALUES (?, ?)";
+ String select = "SELECT * FROM test_simple.names WHERE key = ?";
+
+ new TestCase()
+ .nodes(3)
+ .nodesToUpgrade(1, 2)
+ .upgradesToCurrentFrom(v30)
+ .setup(cluster -> {
+ cluster.schemaChange("CREATE KEYSPACE test_simple WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};");
+ cluster.schemaChange("CREATE TABLE test_simple.names (key int PRIMARY KEY, name text)");
+ })
+ .runAfterNodeUpgrade((cluster, upgraded) -> {
+ List<Long> initialTokens = new ArrayList<>(cluster.size() + 1);
+ initialTokens.add(null); // The first valid token is at 1 to avoid offset math below.
+
+ for (int i = 1; i <= cluster.size(); i++)
+ initialTokens.add(Long.valueOf(cluster.get(i).config().get("initial_token").toString()));
+
+ List<Long> validTokens = initialTokens.subList(1, cluster.size() + 1);
+
+ // Exercise all the coordinators...
+ for (int i = 1; i <= cluster.size(); i++)
+ {
+ // ...and sample enough keys that we cover the ring.
+ for (int j = 0; j < 10; j++)
+ {
+ int key = j + (i * 10);
+ Object[] row = row(key, "Nero");
+ Long token = tokenFrom(key);
+
+ cluster.coordinator(i).execute(insert, ConsistencyLevel.ALL, row);
+
+ int node = primaryReplica(validTokens, token);
+ assertRows(cluster.get(node).executeInternal(select, key), row);
+
+ node = nextNode(node, cluster.size());
+ assertRows(cluster.get(node).executeInternal(select, key), row);
+
+ // At RF=2, this node should not have received the write.
+ node = nextNode(node, cluster.size());
+ assertRows(cluster.get(node).executeInternal(select, key));
+ }
+ }
+ })
+ .run();
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeGossipTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeGossipTest.java
deleted file mode 100644
index f4c9695..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeGossipTest.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiConsumer;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.UpgradeableCluster;
-import org.apache.cassandra.distributed.api.Feature;
-import org.apache.cassandra.distributed.api.IMessageFilters;
-import org.apache.cassandra.net.Verb;
-import org.assertj.core.api.Assertions;
-
-public class MixedModeGossipTest extends UpgradeTestBase
-{
- Pattern expectedNormalStatus = Pattern.compile("STATUS:\\d+:NORMAL,-?\\d+");
- Pattern expectedNormalStatusWithPort = Pattern.compile("STATUS_WITH_PORT:\\d+:NORMAL,-?\\d+");
- Pattern expectedNormalX3 = Pattern.compile("X3:\\d+:NORMAL,-?\\d+");
-
- @Test
- public void testStatusFieldShouldExistInOldVersionNodes() throws Throwable
- {
- new TestCase()
- .withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK))
- .nodes(3)
- .nodesToUpgradeOrdered(1, 2, 3)
- // all upgrades from v30 up, excluding v30->v3X and from v40
- .singleUpgrade(v30)
- .singleUpgrade(v3X)
- .setup(c -> {})
- .runAfterNodeUpgrade((cluster, node) -> {
- if (node == 1) {
- checkPeerGossipInfoShouldContainNormalStatus(cluster, 2);
- checkPeerGossipInfoShouldContainNormalStatus(cluster, 3);
- }
- if (node == 2) {
- checkPeerGossipInfoShouldContainNormalStatus(cluster, 3);
- }
- })
- .runAfterClusterUpgrade(cluster -> {
- // wait 1 minute for `org.apache.cassandra.gms.Gossiper.upgradeFromVersionSupplier` to update
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MINUTES);
- checkPeerGossipInfoOfAllNodesShouldContainNewStatusAfterUpgrade(cluster);
- })
- .run();
- }
-
- /**
- * Similar to {@link #testStatusFieldShouldExistInOldVersionNodes}, but in an edge case that
- * 1) node2 and node3 cannot gossip with each other.
- * 2) node2 sends SYN to node1 first when upgrading.
- * 3) node3 is at the lower version during the cluster upgrade
- * In this case, node3 gossip info does not contain STATUS field for node2
- */
- @Test
- public void testStatusFieldShouldExistInOldVersionNodesEdgeCase() throws Throwable
- {
- AtomicReference<IMessageFilters.Filter> n1GossipSynBlocker = new AtomicReference<>();
- new TestCase()
- .withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK))
- .nodes(3)
- .nodesToUpgradeOrdered(1, 2, 3)
- // all upgrades from v30 up, excluding v30->v3X and from v40
- .singleUpgrade(v30)
- .singleUpgrade(v3X)
- .setup(cluster -> {
- // node2 and node3 gossiper cannot talk with each other
- cluster.filters().verbs(Verb.GOSSIP_DIGEST_SYN.id).from(2).to(3).drop();
- cluster.filters().verbs(Verb.GOSSIP_DIGEST_SYN.id).from(3).to(2).drop();
- })
- .runAfterNodeUpgrade((cluster, node) -> {
- // let node2 sends the SYN to node1 first
- if (node == 1)
- {
- IMessageFilters.Filter filter = cluster.filters().verbs(Verb.GOSSIP_DIGEST_SYN.id).from(1).to(2).drop();
- n1GossipSynBlocker.set(filter);
- }
- else if (node == 2)
- {
- n1GossipSynBlocker.get().off();
- String node3GossipView = cluster.get(3).nodetoolResult("gossipinfo").getStdout();
- String node2GossipState = getGossipStateOfNode(node3GossipView, "/127.0.0.2");
- Assertions.assertThat(node2GossipState)
- .as("The node2's gossip state from node3's perspective should contain status. " +
- "And it should carry an unrecognized field X3 with NORMAL.")
- .containsPattern(expectedNormalStatus)
- .containsPattern(expectedNormalX3);
- }
- })
- .runAfterClusterUpgrade(cluster -> {
- // wait 1 minute for `org.apache.cassandra.gms.Gossiper.upgradeFromVersionSupplier` to update
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MINUTES);
- checkPeerGossipInfoOfAllNodesShouldContainNewStatusAfterUpgrade(cluster);
- })
- .run();
- }
-
- private void checkPeerGossipInfoOfAllNodesShouldContainNewStatusAfterUpgrade(UpgradeableCluster cluster)
- {
- for (int i = 1; i <= 3; i++)
- {
- int n = i;
- checkPeerGossipInfo(cluster, i, (gossipInfo, peers) -> {
- for (String p : peers)
- {
- Assertions.assertThat(getGossipStateOfNode(gossipInfo, p))
- .as(String.format("%s gossip state in node%s should not contain STATUS " +
- "and should contain STATUS_WITH_PORT.", p, n))
- .doesNotContain("STATUS:")
- .containsPattern(expectedNormalStatusWithPort);
- }
- });
- }
- }
-
- private void checkPeerGossipInfoShouldContainNormalStatus(UpgradeableCluster cluster, int node)
- {
- checkPeerGossipInfo(cluster, node, (gossipInfo, peers) -> {
- for (String n : peers)
- {
- Assertions.assertThat(getGossipStateOfNode(gossipInfo, n))
- .containsPattern(expectedNormalStatus);
- }
- });
- }
-
- private void checkPeerGossipInfo(UpgradeableCluster cluster, int node, BiConsumer<String, Set<String>> verifier)
- {
- Set<Integer> peers = new HashSet<>(Arrays.asList(1, 2, 3));
- peers.remove(node);
- String gossipInfo = cluster.get(node).nodetoolResult("gossipinfo").getStdout();
- verifier.accept(gossipInfo, peers.stream().map(i -> "127.0.0." + i).collect(Collectors.toSet()));
- }
-
- private String getGossipStateOfNode(String rawOutput, String nodeInterested)
- {
- String temp = rawOutput.substring(rawOutput.indexOf(nodeInterested));
- int nextSlashIndex = temp.indexOf('/', 1);
- if (nextSlashIndex != -1)
- return temp.substring(0, nextSlashIndex);
- else
- return temp;
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeMessageForwardTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeMessageForwardTest.java
index d1551fb..55092ab 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeMessageForwardTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeMessageForwardTest.java
@@ -103,7 +103,7 @@
.withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK))
.withBuilder(b -> b.withRacks(numDCs, 1, nodesPerDc))
.nodes(numDCs * nodesPerDc)
- .singleUpgrade(v30)
+ .upgradesToCurrentFrom(v40)
.setup(cluster -> {
cluster.schemaChange("ALTER KEYSPACE " + KEYSPACE +
" WITH replication = {'class': 'NetworkTopologyStrategy', " + ntsArgs + " };");
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairDeleteTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairDeleteTest.java
index 01955c5..e603778 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairDeleteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairDeleteTest.java
@@ -46,7 +46,8 @@
allUpgrades(2, 1)
.setup(cluster -> {
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, s int static, PRIMARY KEY (k, c))"));
-
+ })
+ .runBeforeClusterUpgrade(cluster -> {
// insert the rows in all the nodes
String insert = withKeyspace("INSERT INTO %s.t (k, c, v, s) VALUES (?, ?, ?, ?)");
cluster.coordinator(1).execute(insert, ConsistencyLevel.ALL, row1);
@@ -85,7 +86,8 @@
allUpgrades(2, 1)
.setup(cluster -> {
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, s int static, PRIMARY KEY (k, c))"));
-
+ })
+ .runBeforeClusterUpgrade(cluster -> {
// insert half partition in each node
String insert = withKeyspace("INSERT INTO %s.t (k, c, v, s) VALUES (?, ?, ?, ?)");
cluster.coordinator(1).execute(insert, ConsistencyLevel.ALL, partition1[0]);
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairWriteTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairWriteTest.java
index fcb0482..4966e5c 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairWriteTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadRepairWriteTest.java
@@ -45,6 +45,7 @@
allUpgrades(2, 1)
.setup(c -> c.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, PRIMARY KEY (k, c))")))
+ .runBeforeClusterUpgrade(cluster -> cluster.coordinator(1).execute(withKeyspace("TRUNCATE %s.t"), ConsistencyLevel.ALL))
.runAfterClusterUpgrade(cluster -> {
// insert rows internally in each node
@@ -77,7 +78,8 @@
allUpgrades(2, 1)
.setup(cluster -> {
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, PRIMARY KEY (k, c))"));
-
+ })
+ .runBeforeClusterUpgrade(cluster -> {
// insert the initial version of the rows in all the nodes
String insert = withKeyspace("INSERT INTO %s.t (k, c, v) VALUES (?, ?, ?)");
cluster.coordinator(1).execute(insert, ConsistencyLevel.ALL, row1);
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadTest.java
deleted file mode 100644
index cbbcff0..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReadTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.api.Feature;
-import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.utils.CassandraVersion;
-
-import static org.apache.cassandra.distributed.test.ReadDigestConsistencyTest.CREATE_TABLE;
-import static org.apache.cassandra.distributed.test.ReadDigestConsistencyTest.insertData;
-import static org.apache.cassandra.distributed.test.ReadDigestConsistencyTest.testDigestConsistency;
-
-public class MixedModeReadTest extends UpgradeTestBase
-{
- @Test
- public void mixedModeReadColumnSubsetDigestCheck() throws Throwable
- {
- new TestCase()
- .withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK))
- .nodes(2)
- .nodesToUpgrade(1)
- // all upgrades from v30 up, excluding v30->v3X and from v40
- .singleUpgrade(v30)
- .singleUpgrade(v3X)
- .setup(cluster -> {
- cluster.schemaChange(CREATE_TABLE);
- insertData(cluster.coordinator(1));
- testDigestConsistency(cluster.coordinator(1));
- testDigestConsistency(cluster.coordinator(2));
- })
- .runAfterClusterUpgrade(cluster -> {
- // we need to let gossip settle or the test will fail
- int attempts = 1;
- //noinspection Convert2MethodRef
- while (!((IInvokableInstance) cluster.get(1)).callOnInstance(() -> Gossiper.instance.isUpgradingFromVersionLowerThan(CassandraVersion.CASSANDRA_4_0) &&
- !Gossiper.instance.isUpgradingFromVersionLowerThan(new CassandraVersion(("3.0")).familyLowerBound.get())))
- {
- if (attempts++ > 90)
- throw new RuntimeException("Gossiper.instance.haveMajorVersion3Nodes() continually returns false despite expecting to be true");
- Thread.sleep(1000);
- }
-
- // should not cause a disgest mismatch in mixed mode
- testDigestConsistency(cluster.coordinator(1));
- testDigestConsistency(cluster.coordinator(2));
- })
- .run();
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRepairTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRepairTest.java
index 813d9f2..6e22be5 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeRepairTest.java
@@ -54,7 +54,7 @@
new UpgradeTestBase.TestCase()
.nodes(2)
.nodesToUpgrade(UPGRADED_NODE)
- .singleUpgrade(v3X)
+ .upgradesToCurrentFrom(v40)
.withConfig(config -> config.with(NETWORK, GOSSIP))
.setup(cluster -> {
cluster.schemaChange(CREATE_TABLE);
@@ -78,39 +78,51 @@
cluster.get(2).flush(KEYSPACE);
// in case of repairing the upgraded node the repair should be rejected with a decriptive error in both
- // nodetool output and logs (see CASSANDRA-13944)
- if (repairedNode == UPGRADED_NODE)
+ // nodetool output and logs (see CASSANDRA-13944), if MessagingService.currentVersion has changed
+ if (cluster.get(1).getMessagingVersion() != cluster.get(2).getMessagingVersion())
{
- String errorMessage = "Repair is not supported in mixed major version clusters";
- cluster.get(repairedNode)
- .nodetoolResult("repair", "--full", KEYSPACE)
- .asserts()
- .errorContains(errorMessage);
- assertLogHas(cluster, repairedNode, errorMessage);
+ if (repairedNode == UPGRADED_NODE)
+ {
+ // Repair is only not supported when MessagingService.current_version don't match
+ // but we keep the error message simple and user-facing
+ String errorMessage = "Repair is not supported in mixed major version clusters";
+ cluster.get(repairedNode)
+ .nodetoolResult("repair", "--full", KEYSPACE)
+ .asserts()
+ .errorContains(errorMessage);
+ assertLogHas(cluster, repairedNode, errorMessage);
+ }
+ // if the node issuing the repair is the not updated node we don't have specific error management,
+ // so the repair will produce a failure in the upgraded node, and it will take one hour to time out in
+ // the not upgraded node. Since we don't want to wait that long, we only wait a few seconds for the
+ // repair before verifying the "unknown verb id" error in the upgraded node.
+ else
+ {
+ try
+ {
+ IUpgradeableInstance instance = cluster.get(repairedNode);
+ CompletableFuture.supplyAsync(() -> instance.nodetoolResult("repair", "--full", KEYSPACE))
+ .get(10, TimeUnit.SECONDS);
+ fail("Repair in the not upgraded node should have timed out");
+ }
+ catch (TimeoutException e)
+ {
+ assertLogHas(cluster, UPGRADED_NODE, "unexpected exception caught while processing inbound messages");
+ assertLogHas(cluster, UPGRADED_NODE, "java.lang.IllegalArgumentException: Unknown verb id");
+ }
+ }
+
+ // verify that the previous failed repair hasn't repaired the data
+ assertRows(cluster.get(1).executeInternal(SELECT, key), row1);
+ assertRows(cluster.get(2).executeInternal(SELECT, key), row2);
}
- // if the node issuing the repair is the not updated node we don't have specific error management,
- // so the repair will produce a failure in the upgraded node, and it will take one hour to time out in
- // the not upgraded node. Since we don't want to wait that long, we only wait a few seconds for the
- // repair before verifying the "unknown verb id" error in the upgraded node.
else
{
- try
- {
- IUpgradeableInstance instance = cluster.get(repairedNode);
- CompletableFuture.supplyAsync(() -> instance.nodetoolResult("repair", "--full", KEYSPACE))
- .get(10, TimeUnit.SECONDS);
- fail("Repair in the not upgraded node should have timed out");
- }
- catch (TimeoutException e)
- {
- assertLogHas(cluster, UPGRADED_NODE, "unexpected exception caught while processing inbound messages");
- assertLogHas(cluster, UPGRADED_NODE, "java.lang.IllegalArgumentException: Unknown verb id");
- }
+ cluster.get(repairedNode).nodetoolResult("repair", "--full", KEYSPACE);
+ // verify that the repair repaired the data
+ assertRows(cluster.get(1).executeInternal(SELECT, key), row1, row2);
+ assertRows(cluster.get(2).executeInternal(SELECT, key), row1, row2);
}
-
- // verify that the previous failed repair hasn't repaired the data
- assertRows(cluster.get(1).executeInternal(SELECT, key), row1);
- assertRows(cluster.get(2).executeInternal(SELECT, key), row2);
}
})
.run();
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReplicationTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReplicationTestBase.java
deleted file mode 100644
index 3f2da7a..0000000
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeReplicationTestBase.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.upgrade;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.vdurmont.semver4j.Semver;
-
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
-
-import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
-import static org.apache.cassandra.distributed.shared.AssertUtils.row;
-
-/**
- * A base class for testing basic replication on mixed-version clusters.
- */
-public class MixedModeReplicationTestBase extends UpgradeTestBase
-{
- protected void testSimpleStrategy(Semver from) throws Throwable
- {
- testSimpleStrategy(from, UpgradeTestBase.CURRENT);
- }
-
- protected void testSimpleStrategy(Semver from, Semver to) throws Throwable
- {
- String insert = "INSERT INTO test_simple.names (key, name) VALUES (?, ?)";
- String select = "SELECT * FROM test_simple.names WHERE key = ?";
-
- new TestCase()
- .nodes(3)
- .nodesToUpgrade(1, 2)
- .upgrades(from, to)
- .setup(cluster -> {
- cluster.schemaChange("CREATE KEYSPACE test_simple WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};");
- cluster.schemaChange("CREATE TABLE test_simple.names (key int PRIMARY KEY, name text)");
- })
- .runAfterNodeUpgrade((cluster, upgraded) -> {
- List<Long> initialTokens = new ArrayList<>(cluster.size() + 1);
- initialTokens.add(null); // The first valid token is at 1 to avoid offset math below.
-
- for (int i = 1; i <= cluster.size(); i++)
- initialTokens.add(Long.valueOf(cluster.get(i).config().get("initial_token").toString()));
-
- List<Long> validTokens = initialTokens.subList(1, cluster.size() + 1);
-
- // Exercise all the coordinators...
- for (int i = 1; i <= cluster.size(); i++)
- {
- // ...and sample enough keys that we cover the ring.
- for (int j = 0; j < 10; j++)
- {
- int key = j + (i * 10);
- Object[] row = row(key, "Nero");
- Long token = tokenFrom(key);
-
- cluster.coordinator(i).execute(insert, ConsistencyLevel.ALL, row);
-
- int node = primaryReplica(validTokens, token);
- assertRows(cluster.get(node).executeInternal(select, key), row);
-
- node = nextNode(node, cluster.size());
- assertRows(cluster.get(node).executeInternal(select, key), row);
-
- // At RF=2, this node should not have received the write.
- node = nextNode(node, cluster.size());
- assertRows(cluster.get(node).executeInternal(select, key));
- }
- }
- })
- .run();
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeWritetimeOrTTLTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeWritetimeOrTTLTest.java
new file mode 100644
index 0000000..295ffd2
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeWritetimeOrTTLTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.upgrade;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+/**
+ * Tests the CQL functions {@code writetime}, {@code maxwritetime} and {@code ttl} on rolling upgrade.
+ *
+ * {@code writetime} and {@code ttl} on single-cell columns is always supported, even in mixed clusters.
+ * {@code writetime} and {@code ttl} on multi-cell columns is not supported in coordinator nodes < 4.2.
+ * {@code maxwritetime} is not supported in coordinator nodes < 4.2.
+ */
+public class MixedModeWritetimeOrTTLTest extends UpgradeTestBase
+{
+ @Test
+ public void testWritetimeOrTTLDuringUpgrade() throws Throwable
+ {
+ new TestCase()
+ .nodes(2)
+ .nodesToUpgradeOrdered(1, 2)
+ .upgradesToCurrentFrom(v30)
+ .setup(cluster -> {
+
+ ICoordinator coordinator = cluster.coordinator(1);
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int PRIMARY KEY, v int, s set<int>, fs frozen<set<int>>)"));
+ coordinator.execute(withKeyspace("INSERT INTO %s.t (k, v, s, fs) VALUES (0, 0, {0, 1}, {0, 1, 2, 3}) USING TIMESTAMP 1 AND TTL 1000"), ALL);
+ coordinator.execute(withKeyspace("UPDATE %s.t USING TIMESTAMP 2 AND TTL 2000 SET v = 1, s = s + {2, 3} WHERE k = 0"), ALL);
+
+ assertPre42Behaviour(cluster.coordinator(1));
+ assertPre42Behaviour(cluster.coordinator(2));
+ })
+ .runAfterNodeUpgrade((cluster, node) -> {
+ if (node == 1) // only node1 is upgraded, and the cluster is in mixed mode
+ {
+ assertPost42Behaviour(cluster.coordinator(1));
+ assertPre42Behaviour(cluster.coordinator(2));
+ }
+ else // both nodes have been upgraded, and the cluster isn't in mixed mode anymore
+ {
+ assertPost42Behaviour(cluster.coordinator(1));
+ assertPost42Behaviour(cluster.coordinator(2));
+ }
+ })
+ .run();
+ }
+
+ private void assertPre42Behaviour(ICoordinator coordinator)
+ {
+ // regular column, supported except for maxwritetime
+ assertRows(coordinator.execute(withKeyspace("SELECT writetime(v) FROM %s.t"), ALL), row(2L));
+ Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("SELECT maxwritetime(v) FROM %s.t"), ALL))
+ .hasMessageContaining("Unknown function 'maxwritetime'");
+ Assertions.assertThat((Integer) coordinator.execute(withKeyspace("SELECT ttl(v) FROM %s.t"), ALL)[0][0])
+ .isLessThanOrEqualTo(2000).isGreaterThan(2000 - 300); // margin of error of 5 minutes since TTLs decrease
+
+ // frozen collection, supported except for maxwritetime
+ assertRows(coordinator.execute(withKeyspace("SELECT writetime(fs) FROM %s.t"), ALL), row(1L));
+ Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("SELECT maxwritetime(fs) FROM %s.t"), ALL))
+ .hasMessageContaining("Unknown function 'maxwritetime'");
+ Assertions.assertThat((Integer) coordinator.execute(withKeyspace("SELECT ttl(fs) FROM %s.t"), ALL)[0][0])
+ .isLessThanOrEqualTo(1000).isGreaterThan(1000 - 300); // margin of error of 5 minutes since TTLs decrease
+
+ // not-frozen collection, not supported
+ Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("SELECT writetime(s) FROM %s.t"), ALL))
+ .hasMessageContaining("Cannot use selection function writeTime on non-frozen collection s");
+ Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("SELECT maxwritetime(s) FROM %s.t"), ALL))
+ .hasMessageContaining("Unknown function 'maxwritetime'");
+ Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("SELECT ttl(s) FROM %s.t"), ALL))
+ .hasMessageContaining("Cannot use selection function ttl on non-frozen collection s");
+ }
+
+ private void assertPost42Behaviour(ICoordinator coordinator)
+ {
+ // regular column, fully supported
+ assertRows(coordinator.execute(withKeyspace("SELECT writetime(v) FROM %s.t"), ALL), row(2L));
+ assertRows(coordinator.execute(withKeyspace("SELECT maxwritetime(v) FROM %s.t"), ALL), row(2L));
+ Assertions.assertThat((Integer) coordinator.execute(withKeyspace("SELECT ttl(v) FROM %s.t"), ALL)[0][0])
+ .isLessThanOrEqualTo(2000).isGreaterThan(2000 - 300); // margin of error of 5 minutes since TTLs decrease
+
+ // frozen collection, fully supported
+ assertRows(coordinator.execute(withKeyspace("SELECT writetime(fs) FROM %s.t"), ALL), row(1L));
+ assertRows(coordinator.execute(withKeyspace("SELECT maxwritetime(fs) FROM %s.t"), ALL), row(1L));
+ Assertions.assertThat((Integer) coordinator.execute(withKeyspace("SELECT ttl(fs) FROM %s.t"), ALL)[0][0])
+ .isLessThanOrEqualTo(1000).isGreaterThan(1000 - 300); // margin of error of 5 minutes since TTLs decrease
+
+ // not-frozen collection, fully supported
+ assertRows(coordinator.execute(withKeyspace("SELECT writetime(s) FROM %s.t"), ALL), row(Arrays.asList(1L, 1L, 2L, 2L)));
+ assertRows(coordinator.execute(withKeyspace("SELECT maxwritetime(s) FROM %s.t"), ALL), row(2L));
+ Assertions.assertThat(coordinator.execute(withKeyspace("SELECT ttl(s) FROM %s.t"), ALL)[0][0])
+ .matches(l -> l instanceof List && ((List<?>) l).size() == 4);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/Pre40MessageFilterTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/Pre40MessageFilterTest.java
index 4cca7b9..9030fdc 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/Pre40MessageFilterTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/Pre40MessageFilterTest.java
@@ -34,9 +34,7 @@
.nodes(2)
.withConfig(configConsumer)
.nodesToUpgrade(1)
- // all upgrades from v30 up, excluding v30->v3X
- .singleUpgrade(v30)
- .upgradesFrom(v3X)
+ .upgradesToCurrentFrom(v40)
.setup((cluster) -> {
cluster.filters().outbound().allVerbs().messagesMatching((f,t,m) -> false).drop();
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java
index b1692b9..55e1f1e 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTest.java
@@ -22,6 +22,7 @@
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
+
import static org.apache.cassandra.distributed.shared.AssertUtils.*;
public class UpgradeTest extends UpgradeTestBase
@@ -33,7 +34,7 @@
.nodes(2)
.nodesToUpgrade(1)
.withConfig((cfg) -> cfg.with(Feature.NETWORK, Feature.GOSSIP))
- .upgradesFrom(v3X)
+ .upgradesToCurrentFrom(v3X)
.setup((cluster) -> {
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)", ConsistencyLevel.ALL);
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java
index 5c32fcd..e44ecc7 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java
@@ -19,17 +19,22 @@
package org.apache.cassandra.distributed.upgrade;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.NavigableSet;
+import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
-import com.google.common.collect.ImmutableList;
import com.vdurmont.semver4j.Semver;
import com.vdurmont.semver4j.Semver.SemverType;
import org.junit.After;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@@ -44,10 +49,11 @@
import org.apache.cassandra.distributed.shared.ThrowingRunnable;
import org.apache.cassandra.distributed.shared.Versions;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.SimpleGraph;
import static org.apache.cassandra.distributed.shared.Versions.Version;
import static org.apache.cassandra.distributed.shared.Versions.find;
+import static org.apache.cassandra.utils.SimpleGraph.sortedVertices;
public class UpgradeTestBase extends DistributedTestBase
{
@@ -56,8 +62,7 @@
@After
public void afterEach()
{
- System.runFinalization();
- System.gc();
+ triggerGC();
}
@BeforeClass
@@ -86,27 +91,58 @@
public static final Semver v3X = new Semver("3.11.0", SemverType.LOOSE);
public static final Semver v40 = new Semver("4.0-alpha1", SemverType.LOOSE);
public static final Semver v41 = new Semver("4.1-alpha1", SemverType.LOOSE);
+ public static final Semver v50 = new Semver("5.0-alpha1", SemverType.LOOSE);
- protected static final List<Pair<Semver,Semver>> SUPPORTED_UPGRADE_PATHS = ImmutableList.of(
- Pair.create(v30, v3X),
- Pair.create(v30, v40),
- Pair.create(v30, v41),
- Pair.create(v3X, v40),
- Pair.create(v3X, v41),
- Pair.create(v40, v41));
+ protected static final SimpleGraph<Semver> SUPPORTED_UPGRADE_PATHS = new SimpleGraph.Builder<Semver>()
+ .addEdge(v30, v3X)
+ .addEdge(v30, v40)
+ .addEdge(v30, v41)
+ .addEdge(v3X, v40)
+ .addEdge(v3X, v41)
+ .addEdge(v40, v41)
+ .addEdge(v40, v50)
+ .addEdge(v41, v50)
+ .build();
// the last is always the current
- public static final Semver CURRENT = SUPPORTED_UPGRADE_PATHS.get(SUPPORTED_UPGRADE_PATHS.size() - 1).right;
+ public static final Semver CURRENT = SimpleGraph.max(SUPPORTED_UPGRADE_PATHS);
+ public static final Semver OLDEST = SimpleGraph.min(SUPPORTED_UPGRADE_PATHS);
public static class TestVersions
{
final Version initial;
- final Version upgrade;
+ final List<Version> upgrade;
+ final List<Semver> upgradeVersions;
- public TestVersions(Version initial, Version upgrade)
+ public TestVersions(Version initial, List<Version> upgrade)
{
this.initial = initial;
this.upgrade = upgrade;
+ this.upgradeVersions = upgrade.stream().map(v -> v.version).collect(Collectors.toList());
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ TestVersions that = (TestVersions) o;
+ return Objects.equals(initial.version, that.initial.version) && Objects.equals(upgradeVersions, that.upgradeVersions);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(initial.version, upgradeVersions);
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append(initial.version).append(" -> ");
+ sb.append(upgradeVersions);
+ return sb.toString();
}
}
@@ -118,6 +154,7 @@
private RunOnCluster setup;
private RunOnClusterAndNode runBeforeNodeRestart;
private RunOnClusterAndNode runAfterNodeUpgrade;
+ private RunOnCluster runBeforeClusterUpgrade;
private RunOnCluster runAfterClusterUpgrade;
private final Set<Integer> nodesToUpgrade = new LinkedHashSet<>();
private Consumer<IInstanceConfig> configConsumer;
@@ -139,29 +176,80 @@
return this;
}
- /** performs all supported upgrade paths that exist in between from and CURRENT (inclusive) **/
- public TestCase upgradesFrom(Semver from)
+ /** performs all supported upgrade paths that exist in between from and end on CURRENT (inclusive)
+ * {@code upgradesToCurrentFrom(3.0); // produces: 3.0 -> CURRENT, 3.11 -> CURRENT, …}
+ **/
+ public TestCase upgradesToCurrentFrom(Semver from)
{
- return upgrades(from, CURRENT);
+ return upgradesTo(from, CURRENT);
}
- /** performs all supported upgrade paths that exist in between from and to (inclusive) **/
- public TestCase upgrades(Semver from, Semver to)
+ /**
+ * performs all supported upgrade paths to the "to" target; example
+ * {@code upgradesTo(3.0, 4.0); // produces: 3.0 -> 4.0, 3.11 -> 4.0}
+ */
+ public TestCase upgradesTo(Semver from, Semver to)
{
- SUPPORTED_UPGRADE_PATHS.stream()
- .filter(upgradePath -> (upgradePath.left.compareTo(from) >= 0 && upgradePath.right.compareTo(to) <= 0))
- .forEachOrdered(upgradePath ->
- {
- this.upgrade.add(
- new TestVersions(versions.getLatest(upgradePath.left), versions.getLatest(upgradePath.right)));
- });
+ List<TestVersions> upgrade = new ArrayList<>();
+ NavigableSet<Semver> vertices = sortedVertices(SUPPORTED_UPGRADE_PATHS);
+ for (Semver start : vertices.subSet(from, true, to, false))
+ {
+ // only include pairs that are allowed, and start or end on CURRENT
+ if (SUPPORTED_UPGRADE_PATHS.hasEdge(start, to) && contains(start, to, CURRENT))
+ upgrade.add(new TestVersions(versions.getLatest(start), Collections.singletonList(versions.getLatest(to))));
+ }
+ logger.info("Adding upgrades of\n{}", upgrade.stream().map(TestVersions::toString).collect(Collectors.joining("\n")));
+ this.upgrade.addAll(upgrade);
return this;
}
- /** Will test this specific upgrade path **/
- public TestCase singleUpgrade(Semver from)
+ /**
+ * performs all supported upgrade paths from the "from" target; example
+ * {@code upgradesFrom(4.0, 4.2); // produces: 4.0 -> 4.1, 4.0 -> 4.2}
+ */
+ public TestCase upgradesFrom(Semver from, Semver to)
{
- this.upgrade.add(new TestVersions(versions.getLatest(from), versions.getLatest(CURRENT)));
+ List<TestVersions> upgrade = new ArrayList<>();
+ NavigableSet<Semver> vertices = sortedVertices(SUPPORTED_UPGRADE_PATHS);
+ for (Semver end : vertices.subSet(from, false, to, true))
+ {
+ // only include pairs that are allowed, and start or end on CURRENT
+ if (SUPPORTED_UPGRADE_PATHS.hasEdge(from, end) && contains(from, end, CURRENT))
+ upgrade.add(new TestVersions(versions.getLatest(from), Collections.singletonList(versions.getLatest(end))));
+ }
+ logger.info("Adding upgrades of\n{}", upgrade.stream().map(TestVersions::toString).collect(Collectors.joining("\n")));
+ this.upgrade.addAll(upgrade);
+ return this;
+ }
+
+ /**
+ * performs all supported upgrade paths that exist in between from and to that include the current version.
+ * This call is equivalent to calling {@code upgradesTo(from, CURRENT).upgradesFrom(CURRENT, to)}.
+ **/
+ public TestCase upgrades(Semver from, Semver to)
+ {
+ Assume.assumeTrue("Unable to do upgrades(" + from + ", " + to + "); does not contain CURRENT=" + CURRENT, contains(from, to, CURRENT));
+ if (from.compareTo(CURRENT) < 0)
+ upgradesTo(from, CURRENT);
+ if (CURRENT.compareTo(to) < 0)
+ upgradesFrom(CURRENT, to);
+ return this;
+ }
+
+ private static boolean contains(Semver from, Semver to, Semver target)
+ {
+ // target >= from && target <= to
+ return target.compareTo(from) >= 0 && target.compareTo(to) <= 0;
+ }
+
+ /** Will test this specific upgrade path **/
+ public TestCase singleUpgradeToCurrentFrom(Semver from)
+ {
+ if (!SUPPORTED_UPGRADE_PATHS.hasEdge(from, CURRENT))
+ throw new AssertionError("Upgrading from " + from + " to " + CURRENT + " isn't directly supported and must go through other versions first; supported paths: " + SUPPORTED_UPGRADE_PATHS.findPaths(from, CURRENT));
+ TestVersions tests = new TestVersions(this.versions.getLatest(from), Arrays.asList(this.versions.getLatest(CURRENT)));
+ logger.info("Adding upgrade of {}", tests);
+ this.upgrade.add(tests);
return this;
}
@@ -183,6 +271,12 @@
return this;
}
+ public TestCase runBeforeClusterUpgrade(RunOnCluster runBeforeClusterUpgrade)
+ {
+ this.runBeforeClusterUpgrade = runBeforeClusterUpgrade;
+ return this;
+ }
+
public TestCase runAfterClusterUpgrade(RunOnCluster runAfterClusterUpgrade)
{
this.runAfterClusterUpgrade = runAfterClusterUpgrade;
@@ -211,6 +305,8 @@
throw new AssertionError();
if (runBeforeNodeRestart == null)
runBeforeNodeRestart = (c, n) -> {};
+ if (runBeforeClusterUpgrade == null)
+ runBeforeClusterUpgrade = (c) -> {};
if (runAfterClusterUpgrade == null)
runAfterClusterUpgrade = (c) -> {};
if (runAfterNodeUpgrade == null)
@@ -219,26 +315,44 @@
for (int n = 1; n <= nodeCount; n++)
nodesToUpgrade.add(n);
+ int offset = 0;
for (TestVersions upgrade : this.upgrade)
{
- logger.info("testing upgrade from {} to {}", upgrade.initial.version, upgrade.upgrade.version);
+ logger.info("testing upgrade from {} to {}", upgrade.initial.version, upgrade.upgradeVersions);
try (UpgradeableCluster cluster = init(UpgradeableCluster.create(nodeCount, upgrade.initial, configConsumer, builderConsumer)))
{
setup.run(cluster);
- for (int n : nodesToUpgrade)
+ for (Version nextVersion : upgrade.upgrade)
{
- cluster.get(n).shutdown().get();
- cluster.get(n).setVersion(upgrade.upgrade);
- runBeforeNodeRestart.run(cluster, n);
- cluster.get(n).startup();
- runAfterNodeUpgrade.run(cluster, n);
- }
+ try
+ {
+ runBeforeClusterUpgrade.run(cluster);
- runAfterClusterUpgrade.run(cluster);
+ for (int n : nodesToUpgrade)
+ {
+ cluster.get(n).shutdown().get();
+ triggerGC();
+ cluster.get(n).setVersion(nextVersion);
+ runBeforeNodeRestart.run(cluster, n);
+ cluster.get(n).startup();
+ runAfterNodeUpgrade.run(cluster, n);
+ }
+
+ runAfterClusterUpgrade.run(cluster);
+
+ cluster.checkAndResetUncaughtExceptions();
+ }
+ catch (Throwable t)
+ {
+ throw new AssertionError(String.format("Error in test '%s' while upgrading to '%s'; successful upgrades %s", upgrade, nextVersion.version, this.upgrade.stream().limit(offset).collect(Collectors.toList())), t);
+ }
+ }
}
+ offset++;
}
}
+
public TestCase nodesToUpgrade(int ... nodes)
{
Set<Integer> set = new HashSet<>(nodes.length);
@@ -260,10 +374,16 @@
}
}
+ private static void triggerGC()
+ {
+ System.runFinalization();
+ System.gc();
+ }
+
protected TestCase allUpgrades(int nodes, int... toUpgrade)
{
return new TestCase().nodes(nodes)
- .upgradesFrom(v30)
+ .upgradesToCurrentFrom(v30)
.nodesToUpgrade(toUpgrade);
}
diff --git a/test/distributed/org/apache/cassandra/distributed/util/Coordinators.java b/test/distributed/org/apache/cassandra/distributed/util/Coordinators.java
new file mode 100644
index 0000000..fd26e60
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/Coordinators.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util;
+
+import java.util.UUID;
+
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.utils.TimeUUID;
+
+public class Coordinators
+{
+ public static class WithTrace
+ {
+ public final SimpleQueryResult result, trace;
+
+ public WithTrace(SimpleQueryResult result, SimpleQueryResult trace)
+ {
+ this.result = result;
+ this.trace = trace;
+ }
+ }
+
+ public static WithTrace withTracing(ICoordinator coordinator, String query, ConsistencyLevel consistencyLevel, Object... boundValues)
+ throws WithTraceException
+ {
+ UUID session = TimeUUID.Generator.nextTimeAsUUID();
+ try
+ {
+ SimpleQueryResult result = coordinator.executeWithTracingWithResult(session, query, consistencyLevel, boundValues);
+ return new WithTrace(result, getTrace(coordinator, session));
+ }
+ catch (Throwable t)
+ {
+ throw new WithTraceException(t, getTrace(coordinator, session));
+ }
+ }
+
+ public static SimpleQueryResult getTrace(ICoordinator coordinator, UUID session)
+ {
+ return coordinator.executeWithResult("SELECT * FROM system_traces.events WHERE session_id=?", ConsistencyLevel.LOCAL_QUORUM, session);
+ }
+
+ public static class WithTraceException extends RuntimeException
+ {
+ public final SimpleQueryResult trace;
+
+ public WithTraceException(Throwable cause, SimpleQueryResult trace)
+ {
+ super(cause);
+ this.trace = trace;
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/util/QueryResultUtil.java b/test/distributed/org/apache/cassandra/distributed/util/QueryResultUtil.java
index a502e8c..23d88e6 100644
--- a/test/distributed/org/apache/cassandra/distributed/util/QueryResultUtil.java
+++ b/test/distributed/org/apache/cassandra/distributed/util/QueryResultUtil.java
@@ -17,11 +17,14 @@
*/
package org.apache.cassandra.distributed.util;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.Objects;
import java.util.function.Predicate;
import com.google.monitoring.runtime.instrumentation.common.collect.Iterators;
+import org.apache.cassandra.distributed.api.QueryResults;
import org.apache.cassandra.distributed.api.Row;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
@@ -33,6 +36,38 @@
{
}
+ public static void orderBy(SimpleQueryResult qr, String... columns)
+ {
+ if (columns == null || columns.length == 0)
+ throw new IllegalArgumentException("No columns defined");
+ int[] index = new int[columns.length];
+ {
+ int offset = 0;
+ for (String name : columns)
+ {
+ int idx = qr.names().indexOf(name);
+ if (idx == -1)
+ throw new IllegalArgumentException("Unknown column " + name);
+ index[offset++] = idx;
+ }
+ }
+ qr.reset();
+ Arrays.sort(qr.toObjectArrays(), (a, b) -> {
+ for (int i = 0; i < index.length; i++)
+ {
+ int idx = index[i];
+ Object ao = a[idx];
+ Object bo = b[idx];
+ if (ao == null && bo == null) return 0;
+ if (ao == null) return 1;
+ if (bo == null) return -1;
+ int rc = ((Comparable) ao).compareTo(bo);
+ if (rc != 0) return rc;
+ }
+ return 0;
+ });
+ }
+
public static boolean contains(SimpleQueryResult qr, Object... values)
{
return contains(qr, a -> equals(a, values));
@@ -100,6 +135,11 @@
return sb.toString();
}
+ public static QueryBuilder query(SimpleQueryResult result)
+ {
+ return new QueryBuilder(result);
+ }
+
public static class RowAssertHelper
{
private final Row row;
@@ -184,4 +224,73 @@
Assertions.assertThat(size).describedAs("QueryResult is not empty").isEqualTo(0);
}
}
+
+ public static class QueryBuilder
+ {
+ private final SimpleQueryResult input;
+ private final List<String> names = new ArrayList<>();
+ private Predicate<Row> filter = ignore -> true;
+
+ public QueryBuilder(SimpleQueryResult input)
+ {
+ this.input = input;
+ }
+
+ public QueryBuilder select(String... names)
+ {
+ for (String name : names)
+ {
+ if (!input.names().contains(name))
+ throw new IllegalArgumentException("Unknown column " + name);
+ }
+ this.names.clear();
+ // if names is empty, then this becomes "SELECT *"
+ this.names.addAll(Arrays.asList(names));
+ return this;
+ }
+
+ public QueryBuilder filter(Predicate<Row> fn)
+ {
+ this.filter = fn;
+ return this;
+ }
+
+ public SimpleQueryResult build()
+ {
+ QueryResults.Builder builder = QueryResults.builder();
+ if (names.isEmpty())
+ {
+ builder.columns(input.names().toArray(new String[0]));
+ while (input.hasNext())
+ {
+ Row row = input.next();
+ if (filter.test(row))
+ builder.row(row.toObjectArray());
+ }
+ }
+ else
+ {
+ String[] names = this.names.toArray(new String[0]);
+ builder.columns(names);
+ int[] index = new int[names.length];
+ {
+ int offset = 0;
+ for (String name : names)
+ index[offset++] = input.names().indexOf(name);
+ }
+ Row row = new Row(names);
+ while (input.hasNext())
+ {
+ Object[] raw = input.next().toObjectArray();
+ Object[] updated = new Object[index.length];
+ for (int i = 0; i < index.length; i++)
+ updated[i] = raw[index[i]];
+ row.setResults(updated);
+ if (filter.test(row))
+ builder.row(updated);
+ }
+ }
+ return builder.build();
+ }
+ }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/util/SingleHostLoadBalancingPolicy.java b/test/distributed/org/apache/cassandra/distributed/util/SingleHostLoadBalancingPolicy.java
new file mode 100644
index 0000000..198baae
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/SingleHostLoadBalancingPolicy.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util;
+
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import com.google.common.collect.Iterators;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.HostDistance;
+import com.datastax.driver.core.Statement;
+import com.datastax.driver.core.policies.LoadBalancingPolicy;
+
+/**
+ * A CQL driver {@link LoadBalancingPolicy} that only connects to a fixed host.
+ */
+public class SingleHostLoadBalancingPolicy implements LoadBalancingPolicy
+{
+ private final InetAddress address;
+ private Host host;
+
+ public SingleHostLoadBalancingPolicy(InetAddress address)
+ {
+ this.address = address;
+ }
+
+ protected final List<Host> hosts = new CopyOnWriteArrayList<>();
+
+ @Override
+ public void init(Cluster cluster, Collection<Host> hosts)
+ {
+ host = hosts.stream()
+ .filter(h -> h.getBroadcastAddress().equals(address)).findFirst()
+ .orElseThrow(() -> new AssertionError("The host should be a contact point"));
+ this.hosts.add(host);
+ }
+
+ @Override
+ public HostDistance distance(Host host)
+ {
+ return HostDistance.LOCAL;
+ }
+
+ @Override
+ public Iterator<Host> newQueryPlan(String loggedKeyspace, Statement statement)
+ {
+ return Iterators.singletonIterator(host);
+ }
+
+ @Override
+ public void onAdd(Host host)
+ {
+ // no-op
+ }
+
+ @Override
+ public void onUp(Host host)
+ {
+ // no-op
+ }
+
+ @Override
+ public void onDown(Host host)
+ {
+ // no-op
+ }
+
+ @Override
+ public void onRemove(Host host)
+ {
+ // no-op
+ }
+
+ @Override
+ public void close()
+ {
+ // no-op
+ }
+}
\ No newline at end of file
diff --git a/test/distributed/org/apache/cassandra/distributed/util/TwoWay.java b/test/distributed/org/apache/cassandra/distributed/util/TwoWay.java
new file mode 100644
index 0000000..c577cc9
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/TwoWay.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+
+import org.apache.cassandra.utils.Shared;
+import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+
+@Shared
+public class TwoWay implements AutoCloseable
+{
+ private volatile CountDownLatch first = new CountDownLatch(1);
+ private final CountDownLatch second = new CountDownLatch(1);
+
+ public void awaitAndEnter()
+ {
+ await();
+ enter();
+ }
+
+ public void await()
+ {
+ await(first);
+ first = null;
+ }
+
+ public void enter()
+ {
+ if (first != null)
+ {
+ first.countDown();
+ await(second);
+ }
+ else
+ {
+ second.countDown();
+ }
+ }
+
+ private void await(CountDownLatch latch)
+ {
+ try
+ {
+ if (!latch.await(1, TimeUnit.MINUTES))
+ throw new UncheckedTimeoutException("Timeout waiting on " + (latch == second ? "second" : "first") + " latch");
+ }
+ catch (InterruptedException e)
+ {
+ throw new UncheckedInterruptedException(e);
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ if (first != null)
+ first.countDown();
+ first = null;
+ second.countDown();
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/util/byterewrite/StatusChangeListener.java b/test/distributed/org/apache/cassandra/distributed/util/byterewrite/StatusChangeListener.java
new file mode 100644
index 0000000..b6b8776
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/byterewrite/StatusChangeListener.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util.byterewrite;
+
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.DynamicType;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.distributed.impl.InstanceIDDefiner;
+import org.apache.cassandra.distributed.util.TwoWay;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.Shared;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+
+public class StatusChangeListener
+{
+ public enum Status
+ {
+ LEAVING("startLeaving"),
+ LEAVE("leaveRing");
+
+ private final String method;
+
+ Status(String method)
+ {
+ this.method = method;
+ }
+ }
+
+ public static void install(ClassLoader cl, int node, Status first, Status... rest)
+ {
+ install(cl, node, EnumSet.of(first, rest));
+ }
+
+ public static void install(ClassLoader cl, int node, Set<Status> statuses)
+ {
+ if (statuses.isEmpty()) throw new IllegalStateException("Need a set of status to listen to");
+
+ State.hooks.put(node, new Hooks());
+ DynamicType.Builder<StorageService> builder = new ByteBuddy().rebase(StorageService.class);
+ for (Status s : statuses)
+ builder = builder.method(named(s.method)).intercept(MethodDelegation.to(BB.class));
+ builder.make().load(cl, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static void close()
+ {
+ for (Hooks hook : State.hooks.values())
+ hook.close();
+ }
+
+ public static Hooks hooks(int node)
+ {
+ return Objects.requireNonNull(State.hooks.get(node), "Unknown node" + node);
+ }
+
+ @Shared
+ public static class State
+ {
+ public static final Map<Integer, Hooks> hooks = new ConcurrentHashMap<>();
+ }
+
+ @Shared
+ public static class Hooks implements AutoCloseable
+ {
+ public static final TwoWay leaving = new TwoWay();
+ public static final TwoWay leave = new TwoWay();
+
+ @Override
+ public void close()
+ {
+ for (TwoWay condition: Arrays.asList(leaving, leave))
+ condition.close();
+ }
+ }
+
+ public static class BB
+ {
+ private static volatile int NODE = -1;
+
+ public static void startLeaving(@SuperCall Runnable zuper)
+ {
+ // see org.apache.cassandra.service.StorageService.startLeaving
+ hooks().leaving.enter();
+ zuper.run();
+ }
+
+ public static void leaveRing(@SuperCall Runnable zuper)
+ {
+ // see org.apache.cassandra.service.StorageService.leaveRing
+ hooks().leave.enter();
+ zuper.run();
+ }
+
+ private static Hooks hooks()
+ {
+ return State.hooks.get(node());
+ }
+
+ private static int node()
+ {
+ int node = NODE;
+ if (node == -1)
+ node = NODE = Integer.parseInt(InstanceIDDefiner.getInstanceId().replace("node", ""));
+ return node;
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/util/byterewrite/Undead.java b/test/distributed/org/apache/cassandra/distributed/util/byterewrite/Undead.java
new file mode 100644
index 0000000..661d65f
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/util/byterewrite/Undead.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.util.byterewrite;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.utils.Shared;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+
+public class Undead
+{
+ public static void install(ClassLoader cl)
+ {
+ new ByteBuddy().rebase(EndpointState.class)
+ .method(named("markDead")).intercept(MethodDelegation.to(BB.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static void close()
+ {
+ State.enabled = false;
+ }
+
+ @Shared
+ public static class State
+ {
+ public static volatile boolean enabled = true;
+ }
+
+ public static class BB
+ {
+ public static void markDead(@SuperCall Runnable real)
+ {
+ if (State.enabled)
+ {
+ // don't let anything get marked dead...
+ return;
+ }
+ real.run();
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
index 9012dcc..91d41fe 100644
--- a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
+++ b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
@@ -23,16 +23,14 @@
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import com.google.common.util.concurrent.RateLimiter;
-import org.apache.cassandra.cache.InstrumentingCache;
-import org.apache.cassandra.cache.KeyCacheKey;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DataRange;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -43,19 +41,24 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.CheckedFunction;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.EstimatedHistogram;
-import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Ref;
@@ -63,14 +66,25 @@
{
private final SSTableReader delegate;
+ private static class Builder extends SSTableReader.Builder<ForwardingSSTableReader, Builder>
+ {
+ public Builder(SSTableReader delegate)
+ {
+ super(delegate.descriptor);
+ delegate.unbuildTo(this, false);
+ }
+
+ @Override
+ public ForwardingSSTableReader buildInternal(Owner owner)
+ {
+ throw new UnsupportedOperationException();
+ }
+ }
+
public ForwardingSSTableReader(SSTableReader delegate)
{
- super(delegate.descriptor, SSTable.componentsFor(delegate.descriptor),
- TableMetadataRef.forOfflineTools(delegate.metadata()), delegate.maxDataAge, delegate.getSSTableMetadata(),
- delegate.openReason, delegate.header, delegate.indexSummary, delegate.dfile, delegate.ifile, delegate.bf);
+ super(new Builder(delegate), delegate.owner().orElse(null));
this.delegate = delegate;
- this.first = delegate.first;
- this.last = delegate.last;
}
@Override
@@ -92,12 +106,24 @@
}
@Override
+ public boolean mayContainAssumingKeyIsInRange(DecoratedKey key)
+ {
+ return delegate.mayContainAssumingKeyIsInRange(key);
+ }
+
+ @Override
public void setupOnline()
{
delegate.setupOnline();
}
@Override
+ public <R, E extends Exception> R runWithLock(CheckedFunction<Descriptor, R, E> task) throws E
+ {
+ return delegate.runWithLock(task);
+ }
+
+ @Override
public void setReplaced()
{
delegate.setReplaced();
@@ -122,15 +148,9 @@
}
@Override
- public SSTableReader cloneWithNewStart(DecoratedKey newStart, Runnable runOnClose)
+ public SSTableReader cloneWithNewStart(DecoratedKey newStart)
{
- return delegate.cloneWithNewStart(newStart, runOnClose);
- }
-
- @Override
- public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
- {
- return delegate.cloneWithNewSummarySamplingLevel(parent, samplingLevel);
+ return delegate.cloneWithNewStart(newStart);
}
@Override
@@ -140,39 +160,21 @@
}
@Override
- public int getIndexSummarySamplingLevel()
+ public void releaseInMemoryComponents()
{
- return delegate.getIndexSummarySamplingLevel();
+ delegate.releaseInMemoryComponents();
}
@Override
- public long getIndexSummaryOffHeapSize()
+ public void validate()
{
- return delegate.getIndexSummaryOffHeapSize();
+ delegate.validate();
}
@Override
- public int getMinIndexInterval()
+ protected void closeInternalComponent(AutoCloseable closeable)
{
- return delegate.getMinIndexInterval();
- }
-
- @Override
- public double getEffectiveIndexInterval()
- {
- return delegate.getEffectiveIndexInterval();
- }
-
- @Override
- public void releaseSummary()
- {
- delegate.releaseSummary();
- }
-
- @Override
- public long getIndexScanPosition(PartitionPosition key)
- {
- return delegate.getIndexScanPosition(key);
+ delegate.closeInternalComponent(closeable);
}
@Override
@@ -188,24 +190,6 @@
}
@Override
- public IFilter getBloomFilter()
- {
- return delegate.getBloomFilter();
- }
-
- @Override
- public long getBloomFilterSerializedSize()
- {
- return delegate.getBloomFilterSerializedSize();
- }
-
- @Override
- public long getBloomFilterOffHeapSize()
- {
- return delegate.getBloomFilterOffHeapSize();
- }
-
- @Override
public long estimatedKeys()
{
return delegate.estimatedKeys();
@@ -218,24 +202,6 @@
}
@Override
- public int getIndexSummarySize()
- {
- return delegate.getIndexSummarySize();
- }
-
- @Override
- public int getMaxIndexSummarySize()
- {
- return delegate.getMaxIndexSummarySize();
- }
-
- @Override
- public byte[] getIndexSummaryKey(int index)
- {
- return delegate.getIndexSummaryKey(index);
- }
-
- @Override
public Iterable<DecoratedKey> getKeySamples(Range<Token> range)
{
return delegate.getKeySamples(range);
@@ -248,39 +214,15 @@
}
@Override
- public KeyCacheKey getCacheKey(DecoratedKey key)
+ protected long getPosition(PartitionPosition key, Operator op, boolean updateStats, SSTableReadsListener listener)
{
- return delegate.getCacheKey(key);
+ return delegate.getPosition(key, op, updateStats, listener);
}
@Override
- public void cacheKey(DecoratedKey key, RowIndexEntry info)
+ protected AbstractRowIndexEntry getRowIndexEntry(PartitionPosition key, Operator op, boolean updateCacheAndStats, SSTableReadsListener listener)
{
- delegate.cacheKey(key, info);
- }
-
- @Override
- public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
- {
- return delegate.getCachedPosition(key, updateStats);
- }
-
- @Override
- protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
- {
- return delegate.getCachedPosition(unifiedKey, updateStats);
- }
-
- @Override
- public boolean isKeyCacheEnabled()
- {
- return delegate.isKeyCacheEnabled();
- }
-
- @Override
- protected RowIndexEntry getPosition(PartitionPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast, SSTableReadsListener listener)
- {
- return delegate.getPosition(key, op, updateCacheAndStats, permitMatchPastLast, listener);
+ return delegate.getRowIndexEntry(key, op, updateCacheAndStats, listener);
}
@Override
@@ -290,15 +232,21 @@
}
@Override
- public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
+ public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, long dataPosition, boolean tombstoneOnly)
{
- return delegate.rowIterator(file, key, indexEntry, slices, selectedColumns, reversed);
+ return delegate.simpleIterator(file, key, dataPosition, tombstoneOnly);
}
@Override
- public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly)
+ public KeyReader keyReader() throws IOException
{
- return delegate.simpleIterator(file, key, indexEntry, tombstoneOnly);
+ return delegate.keyReader();
+ }
+
+ @Override
+ public KeyIterator keyIterator() throws IOException
+ {
+ return delegate.keyIterator();
}
@Override
@@ -398,9 +346,9 @@
}
@Override
- public boolean newSince(long age)
+ public boolean newSince(long timestampMillis)
{
- return delegate.newSince(age);
+ return delegate.newSince(timestampMillis);
}
@Override
@@ -410,15 +358,21 @@
}
@Override
+ public void createLinks(String snapshotDirectoryPath, RateLimiter rateLimiter)
+ {
+ delegate.createLinks(snapshotDirectoryPath, rateLimiter);
+ }
+
+ @Override
public boolean isRepaired()
{
return delegate.isRepaired();
}
@Override
- public DecoratedKey keyAt(long indexPosition) throws IOException
+ public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException
{
- return delegate.keyAt(indexPosition);
+ return delegate.keyAtPositionFromSecondaryIndex(keyPositionFromSecondaryIndex);
}
@Override
@@ -452,36 +406,6 @@
}
@Override
- public long getBloomFilterFalsePositiveCount()
- {
- return delegate.getBloomFilterFalsePositiveCount();
- }
-
- @Override
- public long getRecentBloomFilterFalsePositiveCount()
- {
- return delegate.getRecentBloomFilterFalsePositiveCount();
- }
-
- @Override
- public long getBloomFilterTruePositiveCount()
- {
- return delegate.getBloomFilterTruePositiveCount();
- }
-
- @Override
- public long getRecentBloomFilterTruePositiveCount()
- {
- return delegate.getRecentBloomFilterTruePositiveCount();
- }
-
- @Override
- public InstrumentingCache<KeyCacheKey, RowIndexEntry> getKeyCache()
- {
- return delegate.getKeyCache();
- }
-
- @Override
public EstimatedHistogram getEstimatedPartitionSize()
{
return delegate.getEstimatedPartitionSize();
@@ -578,6 +502,18 @@
}
@Override
+ public void mutateLevelAndReload(int newLevel) throws IOException
+ {
+ delegate.mutateLevelAndReload(newLevel);
+ }
+
+ @Override
+ public void mutateRepairedAndReload(long newRepairedAt, TimeUUID newPendingRepair, boolean isTransient) throws IOException
+ {
+ delegate.mutateRepairedAndReload(newRepairedAt, newPendingRepair, isTransient);
+ }
+
+ @Override
public void reloadSSTableMetadata() throws IOException
{
delegate.reloadSSTableMetadata();
@@ -602,9 +538,9 @@
}
@Override
- public RandomAccessReader openIndexReader()
+ public void trySkipFileCacheBefore(DecoratedKey key)
{
- return delegate.openIndexReader();
+ delegate.trySkipFileCacheBefore(key);
}
@Override
@@ -614,33 +550,9 @@
}
@Override
- public ChannelProxy getIndexChannel()
+ public long getDataCreationTime()
{
- return delegate.getIndexChannel();
- }
-
- @Override
- public FileHandle getIndexFile()
- {
- return delegate.getIndexFile();
- }
-
- @Override
- public long getCreationTimeFor(Component component)
- {
- return delegate.getCreationTimeFor(component);
- }
-
- @Override
- public long getKeyCacheHit()
- {
- return delegate.getKeyCacheHit();
- }
-
- @Override
- public long getKeyCacheRequest()
- {
- return delegate.getKeyCacheRequest();
+ return delegate.getDataCreationTime();
}
@Override
@@ -674,7 +586,13 @@
}
@Override
- void setup(boolean trackHotness)
+ protected List<AutoCloseable> setupInstance(boolean trackHotness)
+ {
+ return delegate.setupInstance(trackHotness);
+ }
+
+ @Override
+ public void setup(boolean trackHotness)
{
delegate.setup(trackHotness);
}
@@ -710,12 +628,6 @@
}
@Override
- public String getIndexFilename()
- {
- return delegate.getIndexFilename();
- }
-
- @Override
public String getColumnFamilyName()
{
return delegate.getColumnFamilyName();
@@ -740,6 +652,30 @@
}
@Override
+ public long logicalBytesOnDisk()
+ {
+ return delegate.logicalBytesOnDisk();
+ }
+
+ @Override
+ public Set<Component> getComponents()
+ {
+ return delegate.getComponents();
+ }
+
+ @Override
+ public UnfilteredRowIterator rowIterator(DecoratedKey key)
+ {
+ return delegate.rowIterator(key);
+ }
+
+ @Override
+ public void maybePersistSSTableReadMeter()
+ {
+ delegate.maybePersistSSTableReadMeter();
+ }
+
+ @Override
public String toString()
{
return delegate.toString();
@@ -756,4 +692,40 @@
{
return delegate.getBounds();
}
+
+ @Override
+ public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options)
+ {
+ return delegate.getVerifier(cfs, outputHandler, isOffline, options);
+ }
+
+ @Override
+ protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry)
+ {
+ delegate.notifySelected(reason, localListener, op, updateStats, entry);
+ }
+
+ @Override
+ protected void notifySkipped(SSTableReadsListener.SkippingReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats)
+ {
+ delegate.notifySkipped(reason, localListener, op, updateStats);
+ }
+
+ @Override
+ public boolean isEstimationInformative()
+ {
+ return delegate.isEstimationInformative();
+ }
+
+ @Override
+ public DataIntegrityMetadata.ChecksumValidator maybeGetChecksumValidator() throws IOException
+ {
+ return delegate.maybeGetChecksumValidator();
+ }
+
+ @Override
+ public DataIntegrityMetadata.FileDigestValidator maybeGetDigestValidator() throws IOException
+ {
+ return delegate.maybeGetDigestValidator();
+ }
}
diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java
index 1df7e7c..ad18d9f 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java
@@ -29,13 +29,13 @@
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Hex;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_STRICT_LCS_CHECKS;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class LongLeveledCompactionStrategyCQLTest extends CQLTester
@@ -44,7 +44,7 @@
@Test
public void stressTestCompactionStrategyManager() throws ExecutionException, InterruptedException
{
- System.setProperty(Config.PROPERTY_PREFIX + "test.strict_lcs_checks", "true");
+ TEST_STRICT_LCS_CHECKS.setBoolean(true);
// flush/compact tons of sstables, invalidate token metadata in a loop to make CSM reload the strategies
createTable("create table %s (id int primary key, i text) with compaction = {'class':'LeveledCompactionStrategy', 'sstable_size_in_mb':1}");
ExecutorService es = Executors.newSingleThreadExecutor();
diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
index 733e46f..a780cf1 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
@@ -198,9 +198,7 @@
}
return null;
}
- }, true, true);
-
-
+ }, OperationType.COMPACTION, true, true);
}
@Test
diff --git a/test/long/org/apache/cassandra/utils/HeapUtilsTest.java b/test/long/org/apache/cassandra/utils/HeapUtilsTest.java
new file mode 100644
index 0000000..18c91c4
--- /dev/null
+++ b/test/long/org/apache/cassandra/utils/HeapUtilsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.nio.file.Paths;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class HeapUtilsTest
+{
+ @BeforeClass
+ public static void setup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void shouldDumpHeapWithPathArgSpecified()
+ {
+ DatabaseDescriptor.setDumpHeapOnUncaughtException(true);
+ String path = HeapUtils.maybeCreateHeapDump();
+ assertNotNull(path);
+ assertTrue(Paths.get(path).toFile().exists());
+ assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException());
+
+ // After the first dump, subsequent requests should be no-ops...
+ path = HeapUtils.maybeCreateHeapDump();
+ assertNull(path);
+
+ // ...until creation is manually re-enabled.
+ DatabaseDescriptor.setDumpHeapOnUncaughtException(true);
+ assertTrue(DatabaseDescriptor.getDumpHeapOnUncaughtException());
+ assertNotNull(DatabaseDescriptor.getHeapDumpPath());
+ path = HeapUtils.maybeCreateHeapDump();
+ assertNotNull(path);
+ assertTrue(Paths.get(path).toFile().exists());
+ assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException());
+ }
+}
\ No newline at end of file
diff --git a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
index 4d652b7..e2a4ede 100644
--- a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
+++ b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java
@@ -393,7 +393,7 @@
Measurement measurement = createMeasurement();
measurement.start();
for (int i=0; i<numAlloc; i++)
- new Integer(i);
+ Integer.valueOf(i);
measurement.stop();
logger.info(" ** {}", measurement.prettyBytes());
diff --git a/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java b/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java
new file mode 100644
index 0000000..9496172
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/AbstractTypeByteSourceDecodingBench.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+
+import net.nicoulaj.compilecommand.annotations.Inline;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.TypeParser;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 5, time = 2)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx4G", "-Xms4G", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(1)
+@State(Scope.Benchmark)
+public class AbstractTypeByteSourceDecodingBench
+{
+
+ private static final ByteComparable.Version LATEST = ByteComparable.Version.OSS50;
+
+ private static final Map<AbstractType, BiFunction<Random, Integer, ByteSource.Peekable>> PEEKABLE_GENERATOR_BY_TYPE = new HashMap<>();
+ static
+ {
+ PEEKABLE_GENERATOR_BY_TYPE.put(UTF8Type.instance, (prng, length) ->
+ {
+ byte[] randomBytes = new byte[length];
+ prng.nextBytes(randomBytes);
+ return ByteSource.peekable(ByteSource.of(new String(randomBytes, StandardCharsets.UTF_8), LATEST));
+ });
+ PEEKABLE_GENERATOR_BY_TYPE.put(BytesType.instance, (prng, length) ->
+ {
+ byte[] randomBytes = new byte[length];
+ prng.nextBytes(randomBytes);
+ return ByteSource.peekable(ByteSource.of(randomBytes, LATEST));
+ });
+ PEEKABLE_GENERATOR_BY_TYPE.put(IntegerType.instance, (prng, length) ->
+ {
+ BigInteger randomVarint = BigInteger.valueOf(prng.nextLong());
+ for (int i = 1; i < length / 8; ++i)
+ randomVarint = randomVarint.multiply(BigInteger.valueOf(prng.nextLong()));
+ return ByteSource.peekable(IntegerType.instance.asComparableBytes(IntegerType.instance.decompose(randomVarint), LATEST));
+ });
+ PEEKABLE_GENERATOR_BY_TYPE.put(DecimalType.instance, (prng, length) ->
+ {
+ BigInteger randomMantissa = BigInteger.valueOf(prng.nextLong());
+ for (int i = 1; i < length / 8; ++i)
+ randomMantissa = randomMantissa.multiply(BigInteger.valueOf(prng.nextLong()));
+ int randomScale = prng.nextInt(Integer.MAX_VALUE >> 1) + Integer.MAX_VALUE >> 1;
+ BigDecimal randomDecimal = new BigDecimal(randomMantissa, randomScale);
+ return ByteSource.peekable(DecimalType.instance.asComparableBytes(DecimalType.instance.decompose(randomDecimal), LATEST));
+ });
+ }
+
+ private Random prng = new Random();
+
+ @Param({"32", "128", "512"})
+ private int length;
+
+ @Param({"UTF8Type", "BytesType", "IntegerType", "DecimalType"})
+ private String abstractTypeName;
+
+ private AbstractType abstractType;
+ private BiFunction<Random, Integer, ByteSource.Peekable> peekableGenerator;
+
+ @Setup(Level.Trial)
+ public void setup()
+ {
+ abstractType = TypeParser.parse(abstractTypeName);
+ peekableGenerator = PEEKABLE_GENERATOR_BY_TYPE.get(abstractType);
+ }
+
+ @Inline
+ private ByteSource.Peekable randomPeekableBytes()
+ {
+ return peekableGenerator.apply(prng, length);
+ }
+
+ @Benchmark
+ public int baseline()
+ {
+ // Getting the source is not enough as its content is produced on next() calls.
+ ByteSource.Peekable source = randomPeekableBytes();
+ int count = 0;
+ while (source.next() != ByteSource.END_OF_STREAM)
+ ++count;
+ return count;
+ }
+
+ @Benchmark
+ public ByteBuffer fromComparableBytes()
+ {
+ ByteSource.Peekable peekableBytes = randomPeekableBytes();
+ return abstractType.fromComparableBytes(peekableBytes, ByteComparable.Version.OSS50);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
index 32e048d..b03372e 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.test.microbench;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.TimeUnit;
@@ -53,7 +52,6 @@
@State(Scope.Benchmark)
public class BloomFilterSerializerBench
{
-
@Param({"1", "10", "100", "1024"})
private long numElemsInK;
@@ -67,6 +65,8 @@
private ByteBuffer testVal = ByteBuffer.wrap(new byte[] { 0, 1});
+ private static final BloomFilterSerializer serializer = BloomFilterSerializer.forVersion(false);
+
@Benchmark
public void serializationTest() throws IOException
{
@@ -75,16 +75,16 @@
{
BloomFilter filter = (BloomFilter) FilterFactory.getFilter(numElemsInK * 1024, 0.01d);
filter.add(wrap(testVal));
- DataOutputStreamPlus out = new FileOutputStreamPlus(file);
+ FileOutputStreamPlus out = new FileOutputStreamPlus(file);
if (oldBfFormat)
SerializationsTest.serializeOldBfFormat(filter, out);
else
- BloomFilterSerializer.serialize(filter, out);
+ serializer.serialize(filter, out);
out.close();
filter.close();
FileInputStreamPlus in = new FileInputStreamPlus(file);
- BloomFilter filter2 = BloomFilterSerializer.deserialize(in, oldBfFormat);
+ BloomFilter filter2 = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(in);
FileUtils.closeQuietly(in);
filter2.close();
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java
index de788b7..afafad6 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java
@@ -25,15 +25,21 @@
import org.junit.Assert;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.Util;
import org.apache.cassandra.cache.AutoSavingCache;
import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.service.CacheService;
@@ -45,6 +51,7 @@
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
@@ -55,24 +62,40 @@
@SuppressWarnings("unused")
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 2, time = 2, timeUnit = TimeUnit.SECONDS)
+@Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS)
@Fork(value = 1)
@Threads(1)
@State(Scope.Benchmark)
-public class CacheLoaderBench extends CQLTester
+public class CacheLoaderBench
{
- private static final int numSSTables = 1000;
+ private static final int numSSTables = 100;
+ private static final int numKeysPerTable = 10000;
+
private final Random random = new Random();
+ @Param({ "true", "false" })
+ boolean useUUIDGenerationIdentifiers;
+
@Setup(Level.Trial)
public void setup() throws Throwable
{
- CQLTester.prepareServer();
- String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
- String table1 = createTable(keyspace, "CREATE TABLE %s (key text PRIMARY KEY, val text)");
- String table2 = createTable(keyspace, "CREATE TABLE %s (key text PRIMARY KEY, val text)");
+ DatabaseDescriptor.daemonInitialization(() -> {
+ Config config = DatabaseDescriptor.loadConfig();
+ config.key_cache_size = new DataStorageSpec.LongMebibytesBound(256);
+ config.uuid_sstable_identifiers_enabled = useUUIDGenerationIdentifiers;
+ config.dump_heap_on_uncaught_exception = false;
+ return config;
+ });
+ ServerTestUtils.prepareServer();
+ String keyspace = "ks";
+ String table1 = "tab1";
+ String table2 = "tab2";
+
+ QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false", keyspace));
+ QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (key text PRIMARY KEY, val text)", keyspace, table1));
+ QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (key text PRIMARY KEY, val text)", keyspace, table2));
Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
@@ -87,16 +110,18 @@
columnFamilyStores.add(cfs1);
columnFamilyStores.add(cfs2);
- logger.info("Creating {} sstables", numSSTables);
for (ColumnFamilyStore cfs: columnFamilyStores)
{
cfs.truncateBlocking();
for (int i = 0; i < numSSTables ; i++)
{
ColumnMetadata colDef = ColumnMetadata.regularColumn(cfs.metadata(), ByteBufferUtil.bytes("val"), AsciiType.instance);
- RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis() + random.nextInt(), "key");
- rowBuilder.add(colDef, "val1");
- rowBuilder.build().apply();
+ for (int k = 0; k < numKeysPerTable; k++)
+ {
+ RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis() + random.nextInt(), "key" + k);
+ rowBuilder.add(colDef, "val1");
+ rowBuilder.build().apply();
+ }
cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
}
@@ -104,10 +129,15 @@
// preheat key cache
for (SSTableReader sstable : cfs.getLiveSSTables())
- sstable.getPosition(Util.dk("key"), SSTableReader.Operator.EQ);
+ {
+ for (int k = 0; k < numKeysPerTable; k++)
+ {
+ sstable.getPosition(Util.dk("key" + k), SSTableReader.Operator.EQ);
+ }
+ }
}
- AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = CacheService.instance.keyCache;
+ AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = CacheService.instance.keyCache;
// serialize to file
keyCache.submitWrite(keyCache.size()).get();
@@ -116,21 +146,22 @@
@Setup(Level.Invocation)
public void setupKeyCache()
{
- AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = CacheService.instance.keyCache;
+ AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = CacheService.instance.keyCache;
keyCache.clear();
}
@TearDown(Level.Trial)
public void teardown()
{
- CQLTester.cleanup();
CQLTester.tearDownClass();
+ CommitLog.instance.stopUnsafe(true);
+ CQLTester.cleanup();
}
@Benchmark
public void keyCacheLoadTest() throws Throwable
{
- AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = CacheService.instance.keyCache;
+ AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = CacheService.instance.keyCache;
keyCache.loadSavedAsync().get();
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
index c589ca5..b1f8870 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
@@ -29,15 +29,14 @@
import java.util.function.Predicate;
import com.google.common.collect.Iterables;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.Assert;
-
+import org.apache.cassandra.cache.ChunkCache;
import org.apache.cassandra.config.Config.CommitLogSync;
import org.apache.cassandra.config.Config.DiskAccessMode;
-import org.apache.cassandra.cache.ChunkCache;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -201,7 +200,7 @@
{
id.set(0);
compactionTimeNanos = 0;
- ChunkCache.instance.enable(cacheEnabled);
+ DatabaseDescriptor.setFileCacheEnabled(cacheEnabled);
DatabaseDescriptor.setDiskAccessMode(mode);
alterTable("ALTER TABLE %s WITH compaction = { 'class' : '" + compactionClass + "' };");
alterTable("ALTER TABLE %s WITH compression = { 'sstable_compression' : '" + compressorClass + "' };");
@@ -385,4 +384,4 @@
}
return instances;
}
-}
+}
\ No newline at end of file
diff --git a/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java
new file mode 100644
index 0000000..85cffaf
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringBoundary;
+import org.apache.cassandra.db.ClusteringPrefix.Kind;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.jctools.util.Pow2;
+import org.openjdk.jmh.annotations.*;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 10, time = 1)
+@Measurement(iterations = 10, time = 1)
+@Fork(1)
+@State(Scope.Benchmark)
+public class MetadataCollectorBench
+{
+ @Param({ "10" })
+ int clusteringKeyNum;
+
+ @Param({ "10000" })
+ int datasetSize;
+ int datumIndex;
+ Cell<?>[] cells;
+ Clustering<?>[] clusterings;
+ ClusteringBound<?>[] clusteringBounds;
+ ClusteringBoundary<?>[] clusteringBoundaries;
+ MetadataCollector collector;
+
+ @Setup
+ public void setup()
+ {
+ TableMetadata.Builder tableMetadataBuilder = TableMetadata.builder("k", "t")
+ .addPartitionKeyColumn("pk", LongType.instance)
+ .addRegularColumn("rc", LongType.instance);
+ for (int i = 0; i < clusteringKeyNum; i++)
+ tableMetadataBuilder.addClusteringColumn("ck" + i, LongType.instance);
+ TableMetadata tableMetadata = tableMetadataBuilder.build();
+ collector = new MetadataCollector(tableMetadata.comparator);
+
+ ColumnMetadata columnMetadata = tableMetadata.regularColumns().iterator().next();
+ ThreadLocalRandom current = ThreadLocalRandom.current();
+ datasetSize = Pow2.roundToPowerOfTwo(datasetSize);
+ cells = new Cell[datasetSize];
+ for (int i = 0; i < datasetSize; i++)
+ {
+ cells[i] = new BufferCell(columnMetadata, current.nextLong(0, Long.MAX_VALUE), current.nextInt(1, Integer.MAX_VALUE), Cell.NO_DELETION_TIME, null, null);
+ }
+ clusterings = new Clustering[datasetSize];
+ clusteringBounds = new ClusteringBound[datasetSize];
+ clusteringBoundaries = new ClusteringBoundary[datasetSize];
+ ByteBuffer[] cks = new ByteBuffer[clusteringKeyNum];
+ Kind[] clusteringBoundKinds = new Kind[]{ Kind.INCL_START_BOUND, Kind.INCL_END_BOUND, Kind.EXCL_START_BOUND, Kind.EXCL_END_BOUND };
+ Kind[] clusteringBoundaryKinds = new Kind[]{ Kind.INCL_END_EXCL_START_BOUNDARY, Kind.EXCL_END_INCL_START_BOUNDARY };
+ for (int i = 0; i < datasetSize; i++)
+ {
+ for (int j = 0; j < clusteringKeyNum; j++)
+ cks[j] = LongType.instance.decompose(current.nextLong());
+ clusterings[i] = Clustering.make(Arrays.copyOf(cks, cks.length));
+ clusteringBounds[i] = ClusteringBound.create(clusteringBoundKinds[i % clusteringBoundKinds.length], clusterings[i]);
+ clusteringBoundaries[i] = ClusteringBoundary.create(clusteringBoundaryKinds[i % clusteringBoundaryKinds.length], clusterings[i]);
+ }
+
+ System.gc();
+ // shuffle array contents to ensure a more 'natural' layout
+ for (int i = 0; i < datasetSize; i++)
+ {
+ int to = current.nextInt(0, datasetSize);
+ Cell<?> temp = cells[i];
+ cells[i] = cells[to];
+ cells[to] = temp;
+ }
+
+ for (int i = 0; i < datasetSize; i++)
+ {
+ int to = current.nextInt(0, datasetSize);
+ Clustering<?> temp = clusterings[i];
+ clusterings[i] = clusterings[to];
+ clusterings[to] = temp;
+ }
+ }
+
+ @Benchmark
+ public void updateCell()
+ {
+ collector.update(nextCell());
+ }
+
+ @Benchmark
+ public void updateClustering()
+ {
+ collector.updateClusteringValues(nextClustering());
+ }
+
+ @Benchmark
+ public void updateClusteringBound()
+ {
+ collector.updateClusteringValuesByBoundOrBoundary(nextClusteringBound());
+ }
+
+ @Benchmark
+ public void updateClusteringBoundary()
+ {
+ collector.updateClusteringValuesByBoundOrBoundary(nextClusteringBoundary());
+ }
+
+ public Cell<?> nextCell()
+ {
+ return cells[datumIndex++ & (cells.length - 1)];
+ }
+
+ public Clustering<?> nextClustering()
+ {
+ return clusterings[datumIndex++ & (clusterings.length - 1)];
+ }
+
+ public ClusteringBound<?> nextClusteringBound()
+ {
+ return clusteringBounds[datumIndex++ & (clusteringBounds.length - 1)];
+ }
+
+ public ClusteringBoundary<?> nextClusteringBoundary()
+ {
+ return clusteringBoundaries[datumIndex++ & (clusteringBoundaries.length - 1)];
+ }
+}
\ No newline at end of file
diff --git a/test/microbench/org/apache/cassandra/test/microbench/VIntCodingBench.java b/test/microbench/org/apache/cassandra/test/microbench/VIntCodingBench.java
index 9c82236..a60d9f2 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/VIntCodingBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/VIntCodingBench.java
@@ -18,6 +18,12 @@
package org.apache.cassandra.test.microbench;
+import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.vint.VIntCoding;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
@@ -28,23 +34,6 @@
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.utils.vint.VIntCoding;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
-
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java
index b5bb40c..59acfc6 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java
@@ -54,7 +54,6 @@
import org.apache.cassandra.net.AsyncStreamingOutputPlus;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionInfo;
import org.apache.cassandra.streaming.StreamCoordinator;
@@ -63,6 +62,7 @@
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.streaming.StreamSummary;
+import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.messages.StreamMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -134,7 +134,6 @@
CassandraStreamHeader entireSSTableStreamHeader =
CassandraStreamHeader.builder()
- .withSSTableFormat(sstable.descriptor.formatType)
.withSSTableVersion(sstable.descriptor.version)
.withSSTableLevel(0)
.withEstimatedKeys(sstable.estimatedKeys())
@@ -154,7 +153,6 @@
List<Range<Token>> requestedRanges = Arrays.asList(new Range<>(sstable.first.minValue().getToken(), sstable.last.getToken()));
CassandraStreamHeader partialSSTableStreamHeader =
CassandraStreamHeader.builder()
- .withSSTableFormat(sstable.descriptor.formatType)
.withSSTableVersion(sstable.descriptor.version)
.withSSTableLevel(0)
.withEstimatedKeys(sstable.estimatedKeys())
@@ -221,7 +219,7 @@
StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
- streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED));
+ streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, null));
StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer);
session.init(future);
diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java
index 34ec29a..7083832 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java
@@ -50,8 +50,8 @@
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.partitions.AbstractBTreePartition;
import org.apache.cassandra.db.partitions.AtomicBTreePartition;
+import org.apache.cassandra.db.partitions.BTreePartitionData;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
@@ -240,7 +240,7 @@
try (BulkIterator<Row> iter = BulkIterator.of(insertBuffer))
{
Object[] tree = BTree.build(iter, rowCount, UpdateFunction.noOp());
- return PartitionUpdate.unsafeConstruct(metadata, decoratedKey, AbstractBTreePartition.unsafeConstructHolder(partitionColumns, tree, DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS), NO_DELETION_INFO, false);
+ return PartitionUpdate.unsafeConstruct(metadata, decoratedKey, BTreePartitionData.unsafeConstruct(partitionColumns, tree, DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS), NO_DELETION_INFO, false);
}
}
@@ -342,12 +342,7 @@
public ByteBuffer allocate(int size)
{
if (invalidateOn > 0 && --invalidateOn == 0)
- {
- AbstractBTreePartition.Holder holder = update.unsafeGetHolder();
- if (!BTree.isEmpty(holder.tree))
- update.unsafeSetHolder(AbstractBTreePartition.unsafeConstructHolder(
- holder.columns, Arrays.copyOf(holder.tree, holder.tree.length), holder.deletionInfo, holder.staticRow, holder.stats));
- }
+ BTreePartitionData.unsafeInvalidate(update);
return ByteBuffer.allocate(size);
}
};
@@ -397,7 +392,7 @@
ThreadLocalRandom.current().nextLong();
}
invokeBefore.accept(this);
- update.addAllWithSizeDelta(insert[index], cloner, NO_ORDER.getCurrent(), UpdateTransaction.NO_OP);
+ update.addAll(insert[index], cloner, NO_ORDER.getCurrent(), UpdateTransaction.NO_OP);
return true;
}
finally
@@ -405,7 +400,7 @@
if (state.addAndGet(0x100000L) == ((((long)ifGeneration) << 40) | (((long)insert.length) << 20) | insert.length))
{
activeThreads.set(0);
- update.unsafeSetHolder(AbstractBTreePartition.unsafeGetEmptyHolder());
+ update.unsafeSetHolder(BTreePartitionData.unsafeGetEmpty());
// reset the state and rollover the generation
state.set((ifGeneration + 1L) << 40);
}
@@ -566,7 +561,13 @@
private static ColumnMetadata[] columns(AbstractType<?> type, ColumnMetadata.Kind kind, int count, String prefix)
{
return IntStream.range(0, count)
- .mapToObj(i -> new ColumnMetadata("", "", new ColumnIdentifier(prefix + i, true), type, kind != ColumnMetadata.Kind.REGULAR ? i : ColumnMetadata.NO_POSITION, kind))
+ .mapToObj(i -> new ColumnMetadata("",
+ "",
+ new ColumnIdentifier(prefix + i, true),
+ type,
+ kind != ColumnMetadata.Kind.REGULAR ? i : ColumnMetadata.NO_POSITION,
+ kind,
+ null))
.toArray(ColumnMetadata[]::new);
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java
index 4f2fa78..a459316 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java
@@ -70,7 +70,7 @@
setup(2 * dataSize);
data2 = data.clone();
for (int i = 0 ; i < data2.length ; ++i)
- data2[i] = new Integer(data2[i]);
+ data2[i] = Integer.valueOf(data2[i]);
}
@State(Scope.Thread)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java
index f2be859..c7ad81c 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java
@@ -83,7 +83,7 @@
setup(2 * (dataSize + insertSize));
data2 = data.clone();
for (int i = 0 ; i < data2.length ; ++i)
- data2[i] = new Integer(data2[i]);
+ data2[i] = Integer.valueOf(data2[i]);
}
@State(Scope.Thread)
diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java
index 54f721b..5a86842 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java
@@ -80,11 +80,21 @@
// don't flush
}
+ System.err.format("%s sstables, total %s, %,d partitions. Mean write latency %.2f ms\n",
+ cfs.getLiveSSTables().size(),
+ FBUtilities.prettyPrintMemory(cfs.metric.liveDiskSpaceUsed.getCount()),
+ cfs.metric.estimatedPartitionCount.getValue(),
+ cfs.metric.writeLatency.latency.getSnapshot().getMean());
// Needed to stabilize sstable count for off-cache sized tests (e.g. count = 100_000_000)
while (cfs.getLiveSSTables().size() >= 15)
{
cfs.enableAutoCompaction(true);
cfs.disableAutoCompaction();
+ System.err.format("%s sstables, total %s, %,d partitions. Mean write latency %.2f ms\n",
+ cfs.getLiveSSTables().size(),
+ FBUtilities.prettyPrintMemory(cfs.metric.liveDiskSpaceUsed.getCount()),
+ cfs.metric.estimatedPartitionCount.getValue(),
+ cfs.metric.writeLatency.latency.getSnapshot().getMean());
}
}
@@ -109,7 +119,8 @@
}
catch (Throwable throwable)
{
- throw Throwables.propagate(throwable);
+ Throwables.throwIfUnchecked(throwable);
+ throw new RuntimeException(throwable);
}
}));
}
@@ -142,7 +153,8 @@
}
catch (Throwable throwable)
{
- throw Throwables.propagate(throwable);
+ Throwables.throwIfUnchecked(throwable);
+ throw new RuntimeException(throwable);
}
}));
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java b/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java
index fba8d16..cc78e03 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java
@@ -138,7 +138,8 @@
}
catch (Throwable throwable)
{
- throw Throwables.propagate(throwable);
+ Throwables.throwIfUnchecked(throwable);
+ throw new RuntimeException(throwable);
}
}));
}
@@ -169,7 +170,8 @@
}
catch (Throwable throwable)
{
- throw Throwables.propagate(throwable);
+ Throwables.throwIfUnchecked(throwable);
+ throw new RuntimeException(throwable);
}
}));
}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java
new file mode 100644
index 0000000..1250512
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/ComparisonReadBench.java
@@ -0,0 +1,517 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.test.microbench.tries;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.github.jamm.MemoryMeter;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 2, time = 1)
+@Measurement(iterations = 3, time = 1)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx4G", "-Xms4G", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(1) // no concurrent writes
+@State(Scope.Benchmark)
+public class ComparisonReadBench
+{
+ // Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of
+ // partitions).
+ static MemoryMeter meter = new MemoryMeter().ignoreKnownSingletons()
+ .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE);
+
+ @Param({"ON_HEAP"})
+ BufferType bufferType = BufferType.OFF_HEAP;
+
+ @Param({"1000", "100000", "10000000"})
+ int count = 1000;
+
+ @Param({"TREE_MAP", "CSLM", "TRIE"})
+ MapOption map = MapOption.TRIE;
+
+ @Param({"LONG"})
+ TypeOption type = TypeOption.LONG;
+
+ final static InMemoryTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
+
+ Access<?> access;
+
+ @Setup(Level.Trial)
+ public void setup() throws Throwable
+ {
+ switch (map)
+ {
+ case TREE_MAP:
+ access = new NavigableMapAccess(new TreeMap(type.type.comparator()), type.type);
+ break;
+ case CSLM:
+ access = new NavigableMapAccess(new ConcurrentSkipListMap(type.type.comparator()), type.type);
+ break;
+ case TRIE:
+ access = new TrieAccess(type.type);
+ break;
+ }
+ Random rand = new Random(1);
+
+ System.out.format("Putting %,d\n", count);
+ long time = System.currentTimeMillis();
+ for (long current = 0; current < count; ++current)
+ {
+ long l = rand.nextLong();
+ access.put(l, Byte.valueOf((byte) (l >> 56)));
+ }
+ time = System.currentTimeMillis() - time;
+ System.out.format("Took %.3f seconds\n", time * 0.001);
+ access.printSize();
+ }
+
+ interface Type<T>
+ {
+ T fromLong(long l);
+ T fromByteComparable(ByteComparable bc);
+ ByteComparable longToByteComparable(long l);
+ Comparator<T> comparator();
+ }
+
+ public enum TypeOption
+ {
+ LONG(new LongType()),
+ BIGINT(new BigIntType()),
+ DECIMAL(new BigDecimalType()),
+ STRING_BASE16(new StringType(16)),
+ STRING_BASE10(new StringType(10)),
+ STRING_BASE4(new StringType(4)),
+ ARRAY(new ArrayType(1)),
+ ARRAY_REP3(new ArrayType(3)),
+ ARRAY_REP7(new ArrayType(7));
+
+ final Type<?> type;
+
+ TypeOption(Type<?> type)
+ {
+ this.type = type;
+ }
+ }
+
+ static class LongType implements Type<Long>
+ {
+ public Long fromLong(long l)
+ {
+ return l;
+ }
+
+ public Long fromByteComparable(ByteComparable bc)
+ {
+ return ByteSourceInverse.getSignedLong(bc.asComparableBytes(ByteComparable.Version.OSS50));
+ }
+
+ public ByteComparable longToByteComparable(long l)
+ {
+ return ByteComparable.of(l);
+ }
+
+ public Comparator<Long> comparator()
+ {
+ return Comparator.naturalOrder();
+ }
+ }
+
+ static class BigIntType implements Type<BigInteger>
+ {
+ public BigInteger fromLong(long l)
+ {
+ return BigInteger.valueOf(l);
+ }
+
+ public BigInteger fromByteComparable(ByteComparable bc)
+ {
+ return IntegerType.instance.compose(IntegerType.instance.fromComparableBytes(ByteSource.peekable(bc.asComparableBytes(ByteComparable.Version.OSS50)),
+ ByteComparable.Version.OSS50));
+ }
+
+ public ByteComparable longToByteComparable(long l)
+ {
+ return v -> IntegerType.instance.asComparableBytes(IntegerType.instance.decompose(fromLong(l)), v);
+ }
+
+ public Comparator<BigInteger> comparator()
+ {
+ return Comparator.naturalOrder();
+ }
+ }
+
+ static class BigDecimalType implements Type<BigDecimal>
+ {
+ public BigDecimal fromLong(long l)
+ {
+ return BigDecimal.valueOf(l);
+ }
+
+ public BigDecimal fromByteComparable(ByteComparable bc)
+ {
+ return DecimalType.instance.compose(DecimalType.instance.fromComparableBytes(ByteSource.peekable(bc.asComparableBytes(ByteComparable.Version.OSS50)),
+ ByteComparable.Version.OSS50));
+ }
+
+ public ByteComparable longToByteComparable(long l)
+ {
+ return v -> DecimalType.instance.asComparableBytes(DecimalType.instance.decompose(fromLong(l)), v);
+ }
+
+ public Comparator<BigDecimal> comparator()
+ {
+ return Comparator.naturalOrder();
+ }
+ }
+
+ static class StringType implements Type<String>
+ {
+ final int base;
+
+ StringType(int base)
+ {
+ this.base = base;
+ }
+
+ public String fromLong(long l)
+ {
+ return Long.toString(l, base);
+ }
+
+ public String fromByteComparable(ByteComparable bc)
+ {
+ return new String(ByteSourceInverse.readBytes(bc.asComparableBytes(ByteComparable.Version.OSS50)), StandardCharsets.UTF_8);
+ }
+
+ public ByteComparable longToByteComparable(long l)
+ {
+ return ByteComparable.fixedLength(fromLong(l).getBytes(StandardCharsets.UTF_8));
+ }
+
+ public Comparator<String> comparator()
+ {
+ return Comparator.naturalOrder();
+ }
+ }
+
+ static class ArrayType implements Type<byte[]>
+ {
+ final int reps;
+
+ ArrayType(int reps)
+ {
+ this.reps = reps;
+ }
+
+ public byte[] fromLong(long l)
+ {
+ byte[] value = new byte[8 * reps];
+ for (int i = 0; i < 8; ++i)
+ {
+ for (int j = 0; j < reps; ++j)
+ value[i * reps + j] = (byte)(l >> (56 - i * 8));
+ }
+ return value;
+ }
+
+ public byte[] fromByteComparable(ByteComparable bc)
+ {
+ return ByteSourceInverse.readBytes(bc.asComparableBytes(ByteComparable.Version.OSS50));
+ }
+
+ public ByteComparable longToByteComparable(long l)
+ {
+ return ByteComparable.fixedLength(fromLong(l));
+ }
+
+ public Comparator<byte[]> comparator()
+ {
+ return ByteArrayUtil::compareUnsigned;
+ }
+ }
+
+ interface Access<T>
+ {
+ void put(long v, byte b);
+ byte get(long v);
+ Iterable<Byte> values();
+ Iterable<Byte> valuesSlice(long left, boolean includeLeft, long right, boolean includeRight);
+ Iterable<Map.Entry<T, Byte>> entrySet();
+ void consumeValues(Consumer<Byte> consumer);
+ void consumeEntries(BiConsumer<T, Byte> consumer);
+ void printSize();
+ }
+
+ public enum MapOption
+ {
+ TREE_MAP,
+ CSLM,
+ TRIE
+ }
+
+ class TrieAccess<T> implements Access<T>
+ {
+ final InMemoryTrie<Byte> trie;
+ final Type<T> type;
+
+ TrieAccess(Type<T> type)
+ {
+ this.type = type;
+ trie = new InMemoryTrie<>(bufferType);
+ }
+
+ public void put(long v, byte b)
+ {
+ try
+ {
+ trie.putRecursive(type.longToByteComparable(v), b, resolver);
+ }
+ catch (InMemoryTrie.SpaceExhaustedException e)
+ {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ public byte get(long v)
+ {
+ return trie.get(type.longToByteComparable(v));
+ }
+
+ public Iterable<Byte> values()
+ {
+ return trie.values();
+ }
+
+ public Iterable<Byte> valuesSlice(long left, boolean includeLeft, long right, boolean includeRight)
+ {
+ return trie.subtrie(type.longToByteComparable(left), includeLeft, type.longToByteComparable(right), includeRight)
+ .values();
+ }
+
+ public Iterable<Map.Entry<T, Byte>> entrySet()
+ {
+ return Iterables.transform(trie.entrySet(),
+ en -> new AbstractMap.SimpleEntry<>(type.fromByteComparable(en.getKey()),
+ en.getValue()));
+ }
+
+ public void consumeValues(Consumer<Byte> consumer)
+ {
+ trie.forEachValue(consumer::accept);
+ }
+
+ public void consumeEntries(BiConsumer<T, Byte> consumer)
+ {
+ trie.forEachEntry((key, value) -> consumer.accept(type.fromByteComparable(key), value));
+ }
+
+ public void printSize()
+ {
+ long deepsize = meter.measureDeep(trie);
+ System.out.format("Trie size on heap %,d off heap %,d deep size %,d\n",
+ trie.sizeOnHeap(), trie.sizeOffHeap(), deepsize);
+ System.out.format("per entry on heap %.2f off heap %.2f deep size %.2f\n",
+ trie.sizeOnHeap() * 1.0 / count, trie.sizeOffHeap() * 1.0 / count, deepsize * 1.0 / count);
+ }
+ }
+
+ class NavigableMapAccess<T> implements Access<T>
+ {
+ final NavigableMap<T, Byte> navigableMap;
+ final Type<T> type;
+
+ NavigableMapAccess(NavigableMap<T, Byte> navigableMap, Type<T> type)
+ {
+ this.navigableMap = navigableMap;
+ this.type = type;
+ }
+
+ public void put(long v, byte b)
+ {
+ navigableMap.put(type.fromLong(v), b);
+ }
+
+ public byte get(long v)
+ {
+ return navigableMap.get(type.fromLong(v));
+ }
+
+ public Iterable<Byte> values()
+ {
+ return navigableMap.values();
+ }
+
+ public Iterable<Byte> valuesSlice(long left, boolean includeLeft, long right, boolean includeRight)
+ {
+ return navigableMap.subMap(type.fromLong(left), includeLeft, type.fromLong(right), includeRight)
+ .values();
+ }
+
+ public Iterable<Map.Entry<T, Byte>> entrySet()
+ {
+ return navigableMap.entrySet();
+ }
+
+ public void consumeValues(Consumer<Byte> consumer)
+ {
+ navigableMap.values().forEach(consumer);
+ }
+
+ public void consumeEntries(BiConsumer<T, Byte> consumer)
+ {
+ navigableMap.forEach(consumer);
+ }
+
+ public void printSize()
+ {
+ long size = meter.measureDeep(navigableMap);
+ System.out.format(map + " size on heap %,d\n", size);
+ System.out.format("per entry on heap %.2f\n", size * 1.0 / count);
+ }
+ }
+
+ @Benchmark
+ public void getRandom()
+ {
+ Random rand = new Random(1);
+
+ for (long current = 0; current < count; ++current)
+ {
+ long l = rand.nextLong();
+ Byte res = access.get(l);
+ if (res.byteValue() != l >> 56)
+ throw new AssertionError();
+ }
+ }
+
+ @Benchmark
+ public int iterateValues()
+ {
+ int sum = 0;
+ for (byte b : access.values())
+ sum += b;
+ return sum;
+ }
+
+ @Benchmark
+ public int consumeValues()
+ {
+ class Counter implements Consumer<Byte>
+ {
+ int sum = 0;
+
+ @Override
+ public void accept(Byte aByte)
+ {
+ sum += aByte;
+ }
+ }
+ Counter counter = new Counter();
+ access.consumeValues(counter);
+ return counter.sum;
+ }
+
+ @Benchmark
+ public int consumeEntries()
+ {
+ class Counter<T> implements BiConsumer<T, Byte>
+ {
+ int sum = 0;
+
+ @Override
+ public void accept(T key, Byte aByte)
+ {
+ sum += aByte;
+ }
+ }
+ Counter counter = new Counter();
+ access.consumeEntries(counter);
+ return counter.sum;
+ }
+
+ @Benchmark
+ public int iterateEntries()
+ {
+ int sum = 0;
+ for (Map.Entry<?, Byte> en : access.entrySet())
+ sum += en.getValue();
+ return sum;
+ }
+
+
+ @Benchmark
+ public int getByIterateValueSlice()
+ {
+ Random rand = new Random(1);
+ int sum = 0;
+ for (int i = 0; i < count; ++i)
+ {
+ long v = rand.nextLong();
+ Iterable<Byte> values = access.valuesSlice(v, true, v, true);
+ for (byte b : values)
+ sum += b;
+ }
+ return sum;
+ }
+
+ @Benchmark
+ public int iterateValuesLimited()
+ {
+ int sum = 0;
+ Iterable<Byte> values = access.valuesSlice(0L, false, Long.MAX_VALUE / 2, true); // 1/4
+ for (byte b : values)
+ sum += b;
+ return sum;
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java
new file mode 100644
index 0000000..cff2e4a
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.test.microbench.tries;
+
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.db.tries.TrieEntriesWalker;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.openjdk.jmh.annotations.*;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx4G", "-Xms4G", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(1) // no concurrent writes
+@State(Scope.Benchmark)
+public class InMemoryTrieReadBench
+{
+ @Param({"ON_HEAP", "OFF_HEAP"})
+ BufferType bufferType = BufferType.OFF_HEAP;
+
+ @Param({"1000", "100000", "10000000"})
+ int count = 1000;
+
+ final static InMemoryTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
+
+ InMemoryTrie<Byte> trie;
+
+ @Setup(Level.Trial)
+ public void setup() throws Throwable
+ {
+ trie = new InMemoryTrie<>(bufferType);
+ Random rand = new Random(1);
+
+ System.out.format("Putting %,d\n", count);
+ for (long current = 0; current < count; ++current)
+ {
+ long l = rand.nextLong();
+ trie.putRecursive(ByteComparable.of(l), Byte.valueOf((byte) (l >> 56)), resolver);
+ }
+ System.out.format("Trie size on heap %,d off heap %,d\n",
+ trie.sizeOnHeap(), trie.sizeOffHeap());
+ System.out.format("per entry on heap %.2f off heap %.2f\n",
+ trie.sizeOnHeap() * 1.0 / count, trie.sizeOffHeap() * 1.0 / count);
+ }
+
+ @Benchmark
+ public void getRandom()
+ {
+ Random rand = new Random(1);
+
+ for (long current = 0; current < count; ++current)
+ {
+ long l = rand.nextLong();
+ Byte res = trie.get(ByteComparable.of(l));
+ if (res.byteValue() != l >> 56)
+ throw new AssertionError();
+ }
+ }
+
+ @Benchmark
+ public int iterateValues()
+ {
+ int sum = 0;
+ for (byte b : trie.values())
+ sum += b;
+ return sum;
+ }
+
+ @Benchmark
+ public int consumeValues()
+ {
+ class Counter implements Trie.ValueConsumer<Byte>
+ {
+ int sum = 0;
+
+ @Override
+ public void accept(Byte aByte)
+ {
+ sum += aByte;
+ }
+ }
+ Counter counter = new Counter();
+ trie.forEachValue(counter);
+ return counter.sum;
+ }
+
+ @Benchmark
+ public int consumeEntries()
+ {
+ class Counter implements BiConsumer<ByteComparable, Byte>
+ {
+ int sum = 0;
+
+ @Override
+ public void accept(ByteComparable byteComparable, Byte aByte)
+ {
+ sum += aByte;
+ }
+ }
+ Counter counter = new Counter();
+ trie.forEachEntry(counter);
+ return counter.sum;
+ }
+
+ @Benchmark
+ public int processEntries()
+ {
+ class Counter extends TrieEntriesWalker<Byte, Void>
+ {
+ int sum = 0;
+
+ @Override
+ protected void content(Byte content, byte[] bytes, int byteLength)
+ {
+ sum += content;
+ }
+
+ @Override
+ public Void complete()
+ {
+ return null;
+ }
+ }
+ Counter counter = new Counter();
+ trie.process(counter);
+ return counter.sum;
+ }
+
+ @Benchmark
+ public int iterateValuesUnordered()
+ {
+ int sum = 0;
+ for (byte b : trie.valuesUnordered())
+ sum += b;
+ return sum;
+ }
+
+ @Benchmark
+ public int iterateEntries()
+ {
+ int sum = 0;
+ for (Map.Entry<ByteComparable, Byte> en : trie.entrySet())
+ sum += en.getValue();
+ return sum;
+ }
+
+ @Benchmark
+ public int iterateValuesLimited()
+ {
+ Iterable<Byte> values = trie.subtrie(ByteComparable.of(0L),
+ true,
+ ByteComparable.of(Long.MAX_VALUE / 2), // 1/4 of all
+ false)
+ .values();
+ int sum = 0;
+ for (byte b : values)
+ sum += b;
+ return sum;
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java
new file mode 100644
index 0000000..e32e20f
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.test.microbench.tries;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.openjdk.jmh.annotations.*;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx4G", "-Xms4G", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(1) // no concurrent writes
+@State(Scope.Benchmark)
+public class InMemoryTrieUnionBench
+{
+ @Param({"ON_HEAP", "OFF_HEAP"})
+ BufferType bufferType = BufferType.OFF_HEAP;
+
+ @Param({"1000", "100000", "10000000"})
+ int count = 1000;
+
+ @Param({"2", "3", "8"})
+ int sources = 2;
+
+ @Param({"false", "true"})
+ boolean sequential = true;
+
+ final static InMemoryTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
+
+ Trie<Byte> trie;
+
+ @Setup(Level.Trial)
+ public void setup() throws Throwable
+ {
+ List<InMemoryTrie<Byte>> tries = new ArrayList<>(sources);
+ System.out.format("Putting %,d among %d tries\n", count, sources);
+ Random rand = new Random(1);
+ if (sequential)
+ {
+ long sz = 65536 / sources;
+ for (int i = 0; i < sources; ++i)
+ tries.add(new InMemoryTrie<>(bufferType));
+
+ for (long current = 0; current < count; ++current)
+ {
+ long l = rand.nextLong();
+ InMemoryTrie<Byte> tt = tries.get(Math.min((int) (((l >> 48) + 32768) / sz), sources - 1));
+ tt.putRecursive(ByteComparable.of(l), (byte) (l >> 56), resolver);
+ }
+
+ }
+ else
+ {
+ long current = 0;
+ for (int i = 0; i < sources; ++i)
+ {
+ InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+ int currMax = this.count * (i + 1) / sources;
+
+ for (; current < currMax; ++current)
+ {
+ long l = rand.nextLong();
+ trie.putRecursive(ByteComparable.of(l), (byte) (l >> 56), resolver);
+ }
+ tries.add(trie);
+ }
+ }
+
+ for (InMemoryTrie<Byte> trie : tries)
+ {
+ System.out.format("Trie size on heap %,d off heap %,d\n",
+ trie.sizeOnHeap(), trie.sizeOffHeap());
+ }
+ trie = Trie.mergeDistinct(tries);
+
+ System.out.format("Actual count %,d\n", Iterables.size(trie.values()));
+ }
+
+ @Benchmark
+ public int iterateValues()
+ {
+ int sum = 0;
+ for (byte b : trie.values())
+ sum += b;
+ return sum;
+ }
+
+ @Benchmark
+ public int iterateValuesUnordered()
+ {
+ int sum = 0;
+ for (byte b : trie.valuesUnordered())
+ sum += b;
+ return sum;
+ }
+
+ @Benchmark
+ public int iterateEntries()
+ {
+ int sum = 0;
+ for (Map.Entry<ByteComparable, Byte> en : trie.entrySet())
+ sum += en.getValue();
+ return sum;
+ }
+
+ @Benchmark
+ public int iterateValuesLimited()
+ {
+ Iterable<Byte> values = trie.subtrie(ByteComparable.of(0L),
+ true,
+ ByteComparable.of(Long.MAX_VALUE / 2), // 1/4 of all
+ false)
+ .values();
+ int sum = 0;
+ for (byte b : values)
+ sum += b;
+ return sum;
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java
new file mode 100644
index 0000000..f2be11d
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieWriteBench.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.test.microbench.tries;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(value = 1,jvmArgsAppend = { "-Xmx4G", "-Xms4G", "-Djmh.executor=CUSTOM", "-Djmh.executor.class=org.apache.cassandra.test.microbench.FastThreadExecutor"})
+@Threads(1) // no concurrent writes
+@State(Scope.Benchmark)
+public class InMemoryTrieWriteBench
+{
+ @Param({"ON_HEAP", "OFF_HEAP"})
+ BufferType bufferType = BufferType.OFF_HEAP;
+
+ @Param({"1000", "100000", "10000000"})
+ int count = 1000;
+
+ @Param({"8"})
+ int keyLength = 8;
+
+ final static InMemoryTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
+
+ // Set this to true to print the trie sizes after insertions for sanity checking.
+ // This might affect the timings, do not commit with this set to true.
+ final static boolean PRINT_SIZES = false;
+
+ @Benchmark
+ public void putSequential(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+ {
+ InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+ ByteBuffer buf = ByteBuffer.allocate(keyLength);
+
+ for (long current = 0; current < count; ++current)
+ {
+ long l = current;
+ buf.putLong(keyLength - 8, l);
+ trie.putRecursive(ByteComparable.fixedLength(buf), Byte.valueOf((byte) (l >> 56)), resolver);
+ }
+ if (PRINT_SIZES)
+ System.out.println(trie.valuesCount());
+ bh.consume(trie);
+ }
+
+ @Benchmark
+ public void putRandom(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+ {
+ InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+ Random rand = new Random(1);
+ byte[] buf = new byte[keyLength];
+
+ for (long current = 0; current < count; ++current)
+ {
+ rand.nextBytes(buf);
+ trie.putRecursive(ByteComparable.fixedLength(buf), Byte.valueOf(buf[0]), resolver);
+ }
+ if (PRINT_SIZES)
+ System.out.println(trie.valuesCount());
+ bh.consume(trie);
+ }
+
+ @Benchmark
+ public void applySequential(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+ {
+ InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+ ByteBuffer buf = ByteBuffer.allocate(keyLength);
+
+ for (long current = 0; current < count; ++current)
+ {
+ long l = current;
+ buf.putLong(keyLength - 8, l);
+ trie.putSingleton(ByteComparable.fixedLength(buf), Byte.valueOf((byte) (l >> 56)), resolver);
+ }
+ if (PRINT_SIZES)
+ System.out.println(trie.valuesCount());
+ bh.consume(trie);
+ }
+
+ @Benchmark
+ public void applyRandom(Blackhole bh) throws InMemoryTrie.SpaceExhaustedException
+ {
+ InMemoryTrie<Byte> trie = new InMemoryTrie(bufferType);
+ Random rand = new Random(1);
+ byte[] buf = new byte[keyLength];
+
+ for (long current = 0; current < count; ++current)
+ {
+ rand.nextBytes(buf);
+ trie.putSingleton(ByteComparable.fixedLength(buf), Byte.valueOf(buf[0]), resolver);
+ }
+ if (PRINT_SIZES)
+ System.out.println(trie.valuesCount());
+ bh.consume(trie);
+ }
+}
diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java
index 6e6b0d3..778e44c 100644
--- a/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java
+++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java
@@ -110,6 +110,8 @@
private final EnumSet<Flag> flags;
private final Consumer<String> dependentTypes;
+ private boolean updateVisibility = false;
+
ClassTransformer(int api, String className, EnumSet<Flag> flags, Consumer<String> dependentTypes)
{
this(api, new ClassWriter(0), className, flags, null, null, null, null, dependentTypes);
@@ -137,12 +139,58 @@
this.methodLogger = MethodLogger.log(api, className);
}
+ public void setUpdateVisibility(boolean updateVisibility)
+ {
+ this.updateVisibility = updateVisibility;
+ }
+
+ /**
+ * Java 11 changed the way that classes defined in the same source file get access to private state (see https://openjdk.org/jeps/181),
+ * rather than trying to adapt to this, this method attempts to make the field/method/class public so that access
+ * is not restricted.
+ */
+ private int makePublic(int access)
+ {
+ if (!updateVisibility)
+ return access;
+ // leave non-user created methods/fields/etc. alone
+ if (contains(access, Opcodes.ACC_BRIDGE) || contains(access, Opcodes.ACC_SYNTHETIC))
+ return access;
+ if (contains(access, Opcodes.ACC_PRIVATE))
+ {
+ access &= ~Opcodes.ACC_PRIVATE;
+ access |= Opcodes.ACC_PUBLIC;
+ }
+ else if (contains(access, Opcodes.ACC_PROTECTED))
+ {
+ access &= ~Opcodes.ACC_PROTECTED;
+ access |= Opcodes.ACC_PUBLIC;
+ }
+ else if (!contains(access, Opcodes.ACC_PUBLIC)) // package-protected
+ {
+ access |= Opcodes.ACC_PUBLIC;
+ }
+ return access;
+ }
+
+ private static boolean contains(int value, int mask)
+ {
+ return (value & mask) != 0;
+ }
+
+ @Override
+ public void visit(int version, int access, String name, String signature, String superName, String[] interfaces)
+ {
+ super.visit(version, makePublic(access), name, signature, superName, interfaces);
+
+ }
+
@Override
public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value)
{
if (dependentTypes != null)
Utils.visitIfRefType(descriptor, dependentTypes);
- return super.visitField(access, name, descriptor, signature, value);
+ return super.visitField(makePublic(access), name, descriptor, signature, value);
}
@Override
@@ -176,6 +224,7 @@
isToString = true;
}
+ access = makePublic(access);
MethodVisitor visitor;
if (flags.contains(MONITORS) && (access & Opcodes.ACC_SYNCHRONIZED) != 0)
{
diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java
index 87cfab0..c4beaf9 100644
--- a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java
+++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java
@@ -54,6 +54,8 @@
import static org.objectweb.asm.Opcodes.IRETURN;
import static org.objectweb.asm.Opcodes.RETURN;
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
/**
* A mechanism for weaving classes loaded by the bootstrap classloader that we cannot override.
* The design supports weaving of the internals of these classes, and in future we may want to
@@ -296,12 +298,14 @@
visitor.visitMethodInsn(INVOKESTATIC, "java/lang/Thread", "currentThread", "()Ljava/lang/Thread;", false);
visitor.visitFieldInsn(GETSTATIC, "java/util/concurrent/ThreadLocalRandom", "SEED", "J");
visitor.visitMethodInsn(INVOKESTATIC, "org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods$Global", "randomSeed", "()J", false);
- visitor.visitMethodInsn(INVOKEVIRTUAL, "sun/misc/Unsafe", "putLong", "(Ljava/lang/Object;JJ)V", false);
+
+ String unsafeClass = Utils.descriptorToClassName(unsafeDescriptor);
+ visitor.visitMethodInsn(INVOKEVIRTUAL, unsafeClass, "putLong", "(Ljava/lang/Object;JJ)V", false);
visitor.visitFieldInsn(GETSTATIC, "java/util/concurrent/ThreadLocalRandom", unsafeFieldName, unsafeDescriptor);
visitor.visitMethodInsn(INVOKESTATIC, "java/lang/Thread", "currentThread", "()Ljava/lang/Thread;", false);
visitor.visitFieldInsn(GETSTATIC, "java/util/concurrent/ThreadLocalRandom", "PROBE", "J");
visitor.visitLdcInsn(0);
- visitor.visitMethodInsn(INVOKEVIRTUAL, "sun/misc/Unsafe", "putInt", "(Ljava/lang/Object;JI)V", false);
+ visitor.visitMethodInsn(INVOKEVIRTUAL, unsafeClass, "putInt", "(Ljava/lang/Object;JI)V", false);
visitor.visitInsn(RETURN);
visitor.visitLabel(new Label());
visitor.visitMaxs(6, 1);
diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java
index a57074d..473cc27 100644
--- a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java
+++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java
@@ -246,6 +246,7 @@
}
ClassTransformer transformer = new ClassTransformer(api, internalName, flags, monitorDelayChance, new NemesisGenerator(api, internalName, nemesisChance), nemesisFieldSelector, hashcode, dependentTypes);
+ transformer.setUpdateVisibility(true);
transformer.readAndTransform(input);
if (!transformer.isTransformed())
diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java
index 1e6a844..7e6a689 100644
--- a/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java
+++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java
@@ -37,6 +37,8 @@
import static org.apache.cassandra.simulator.asm.MethodLogger.Level.NONE;
import static org.apache.cassandra.simulator.asm.MethodLogger.Level.valueOf;
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
// TODO (config): support logging only for packages/classes matching a pattern
interface MethodLogger
{
diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java
index be2ef6c..d710ffe 100644
--- a/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java
+++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java
@@ -287,4 +287,12 @@
forEach.accept(descriptor.substring(i + 1, descriptor.length() - 1));
}
}
+
+ public static String descriptorToClassName(String desc)
+ {
+ // samples: "Ljdk/internal/misc/Unsafe;", "Lsun/misc/Unsafe;"
+ if (!(desc.startsWith("L") && desc.endsWith(";")))
+ throw new IllegalArgumentException("Unable to parse descriptor: " + desc);
+ return desc.substring(1, desc.length() - 1);
+ }
}
diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java
index 18fc877..6119e47 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java
@@ -43,6 +43,7 @@
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.Throwables;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_DEBUG;
import static org.apache.cassandra.simulator.Action.Modifier.DAEMON;
import static org.apache.cassandra.simulator.Action.Modifier.STREAM;
import static org.apache.cassandra.simulator.Action.Phase.CONSEQUENCE;
@@ -300,7 +301,7 @@
}
else
{
- logger.error("Simulation failed to make progress. Run with -Dcassandra.test.simulator.debug=true to see the blocked task graph. Blocked tasks:");
+ logger.error("Simulation failed to make progress. Run with -D{}=true to see the blocked task graph. Blocked tasks:", TEST_SIMULATOR_DEBUG.getKey());
actions = sequences.values()
.stream()
.filter(s -> s.on instanceof OrderOnId)
diff --git a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
index caf642e..892e904 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
@@ -21,8 +21,8 @@
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
-import java.nio.file.FileSystem;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
@@ -35,8 +35,6 @@
import java.util.function.Predicate;
import java.util.function.Supplier;
-import com.google.common.jimfs.Configuration;
-import com.google.common.jimfs.Jimfs;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.FutureCallback;
@@ -55,6 +53,7 @@
import org.apache.cassandra.distributed.impl.DirectStreamingConnectionFactory;
import org.apache.cassandra.distributed.impl.IsolatedExecutor;
import org.apache.cassandra.io.compress.LZ4Compressor;
+import org.apache.cassandra.io.util.FileSystems;
import org.apache.cassandra.service.paxos.BallotGenerator;
import org.apache.cassandra.service.paxos.PaxosPrepare;
import org.apache.cassandra.simulator.RandomSource.Choices;
@@ -62,6 +61,7 @@
import org.apache.cassandra.simulator.asm.NemesisFieldSelectors;
import org.apache.cassandra.simulator.cluster.ClusterActions;
import org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange;
+import org.apache.cassandra.io.filesystem.ListenableFileSystem;
import org.apache.cassandra.simulator.systems.Failures;
import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture;
import org.apache.cassandra.simulator.systems.InterceptibleThread;
@@ -607,7 +607,7 @@
public final SimulatedSystems simulated;
public final Cluster cluster;
public final S simulation;
- private final FileSystem jimfs;
+ private final ListenableFileSystem fs;
protected final Map<Integer, List<Closeable>> onUnexpectedShutdown = new TreeMap<>();
protected final List<Callable<Void>> onShutdown = new CopyOnWriteArrayList<>();
protected final ThreadLocalRandomCheck threadLocalRandomCheck;
@@ -618,9 +618,7 @@
SimulationFactory<S> factory) throws IOException
{
this.random = random;
- this.jimfs = Jimfs.newFileSystem(Long.toHexString(seed) + 'x' + uniqueNum, Configuration.unix().toBuilder()
- .setMaxSize(4L << 30).setBlockSize(512)
- .build());
+ this.fs = new ListenableFileSystem(FileSystems.jimfs(Long.toHexString(seed) + 'x' + uniqueNum));
final SimulatedMessageDelivery delivery;
final SimulatedExecution execution;
@@ -670,23 +668,28 @@
Failures failures = new Failures();
ThreadAllocator threadAllocator = new ThreadAllocator(random, builder.threadCount, numOfNodes);
+ List<String> allowedDiskAccessModes = Arrays.asList("mmap", "mmap_index_only", "standard");
+ String disk_access_mode = allowedDiskAccessModes.get(random.uniform(0, allowedDiskAccessModes.size() - 1));
+ boolean commitlogCompressed = random.decide(.5f);
cluster = snitch.setup(Cluster.build(numOfNodes)
- .withRoot(jimfs.getPath("/cassandra"))
+ .withRoot(fs.getPath("/cassandra"))
.withSharedClasses(sharedClassPredicate)
- .withConfig(config -> configUpdater.accept(threadAllocator.update(config
- .with(Feature.BLANK_GOSSIP)
- .set("read_request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.readTimeoutNanos)))
- .set("write_request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.writeTimeoutNanos)))
- .set("cas_contention_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.contentionTimeoutNanos)))
- .set("request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.requestTimeoutNanos)))
- .set("memtable_heap_space", "1MiB")
- .set("memtable_allocation_type", builder.memoryListener != null ? "unslabbed_heap_buffers_logged" : "heap_buffers")
- .set("file_cache_size", "16MiB")
- .set("use_deterministic_table_id", true)
- .set("disk_access_mode", "standard")
- .set("failure_detector", SimulatedFailureDetector.Instance.class.getName())
- .set("commitlog_compression", new ParameterizedClass(LZ4Compressor.class.getName(), emptyMap()))
- )))
+ .withConfig(config -> {
+ config.with(Feature.BLANK_GOSSIP)
+ .set("read_request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.readTimeoutNanos)))
+ .set("write_request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.writeTimeoutNanos)))
+ .set("cas_contention_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.contentionTimeoutNanos)))
+ .set("request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.requestTimeoutNanos)))
+ .set("memtable_heap_space", "1MiB")
+ .set("memtable_allocation_type", builder.memoryListener != null ? "unslabbed_heap_buffers_logged" : "heap_buffers")
+ .set("file_cache_size", "16MiB")
+ .set("use_deterministic_table_id", true)
+ .set("disk_access_mode", disk_access_mode)
+ .set("failure_detector", SimulatedFailureDetector.Instance.class.getName());
+ if (commitlogCompressed)
+ config.set("commitlog_compression", new ParameterizedClass(LZ4Compressor.class.getName(), emptyMap()));
+ configUpdater.accept(threadAllocator.update(config));
+ })
.withInstanceInitializer(new IInstanceInitializer()
{
@Override
diff --git a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
index 6f1eb12..32888f4 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
@@ -54,18 +54,21 @@
import static java.util.Arrays.stream;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT;
import static org.apache.cassandra.config.CassandraRelevantProperties.BATCH_COMMIT_LOG_SYNC_INTERVAL;
import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_REMOTE_PORT;
import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_GLOBAL;
import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_APPROX;
import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_PRECISE;
-import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_CONSISTENT_DIRECTORY_LISTINGS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_DIRECTORY_LISTINGS;
import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_UNSAFE_UUID_NODE;
import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_SSTABLE_ACTIVITY_TRACKING;
import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_SSTABLE_COMPRESSION_DEFAULT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_API_LOG_TOPOLOGY;
import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_SKIP_WAITING_TO_SETTLE;
import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_MISSING_NATIVE_FILE_HINTS;
-import static org.apache.cassandra.config.CassandraRelevantProperties.IS_DISABLED_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.LIBJEMALLOC;
import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE;
import static org.apache.cassandra.config.CassandraRelevantProperties.MIGRATION_DELAY;
import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS;
@@ -99,12 +102,12 @@
try { Clock.Global.nanoTime(); } catch (IllegalStateException e) {} // make sure static initializer gets called
// TODO (cleanup): disable unnecessary things like compaction logger threads etc
- System.setProperty("cassandra.libjemalloc", "-");
- System.setProperty("cassandra.dtest.api.log.topology", "false");
+ LIBJEMALLOC.setString("-");
+ DTEST_API_LOG_TOPOLOGY.setBoolean(false);
// this property is used to allow non-members of the ring to exist in gossip without breaking RF changes
// it would be nice not to rely on this, but hopefully we'll have consistent range movements before it matters
- System.setProperty("cassandra.allow_alter_rf_during_range_movement", "true");
+ ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT.setBoolean(true);
for (CassandraRelevantProperties property : Arrays.asList(CLOCK_GLOBAL, CLOCK_MONOTONIC_APPROX, CLOCK_MONOTONIC_PRECISE))
property.setString("org.apache.cassandra.simulator.systems.SimulatedTime$Global");
@@ -118,14 +121,14 @@
BATCH_COMMIT_LOG_SYNC_INTERVAL.setInt(-1);
DISABLE_SSTABLE_ACTIVITY_TRACKING.setBoolean(false);
DETERMINISM_SSTABLE_COMPRESSION_DEFAULT.setBoolean(false); // compression causes variation in file size for e.g. UUIDs, IP addresses, random file paths
- DETERMINISM_CONSISTENT_DIRECTORY_LISTINGS.setBoolean(true);
+ CONSISTENT_DIRECTORY_LISTINGS.setBoolean(true);
TEST_IGNORE_SIGAR.setBoolean(true);
SYSTEM_AUTH_DEFAULT_RF.setInt(3);
MIGRATION_DELAY.setInt(Integer.MAX_VALUE);
DISABLE_GOSSIP_ENDPOINT_REMOVAL.setBoolean(true);
MEMTABLE_OVERHEAD_SIZE.setInt(100);
IGNORE_MISSING_NATIVE_FILE_HINTS.setBoolean(true);
- IS_DISABLED_MBEAN_REGISTRATION.setBoolean(true);
+ ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true);
TEST_JVM_DTEST_DISABLE_SSL.setBoolean(true); // to support easily running without netty from dtest-jar
if (Thread.currentThread() instanceof InterceptibleThread); // load InterceptibleThread class to avoid infinite loop in InterceptorOfGlobalMethods
diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java
index 5acf764..8d0ed46 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java
@@ -35,7 +35,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.Util;
import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.simulator.ClusterSimulation;
import org.apache.cassandra.simulator.RandomSource;
@@ -80,7 +82,7 @@
String readInterned() throws IOException
{
- int id = (int) in.readVInt();
+ int id = in.readVInt32();
if (id == strings.size()) strings.add(in.readUTF());
else if (id > strings.size()) throw failWithOOM();
return strings.get(id);
@@ -210,7 +212,7 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
long v = in.readLong();
threads.checkThread();
if (type != 7 || c != count || value != v)
@@ -240,11 +242,11 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
threads.checkThread();
- int min1 = (int) in.readVInt();
- int max1 = (int) in.readVInt() + min1;
- int v1 = (int) in.readVInt() + min1;
+ int min1 = in.readVInt32();
+ int max1 = in.readVInt32() + min1;
+ int v1 = in.readVInt32() + min1;
if (type != 1 || min != min1 || max != max1 || v != v1 || c != count)
{
logger.error(String.format("(%d,%d,%d[%d,%d]) != (%d,%d,%d[%d,%d])", 1, count, v, min, max, type, c, v1, min1, max1));
@@ -273,7 +275,7 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
threads.checkThread();
long min1 = in.readVInt();
long max1 = in.readVInt() + min1;
@@ -306,7 +308,7 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
threads.checkThread();
float v1 = in.readFloat();
if (type != 3 || v != v1 || c != count)
@@ -338,7 +340,7 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
threads.checkThread();
double v1 = in.readDouble();
if (type != 6 || v != v1 || c != count)
@@ -369,7 +371,7 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
long v1 = in.readVInt();
if (type != 4 || seed != v1 || c != count)
throw failWithOOM();
@@ -395,7 +397,7 @@
try
{
byte type = in.readByte();
- int c = (int) in.readVInt();
+ int c = in.readVInt32();
long v1 = in.readVInt();
if (type != 5 || v != v1 || c != count)
throw failWithOOM();
@@ -433,8 +435,8 @@
File timeFile = new File(new File(loadFromDir), Long.toHexString(seed) + ".time.gz");
try (BufferedReader eventIn = new BufferedReader(new InputStreamReader(new GZIPInputStream(eventFile.newInputStream())));
- DataInputPlus.DataInputStreamPlus rngIn = new DataInputPlus.DataInputStreamPlus(rngFile.exists() && withRng != NONE ? new GZIPInputStream(rngFile.newInputStream()) : new ByteArrayInputStream(new byte[0]));
- DataInputPlus.DataInputStreamPlus timeIn = new DataInputPlus.DataInputStreamPlus(timeFile.exists() && withTime != NONE ? new GZIPInputStream(timeFile.newInputStream()) : new ByteArrayInputStream(new byte[0])))
+ DataInputStreamPlus rngIn = Util.DataInputStreamPlusImpl.wrap(rngFile.exists() && withRng != NONE ? new GZIPInputStream(rngFile.newInputStream()) : new ByteArrayInputStream(new byte[0]));
+ DataInputStreamPlus timeIn = Util.DataInputStreamPlusImpl.wrap(timeFile.exists() && withTime != NONE ? new GZIPInputStream(timeFile.newInputStream()) : new ByteArrayInputStream(new byte[0])))
{
boolean inputHasWaitSites, inputHasWakeSites, inputHasRngCallSites, inputHasTimeCallSites;
{
diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java b/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java
index 7ca7a20..42c7b08 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java
@@ -18,24 +18,6 @@
package org.apache.cassandra.simulator.debug;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.nio.channels.Channels;
-import java.util.Arrays;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-import java.util.function.Supplier;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
-import java.util.zip.GZIPOutputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.File;
@@ -46,11 +28,22 @@
import org.apache.cassandra.utils.Closeable;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.concurrent.Threads;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.channels.Channels;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+import java.util.zip.GZIPOutputStream;
import static org.apache.cassandra.io.util.File.WriteMode.OVERWRITE;
-import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.NONE;
-import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.VALUE;
-import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.WITH_CALLSITES;
+import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.*;
import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM;
public class Record
@@ -282,7 +275,7 @@
synchronized (this)
{
out.writeByte(7);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
out.writeLong(value);
threads.writeThread();
}
@@ -309,11 +302,11 @@
synchronized (this)
{
out.writeByte(1);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
threads.writeThread();
- out.writeVInt(min);
- out.writeVInt(max - min);
- out.writeVInt(v - min);
+ out.writeVInt32(min);
+ out.writeVInt32(max - min);
+ out.writeVInt32(v - min);
}
}
catch (IOException e)
@@ -339,7 +332,7 @@
synchronized (this)
{
out.writeByte(2);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
threads.writeThread();
out.writeVInt(min);
out.writeVInt(max - min);
@@ -369,7 +362,7 @@
synchronized (this)
{
out.writeByte(3);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
threads.writeThread();
out.writeFloat(v);
}
@@ -397,7 +390,7 @@
synchronized (this)
{
out.writeByte(6);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
threads.writeThread();
out.writeDouble(v);
}
@@ -425,7 +418,7 @@
synchronized (this)
{
out.writeByte(4);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
out.writeVInt(seed);
}
}
@@ -451,7 +444,7 @@
synchronized (this)
{
out.writeByte(5);
- out.writeVInt(count++);
+ out.writeVInt32(count++);
out.writeFloat(v);
}
}
@@ -514,11 +507,11 @@
Integer id = objects.get(o);
if (id != null)
{
- out.writeVInt(id);
+ out.writeVInt32(id);
}
else
{
- out.writeVInt(objects.size());
+ out.writeVInt32(objects.size());
out.writeUTF(o.toString());
objects.put(o, objects.size());
}
diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcilingRandom.java b/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcilingRandom.java
index 444ec44..99df4b9 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcilingRandom.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcilingRandom.java
@@ -20,24 +20,29 @@
import java.util.function.Supplier;
+import org.agrona.collections.Long2LongHashMap;
import org.apache.cassandra.simulator.RandomSource;
-import org.hsqldb.lib.IntKeyLongValueHashMap;
import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM;
public class SelfReconcilingRandom implements Supplier<RandomSource>
{
- static class Map extends IntKeyLongValueHashMap
+ static class Map extends Long2LongHashMap
{
+ public Map(long missingValue)
+ {
+ super(missingValue);
+ }
+
public boolean put(int i, long v)
{
int size = this.size();
- super.addOrRemove((long)i, (long)v, (Object)null, (Object)null, false);
+ super.put(i, v);
return size != this.size();
}
}
- final Map map = new Map();
- long[] tmp = new long[1];
+
+ final Map map = new Map(Long.MIN_VALUE);
boolean isNextPrimary = true;
static abstract class AbstractVerifying extends RandomSource.Abstract
@@ -113,10 +118,11 @@
{
void next(long v)
{
- if (!map.get(++cur, tmp))
+ long value = map.get(++cur);
+ if (value == Long.MIN_VALUE)
throw failWithOOM();
map.remove(cur);
- if (tmp[0] != v)
+ if (value != v)
throw failWithOOM();
}
diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
index 08e1b2e..60ebf47 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
@@ -42,17 +42,17 @@
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.simulator.systems.NonInterceptible;
-import org.apache.cassandra.simulator.systems.NonInterceptible.Permit;
-import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.service.paxos.Commit;
import org.apache.cassandra.service.paxos.PaxosState;
+import org.apache.cassandra.simulator.systems.NonInterceptible;
+import org.apache.cassandra.simulator.systems.NonInterceptible.Permit;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Shared;
+import org.apache.cassandra.utils.TimeUUID;
import static java.lang.Long.max;
import static java.util.Arrays.stream;
diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java
index 63e84dc..a6fbc44 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java
@@ -26,17 +26,14 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.LongSupplier;
-
import javax.annotation.Nullable;
import com.google.common.base.Throwables;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ExecutorFactory;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
-import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInvokableInstance;
@@ -55,8 +52,9 @@
import org.apache.cassandra.utils.concurrent.Threads;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
-import static org.apache.cassandra.simulator.Action.Modifiers.NONE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_LIVENESS_CHECK;
import static org.apache.cassandra.simulator.Action.Modifiers.DISPLAY_ORIGIN;
+import static org.apache.cassandra.simulator.Action.Modifiers.NONE;
import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM;
import static org.apache.cassandra.simulator.paxos.HistoryChecker.causedBy;
@@ -133,7 +131,7 @@
AtomicLong counter = new AtomicLong();
ScheduledExecutorPlus livenessChecker = null;
ScheduledFuture<?> liveness = null;
- if (CassandraRelevantProperties.TEST_SIMULATOR_LIVENESS_CHECK.getBoolean())
+ if (TEST_SIMULATOR_LIVENESS_CHECK.getBoolean())
{
livenessChecker = ExecutorFactory.Global.executorFactory().scheduled("SimulationLiveness");
liveness = livenessChecker.scheduleWithFixedDelay(new Runnable()
@@ -170,7 +168,7 @@
long cur = counter.get();
if (cur == prev)
{
- logger.error("Simulation appears to have stalled; terminating. To disable set -Dcassandra.test.simulator.livenesscheck=false");
+ logger.error("Simulation appears to have stalled; terminating. To disable set -D{}=false", TEST_SIMULATOR_LIVENESS_CHECK.getKey());
shutdown.set(1);
throw failWithOOM();
}
@@ -262,8 +260,9 @@
}
log(causedByPrimaryKey);
- if (causedByPrimaryKey != null) throw Throwables.propagate(causedByThrowable);
- else throw Throwables.propagate(simulated.failures.get().get(0));
+ Throwable t = (causedByPrimaryKey != null) ? causedByThrowable : simulated.failures.get().get(0);
+ Throwables.throwIfUnchecked(t);
+ throw new RuntimeException(t);
}
public void close()
diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java
index 19d6601..f195f1b 100644
--- a/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java
+++ b/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java
@@ -40,3 +40,4 @@
PaxosSimulationRunner.main(new String[] { "reconcile", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30", "--with-self" });
}
}
+
diff --git a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java
index 6503707..ebb74a7 100644
--- a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java
+++ b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java
@@ -27,17 +27,19 @@
import java.util.HashMap;
import java.util.Map;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_VERSION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SERIALIZATION_WRITES;
+
public class AbstractSerializationsTester
{
- protected static final String CUR_VER = System.getProperty("cassandra.version", "4.1");
+ protected static final String CUR_VER = CASSANDRA_VERSION.getString("4.0");
protected static final Map<String, Integer> VERSION_MAP = new HashMap<String, Integer> ()
{{
put("3.0", MessagingService.VERSION_30);
put("4.0", MessagingService.VERSION_40);
- put("4.1", MessagingService.VERSION_41);
}};
- protected static final boolean EXECUTE_WRITES = Boolean.getBoolean("cassandra.test-serialization-writes");
+ protected static final boolean EXECUTE_WRITES = TEST_SERIALIZATION_WRITES.getBoolean();
protected static int getVersion()
{
diff --git a/test/unit/org/apache/cassandra/CassandraBriefJUnitResultFormatter.java b/test/unit/org/apache/cassandra/CassandraBriefJUnitResultFormatter.java
index 2befb5c..dc9b8f8 100644
--- a/test/unit/org/apache/cassandra/CassandraBriefJUnitResultFormatter.java
+++ b/test/unit/org/apache/cassandra/CassandraBriefJUnitResultFormatter.java
@@ -36,6 +36,9 @@
import org.apache.tools.ant.util.FileUtils;
import org.apache.tools.ant.util.StringUtils;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_KEEPBRIEFBRIEF;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_TESTTAG;
+
/**
* Prints plain text output of the test to a specified Writer.
* Inspired by the PlainJUnitResultFormatter.
@@ -47,9 +50,9 @@
private static final double ONE_SECOND = 1000.0;
- private static final String tag = System.getProperty("cassandra.testtag", "");
+ private static final String tag = TEST_CASSANDRA_TESTTAG.getString();
- private static final Boolean keepBriefBrief = Boolean.getBoolean("cassandra.keepBriefBrief");
+ private static final Boolean keepBriefBrief = TEST_CASSANDRA_KEEPBRIEFBRIEF.getBoolean();
/**
* Where to write the log to.
diff --git a/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java b/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java
index de8fb4e..6f821c3 100644
--- a/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java
+++ b/test/unit/org/apache/cassandra/CassandraXMLJUnitResultFormatter.java
@@ -50,6 +50,9 @@
import org.w3c.dom.Element;
import org.w3c.dom.Text;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_SUITENAME;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_TESTTAG;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_JAVA_COMMAND;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
@@ -73,7 +76,7 @@
}
}
- private static final String tag = System.getProperty("cassandra.testtag", "");
+ private static final String tag = TEST_CASSANDRA_TESTTAG.getString();
/*
* Set the property for the test suite name so that log configuration can pick it up
@@ -81,9 +84,9 @@
*/
static
{
- String command = System.getProperty("sun.java.command");
+ String command = SUN_JAVA_COMMAND.getString();
String args[] = command.split(" ");
- System.setProperty("suitename", args[1]);
+ TEST_CASSANDRA_SUITENAME.setString(args[1]);
}
/**
diff --git a/test/unit/org/apache/cassandra/LogbackStatusListener.java b/test/unit/org/apache/cassandra/LogbackStatusListener.java
index 719fada..834d51f 100644
--- a/test/unit/org/apache/cassandra/LogbackStatusListener.java
+++ b/test/unit/org/apache/cassandra/LogbackStatusListener.java
@@ -33,8 +33,12 @@
import ch.qos.logback.classic.spi.LoggerContextListener;
import ch.qos.logback.core.status.Status;
import ch.qos.logback.core.status.StatusListener;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.distributed.shared.InstanceClassLoader;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_STDERR_ENCODING;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_STDOUT_ENCODING;
+
/*
* Listen for logback readiness and then redirect stdout/stderr to logback
*/
@@ -84,9 +88,9 @@
}
}
- private static PrintStream wrapLogger(Logger logger, PrintStream original, String encodingProperty, boolean error) throws Exception
+ private static PrintStream wrapLogger(Logger logger, PrintStream original, CassandraRelevantProperties encodingProperty, boolean error) throws Exception
{
- final String encoding = System.getProperty(encodingProperty);
+ final String encoding = encodingProperty.getString();
OutputStream os = new ToLoggerOutputStream(logger, encoding, error);
return encoding != null ? new WrappedPrintStream(os, true, encoding, original)
: new WrappedPrintStream(os, true, original);
@@ -475,9 +479,9 @@
Logger stdoutLogger = LoggerFactory.getLogger("stdout");
Logger stderrLogger = LoggerFactory.getLogger("stderr");
- replacementOut = wrapLogger(stdoutLogger, originalOut, "sun.stdout.encoding", false);
+ replacementOut = wrapLogger(stdoutLogger, originalOut, SUN_STDOUT_ENCODING, false);
System.setOut(replacementOut);
- replacementErr = wrapLogger(stderrLogger, originalErr, "sun.stderr.encoding", true);
+ replacementErr = wrapLogger(stderrLogger, originalErr, SUN_STDERR_ENCODING, true);
System.setErr(replacementErr);
}
catch (Exception e)
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 3279490..f0178d1 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -50,6 +50,9 @@
import org.junit.After;
import org.junit.BeforeClass;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_JOIN;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_COMPRESSION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_COMPRESSION_ALGO;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class SchemaLoader
@@ -82,7 +85,7 @@
public static void startGossiper()
{
// skip shadow round and endpoint collision check in tests
- System.setProperty("cassandra.allow_unsafe_join", "true");
+ ALLOW_UNSAFE_JOIN.setBoolean(true);
if (!Gossiper.instance.isEnabled())
Gossiper.instance.start((int) (currentTimeMillis() / 1000));
}
@@ -249,7 +252,7 @@
for (KeyspaceMetadata ksm : schema)
SchemaTestUtil.announceNewKeyspace(ksm);
- if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
+ if (TEST_COMPRESSION.getBoolean())
useCompression(schema, compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH));
}
@@ -274,7 +277,7 @@
public static void createKeyspace(String name, KeyspaceParams params, Tables tables, Types types)
{
- SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables, Views.none(), types, Functions.none()));
+ SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables, Views.none(), types, UserFunctions.none()));
}
public static void setupAuth(IRoleManager roleManager, IAuthenticator authenticator, IAuthorizer authorizer, INetworkAuthorizer networkAuthorizer)
@@ -286,6 +289,7 @@
SchemaTestUtil.announceNewKeyspace(AuthKeyspace.metadata());
DatabaseDescriptor.getRoleManager().setup();
DatabaseDescriptor.getAuthenticator().setup();
+ DatabaseDescriptor.getInternodeAuthenticator().setupInternode();
DatabaseDescriptor.getAuthorizer().setup();
DatabaseDescriptor.getNetworkAuthorizer().setup();
Schema.instance.registerListener(new AuthSchemaChangeListener());
@@ -298,7 +302,8 @@
ColumnIdentifier.getInterned(IntegerType.instance.fromString("42"), IntegerType.instance),
UTF8Type.instance,
ColumnMetadata.NO_POSITION,
- ColumnMetadata.Kind.REGULAR);
+ ColumnMetadata.Kind.REGULAR,
+ null);
}
public static ColumnMetadata utf8Column(String ksName, String cfName)
@@ -308,7 +313,8 @@
ColumnIdentifier.getInterned("fortytwo", true),
UTF8Type.instance,
ColumnMetadata.NO_POSITION,
- ColumnMetadata.Kind.REGULAR);
+ ColumnMetadata.Kind.REGULAR,
+ null);
}
public static TableMetadata perRowIndexedCFMD(String ksName, String cfName)
@@ -722,7 +728,7 @@
public static CompressionParams getCompressionParameters(Integer chunkSize)
{
- if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
+ if (TEST_COMPRESSION.getBoolean())
return chunkSize != null ? compressionParams(chunkSize) : compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH);
return CompressionParams.noCompression();
@@ -756,7 +762,7 @@
private static CompressionParams compressionParams(int chunkLength)
{
- String algo = System.getProperty("cassandra.test.compression.algo", "lz4").toLowerCase();
+ String algo = TEST_COMPRESSION_ALGO.getString().toLowerCase();
switch (algo)
{
case "deflate":
diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java
index 10cb082..fa59f0d 100644
--- a/test/unit/org/apache/cassandra/ServerTestUtils.java
+++ b/test/unit/org/apache/cassandra/ServerTestUtils.java
@@ -21,16 +21,22 @@
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.audit.AuditLogManager;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.AbstractEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -203,4 +209,22 @@
private ServerTestUtils()
{
}
+
+ public static List<BigTableReader> getLiveBigTableReaders(ColumnFamilyStore cfs)
+ {
+ return cfs.getLiveSSTables()
+ .stream()
+ .filter(BigTableReader.class::isInstance)
+ .map(BigTableReader.class::cast)
+ .collect(Collectors.toList());
+ }
+
+ public static <R extends SSTableReader & IndexSummarySupport<R>> List<R> getLiveIndexSummarySupportingReaders(ColumnFamilyStore cfs)
+ {
+ return cfs.getLiveSSTables()
+ .stream()
+ .filter(IndexSummarySupport.class::isInstance)
+ .map(r -> (R) r)
+ .collect(Collectors.toList());
+ }
}
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index 0459cb3..3e36692 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -20,9 +20,12 @@
*/
import java.io.Closeable;
+import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOError;
import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigInteger;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
@@ -113,6 +116,8 @@
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
@@ -136,6 +141,8 @@
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Throwables;
import org.awaitility.Awaitility;
+import org.mockito.Mockito;
+import org.mockito.internal.stubbing.defaultanswers.ForwardsInvocations;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
@@ -464,10 +471,10 @@
assert iterator.hasNext() : "Expecting one row in one partition but got nothing";
try (RowIterator partition = iterator.next())
{
- assert !iterator.hasNext() : "Expecting a single partition but got more";
assert partition.hasNext() : "Expecting one row in one partition but got an empty partition";
Row row = partition.next();
assert !partition.hasNext() : "Expecting a single row but got more";
+ assert !iterator.hasNext() : "Expecting a single partition but got more";
return row;
}
}
@@ -850,7 +857,7 @@
{
if (sst.name().contains("Data"))
{
- Descriptor d = Descriptor.fromFilename(sst.absolutePath());
+ Descriptor d = Descriptor.fromFileWithComponent(sst, false).left;
assertTrue(liveIdentifiers.contains(d.id));
fileCount++;
}
@@ -859,6 +866,199 @@
assertEquals(expectedSSTableCount, fileCount);
}
+ public static ByteBuffer generateMurmurCollision(ByteBuffer original, byte... bytesToAdd)
+ {
+ // Round size up to 16, and add another 16 bytes
+ ByteBuffer collision = ByteBuffer.allocate((original.remaining() + bytesToAdd.length + 31) & -16);
+ collision.put(original); // we can use this as a copy of original with 0s appended at the end
+
+ original.flip();
+
+ long c1 = 0x87c37b91114253d5L;
+ long c2 = 0x4cf5ad432745937fL;
+
+ long h1 = 0;
+ long h2 = 0;
+
+ // Get hash of original
+ int index = 0;
+ final int length = original.limit();
+ while (index <= length - 16)
+ {
+ long k1 = Long.reverseBytes(collision.getLong(index + 0));
+ long k2 = Long.reverseBytes(collision.getLong(index + 8));
+
+ // 16 bytes
+ k1 *= c1;
+ k1 = rotl64(k1, 31);
+ k1 *= c2;
+ h1 ^= k1;
+ h1 = rotl64(h1, 27);
+ h1 += h2;
+ h1 = h1 * 5 + 0x52dce729;
+ k2 *= c2;
+ k2 = rotl64(k2, 33);
+ k2 *= c1;
+ h2 ^= k2;
+ h2 = rotl64(h2, 31);
+ h2 += h1;
+ h2 = h2 * 5 + 0x38495ab5;
+
+ index += 16;
+ }
+
+ long oh1 = h1;
+ long oh2 = h2;
+
+ // Process final unfilled chunk, but only adjust the original hash value
+ if (index < length)
+ {
+ long k1 = Long.reverseBytes(collision.getLong(index + 0));
+ long k2 = Long.reverseBytes(collision.getLong(index + 8));
+
+ // 16 bytes
+ k1 *= c1;
+ k1 = rotl64(k1, 31);
+ k1 *= c2;
+ oh1 ^= k1;
+
+ k2 *= c2;
+ k2 = rotl64(k2, 33);
+ k2 *= c1;
+ oh2 ^= k2;
+ }
+
+ // These are the hashes the original would provide, before final mixing
+ oh1 ^= original.capacity();
+ oh2 ^= original.capacity();
+
+ // Fill in the remaining bytes before the last 16 and get their hash
+ collision.put(bytesToAdd);
+ while ((collision.position() & 0x0f) != 0)
+ collision.put((byte) 0);
+
+ while (index < collision.position())
+ {
+ long k1 = Long.reverseBytes(collision.getLong(index + 0));
+ long k2 = Long.reverseBytes(collision.getLong(index + 8));
+
+ // 16 bytes
+ k1 *= c1;
+ k1 = rotl64(k1, 31);
+ k1 *= c2;
+ h1 ^= k1;
+ h1 = rotl64(h1, 27);
+ h1 += h2;
+ h1 = h1 * 5 + 0x52dce729;
+ k2 *= c2;
+ k2 = rotl64(k2, 33);
+ k2 *= c1;
+ h2 ^= k2;
+ h2 = rotl64(h2, 31);
+ h2 += h1;
+ h2 = h2 * 5 + 0x38495ab5;
+
+ index += 16;
+ }
+
+ // Working backwards, we must get this hash pair
+ long th1 = h1;
+ long th2 = h2;
+
+ // adjust ohx with length
+ h1 = oh1 ^ collision.capacity();
+ h2 = oh2 ^ collision.capacity();
+
+ // Get modulo-long inverses of the multipliers used in the computation
+ long i5i = inverse(5L);
+ long c1i = inverse(c1);
+ long c2i = inverse(c2);
+
+ // revert one step
+ h2 -= 0x38495ab5;
+ h2 *= i5i;
+ h2 -= h1;
+ h2 = rotl64(h2, 33);
+
+ h1 -= 0x52dce729;
+ h1 *= i5i;
+ h1 -= th2; // use h2 before it's adjusted with k2
+ h1 = rotl64(h1, 37);
+
+ // extract the required modifiers and applies the inverse of their transformation
+ long k1 = h1 ^ th1;
+ k1 = c2i * k1;
+ k1 = rotl64(k1, 33);
+ k1 = c1i * k1;
+
+ long k2 = h2 ^ th2;
+ k2 = c1i * k2;
+ k2 = rotl64(k2, 31);
+ k2 = c2i * k2;
+
+ collision.putLong(Long.reverseBytes(k1));
+ collision.putLong(Long.reverseBytes(k2));
+ collision.flip();
+
+ return collision;
+ }
+
+ // Assumes a and b are positive
+ private static BigInteger[] xgcd(BigInteger a, BigInteger b) {
+ BigInteger x = a, y = b;
+ BigInteger[] qrem;
+ BigInteger[] result = new BigInteger[3];
+ BigInteger x0 = BigInteger.ONE, x1 = BigInteger.ZERO;
+ BigInteger y0 = BigInteger.ZERO, y1 = BigInteger.ONE;
+ while (true)
+ {
+ qrem = x.divideAndRemainder(y);
+ x = qrem[1];
+ x0 = x0.subtract(y0.multiply(qrem[0]));
+ x1 = x1.subtract(y1.multiply(qrem[0]));
+ if (x.equals(BigInteger.ZERO))
+ {
+ result[0] = y;
+ result[1] = y0;
+ result[2] = y1;
+ return result;
+ }
+
+ qrem = y.divideAndRemainder(x);
+ y = qrem[1];
+ y0 = y0.subtract(x0.multiply(qrem[0]));
+ y1 = y1.subtract(x1.multiply(qrem[0]));
+ if (y.equals(BigInteger.ZERO))
+ {
+ result[0] = x;
+ result[1] = x0;
+ result[2] = x1;
+ return result;
+ }
+ }
+ }
+
+ /**
+ * Find a mupltiplicative inverse for the given multiplier for long, i.e.
+ * such that x * inverse(x) = 1 where * is long multiplication.
+ * In other words, such an integer that x * inverse(x) == 1 (mod 2^64).
+ */
+ public static long inverse(long multiplier)
+ {
+ final BigInteger modulus = BigInteger.ONE.shiftLeft(64);
+ // Add the modulus to the multiplier to avoid problems with negatives (a + m == a (mod m))
+ BigInteger[] gcds = xgcd(BigInteger.valueOf(multiplier).add(modulus), modulus);
+ // xgcd gives g, a and b, such that ax + bm = g
+ // ie, ax = g (mod m). Return a
+ assert gcds[0].equals(BigInteger.ONE) : "Even number " + multiplier + " has no long inverse";
+ return gcds[1].longValueExact();
+ }
+
+ public static long rotl64(long v, int n)
+ {
+ return ((v << n) | (v >>> (64 - n)));
+ }
+
/**
* Disable bloom filter on all sstables of given table
*/
@@ -869,7 +1069,7 @@
{
for (SSTableReader sstable : sstables)
{
- sstable = sstable.cloneAndReplace(FilterFactory.AlwaysPresent);
+ sstable = ((SSTableReaderWithFilter) sstable).cloneAndReplace(FilterFactory.AlwaysPresent);
txn.update(sstable, true);
txn.checkpoint();
}
@@ -877,7 +1077,7 @@
}
for (SSTableReader reader : cfs.getLiveSSTables())
- assertEquals(FilterFactory.AlwaysPresent, reader.getBloomFilter());
+ assertEquals(0, ((SSTableReaderWithFilter) reader).getFilterOffHeapSize());
}
/**
@@ -1042,4 +1242,23 @@
{
view.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
}
+
+ public static class DataInputStreamPlusImpl extends DataInputStream implements DataInputPlus
+ {
+ private DataInputStreamPlusImpl(InputStream in)
+ {
+ super(in);
+ }
+
+ public static DataInputStreamPlus wrap(InputStream in)
+ {
+ DataInputStreamPlusImpl impl = new DataInputStreamPlusImpl(in);
+ return Mockito.mock(DataInputStreamPlus.class, new ForwardsInvocations(impl));
+ }
+ }
+
+ public static RuntimeException testMustBeImplementedForSSTableFormat()
+ {
+ return new UnsupportedOperationException("Test must be implemented for sstable format " + DatabaseDescriptor.getSelectedSSTableFormat().getClass().getName());
+ }
}
diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
index 9a3c605..0ced5a1 100644
--- a/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
+++ b/test/unit/org/apache/cassandra/audit/AuditLoggerAuthTest.java
@@ -42,6 +42,7 @@
import org.apache.cassandra.service.EmbeddedCassandraService;
import org.hamcrest.CoreMatchers;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS;
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
@@ -75,7 +76,7 @@
config.audit_logging_options.logger = new ParameterizedClass("InMemoryAuditLogger", null);
});
- System.setProperty("cassandra.superuser_setup_delay_ms", "0");
+ SUPERUSER_SETUP_DELAY_MS.setLong(0);
embedded = ServerTestUtils.startEmbeddedCassandraService();
executeWithCredentials(
diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerCleanupTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerCleanupTest.java
index 891827c..ebc5ead 100644
--- a/test/unit/org/apache/cassandra/audit/AuditLoggerCleanupTest.java
+++ b/test/unit/org/apache/cassandra/audit/AuditLoggerCleanupTest.java
@@ -41,6 +41,7 @@
import org.apache.cassandra.service.StorageService;
import static java.nio.file.Files.list;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -69,7 +70,8 @@
emptyCq4File = Files.createFile(auditLogDirRoot.toPath().resolve("20220928-12" + SingleChronicleQueue.SUFFIX)).toFile();
emptyMetadataFile = Files.createFile(auditLogDirRoot.toPath().resolve("metadata" + SingleTableStore.SUFFIX)).toFile();
- System.setProperty("cassandra.superuser_setup_delay_ms", "0");
+ SUPERUSER_SETUP_DELAY_MS.setLong(0);
+
embedded = new EmbeddedCassandraService();
embedded.start();
}
diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java
index 1ee66f7..9dfd171 100644
--- a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java
+++ b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java
@@ -496,7 +496,7 @@
{
String tblName = createTableName();
- String cql = "CREATE FUNCTION IF NOT EXISTS " + KEYSPACE + "." + tblName + " (column TEXT,num int) RETURNS NULL ON NULL INPUT RETURNS text LANGUAGE javascript AS $$ column.substring(0,num) $$";
+ String cql = "CREATE FUNCTION IF NOT EXISTS " + KEYSPACE + "." + tblName + " (column TEXT,num int) RETURNS NULL ON NULL INPUT RETURNS text LANGUAGE java AS $$ return column.substring(0,num); $$";
executeAndAssert(cql, AuditLogEntryType.CREATE_FUNCTION);
cql = "DROP FUNCTION " + KEYSPACE + "." + tblName;
diff --git a/test/unit/org/apache/cassandra/auth/AuthCacheTest.java b/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
index 0e22346..50a0a9c 100644
--- a/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
+++ b/test/unit/org/apache/cassandra/auth/AuthCacheTest.java
@@ -33,6 +33,8 @@
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.exceptions.UnavailableException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_CACHE_WARMING_MAX_RETRIES;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_CACHE_WARMING_RETRY_INTERVAL_MS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -264,8 +266,8 @@
@Test
public void handleProviderErrorDuringWarming()
{
- System.setProperty(AuthCache.CACHE_LOAD_RETRIES_PROPERTY, "3");
- System.setProperty(AuthCache.CACHE_LOAD_RETRY_INTERVAL_PROPERTY, "0");
+ AUTH_CACHE_WARMING_MAX_RETRIES.setInt(3);
+ AUTH_CACHE_WARMING_RETRY_INTERVAL_MS.setLong(0);
final AtomicInteger attempts = new AtomicInteger(0);
Supplier<Map<String, Integer>> bulkLoader = () -> {
diff --git a/test/unit/org/apache/cassandra/auth/CreateAndAlterRoleTest.java b/test/unit/org/apache/cassandra/auth/CreateAndAlterRoleTest.java
index 33ae129..6bb83b3 100644
--- a/test/unit/org/apache/cassandra/auth/CreateAndAlterRoleTest.java
+++ b/test/unit/org/apache/cassandra/auth/CreateAndAlterRoleTest.java
@@ -21,11 +21,10 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import com.datastax.driver.core.exceptions.AuthenticationException;
-import org.apache.cassandra.Util;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
-import static org.junit.Assert.assertTrue;
import static org.mindrot.jbcrypt.BCrypt.gensalt;
import static org.mindrot.jbcrypt.BCrypt.hashpw;
@@ -34,6 +33,12 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
+ DatabaseDescriptor.setPermissionsValidity(0);
+ DatabaseDescriptor.setRolesValidity(0);
+ DatabaseDescriptor.setCredentialsValidity(0);
+
CQLTester.setUpClass();
requireAuthentication();
requireNetwork();
@@ -62,11 +67,11 @@
useUser(user1, plainTextPwd);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
useUser(user2, plainTextPwd);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
useSuperUser();
@@ -78,11 +83,11 @@
useUser(user1, plainTextPwd2);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
useUser(user2, plainTextPwd2);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
}
@Test
@@ -105,11 +110,11 @@
useUser(user1, plainTextPwd);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
useUser(user2, plainTextPwd);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
useSuperUser();
@@ -118,32 +123,10 @@
useUser(user1, plainTextPwd2);
- executeNetWithAuthSpin("SELECT key FROM system.local");
+ executeNet("SELECT key FROM system.local");
useUser(user2, plainTextPwd2);
- executeNetWithAuthSpin("SELECT key FROM system.local");
- }
-
- /**
- * Altering or creating auth may take some time to be effective
- *
- * @param query
- */
- void executeNetWithAuthSpin(String query)
- {
- Util.spinAssertEquals(true, () -> {
- try
- {
- executeNet(query);
- return true;
- }
- catch (Throwable e)
- {
- assertTrue("Unexpected exception: " + e, e instanceof AuthenticationException);
- reinitializeNetwork();
- return false;
- }
- }, 10);
+ executeNet("SELECT key FROM system.local");
}
}
diff --git a/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java b/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java
index 5c1c2a0..eeff052 100644
--- a/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java
+++ b/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java
@@ -22,6 +22,7 @@
import org.junit.Test;
import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
@@ -36,6 +37,7 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
DatabaseDescriptor.setPermissionsValidity(0);
CQLTester.setUpClass();
requireAuthentication();
diff --git a/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java b/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java
index e1033ff..0a8efc2 100644
--- a/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java
+++ b/test/unit/org/apache/cassandra/auth/PasswordAuthenticatorTest.java
@@ -26,6 +26,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.mindrot.jbcrypt.BCrypt;
import com.datastax.driver.core.Authenticator;
@@ -45,14 +46,13 @@
import static org.apache.cassandra.auth.CassandraRoleManager.getGensaltLogRounds;
import static org.apache.cassandra.auth.PasswordAuthenticator.SaslNegotiator;
import static org.apache.cassandra.auth.PasswordAuthenticator.checkpw;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_BCRYPT_GENSALT_LOG2_ROUNDS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mindrot.jbcrypt.BCrypt.gensalt;
import static org.mindrot.jbcrypt.BCrypt.hashpw;
-import static org.apache.cassandra.auth.CassandraRoleManager.GENSALT_LOG2_ROUNDS_PROPERTY;
-
public class PasswordAuthenticatorTest extends CQLTester
{
private final static PasswordAuthenticator authenticator = new PasswordAuthenticator();
@@ -119,19 +119,10 @@
private void executeSaltRoundsPropertyTest(Integer rounds)
{
- String oldProperty = System.getProperty(GENSALT_LOG2_ROUNDS_PROPERTY);
- try
+ try (WithProperties properties = new WithProperties().set(AUTH_BCRYPT_GENSALT_LOG2_ROUNDS, rounds))
{
- System.setProperty(GENSALT_LOG2_ROUNDS_PROPERTY, rounds.toString());
getGensaltLogRounds();
- Assert.fail("Property " + GENSALT_LOG2_ROUNDS_PROPERTY + " must be in interval [4,30]");
- }
- finally
- {
- if (oldProperty != null)
- System.setProperty(GENSALT_LOG2_ROUNDS_PROPERTY, oldProperty);
- else
- System.clearProperty(GENSALT_LOG2_ROUNDS_PROPERTY);
+ Assert.fail("Property " + AUTH_BCRYPT_GENSALT_LOG2_ROUNDS.getKey() + " must be in interval [4,30]");
}
}
diff --git a/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java b/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java
index 3bc28a9..fece405 100644
--- a/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java
+++ b/test/unit/org/apache/cassandra/auth/jmx/JMXAuthTest.java
@@ -43,6 +43,10 @@
import org.apache.cassandra.db.ColumnFamilyStoreMBean;
import org.apache.cassandra.utils.JMXServerUtils;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_AUTHORIZER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_REMOTE_LOGIN_CONFIG;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_AUTHENTICATE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_SECURITY_AUTH_LOGIN_CONFIG;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -86,10 +90,10 @@
private static void setupJMXServer() throws Exception
{
String config = Paths.get(ClassLoader.getSystemResource("auth/cassandra-test-jaas.conf").toURI()).toString();
- System.setProperty("com.sun.management.jmxremote.authenticate", "true");
- System.setProperty("java.security.auth.login.config", config);
- System.setProperty("cassandra.jmx.remote.login.config", "TestLogin");
- System.setProperty("cassandra.jmx.authorizer", NoSuperUserAuthorizationProxy.class.getName());
+ COM_SUN_MANAGEMENT_JMXREMOTE_AUTHENTICATE.setBoolean(true);
+ JAVA_SECURITY_AUTH_LOGIN_CONFIG.setString(config);
+ CASSANDRA_JMX_REMOTE_LOGIN_CONFIG.setString("TestLogin");
+ CASSANDRA_JMX_AUTHORIZER.setString(NoSuperUserAuthorizationProxy.class.getName());
jmxServer = JMXServerUtils.createJMXServer(9999, "localhost", true);
jmxServer.start();
diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
index 1fb3735..90a1d73 100644
--- a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
+++ b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java
@@ -170,7 +170,7 @@
for (int i = 0; i < 100; i++)
{
- String query = String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = intAsBlob(%d)", KEYSPACE1, CF_STANDARD1, i);
+ String query = String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = int_as_blob(%d)", KEYSPACE1, CF_STANDARD1, i);
UntypedResultSet result = executeInternal(query);
assertNotNull(result);
if (i < 50)
@@ -248,7 +248,7 @@
// We should see half of Standard2-targeted mutations written after the replay and all of Standard3 mutations applied.
for (int i = 0; i < 1000; i++)
{
- UntypedResultSet result = executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = intAsBlob(%d)", KEYSPACE1, CF_STANDARD2,i));
+ UntypedResultSet result = executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = int_as_blob(%d)", KEYSPACE1, CF_STANDARD2,i));
assertNotNull(result);
if (i >= 500)
{
@@ -264,7 +264,7 @@
for (int i = 0; i < 1000; i++)
{
- UntypedResultSet result = executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = intAsBlob(%d)", KEYSPACE1, CF_STANDARD3, i));
+ UntypedResultSet result = executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE key = int_as_blob(%d)", KEYSPACE1, CF_STANDARD3, i));
assertNotNull(result);
assertEquals(ByteBufferUtil.bytes(i), result.one().getBytes("key"));
assertEquals("name" + i, result.one().getString("name"));
diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
index 040409e..50145e1 100644
--- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
+++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
@@ -17,20 +17,25 @@
*/
package org.apache.cassandra.cache;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -44,6 +49,8 @@
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
+ DatabaseDescriptor.daemonInitialization();
+ Assume.assumeTrue(KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
@@ -85,7 +92,7 @@
for (SSTableReader sstable : cfs.getLiveSSTables())
sstable.getPosition(Util.dk("key1"), SSTableReader.Operator.EQ);
- AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = CacheService.instance.keyCache;
+ AutoSavingCache<KeyCacheKey, AbstractRowIndexEntry> keyCache = CacheService.instance.keyCache;
// serialize to file
keyCache.submitWrite(keyCache.size()).get();
diff --git a/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java b/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java
index de6e0d2..7a3e0cd 100644
--- a/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java
+++ b/test/unit/org/apache/cassandra/config/CassandraRelevantPropertiesTest.java
@@ -18,26 +18,41 @@
package org.apache.cassandra.config;
+import org.junit.Before;
import org.junit.Test;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_CASSANDRA_RELEVANT_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
public class CassandraRelevantPropertiesTest
{
- private static final CassandraRelevantProperties TEST_PROP = CassandraRelevantProperties.ORG_APACHE_CASSANDRA_CONF_CASSANDRA_RELEVANT_PROPERTIES_TEST;
+ @Before
+ public void setup()
+ {
+ System.clearProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey());
+ }
+
+ @Test
+ public void testSystemPropertyisSet() {
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "test"))
+ {
+ Assertions.assertThat(System.getProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey())).isEqualTo("test"); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
+ }
+ }
@Test
public void testString()
{
- try
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "some-string"))
{
- System.setProperty(TEST_PROP.getKey(), "some-string");
- Assertions.assertThat(TEST_PROP.getString()).isEqualTo("some-string");
- }
- finally
- {
- System.clearProperty(TEST_PROP.getKey());
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString()).isEqualTo("some-string");
}
}
@@ -46,100 +61,84 @@
{
try
{
- System.setProperty(TEST_PROP.getKey(), "true");
- Assertions.assertThat(TEST_PROP.getBoolean()).isEqualTo(true);
- System.setProperty(TEST_PROP.getKey(), "false");
- Assertions.assertThat(TEST_PROP.getBoolean()).isEqualTo(false);
- System.setProperty(TEST_PROP.getKey(), "junk");
- Assertions.assertThat(TEST_PROP.getBoolean()).isEqualTo(false);
- System.setProperty(TEST_PROP.getKey(), "");
- Assertions.assertThat(TEST_PROP.getBoolean()).isEqualTo(false);
+ System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "true");
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(true);
+ System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "false");
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(false);
+ System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "junk");
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(false);
+ System.setProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey(), "");
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isEqualTo(false);
}
finally
{
- System.clearProperty(TEST_PROP.getKey());
+ System.clearProperty(TEST_CASSANDRA_RELEVANT_PROPERTIES.getKey());
}
}
@Test
public void testBoolean_null()
{
- try
+ try (WithProperties properties = new WithProperties())
{
- TEST_PROP.getBoolean();
- }
- finally
- {
- System.clearProperty(TEST_PROP.getKey());
+ TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean();
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getBoolean()).isFalse();
}
}
@Test
public void testDecimal()
{
- try
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "123456789"))
{
- System.setProperty(TEST_PROP.getKey(), "123456789");
- Assertions.assertThat(TEST_PROP.getInt()).isEqualTo(123456789);
- }
- finally
- {
- System.clearProperty(TEST_PROP.getKey());
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(123456789);
}
}
@Test
public void testHexadecimal()
{
- try
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "0x1234567a"))
{
- System.setProperty(TEST_PROP.getKey(), "0x1234567a");
- Assertions.assertThat(TEST_PROP.getInt()).isEqualTo(305419898);
- }
- finally
- {
- System.clearProperty(TEST_PROP.getKey());
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(305419898);
}
}
@Test
public void testOctal()
{
- try
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "01234567"))
{
- System.setProperty(TEST_PROP.getKey(), "01234567");
- Assertions.assertThat(TEST_PROP.getInt()).isEqualTo(342391);
- }
- finally
- {
- System.clearProperty(TEST_PROP.getKey());
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(342391);
}
}
@Test(expected = ConfigurationException.class)
public void testInteger_empty()
{
- try
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, ""))
{
- System.setProperty(TEST_PROP.getKey(), "");
- Assertions.assertThat(TEST_PROP.getInt()).isEqualTo(342391);
- }
- finally
- {
- System.clearProperty(TEST_PROP.getKey());
+ Assertions.assertThat(TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt()).isEqualTo(342391);
}
}
- @Test(expected = NullPointerException.class)
+ @Test(expected = ConfigurationException.class)
public void testInteger_null()
{
- try
+ try (WithProperties properties = new WithProperties())
{
- TEST_PROP.getInt();
+ TEST_CASSANDRA_RELEVANT_PROPERTIES.getInt();
}
- finally
+ }
+
+ @Test
+ public void testClearProperty()
+ {
+ assertNull(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString());
+ try (WithProperties properties = new WithProperties().set(TEST_CASSANDRA_RELEVANT_PROPERTIES, "test"))
{
- System.clearProperty(TEST_PROP.getKey());
+ assertEquals("test", TEST_CASSANDRA_RELEVANT_PROPERTIES.getString());
}
+ assertNull(TEST_CASSANDRA_RELEVANT_PROPERTIES.getString());
}
}
diff --git a/test/unit/org/apache/cassandra/config/ConfigCompatabilityTest.java b/test/unit/org/apache/cassandra/config/ConfigCompatabilityTest.java
index 8d1e5e2..4a62033 100644
--- a/test/unit/org/apache/cassandra/config/ConfigCompatabilityTest.java
+++ b/test/unit/org/apache/cassandra/config/ConfigCompatabilityTest.java
@@ -200,14 +200,14 @@
private static ClassTree load(String path) throws IOException
{
- ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); // checkstyle: permit this instantiation
return mapper.readValue(new File(path), ClassTree.class);
}
public static void dump(ClassTree classTree, String path) throws IOException
{
logger.info("Dumping class to {}", path);
- ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); // checkstyle: permit this instantiation
mapper.writeValue(new File(path), classTree);
// validate that load works as expected
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
index 3ee05cf..6e2cbe2 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
@@ -37,7 +37,6 @@
import java.util.Set;
import java.util.stream.Collectors;
-import com.google.common.base.Throwables;
import org.junit.Test;
import org.apache.cassandra.utils.Pair;
@@ -46,7 +45,7 @@
import static org.junit.Assert.fail;
/**
- * Verifies that {@link DatabaseDescriptor#clientInitialization()} } and a couple of <i>apply</i> methods
+ * Verifies that {@link DatabaseDescriptor#clientInitialization()} and a couple of <i>apply</i> methods
* do not somehow lazily initialize any unwanted part of Cassandra like schema, commit log or start
* unexpected threads.
*
@@ -56,6 +55,14 @@
public class DatabaseDescriptorRefTest
{
static final String[] validClasses = {
+ "org.apache.cassandra.ConsoleAppender",
+ "org.apache.cassandra.ConsoleAppender$1",
+ "org.apache.cassandra.ConsoleAppenderBeanInfo",
+ "org.apache.cassandra.ConsoleAppenderCustomizer",
+ "org.apache.cassandra.LogbackStatusListener",
+ "org.apache.cassandra.LogbackStatusListener$ToLoggerOutputStream",
+ "org.apache.cassandra.LogbackStatusListener$WrappedPrintStream",
+ "org.apache.cassandra.TeeingAppender",
"org.apache.cassandra.audit.AuditLogOptions",
"org.apache.cassandra.audit.BinAuditLogger",
"org.apache.cassandra.audit.BinLogAuditLogger",
@@ -63,19 +70,18 @@
"org.apache.cassandra.auth.AllowAllInternodeAuthenticator",
"org.apache.cassandra.auth.AuthCache$BulkLoader",
"org.apache.cassandra.auth.Cacheable",
- "org.apache.cassandra.auth.IInternodeAuthenticator",
"org.apache.cassandra.auth.IAuthenticator",
"org.apache.cassandra.auth.IAuthorizer",
- "org.apache.cassandra.auth.IRoleManager",
+ "org.apache.cassandra.auth.IInternodeAuthenticator",
"org.apache.cassandra.auth.INetworkAuthorizer",
- "org.apache.cassandra.config.DatabaseDescriptor",
+ "org.apache.cassandra.auth.IRoleManager",
"org.apache.cassandra.config.CassandraRelevantProperties",
"org.apache.cassandra.config.CassandraRelevantProperties$PropertyConverter",
- "org.apache.cassandra.config.ConfigurationLoader",
"org.apache.cassandra.config.Config",
"org.apache.cassandra.config.Config$1",
- "org.apache.cassandra.config.Config$CommitLogSync",
"org.apache.cassandra.config.Config$CommitFailurePolicy",
+ "org.apache.cassandra.config.Config$CommitLogSync",
+ "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy",
"org.apache.cassandra.config.Config$DiskAccessMode",
"org.apache.cassandra.config.Config$DiskFailurePolicy",
"org.apache.cassandra.config.Config$DiskOptimizationStrategy",
@@ -86,14 +92,17 @@
"org.apache.cassandra.config.Config$PaxosStatePurging",
"org.apache.cassandra.config.Config$PaxosVariant",
"org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy",
+ "org.apache.cassandra.config.Config$SSTableConfig",
"org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy",
- "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy",
- "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit",
+ "org.apache.cassandra.config.ConfigBeanInfo",
+ "org.apache.cassandra.config.ConfigCustomizer",
+ "org.apache.cassandra.config.ConfigurationLoader",
"org.apache.cassandra.config.DataRateSpec",
"org.apache.cassandra.config.DataRateSpec$DataRateUnit",
"org.apache.cassandra.config.DataRateSpec$DataRateUnit$1",
"org.apache.cassandra.config.DataRateSpec$DataRateUnit$2",
"org.apache.cassandra.config.DataRateSpec$DataRateUnit$3",
+ "org.apache.cassandra.config.DataRateSpec$LongBytesPerSecondBound",
"org.apache.cassandra.config.DataStorageSpec",
"org.apache.cassandra.config.DataStorageSpec$DataStorageUnit",
"org.apache.cassandra.config.DataStorageSpec$DataStorageUnit$1",
@@ -105,52 +114,60 @@
"org.apache.cassandra.config.DataStorageSpec$IntMebibytesBound",
"org.apache.cassandra.config.DataStorageSpec$LongBytesBound",
"org.apache.cassandra.config.DataStorageSpec$LongMebibytesBound",
+ "org.apache.cassandra.config.DatabaseDescriptor",
+ "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit",
"org.apache.cassandra.config.DurationSpec",
- "org.apache.cassandra.config.DataRateSpec$LongBytesPerSecondBound",
+ "org.apache.cassandra.config.DurationSpec$IntMillisecondsBound",
+ "org.apache.cassandra.config.DurationSpec$IntMinutesBound",
+ "org.apache.cassandra.config.DurationSpec$IntSecondsBound",
"org.apache.cassandra.config.DurationSpec$LongMillisecondsBound",
+ "org.apache.cassandra.config.DurationSpec$LongMicrosecondsBound",
"org.apache.cassandra.config.DurationSpec$LongNanosecondsBound",
"org.apache.cassandra.config.DurationSpec$LongSecondsBound",
- "org.apache.cassandra.config.DurationSpec$IntMillisecondsBound",
- "org.apache.cassandra.config.DurationSpec$IntSecondsBound",
- "org.apache.cassandra.config.DurationSpec$IntMinutesBound",
"org.apache.cassandra.config.EncryptionOptions",
"org.apache.cassandra.config.EncryptionOptions$ClientEncryptionOptions",
"org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions",
"org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$InternodeEncryption",
"org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$OutgoingEncryptedPortSource",
+ "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsBeanInfo",
+ "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer",
+ "org.apache.cassandra.config.EncryptionOptionsBeanInfo",
+ "org.apache.cassandra.config.EncryptionOptionsCustomizer",
"org.apache.cassandra.config.GuardrailsOptions",
"org.apache.cassandra.config.GuardrailsOptions$Config",
"org.apache.cassandra.config.GuardrailsOptions$ConsistencyLevels",
"org.apache.cassandra.config.GuardrailsOptions$TableProperties",
"org.apache.cassandra.config.ParameterizedClass",
"org.apache.cassandra.config.ReplicaFilteringProtectionOptions",
- "org.apache.cassandra.config.YamlConfigurationLoader",
- "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker",
- "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker$1",
- "org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor",
- "org.apache.cassandra.config.TransparentDataEncryptionOptions",
"org.apache.cassandra.config.StartupChecksOptions",
"org.apache.cassandra.config.SubnetGroups",
"org.apache.cassandra.config.TrackWarnings",
+ "org.apache.cassandra.config.TransparentDataEncryptionOptions",
+ "org.apache.cassandra.config.YamlConfigurationLoader",
+ "org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor",
+ "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker",
+ "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker$1",
+ "org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement",
+ "org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement",
"org.apache.cassandra.db.ConsistencyLevel",
- "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerFactory",
- "org.apache.cassandra.db.commitlog.DefaultCommitLogSegmentMgrFactory",
"org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager",
- "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC",
- "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard",
"org.apache.cassandra.db.commitlog.CommitLog",
"org.apache.cassandra.db.commitlog.CommitLogMBean",
+ "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC",
+ "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerFactory",
+ "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard",
+ "org.apache.cassandra.db.commitlog.DefaultCommitLogSegmentMgrFactory",
"org.apache.cassandra.db.guardrails.GuardrailsConfig",
- "org.apache.cassandra.db.guardrails.GuardrailsConfigMBean",
"org.apache.cassandra.db.guardrails.GuardrailsConfig$ConsistencyLevels",
"org.apache.cassandra.db.guardrails.GuardrailsConfig$TableProperties",
+ "org.apache.cassandra.db.guardrails.GuardrailsConfigMBean",
"org.apache.cassandra.db.guardrails.Values$Config",
+ "org.apache.cassandra.db.rows.UnfilteredSource",
"org.apache.cassandra.dht.IPartitioner",
"org.apache.cassandra.distributed.api.IInstance",
- "org.apache.cassandra.distributed.api.IIsolatedExecutor",
- "org.apache.cassandra.distributed.shared.InstanceClassLoader",
- "org.apache.cassandra.distributed.impl.InstanceConfig",
"org.apache.cassandra.distributed.api.IInvokableInstance",
+ "org.apache.cassandra.distributed.api.IIsolatedExecutor",
+ "org.apache.cassandra.distributed.impl.InstanceConfig",
"org.apache.cassandra.distributed.impl.InvokableInstance$CallableNoExcept",
"org.apache.cassandra.distributed.impl.InvokableInstance$InstanceFunction",
"org.apache.cassandra.distributed.impl.InvokableInstance$SerializableBiConsumer",
@@ -163,67 +180,96 @@
"org.apache.cassandra.distributed.impl.InvokableInstance$TriFunction",
"org.apache.cassandra.distributed.impl.Message",
"org.apache.cassandra.distributed.impl.NetworkTopology",
- "org.apache.cassandra.exceptions.ConfigurationException",
- "org.apache.cassandra.exceptions.RequestValidationException",
+ "org.apache.cassandra.distributed.shared.InstanceClassLoader",
"org.apache.cassandra.exceptions.CassandraException",
+ "org.apache.cassandra.exceptions.ConfigurationException",
"org.apache.cassandra.exceptions.InvalidRequestException",
+ "org.apache.cassandra.exceptions.RequestValidationException",
"org.apache.cassandra.exceptions.TransportException",
"org.apache.cassandra.fql.FullQueryLogger",
"org.apache.cassandra.fql.FullQueryLoggerOptions",
"org.apache.cassandra.gms.IFailureDetector",
- "org.apache.cassandra.locator.IEndpointSnitch",
- "org.apache.cassandra.io.FSWriteError",
"org.apache.cassandra.io.FSError",
+ "org.apache.cassandra.io.FSWriteError",
"org.apache.cassandra.io.compress.ICompressor",
"org.apache.cassandra.io.compress.ICompressor$Uses",
"org.apache.cassandra.io.compress.LZ4Compressor",
+ "org.apache.cassandra.io.sstable.Component",
+ "org.apache.cassandra.io.sstable.Component$Type",
+ "org.apache.cassandra.io.sstable.IScrubber",
+ "org.apache.cassandra.io.sstable.MetricsProviders",
+ "org.apache.cassandra.io.sstable.SSTable",
+ "org.apache.cassandra.io.sstable.SSTable$Builder",
+ "org.apache.cassandra.io.sstable.format.AbstractSSTableFormat",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$Components",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$Components$Types",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$Factory",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$KeyCacheValueSerializer",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableReaderFactory",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableWriterFactory",
+ "org.apache.cassandra.io.sstable.format.SSTableFormat$Type",
+ "org.apache.cassandra.io.sstable.format.SSTableReader",
+ "org.apache.cassandra.io.sstable.format.SSTableReader$Builder",
+ "org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder",
+ "org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter",
+ "org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter$Builder",
+ "org.apache.cassandra.io.sstable.format.SortedTableReaderLoadingBuilder",
+ "org.apache.cassandra.io.sstable.format.SSTableWriter",
+ "org.apache.cassandra.io.sstable.format.SSTableWriter$Builder",
+ "org.apache.cassandra.io.sstable.format.SortedTableWriter",
+ "org.apache.cassandra.io.sstable.format.SortedTableWriter$Builder",
+ "org.apache.cassandra.io.sstable.format.Version",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat$BigFormatFactory",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableReaderFactory",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableWriterFactory",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat$BigVersion",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat$Components",
+ "org.apache.cassandra.io.sstable.format.big.BigFormat$Components$Types",
+ "org.apache.cassandra.io.sstable.format.big.BigSSTableReaderLoadingBuilder",
+ "org.apache.cassandra.io.sstable.format.big.BigTableReader",
+ "org.apache.cassandra.io.sstable.format.big.BigTableReader$Builder",
+ "org.apache.cassandra.io.sstable.format.big.BigTableWriter",
+ "org.apache.cassandra.io.sstable.format.big.BigTableWriter$Builder",
+ "org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport",
+ "org.apache.cassandra.io.sstable.keycache.KeyCacheSupport",
"org.apache.cassandra.io.sstable.metadata.MetadataType",
"org.apache.cassandra.io.util.BufferedDataOutputStreamPlus",
- "org.apache.cassandra.io.util.RebufferingInputStream",
- "org.apache.cassandra.io.util.FileInputStreamPlus",
- "org.apache.cassandra.io.util.FileOutputStreamPlus",
- "org.apache.cassandra.io.util.File",
+ "org.apache.cassandra.io.util.DataInputPlus",
+ "org.apache.cassandra.io.util.DataInputPlus$DataInputStreamPlus",
"org.apache.cassandra.io.util.DataOutputBuffer",
"org.apache.cassandra.io.util.DataOutputBufferFixed",
- "org.apache.cassandra.io.util.DataOutputStreamPlus",
"org.apache.cassandra.io.util.DataOutputPlus",
- "org.apache.cassandra.io.util.DataInputPlus",
+ "org.apache.cassandra.io.util.DataOutputStreamPlus",
"org.apache.cassandra.io.util.DiskOptimizationStrategy",
- "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy",
+ "org.apache.cassandra.io.util.File",
+ "org.apache.cassandra.io.util.FileInputStreamPlus",
+ "org.apache.cassandra.io.util.FileOutputStreamPlus",
"org.apache.cassandra.io.util.PathUtils$IOToLongFunction",
+ "org.apache.cassandra.io.util.RebufferingInputStream",
+ "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy",
+ "org.apache.cassandra.locator.IEndpointSnitch",
+ "org.apache.cassandra.locator.InetAddressAndPort",
"org.apache.cassandra.locator.Replica",
"org.apache.cassandra.locator.ReplicaCollection",
- "org.apache.cassandra.locator.SimpleSeedProvider",
"org.apache.cassandra.locator.SeedProvider",
+ "org.apache.cassandra.locator.SimpleSeedProvider",
+ "org.apache.cassandra.security.EncryptionContext",
"org.apache.cassandra.security.ISslContextFactory",
"org.apache.cassandra.security.SSLFactory",
- "org.apache.cassandra.security.EncryptionContext",
"org.apache.cassandra.service.CacheService$CacheType",
"org.apache.cassandra.transport.ProtocolException",
- "org.apache.cassandra.utils.binlog.BinLogOptions",
+ "org.apache.cassandra.utils.Closeable",
+ "org.apache.cassandra.utils.CloseableIterator",
"org.apache.cassandra.utils.FBUtilities",
"org.apache.cassandra.utils.FBUtilities$1",
- "org.apache.cassandra.utils.CloseableIterator",
"org.apache.cassandra.utils.Pair",
- "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException",
- "org.apache.cassandra.ConsoleAppender",
- "org.apache.cassandra.ConsoleAppender$1",
- "org.apache.cassandra.LogbackStatusListener",
- "org.apache.cassandra.LogbackStatusListener$ToLoggerOutputStream",
- "org.apache.cassandra.LogbackStatusListener$WrappedPrintStream",
- "org.apache.cassandra.TeeingAppender",
- // generated classes
- "org.apache.cassandra.config.ConfigBeanInfo",
- "org.apache.cassandra.config.ConfigCustomizer",
- "org.apache.cassandra.config.EncryptionOptionsBeanInfo",
- "org.apache.cassandra.config.EncryptionOptionsCustomizer",
- "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsBeanInfo",
- "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer",
- "org.apache.cassandra.ConsoleAppenderBeanInfo",
- "org.apache.cassandra.ConsoleAppenderCustomizer",
- "org.apache.cassandra.locator.InetAddressAndPort",
- "org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement",
- "org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement"
+ "org.apache.cassandra.utils.binlog.BinLogOptions",
+ "org.apache.cassandra.utils.concurrent.RefCounted",
+ "org.apache.cassandra.utils.concurrent.SelfRefCounted",
+ "org.apache.cassandra.utils.concurrent.Transactional",
+ "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException"
};
static final Set<String> checkedClasses = new HashSet<>(Arrays.asList(validClasses));
@@ -370,8 +416,8 @@
StringBuilder sb = new StringBuilder();
for (Pair<String, Exception> violation : new ArrayList<>(violations))
sb.append("\n\n")
- .append("VIOLATION: ").append(violation.left).append('\n')
- .append(Throwables.getStackTraceAsString(violation.right));
+ .append("VIOLATION: ").append(violation.left); //.append('\n')
+ //.append(Throwables.getStackTraceAsString(violation.right));
String msg = sb.toString();
err.println(msg);
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
index 2b72cd7..f1f3958 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
@@ -34,9 +34,13 @@
import org.junit.Test;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_LOADER;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PARTITIONER;
import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.KIBIBYTES;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -63,7 +67,7 @@
// Now try custom loader
ConfigurationLoader testLoader = new TestLoader();
- System.setProperty("cassandra.config.loader", testLoader.getClass().getName());
+ CONFIG_LOADER.setString(testLoader.getClass().getName());
config = DatabaseDescriptor.loadConfig();
assertEquals("ConfigurationLoader Test", config.cluster_name);
@@ -233,11 +237,8 @@
@Test
public void testInvalidPartitionPropertyOverride() throws Exception
{
- String key = Config.PROPERTY_PREFIX + "partitioner";
- String previous = System.getProperty(key);
- try
+ try (WithProperties properties = new WithProperties().set(PARTITIONER, "ThisDoesNotExist"))
{
- System.setProperty(key, "ThisDoesNotExist");
Config testConfig = DatabaseDescriptor.loadConfig();
testConfig.partitioner = "Murmur3Partitioner";
@@ -256,17 +257,6 @@
Assert.assertEquals("org.apache.cassandra.dht.ThisDoesNotExist", cause.getMessage());
}
}
- finally
- {
- if (previous == null)
- {
- System.getProperties().remove(key);
- }
- else
- {
- System.setProperty(key, previous);
- }
- }
}
@Test
@@ -298,19 +288,22 @@
try
{
- DatabaseDescriptor.setColumnIndexSize(-1);
- fail("Should have received a IllegalArgumentException column_index_size = -1");
+ DatabaseDescriptor.setColumnIndexSizeInKiB(-5);
+ fail("Should have received a IllegalArgumentException column_index_size = -5");
}
catch (IllegalArgumentException ignored) { }
- Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize());
+ Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize(0));
try
{
- DatabaseDescriptor.setColumnIndexSize(2 * 1024 * 1024);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(2 * 1024 * 1024);
fail("Should have received a ConfigurationException column_index_size = 2GiB");
}
catch (ConfigurationException ignored) { }
- Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize());
+ Assert.assertEquals(4096, DatabaseDescriptor.getColumnIndexSize(0));
+
+ DatabaseDescriptor.setColumnIndexSizeInKiB(-1); // set undefined
+ Assert.assertEquals(8192, DatabaseDescriptor.getColumnIndexSize(8192));
try
{
@@ -415,7 +408,7 @@
try
{
DatabaseDescriptor.setRepairSessionSpaceInMiB(0);
- fail("Should have received a ConfigurationException for depth of 9");
+ fail("Should have received a ConfigurationException for depth of 0");
}
catch (ConfigurationException ignored) { }
@@ -504,7 +497,7 @@
catch (ConfigurationException e)
{
assertThat(e.getMessage()).isEqualTo("To set concurrent_validations > concurrent_compactors, " +
- "set the system property cassandra.allow_unlimited_concurrent_validations=true");
+ "set the system property -D" + ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS.getKey() + "=true");
}
// unless we disable that check (done with a system property at startup or via JMX)
diff --git a/test/unit/org/apache/cassandra/config/DurationSpecTest.java b/test/unit/org/apache/cassandra/config/DurationSpecTest.java
index 22846fc..b0c73de 100644
--- a/test/unit/org/apache/cassandra/config/DurationSpecTest.java
+++ b/test/unit/org/apache/cassandra/config/DurationSpecTest.java
@@ -213,6 +213,7 @@
assertEquals(new DurationSpec.IntSecondsBound("10s"), DurationSpec.IntSecondsBound.inSecondsString("10"));
assertEquals(new DurationSpec.IntSecondsBound("10s"), DurationSpec.IntSecondsBound.inSecondsString("10s"));
+ assertEquals(10L, new DurationSpec.LongMicrosecondsBound("10us").toMicroseconds());
assertEquals(10L, new DurationSpec.LongMillisecondsBound("10ms").toMilliseconds());
assertEquals(10L, new DurationSpec.LongSecondsBound("10s").toSeconds());
}
@@ -284,6 +285,11 @@
assertThatThrownBy(() -> new DurationSpec.IntMinutesBound("-10s")).isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Invalid duration: -10s");
+ assertThatThrownBy(() -> new DurationSpec.LongMicrosecondsBound("10ns")).isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid duration: 10ns Accepted units");
+ assertThatThrownBy(() -> new DurationSpec.LongMicrosecondsBound(10, NANOSECONDS)).isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid duration: 10 NANOSECONDS Accepted units");
+
assertThatThrownBy(() -> new DurationSpec.LongMillisecondsBound("10ns")).isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Invalid duration: 10ns Accepted units");
assertThatThrownBy(() -> new DurationSpec.LongMillisecondsBound(10, NANOSECONDS)).isInstanceOf(IllegalArgumentException.class)
diff --git a/test/unit/org/apache/cassandra/config/EncryptionOptionsTest.java b/test/unit/org/apache/cassandra/config/EncryptionOptionsTest.java
index a76c24a..0ff2124 100644
--- a/test/unit/org/apache/cassandra/config/EncryptionOptionsTest.java
+++ b/test/unit/org/apache/cassandra/config/EncryptionOptionsTest.java
@@ -125,7 +125,7 @@
EncryptionOptions.TlsEncryptionPolicy expected)
{
return new ServerEncryptionOptionsTestCase(new EncryptionOptions.ServerEncryptionOptions(new ParameterizedClass("org.apache.cassandra.security.DefaultSslContextFactory",
- new HashMap<>()), keystorePath, "dummypass", "dummytruststore", "dummypass",
+ new HashMap<>()), keystorePath, "dummypass", keystorePath, "dummypass", "dummytruststore", "dummypass",
Collections.emptyList(), null, null, null, "JKS", false, false, optional, internodeEncryption, false)
.applyConfig(),
expected,
diff --git a/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java b/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java
index 0e028cb..8554ed4 100644
--- a/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java
+++ b/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java
@@ -23,6 +23,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -33,7 +34,7 @@
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-old.yaml");
+ CASSANDRA_CONFIG.setString("cassandra-old.yaml");
DatabaseDescriptor.daemonInitialization();
}
@@ -105,7 +106,7 @@
assertEquals(new DurationSpec.IntSecondsBound(604800), config.trace_type_repair_ttl);
assertNull(config.prepared_statements_cache_size);
assertTrue(config.user_defined_functions_enabled);
- assertTrue(config.scripted_user_defined_functions_enabled);
+ assertFalse(config.scripted_user_defined_functions_enabled);
assertTrue(config.materialized_views_enabled);
assertFalse(config.transient_replication_enabled);
assertTrue(config.sasi_indexes_enabled);
diff --git a/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java b/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java
index 2ffe91b..7a8b14e 100644
--- a/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java
+++ b/test/unit/org/apache/cassandra/config/OverrideConfigurationLoader.java
@@ -22,6 +22,8 @@
import org.apache.cassandra.exceptions.ConfigurationException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_LOADER;
+
/**
* Helper class for programmatically overriding individual config values before DatabaseDescriptor is bootstrapped.
*/
@@ -40,7 +42,7 @@
public static void override(Consumer<Config> modifier)
{
- System.setProperty(Config.PROPERTY_PREFIX + "config.loader", OverrideConfigurationLoader.class.getName());
+ CONFIG_LOADER.setString(OverrideConfigurationLoader.class.getName());
configModifier = modifier;
}
}
diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
index 06be1dc..b46d2ea 100644
--- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
+++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
@@ -91,21 +91,19 @@
// the reason is that its not a scalar but a complex type (collection type), so the map we use needs to have a collection to match.
// It is possible that we define a common string representation for these types so they can be written to; this
// is an issue that SettingsTable may need to worry about.
- try (WithProperties ignore = new WithProperties(CONFIG_ALLOW_SYSTEM_PROPERTIES.getKey(), "true",
- SYSTEM_PROPERTY_PREFIX + "storage_port", "123",
- SYSTEM_PROPERTY_PREFIX + "commitlog_sync", "batch",
- SYSTEM_PROPERTY_PREFIX + "seed_provider.class_name", "org.apache.cassandra.locator.SimpleSeedProvider",
-// PROPERTY_PREFIX + "client_encryption_options.cipher_suites", "[\"FakeCipher\"]",
- SYSTEM_PROPERTY_PREFIX + "client_encryption_options.optional", "false",
- SYSTEM_PROPERTY_PREFIX + "client_encryption_options.enabled", "true",
- SYSTEM_PROPERTY_PREFIX + "doesnotexist", "true"
- ))
+ try (WithProperties ignore = new WithProperties()
+ .set(CONFIG_ALLOW_SYSTEM_PROPERTIES, true)
+ .with(SYSTEM_PROPERTY_PREFIX + "storage_port", "123",
+ SYSTEM_PROPERTY_PREFIX + "commitlog_sync", "batch",
+ SYSTEM_PROPERTY_PREFIX + "seed_provider.class_name", "org.apache.cassandra.locator.SimpleSeedProvider",
+ SYSTEM_PROPERTY_PREFIX + "client_encryption_options.optional", Boolean.FALSE.toString(),
+ SYSTEM_PROPERTY_PREFIX + "client_encryption_options.enabled", Boolean.TRUE.toString(),
+ SYSTEM_PROPERTY_PREFIX + "doesnotexist", Boolean.TRUE.toString()))
{
Config config = YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
assertThat(config.storage_port).isEqualTo(123);
assertThat(config.commitlog_sync).isEqualTo(Config.CommitLogSync.batch);
assertThat(config.seed_provider.class_name).isEqualTo("org.apache.cassandra.locator.SimpleSeedProvider");
-// assertThat(config.client_encryption_options.cipher_suites).isEqualTo(Collections.singletonList("FakeCipher"));
assertThat(config.client_encryption_options.optional).isFalse();
assertThat(config.client_encryption_options.enabled).isTrue();
}
diff --git a/test/unit/org/apache/cassandra/cql3/AssignmentTestableTest.java b/test/unit/org/apache/cassandra/cql3/AssignmentTestableTest.java
new file mode 100644
index 0000000..46c00f1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/AssignmentTestableTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+public class AssignmentTestableTest
+{
+ @Test
+ public void testGetPreferredCompatibleType()
+ {
+ testGetPreferredCompatibleType(null);
+
+ testGetPreferredCompatibleType(AsciiType.instance, AsciiType.instance);
+ testGetPreferredCompatibleType(UTF8Type.instance, UTF8Type.instance);
+ testGetPreferredCompatibleType(AsciiType.instance, AsciiType.instance, AsciiType.instance);
+ testGetPreferredCompatibleType(UTF8Type.instance, UTF8Type.instance, UTF8Type.instance);
+ testGetPreferredCompatibleType(UTF8Type.instance, AsciiType.instance, UTF8Type.instance);
+ testGetPreferredCompatibleType(UTF8Type.instance, UTF8Type.instance, AsciiType.instance);
+
+ testGetPreferredCompatibleType(Int32Type.instance, Int32Type.instance);
+ testGetPreferredCompatibleType(LongType.instance, LongType.instance);
+ testGetPreferredCompatibleType(Int32Type.instance, Int32Type.instance, Int32Type.instance);
+ testGetPreferredCompatibleType(LongType.instance, LongType.instance, LongType.instance);
+ testGetPreferredCompatibleType(null, Int32Type.instance, LongType.instance); // not assignable
+ testGetPreferredCompatibleType(null, LongType.instance, Int32Type.instance); // not assignable
+ }
+
+ public void testGetPreferredCompatibleType(AbstractType<?> type, AbstractType<?>... types)
+ {
+ Assert.assertEquals(type, AssignmentTestable.getCompatibleTypeIfKnown(Arrays.asList(types)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/CDCStatementTest.java b/test/unit/org/apache/cassandra/cql3/CDCStatementTest.java
index fb24aa9..55865b1 100644
--- a/test/unit/org/apache/cassandra/cql3/CDCStatementTest.java
+++ b/test/unit/org/apache/cassandra/cql3/CDCStatementTest.java
@@ -22,6 +22,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.config.DatabaseDescriptor;
public class CDCStatementTest extends CQLTester
@@ -29,6 +30,7 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
DatabaseDescriptor.setCDCEnabled(true);
CQLTester.setUpClass();
}
diff --git a/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java
index f0d2c19..0d73731 100644
--- a/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java
+++ b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java
@@ -484,7 +484,7 @@
}
}
expected.append(bracketClose);
- buffer = CollectionSerializer.pack(buffers, added.size(), version);
+ buffer = CollectionSerializer.pack(buffers, added.size());
}
return new Value(expected.toString(), collectionType.asCQL3Type(), buffer);
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 23b392f..41b86f1 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -26,7 +26,21 @@
import java.net.ServerSocket;
import java.nio.ByteBuffer;
import java.rmi.server.RMISocketFactory;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -34,7 +48,6 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import javax.management.MBeanServerConnection;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
@@ -46,18 +59,27 @@
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
-
-import org.junit.*;
-
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.codahale.metrics.Gauge;
-import com.datastax.driver.core.*;
+import com.datastax.driver.core.CloseFuture;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ColumnDefinitions;
import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.NettyOptions;
import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.SocketOptions;
+import com.datastax.driver.core.Statement;
import com.datastax.driver.core.exceptions.UnauthorizedException;
-
import com.datastax.shaded.netty.channel.EventLoopGroup;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.ServerTestUtils;
@@ -69,50 +91,85 @@
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.DurationType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.io.filesystem.ListenableFileSystem;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileSystems;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.metrics.ClientMetrics;
-import org.apache.cassandra.schema.*;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.marshal.TupleType;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaTestUtil;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.transport.*;
+import org.apache.cassandra.transport.Event;
+import org.apache.cassandra.transport.Message;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.Server;
+import org.apache.cassandra.transport.SimpleClient;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JMXServerUtils;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.assertj.core.api.Assertions;
-import org.apache.cassandra.utils.Pair;
import org.awaitility.Awaitility;
-import static com.datastax.driver.core.SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS;
-import static com.datastax.driver.core.SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DRIVER_CONNECTION_TIMEOUT_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DRIVER_READ_TIMEOUT_MS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_REUSE_PREPARED;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ROW_CACHE_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_USE_PREPARED;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -130,9 +187,9 @@
public static final String KEYSPACE = "cql_test_keyspace";
public static final String KEYSPACE_PER_TEST = "cql_test_keyspace_alt";
- protected static final boolean USE_PREPARED_VALUES = Boolean.valueOf(System.getProperty("cassandra.test.use_prepared", "true"));
- protected static final boolean REUSE_PREPARED = Boolean.valueOf(System.getProperty("cassandra.test.reuse_prepared", "true"));
- protected static final long ROW_CACHE_SIZE_IN_MIB = new DataStorageSpec.LongMebibytesBound(System.getProperty("cassandra.test.row_cache_size", "0MiB")).toMebibytes();
+ protected static final boolean USE_PREPARED_VALUES = TEST_USE_PREPARED.getBoolean();
+ protected static final boolean REUSE_PREPARED = TEST_REUSE_PREPARED.getBoolean();
+ protected static final long ROW_CACHE_SIZE_IN_MIB = new DataStorageSpec.LongMebibytesBound(TEST_ROW_CACHE_SIZE.getString("0MiB")).toMebibytes();
private static final AtomicInteger seqNumber = new AtomicInteger();
protected static final ByteBuffer TOO_BIG = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT + 1024);
public static final String DATA_CENTER = ServerTestUtils.DATA_CENTER;
@@ -191,8 +248,6 @@
nativeAddr = InetAddress.getLoopbackAddress();
nativePort = getAutomaticallyAllocatedPort(nativeAddr);
-
- ServerTestUtils.daemonInitialization();
}
private List<String> keyspaces = new ArrayList<>();
@@ -318,6 +373,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
if (ROW_CACHE_SIZE_IN_MIB > 0)
DatabaseDescriptor.setRowCacheSizeInMiB(ROW_CACHE_SIZE_IN_MIB);
StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
@@ -440,7 +497,7 @@
public static List<String> buildNodetoolArgs(List<String> args)
{
- int port = jmxPort == 0 ? Integer.getInteger("cassandra.jmx.local.port", 7199) : jmxPort;
+ int port = jmxPort == 0 ? CASSANDRA_JMX_LOCAL_PORT.getInt(7199) : jmxPort;
String host = jmxHost == null ? "127.0.0.1" : jmxHost;
List<String> allArgs = new ArrayList<>();
allArgs.add("bin/nodetool");
@@ -573,10 +630,8 @@
private static Cluster initClientCluster(User user, ProtocolVersion version)
{
SocketOptions socketOptions =
- new SocketOptions().setConnectTimeoutMillis(Integer.getInteger("cassandra.test.driver.connection_timeout_ms",
- DEFAULT_CONNECT_TIMEOUT_MILLIS)) // default is 5000
- .setReadTimeoutMillis(Integer.getInteger("cassandra.test.driver.read_timeout_ms",
- DEFAULT_READ_TIMEOUT_MILLIS)); // default is 12000
+ new SocketOptions().setConnectTimeoutMillis(TEST_DRIVER_CONNECTION_TIMEOUT_MS.getInt()) // default is 5000
+ .setReadTimeoutMillis(TEST_DRIVER_READ_TIMEOUT_MS.getInt()); // default is 12000
logger.info("Timeouts: {} / {}", socketOptions.getConnectTimeoutMillis(), socketOptions.getReadTimeoutMillis());
@@ -1286,6 +1341,11 @@
return sessionNet().execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
}
+ protected com.datastax.driver.core.ResultSet executeNetWithoutPaging(String query)
+ {
+ return executeNetWithPaging(query, Integer.MAX_VALUE);
+ }
+
protected Session sessionNet()
{
return sessionNet(getDefaultVersion());
@@ -1342,7 +1402,7 @@
return QueryProcessor.instance.prepare(formatQuery(query), ClientState.forInternalCalls());
}
- protected UntypedResultSet execute(String query, Object... values) throws Throwable
+ protected UntypedResultSet execute(String query, Object... values)
{
return executeFormattedQuery(formatQuery(query), values);
}
@@ -1352,7 +1412,11 @@
return executeFormattedQuery(formatViewQuery(KEYSPACE, query), values);
}
- protected UntypedResultSet executeFormattedQuery(String query, Object... values) throws Throwable
+ /**
+ * Executes the provided query using the {@link ClientState#forInternalCalls()} as the expected ClientState. Note:
+ * this means permissions checking will not apply and queries will proceed regardless of role or guardrails.
+ */
+ protected UntypedResultSet executeFormattedQuery(String query, Object... values)
{
UntypedResultSet rs;
if (usePrepared)
@@ -1727,6 +1791,24 @@
}
}
+ protected void assertColumnNames(ResultSet result, String... expectedColumnNames)
+ {
+ if (result == null)
+ {
+ Assert.fail("No rows returned by query.");
+ return;
+ }
+
+ ColumnDefinitions columnDefinitions = result.getColumnDefinitions();
+ Assert.assertEquals("Got less columns than expected.", expectedColumnNames.length, columnDefinitions.size());
+
+ for (int i = 0, m = columnDefinitions.size(); i < m; i++)
+ {
+ String columnName = columnDefinitions.getName(i);
+ Assert.assertEquals(expectedColumnNames[i], columnName);
+ }
+ }
+
protected void assertAllRows(Object[]... rows) throws Throwable
{
assertRows(execute("SELECT * FROM %s"), rows);
@@ -1737,6 +1819,11 @@
return expected;
}
+ public static Object[][] rows(Object[]... rows)
+ {
+ return rows;
+ }
+
protected void assertEmpty(UntypedResultSet result) throws Throwable
{
if (result != null && !result.isEmpty())
@@ -1768,8 +1855,15 @@
assertInvalidThrowMessage(Optional.empty(), errorMessage, exception, query, values);
}
- // if a protocol version > Integer.MIN_VALUE is supplied, executes
- // the query via the java driver, mimicking a real client.
+ /**
+ * Asserts that the query provided throws the exceptions provided.
+ *
+ * NOTE: This method uses {@link ClientState#forInternalCalls()} which sets the {@link ClientState#isInternal} value
+ * to true, nullifying any system keyspace or other permissions checking for tables.
+ *
+ * If a protocol version > Integer.MIN_VALUE is supplied, executes
+ * the query via the java driver, mimicking a real client.
+ */
protected void assertInvalidThrowMessage(Optional<ProtocolVersion> protocolVersion,
String errorMessage,
Class<? extends Throwable> exception,
@@ -2148,12 +2242,12 @@
return new UserTypeValue(fieldNames, fieldValues);
}
- protected Object list(Object...values)
+ protected List<Object> list(Object...values)
{
return Arrays.asList(values);
}
- protected Object set(Object...values)
+ protected Set<Object> set(Object...values)
{
return ImmutableSet.copyOf(values);
}
@@ -2403,4 +2497,32 @@
&& Objects.equal(password, u.password);
}
}
+
+ public static abstract class InMemory extends CQLTester
+ {
+ protected static ListenableFileSystem fs = null;
+
+ /**
+ * Used by {@link #cleanupFileSystemListeners()} to know if file system listeners should be removed at the start
+ * of a test; can disable for cases where listeners are needed cross mutliple tests.
+ */
+ protected boolean cleanupFileSystemListeners = true;
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ fs = FileSystems.newGlobalInMemoryFileSystem();
+ CassandraRelevantProperties.IGNORE_MISSING_NATIVE_FILE_HINTS.setBoolean(true);
+ FileSystems.maybeCreateTmp();
+
+ CQLTester.setUpClass();
+ }
+ @Before
+ public void cleanupFileSystemListeners()
+ {
+ if (!cleanupFileSystemListeners)
+ return;
+ fs.clearListeners();
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java b/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
index 8768b77..ce9b103 100644
--- a/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/CustomNowInSecondsTest.java
@@ -25,6 +25,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.statements.BatchStatement;
import org.apache.cassandra.cql3.statements.ModificationStatement;
import org.apache.cassandra.db.ConsistencyLevel;
@@ -43,6 +44,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
prepareServer();
requireNetwork();
}
diff --git a/test/unit/org/apache/cassandra/cql3/DecoratedKeyPrefixesTest.java b/test/unit/org/apache/cassandra/cql3/DecoratedKeyPrefixesTest.java
new file mode 100644
index 0000000..ff26d9f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/DecoratedKeyPrefixesTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.Random;
+
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.distributed.shared.ThrowingRunnable;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.cql3.TombstonesWithIndexedSSTableTest.makeRandomString;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests primary index for prefixes. Exercises paths in the BTI partition iterators that are very hard to reach using
+ * a hashing partitioner.
+ */
+public class DecoratedKeyPrefixesTest extends CQLTester
+{
+ // to force indexing, we need more than 2*column_index_size (4k for tests) bytes per partition.
+ static final int ENTRY_SIZE = 100;
+ static final int WIDE_ROW_COUNT = 100;
+
+ String[] keys;
+ String[] samePrefixBefore;
+ String[] samePrefixAfter;
+
+ Random rand = new Random();
+
+ // Must be called the same as CQLTester's version to ensure it overrides it
+ @BeforeClass
+ public static void setUpClass()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+
+ // Once per-JVM is enough
+ prepareServer();
+ }
+
+ boolean wide(int i)
+ {
+ return i % 2 == 1;
+ }
+
+ private int randomCKey(int i)
+ {
+ return wide(i) ? rand.nextInt(WIDE_ROW_COUNT) : 0;
+ }
+
+ void prepareTable() throws Throwable
+ {
+ Assume.assumeTrue(DatabaseDescriptor.getPartitioner() instanceof ByteOrderedPartitioner);
+
+ createTable("CREATE TABLE %s (pk text, ck int, v1 int, v2 text, primary key(pk, ck));");
+
+ String[] baseKeys = new String[] {"A", "BB", "CCC", "DDDD"};
+ keys = new String[baseKeys.length];
+ samePrefixBefore = new String[baseKeys.length];
+ samePrefixAfter = new String[baseKeys.length];
+
+ // Create keys and bounds before and after it with the same prefix
+ for (int i = 0; i < baseKeys.length; ++i)
+ {
+ String key = baseKeys[i];
+ keys[i] = key + "key";
+ samePrefixBefore[i] = key + "before";
+ samePrefixAfter[i] = key + "larger";
+ addPartition(keys[i], wide(i) ? WIDE_ROW_COUNT : 1);
+ }
+ }
+
+ private void addPartition(String key, int rowCount) throws Throwable
+ {
+ for (int i = 0; i < rowCount; ++i)
+ execute("INSERT INTO %s (pk, ck, v1, v2) values (?, ?, ?, ?)", key, i, 1, makeRandomString(ENTRY_SIZE));
+ }
+
+ void testWithFlush(ThrowingRunnable test) throws Throwable
+ {
+ prepareTable();
+ test.run();
+ flush();
+ test.run();
+ }
+
+ @Test
+ public void testPrefixLookupEQ() throws Throwable
+ {
+ testWithFlush(() ->
+ {
+ for (int i = 0; i < keys.length; ++i)
+ {
+ assertRows(execute("SELECT pk FROM %s WHERE pk = ? AND ck = ?", keys[i], randomCKey(i)),
+ row(keys[i]));
+ assertEmpty(execute("SELECT pk FROM %s WHERE pk = ?", samePrefixBefore[i]));
+ assertEmpty(execute("SELECT pk FROM %s WHERE pk = ?", samePrefixAfter[i]));
+ }
+ });
+ }
+
+ @Test
+ public void testPrefixLookupGE() throws Throwable
+ {
+ testWithFlush(() ->
+ {
+ for (int i = 0; i < keys.length; ++i)
+ {
+ assertRows(execute("SELECT pk FROM %s WHERE token(pk) >= token(?) LIMIT 1", keys[i]),
+ row(keys[i]));
+ assertRows(execute("SELECT pk FROM %s WHERE token(pk) >= token(?) LIMIT 1", samePrefixBefore[i]),
+ row(keys[i]));
+ assertReturnsNext(execute("SELECT pk FROM %s WHERE token(pk) >= token(?) LIMIT 1", samePrefixAfter[i]), i);
+ }
+ });
+ }
+
+ void assertReturnsNext(UntypedResultSet result, int i) throws Throwable
+ {
+ if (i == keys.length - 1)
+ assertEmpty(result);
+ else
+ assertRows(result, row(keys[i + 1]));
+ }
+
+ @Test
+ public void testPrefixLookupGT() throws Throwable
+ {
+ testWithFlush(() ->
+ {
+ for (int i = 0; i < keys.length; ++i)
+ {
+ assertReturnsNext(execute("SELECT pk FROM %s WHERE token(pk) > token(?) LIMIT 1", keys[i]), i);
+ assertRows(execute("SELECT pk FROM %s WHERE token(pk) > token(?) LIMIT 1", samePrefixBefore[i]),
+ row(keys[i]));
+ assertReturnsNext(execute("SELECT pk FROM %s WHERE token(pk) > token(?) LIMIT 1", samePrefixAfter[i]), i);
+ }
+ });
+ }
+
+ @Test
+ public void testPrefixKeySliceExact() throws Throwable
+ {
+ testWithFlush(() ->
+ {
+ for (int i = 0; i < keys.length; ++i)
+ {
+ for (int j = i; j < keys.length; ++j)
+ {
+ assertEquals(i + "<>" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", keys[i], keys[j]).size());
+ assertEquals(i + "=<>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", keys[i], keys[j]).size());
+ assertEquals(i + "<>=" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", keys[i], keys[j]).size());
+ assertEquals(i + "=<>=" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", keys[i], keys[j]).size());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testPrefixKeySliceSamePrefix() throws Throwable
+ {
+ testWithFlush(() ->
+ {
+ for (int i = 0; i < keys.length; ++i)
+ {
+ for (int j = i; j < keys.length; ++j)
+ {
+ assertEquals(i + "<ba>" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixAfter[j]).size());
+ assertEquals(i + "=<ba>" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixAfter[j]).size());
+ assertEquals(i + "<ba>=" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixAfter[j]).size());
+ assertEquals(i + "=<ba>=" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixAfter[j]).size());
+
+ assertEquals(i + "<aa>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixAfter[j]).size());
+ assertEquals(i + "=<aa>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixAfter[j]).size());
+ assertEquals(i + "<aa>=" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixAfter[j]).size());
+ assertEquals(i + "=<aa>=" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixAfter[j]).size());
+
+ assertEquals(i + "<bb>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixBefore[j]).size());
+ assertEquals(i + "=<bb>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixBefore[j]).size());
+ assertEquals(i + "<bb>=" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixBefore[j]).size());
+ assertEquals(i + "=<bb>=" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], samePrefixBefore[j]).size());
+
+ assertEquals(i + "<ab>" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixBefore[j]).size());
+ assertEquals(i + "=<ab>" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixBefore[j]).size());
+ assertEquals(i + "<ab>=" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixBefore[j]).size());
+ assertEquals(i + "=<ab>=" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], samePrefixBefore[j]).size());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testPrefixKeySliceMixed() throws Throwable
+ {
+ testWithFlush(() ->
+ {
+ for (int i = 0; i < keys.length; ++i)
+ {
+ for (int j = i; j < keys.length; ++j)
+ {
+ assertEquals(i + "<bk>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], keys[j]).size());
+ assertEquals(i + "=<bk>" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixBefore[i], keys[j]).size());
+
+ assertEquals(i + "<ak>" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], keys[j]).size());
+ assertEquals(i + "=<ak>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) <= token(?) AND ck = 0 ALLOW FILTERING", samePrefixAfter[i], keys[j]).size());
+
+ assertEquals(i + "<kb>" + j, Math.max(0, j - i - 1),
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", keys[i], samePrefixBefore[j]).size());
+ assertEquals(i + "=<kb>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", keys[i], samePrefixBefore[j]).size());
+
+ assertEquals(i + "<ka>" + j, j - i,
+ execute("SELECT pk FROM %s WHERE token(pk) > token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", keys[i], samePrefixAfter[j]).size());
+ assertEquals(i + "=<ka>" + j, j - i + 1,
+ execute("SELECT pk FROM %s WHERE token(pk) >= token(?) AND token(pk) < token(?) AND ck = 0 ALLOW FILTERING", keys[i], samePrefixAfter[j]).size());
+ }
+ }
+ });
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java b/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java
index 3652ac8..ff7f861 100644
--- a/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/EmptyValuesTest.java
@@ -19,7 +19,6 @@
package org.apache.cassandra.cql3;
import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.regex.Pattern;
@@ -73,17 +72,20 @@
ByteArrayOutputStream buf = new ByteArrayOutputStream();
for (SSTableReader ssTable : cfs.getLiveSSTables())
{
- try (PrintStream out = new PrintStream(buf, true))
+ int exitValue = 0;
+ try
{
ProcessBuilder pb = new ProcessBuilder("tools/bin/sstabledump", ssTable.getFilename());
+ pb.redirectErrorStream(true);
Process process = pb.start();
- process.waitFor();
+ exitValue = process.waitFor();
IOUtils.copy(process.getInputStream(), buf);
}
catch (Throwable t)
{
Assert.fail(t.getClass().getName());
}
+ Assert.assertEquals(buf.toString(), 0, exitValue);
}
String outString = new String(buf.toByteArray(), StandardCharsets.UTF_8);
diff --git a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
index eded145..91111d1 100644
--- a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
@@ -55,7 +55,7 @@
}
@Override
- protected UntypedResultSet execute(String query, Object... values) throws Throwable
+ protected UntypedResultSet execute(String query, Object... values)
{
return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
}
diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
index 2b0a3c8..9c7dac1 100644
--- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
+++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
@@ -33,6 +33,8 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
import org.apache.cassandra.metrics.CacheMetrics;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.schema.CachingParams;
@@ -49,6 +51,7 @@
public class KeyCacheCqlTest extends CQLTester
{
+ private static boolean sstableImplCachesKeys;
private static final String commonColumnsDef =
"part_key_a int," +
@@ -98,6 +101,7 @@
{
CachingParams.DEFAULT = CachingParams.CACHE_NOTHING;
CQLTester.setUpClass();
+ sstableImplCachesKeys = KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat());
}
/**
@@ -115,7 +119,7 @@
}
@Override
- protected UntypedResultSet execute(String query, Object... values) throws Throwable
+ protected UntypedResultSet execute(String query, Object... values)
{
return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
}
@@ -268,7 +272,7 @@
long hits = metrics.hits.getCount();
long requests = metrics.requests.getCount();
assertEquals(0, hits);
- assertEquals(expectedRequests, requests);
+ assertEquals(sstableImplCachesKeys ? expectedRequests : 0, requests);
for (int i = 0; i < 10; i++)
{
@@ -285,8 +289,8 @@
metrics = CacheService.instance.keyCache.getMetrics();
hits = metrics.hits.getCount();
requests = metrics.requests.getCount();
- assertEquals(200, hits);
- assertEquals(expectedRequests, requests);
+ assertEquals(sstableImplCachesKeys ? 200 : 0, hits);
+ assertEquals(sstableImplCachesKeys ? expectedRequests : 0, requests);
CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
@@ -362,7 +366,7 @@
long hits = metrics.hits.getCount();
long requests = metrics.requests.getCount();
assertEquals(0, hits);
- assertEquals(expectedNumberOfRequests, requests);
+ assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests);
for (int i = 0; i < 10; i++)
{
@@ -380,8 +384,8 @@
metrics = CacheService.instance.keyCache.getMetrics();
hits = metrics.hits.getCount();
requests = metrics.requests.getCount();
- assertEquals(200, hits);
- assertEquals(expectedNumberOfRequests, requests);
+ assertEquals(sstableImplCachesKeys ? 200 : 0, hits);
+ assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests);
dropTable("DROP TABLE %s");
@@ -428,6 +432,7 @@
long expectedNumberOfRequests = 0;
+ CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
for (int i = 0; i < 10; i++)
{
assertRows(execute("SELECT col_text FROM %s WHERE part_key_a = ? AND part_key_b = ?", i, Integer.toOctalString(i)),
@@ -437,11 +442,10 @@
expectedNumberOfRequests += recentBloomFilterFalsePositives() + 1;
}
- CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();
long hits = metrics.hits.getCount();
long requests = metrics.requests.getCount();
assertEquals(0, hits);
- assertEquals(10, requests);
+ assertEquals(sstableImplCachesKeys ? 10 : 0, requests);
for (int i = 0; i < 100; i++)
{
@@ -454,8 +458,8 @@
hits = metrics.hits.getCount();
requests = metrics.requests.getCount();
- assertEquals(10, hits);
- assertEquals(expectedNumberOfRequests, requests);
+ assertEquals(sstableImplCachesKeys ? 10 : 0, hits);
+ assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests);
}
@Test
@@ -492,7 +496,7 @@
long hits = metrics.hits.getCount();
long requests = metrics.requests.getCount();
assertEquals(0, hits);
- assertEquals(10, requests);
+ assertEquals(sstableImplCachesKeys ? 10 : 0, requests);
// 10 queries, each 50 result rows
for (int i = 0; i < 10; i++)
@@ -503,8 +507,8 @@
metrics = CacheService.instance.keyCache.getMetrics();
hits = metrics.hits.getCount();
requests = metrics.requests.getCount();
- assertEquals(10, hits);
- assertEquals(10 + 10, requests);
+ assertEquals(sstableImplCachesKeys ? 10 : 0, hits);
+ assertEquals(sstableImplCachesKeys ? 20 : 0, requests);
// 100 queries - must get a hit in key-cache
for (int i = 0; i < 10; i++)
@@ -519,8 +523,8 @@
metrics = CacheService.instance.keyCache.getMetrics();
hits = metrics.hits.getCount();
requests = metrics.requests.getCount();
- assertEquals(10 + 100, hits);
- assertEquals(20 + 100, requests);
+ assertEquals(sstableImplCachesKeys ? 10 + 100 : 0, hits);
+ assertEquals(sstableImplCachesKeys ? 20 + 100 : 0, requests);
// 5000 queries - first 10 partitions already in key cache
for (int i = 0; i < 100; i++)
@@ -534,8 +538,8 @@
hits = metrics.hits.getCount();
requests = metrics.requests.getCount();
- assertEquals(110 + 4910, hits);
- assertEquals(120 + 5500, requests);
+ assertEquals(sstableImplCachesKeys ? 110 + 4910 : 0, hits);
+ assertEquals(sstableImplCachesKeys ? 120 + 5500 : 0, requests);
}
// Inserts 100 partitions split over 10 sstables (flush after 10 partitions).
@@ -614,6 +618,9 @@
private long recentBloomFilterFalsePositives()
{
- return getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).metric.recentBloomFilterFalsePositives.getValue();
+ return getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).metric.formatSpecificGauges.get(DatabaseDescriptor.getSelectedSSTableFormat())
+ .get(BloomFilterMetrics.instance.recentBloomFilterFalsePositives.name)
+ .getValue()
+ .longValue();
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/KeywordSplit2Test.java b/test/unit/org/apache/cassandra/cql3/KeywordSplit2Test.java
deleted file mode 100644
index 7be651a..0000000
--- a/test/unit/org/apache/cassandra/cql3/KeywordSplit2Test.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3;
-
-import java.util.Collection;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-/**
- * This base class tests all keywords which took a long time. Hence it was split into multiple
- * KeywordTestSplitN to prevent CI timing out. If timeouts reappear split it further
- */
-@RunWith(Parameterized.class)
-public class KeywordSplit2Test extends KeywordTestBase
-{
- static int SPLIT = 2;
- static int TOTAL_SPLITS = 2;
-
- @Parameterized.Parameters(name = "keyword {0} isReserved {1}")
- public static Collection<Object[]> keywords() {
- return KeywordTestBase.getKeywordsForSplit(SPLIT, TOTAL_SPLITS);
- }
-
- public KeywordSplit2Test(String keyword, boolean isReserved)
- {
- super(keyword, isReserved);
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/KeywordSplitTest.java b/test/unit/org/apache/cassandra/cql3/KeywordSplitTest.java
new file mode 100644
index 0000000..761aae3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/KeywordSplitTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import java.util.Collection;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * This base class tests all keywords which took a long time. Hence it was split into multiple
+ * KeywordTestSplitN to prevent CI timing out. If timeouts reappear split it further
+ */
+@RunWith(Parameterized.class)
+public class KeywordSplitTest extends KeywordTestBase
+{
+ static int SPLIT = 2;
+ static int TOTAL_SPLITS = 2;
+
+ @Parameterized.Parameters(name = "keyword {0} isReserved {1}")
+ public static Collection<Object[]> keywords() {
+ return KeywordTestBase.getKeywordsForSplit(SPLIT, TOTAL_SPLITS);
+ }
+
+ public KeywordSplitTest(String keyword, boolean isReserved)
+ {
+ super(keyword, isReserved);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java b/test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java
deleted file mode 100644
index ee5da92..0000000
--- a/test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3;
-
-import java.util.List;
-
-import com.google.common.collect.ImmutableList;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-
-@RunWith(Parameterized.class)
-public class MemtableQuickTest extends CQLTester
-{
- static final Logger logger = LoggerFactory.getLogger(MemtableQuickTest.class);
-
- static final int partitions = 50_000;
- static final int rowsPerPartition = 4;
-
- static final int deletedPartitionsStart = 20_000;
- static final int deletedPartitionsEnd = deletedPartitionsStart + 10_000;
-
- static final int deletedRowsStart = 40_000;
- static final int deletedRowsEnd = deletedRowsStart + 5_000;
-
- @Parameterized.Parameter()
- public String memtableClass;
-
- @Parameterized.Parameters(name = "{0}")
- public static List<Object> parameters()
- {
- return ImmutableList.of("skiplist",
- "skiplist_sharded",
- "skiplist_sharded_locking");
- }
-
- @BeforeClass
- public static void setUp()
- {
- CQLTester.setUpClass();
- CQLTester.prepareServer();
- CQLTester.disablePreparedReuseForTest();
- logger.info("setupClass done.");
- }
-
- @Test
- public void testMemtable() throws Throwable
- {
-
- String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
- String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" +
- " with compression = {'enabled': false}" +
- " and memtable = '" + memtableClass + "'");
- execute("use " + keyspace + ';');
-
- String writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)";
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
- cfs.disableAutoCompaction();
- Util.flush(cfs);
-
- long i;
- long limit = partitions;
- logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition);
- for (i = 0; i < limit; ++i)
- {
- for (long j = 0; j < rowsPerPartition; ++j)
- execute(writeStatement, i, j, i + j);
- }
-
- logger.info("Deleting partitions between {} and {}", deletedPartitionsStart, deletedPartitionsEnd);
- for (i = deletedPartitionsStart; i < deletedPartitionsEnd; ++i)
- {
- // no partition exists, but we will create a tombstone
- execute("DELETE FROM " + table + " WHERE userid = ?", i);
- }
-
- logger.info("Deleting rows between {} and {}", deletedRowsStart, deletedRowsEnd);
- for (i = deletedRowsStart; i < deletedRowsEnd; ++i)
- {
- // no row exists, but we will create a tombstone (and partition)
- execute("DELETE FROM " + table + " WHERE userid = ? AND picid = ?", i, 0L);
- }
-
- logger.info("Reading {} partitions", partitions);
- for (i = 0; i < limit; ++i)
- {
- UntypedResultSet result = execute("SELECT * FROM " + table + " WHERE userid = ?", i);
- if (i >= deletedPartitionsStart && i < deletedPartitionsEnd)
- assertEmpty(result);
- else
- {
- int start = 0;
- if (i >= deletedRowsStart && i < deletedRowsEnd)
- start = 1;
- Object[][] rows = new Object[rowsPerPartition - start][];
- for (long j = start; j < rowsPerPartition; ++j)
- rows[(int) (j - start)] = row(i, j, i + j);
- assertRows(result, rows);
- }
- }
-
- int deletedPartitions = deletedPartitionsEnd - deletedPartitionsStart;
- int deletedRows = deletedRowsEnd - deletedRowsStart;
- logger.info("Selecting *");
- UntypedResultSet result = execute("SELECT * FROM " + table);
- assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows);
-
- Util.flush(cfs);
-
- logger.info("Selecting *");
- result = execute("SELECT * FROM " + table);
- assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows);
- }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java b/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java
deleted file mode 100644
index 63ff055..0000000
--- a/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3;
-
-import java.util.List;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.ObjectSizes;
-
-@RunWith(Parameterized.class)
-public class MemtableSizeTest extends CQLTester
-{
- static final Logger logger = LoggerFactory.getLogger(MemtableSizeTest.class);
-
- static final int partitions = 50_000;
- static final int rowsPerPartition = 4;
-
- static final int deletedPartitions = 10_000;
- static final int deletedRows = 5_000;
-
- @Parameterized.Parameter(0)
- public String memtableClass;
-
- @Parameterized.Parameter(1)
- public int differencePerPartition;
-
- @Parameterized.Parameters(name = "{0}")
- public static List<Object[]> parameters()
- {
- return ImmutableList.of(new Object[]{"skiplist", 50},
- new Object[]{"skiplist_sharded", 60});
- }
-
- // must be within 50 bytes per partition of the actual size
- final long MAX_DIFFERENCE_PARTITIONS = (partitions + deletedPartitions + deletedRows);
-
- @BeforeClass
- public static void setUp()
- {
- CQLTester.setUpClass();
- CQLTester.prepareServer();
- CQLTester.disablePreparedReuseForTest();
- logger.info("setupClass done.");
- }
-
- @Test
- public void testSize()
- {
- Util.flakyTest(this::testSizeFlaky, 2, "Fails occasionally, see CASSANDRA-16684");
- }
-
- private void testSizeFlaky()
- {
- try
- {
- String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
- String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" +
- " with compression = {'enabled': false}" +
- " and memtable = '" + memtableClass + "'");
- execute("use " + keyspace + ';');
-
- String writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)";
-
- ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
- cfs.disableAutoCompaction();
- Util.flush(cfs);
-
- long deepSizeBefore = ObjectSizes.measureDeep(cfs.getTracker().getView().getCurrentMemtable());
- logger.info("Memtable deep size before {}\n",
- FBUtilities.prettyPrintMemory(deepSizeBefore));
- long i;
- long limit = partitions;
- logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition);
- for (i = 0; i < limit; ++i)
- {
- for (long j = 0; j < rowsPerPartition; ++j)
- execute(writeStatement, i, j, i + j);
- }
-
- logger.info("Deleting {} partitions", deletedPartitions);
- limit += deletedPartitions;
- for (; i < limit; ++i)
- {
- // no partition exists, but we will create a tombstone
- execute("DELETE FROM " + table + " WHERE userid = ?", i);
- }
-
- logger.info("Deleting {} rows", deletedRows);
- limit += deletedRows;
- for (; i < limit; ++i)
- {
- // no row exists, but we will create a tombstone (and partition)
- execute("DELETE FROM " + table + " WHERE userid = ? AND picid = ?", i, 0L);
- }
-
- if (!cfs.getLiveSSTables().isEmpty())
- logger.info("Warning: " + cfs.getLiveSSTables().size() + " sstables created.");
-
- Memtable memtable = cfs.getTracker().getView().getCurrentMemtable();
- Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable);
- long actualHeap = usage.ownsOnHeap;
- logger.info("Memtable in {} mode: {} ops, {} serialized bytes, {}\n",
- DatabaseDescriptor.getMemtableAllocationType(),
- memtable.operationCount(),
- FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()),
- usage);
-
- long deepSizeAfter = ObjectSizes.measureDeep(memtable);
- logger.info("Memtable deep size {}\n",
- FBUtilities.prettyPrintMemory(deepSizeAfter));
-
- long expectedHeap = deepSizeAfter - deepSizeBefore;
- String message = String.format("Expected heap usage close to %s, got %s.\n",
- FBUtilities.prettyPrintMemory(expectedHeap),
- FBUtilities.prettyPrintMemory(actualHeap));
- logger.info(message);
- Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= MAX_DIFFERENCE_PARTITIONS * differencePerPartition);
- }
- catch (Throwable throwable)
- {
- Throwables.propagate(throwable);
- }
- }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/cql3/PagingTest.java b/test/unit/org/apache/cassandra/cql3/PagingTest.java
index a3387c4..75d73e5 100644
--- a/test/unit/org/apache/cassandra/cql3/PagingTest.java
+++ b/test/unit/org/apache/cassandra/cql3/PagingTest.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
@@ -57,7 +58,7 @@
@BeforeClass
public static void setup() throws Exception
{
- System.setProperty("cassandra.config", "cassandra-murmur.yaml");
+ CASSANDRA_CONFIG.setString("cassandra-murmur.yaml");
cassandra = ServerTestUtils.startEmbeddedCassandraService();
diff --git a/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java b/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java
index 01a2afd..63562b7 100644
--- a/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java
+++ b/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java
@@ -17,13 +17,14 @@
*/
package org.apache.cassandra.cql3;
-import java.util.Random;
-
import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.ForwardingSSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -59,7 +60,21 @@
boolean hasIndexed = false;
for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables())
{
- RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ);
+ class IndexEntryAccessor extends ForwardingSSTableReader
+ {
+ public IndexEntryAccessor(SSTableReader delegate)
+ {
+ super(delegate);
+ }
+
+ public AbstractRowIndexEntry getRowIndexEntry(PartitionPosition key, Operator op, boolean updateCacheAndStats, SSTableReadsListener listener)
+ {
+ return super.getRowIndexEntry(key, op, updateCacheAndStats, listener);
+ }
+ }
+
+ IndexEntryAccessor accessor = new IndexEntryAccessor(sstable);
+ AbstractRowIndexEntry indexEntry = accessor.getRowIndexEntry(dk, SSTableReader.Operator.EQ, false, SSTableReadsListener.NOOP_LISTENER);
hasIndexed |= indexEntry != null && indexEntry.isIndexed();
}
assert hasIndexed;
@@ -70,15 +85,4 @@
assertRowCount(execute("SELECT DISTINCT s FROM %s WHERE k = ?", 0), 1);
assertRowCount(execute("SELECT DISTINCT s FROM %s WHERE k = ? ORDER BY t DESC", 0), 1);
}
-
- // Creates a random string
- public static String makeRandomSt(int length)
- {
- Random random = new Random();
- char[] chars = new char[26];
- int i = 0;
- for (char c = 'a'; c <= 'z'; c++)
- chars[i++] = c;
- return new String(chars);
- }
}
diff --git a/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java b/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java
index f9ac8d1..1d888ff 100644
--- a/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java
+++ b/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java
@@ -19,13 +19,18 @@
import java.util.Random;
+import org.junit.Assume;
import org.junit.Test;
import org.apache.cassandra.Util;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
import org.apache.cassandra.utils.ByteBufferUtil;
public class TombstonesWithIndexedSSTableTest extends CQLTester
@@ -44,6 +49,7 @@
public void testTombstoneBoundariesInIndex(String cacheKeys) throws Throwable
{
+ Assume.assumeTrue("This test requires that the default SSTable format is BIG", BigFormat.isSelected());
// That test reproduces the bug from CASSANDRA-11158 where a range tombstone boundary in the column index would
// cause an assertion failure.
@@ -74,19 +80,16 @@
int indexedRow = -1;
for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables())
{
+ BigTableReader reader = (BigTableReader) sstable;
// The line below failed with key caching off (CASSANDRA-11158)
- @SuppressWarnings("unchecked")
- RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ);
+ RowIndexEntry indexEntry = reader.getRowIndexEntry(dk, SSTableReader.Operator.EQ);
if (indexEntry != null && indexEntry.isIndexed())
{
- try (FileDataInput reader = sstable.openIndexReader())
- {
- RowIndexEntry.IndexInfoRetriever infoRetriever = indexEntry.openWithIndex(sstable.getIndexFile());
- ClusteringPrefix<?> firstName = infoRetriever.columnsIndex(1).firstName;
- if (firstName.kind().isBoundary())
- break deletionLoop;
- indexedRow = Int32Type.instance.compose(firstName.bufferAt(0));
- }
+ RowIndexEntry.IndexInfoRetriever infoRetriever = indexEntry.openWithIndex(reader.getIndexFile());
+ ClusteringPrefix<?> firstName = infoRetriever.columnsIndex(1).firstName;
+ if (firstName.kind().isBoundary())
+ break deletionLoop;
+ indexedRow = Int32Type.instance.compose(firstName.bufferAt(0));
}
}
assert indexedRow >= 0;
@@ -108,6 +111,78 @@
assertRowCount(execute("SELECT DISTINCT s FROM %s WHERE k = ? ORDER BY t DESC", 0), 1);
}
+ @Test
+ public void testActiveTombstoneInIndexCached() throws Throwable
+ {
+ Assume.assumeFalse("BTI format does not use key cache", BtiFormat.isSelected());
+ testActiveTombstoneInIndex("ALL");
+ }
+
+ @Test
+ public void testActiveTombstoneInIndexNotCached() throws Throwable
+ {
+ testActiveTombstoneInIndex("NONE");
+ }
+
+ public void testActiveTombstoneInIndex(String cacheKeys) throws Throwable
+ {
+ int ROWS = 1000;
+ int VALUE_LENGTH = 100;
+
+ createTable("CREATE TABLE %s (k int, t int, v1 text, v2 text, v3 text, v4 text, PRIMARY KEY (k, t)) WITH caching = { 'keys' : '" + cacheKeys + "' }");
+ String text = makeRandomString(VALUE_LENGTH);
+
+ // Write a large-enough partition to be indexed.
+ for (int i = 0; i < ROWS; i++)
+ execute("INSERT INTO %s(k, t, v1) VALUES (?, ?, ?) USING TIMESTAMP 1", 0, i, text);
+ // Add v2 that should survive part of the deletion we later insert
+ for (int i = 0; i < ROWS; i++)
+ execute("INSERT INTO %s(k, t, v2) VALUES (?, ?, ?) USING TIMESTAMP 3", 0, i, text);
+ flush();
+
+ // Now delete parts of this partition, but add enough new data to make sure the deletion spans index blocks
+ int minDeleted1 = ROWS/10;
+ int maxDeleted1 = 5 * ROWS/10;
+ execute("DELETE FROM %s USING TIMESTAMP 2 WHERE k = 0 AND t >= ? AND t < ?", minDeleted1, maxDeleted1);
+
+ // Delete again to make a boundary
+ int minDeleted2 = 4 * ROWS/10;
+ int maxDeleted2 = 9 * ROWS/10;
+ execute("DELETE FROM %s USING TIMESTAMP 4 WHERE k = 0 AND t >= ? AND t < ?", minDeleted2, maxDeleted2);
+
+ // Add v3 surviving that deletion too and also ensuring the two deletions span index blocks
+ for (int i = 0; i < ROWS; i++)
+ execute("INSERT INTO %s(k, t, v3) VALUES (?, ?, ?) USING TIMESTAMP 5", 0, i, text);
+ flush();
+
+ // test deletions worked
+ verifyExpectedActiveTombstoneRows(ROWS, text, minDeleted1, minDeleted2, maxDeleted2);
+
+ // Test again compacted. This is much easier to pass and doesn't actually test active tombstones in index
+ compact();
+ verifyExpectedActiveTombstoneRows(ROWS, text, minDeleted1, minDeleted2, maxDeleted2);
+ }
+
+ private void verifyExpectedActiveTombstoneRows(int ROWS, String text, int minDeleted1, int minDeleted2, int maxDeleted2) throws Throwable
+ {
+ assertRowCount(execute("SELECT t FROM %s WHERE k = ? AND v1 = ? ALLOW FILTERING", 0, text), ROWS - (maxDeleted2 - minDeleted1));
+ assertRowCount(execute("SELECT t FROM %s WHERE k = ? AND v1 = ? ORDER BY t DESC ALLOW FILTERING", 0, text), ROWS - (maxDeleted2 - minDeleted1));
+ assertRowCount(execute("SELECT t FROM %s WHERE k = ? AND v2 = ? ALLOW FILTERING", 0, text), ROWS - (maxDeleted2 - minDeleted2));
+ assertRowCount(execute("SELECT t FROM %s WHERE k = ? AND v2 = ? ORDER BY t DESC ALLOW FILTERING", 0, text), ROWS - (maxDeleted2 - minDeleted2));
+ assertRowCount(execute("SELECT t FROM %s WHERE k = ? AND v3 = ? ALLOW FILTERING", 0, text), ROWS);
+ assertRowCount(execute("SELECT t FROM %s WHERE k = ? AND v3 = ? ORDER BY t DESC ALLOW FILTERING", 0, text), ROWS);
+ // test index yields the correct active deletions
+ for (int i = 0; i < ROWS; ++i)
+ {
+ final String v1Expected = i < minDeleted1 || i >= maxDeleted2 ? text : null;
+ final String v2Expected = i < minDeleted2 || i >= maxDeleted2 ? text : null;
+ assertRows(execute("SELECT v1,v2,v3 FROM %s WHERE k = ? AND t >= ? LIMIT 1", 0, i),
+ row(v1Expected, v2Expected, text));
+ assertRows(execute("SELECT v1,v2,v3 FROM %s WHERE k = ? AND t <= ? ORDER BY t DESC LIMIT 1", 0, i),
+ row(v1Expected, v2Expected, text));
+ }
+ }
+
public static String makeRandomString(int length)
{
Random random = new Random();
diff --git a/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java b/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java
index 47d5b8e..93b0924 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java
@@ -31,6 +31,8 @@
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE;
+
/* ViewFilteringTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167)
* Any changes here check if they apply to the other classes
* - ViewFilteringPKTest
@@ -46,13 +48,13 @@
public static void startup()
{
ViewAbstractParameterizedTest.startup();
- System.setProperty("cassandra.mv.allow_filtering_nonkey_columns_unsafe", "true");
+ MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE.setBoolean(true);
}
@AfterClass
public static void tearDown()
{
- System.setProperty("cassandra.mv.allow_filtering_nonkey_columns_unsafe", "false");
+ MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE.setBoolean(false);
}
// TODO will revise the non-pk filter condition in MV, see CASSANDRA-11500
@@ -363,7 +365,7 @@
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a IS NOT NULL AND b IS NOT NULL AND d is NOT NULL PRIMARY KEY ((a, b), c, d)",
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a IS NOT NULL AND b IS NOT NULL AND c is NOT NULL PRIMARY KEY ((a, b), c, d)",
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = ? AND b IS NOT NULL AND c is NOT NULL PRIMARY KEY ((a, b), c, d)",
- "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = blobAsInt(?) AND b IS NOT NULL AND c is NOT NULL PRIMARY KEY ((a, b), c, d)",
+ "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = blob_as_int(?) AND b IS NOT NULL AND c is NOT NULL PRIMARY KEY ((a, b), c, d)",
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s PRIMARY KEY (a, b, c, d)"
);
@@ -391,7 +393,7 @@
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = 1 AND b = 1 AND (c, d) > (1, 1) PRIMARY KEY ((a, b), c, d)",
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = 1 AND b = 1 AND (c, d) IN ((1, 1), (2, 2)) PRIMARY KEY ((a, b), c, d)",
"CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = (int) 1 AND b = 1 AND c = 1 AND d = 1 PRIMARY KEY ((a, b), c, d)",
- "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = blobAsInt(intAsBlob(1)) AND b = 1 AND c = 1 AND d = 1 PRIMARY KEY ((a, b), c, d)"
+ "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE a = blob_as_int(int_as_blob(1)) AND b = 1 AND c = 1 AND d = 1 PRIMARY KEY ((a, b), c, d)"
);
for (int i = 0; i < goodStatements.size(); i++)
@@ -462,7 +464,7 @@
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 1, 1, 3);
createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
- "WHERE a = blobAsInt(intAsBlob(1)) AND b IS NOT NULL " +
+ "WHERE a = blob_as_int(int_as_blob(1)) AND b IS NOT NULL " +
"PRIMARY KEY (a, b)");
assertRows(executeView("SELECT a, b, c FROM %s"),
diff --git a/test/unit/org/apache/cassandra/cql3/ViewPKTest.java b/test/unit/org/apache/cassandra/cql3/ViewPKTest.java
index 6ab27ef..1c2b9a6 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewPKTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewPKTest.java
@@ -253,8 +253,8 @@
}
}
- updateView("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, fromJson(?))", 0, "ascii text", "123123123123");
- updateView("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
+ updateView("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, from_json(?))", 0, "ascii text", "123123123123");
+ updateView("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT bigintval FROM %s WHERE k = ? and asciival = ?", 0, "ascii text"), row(123123123123L));
//Check the MV
@@ -265,7 +265,7 @@
//UPDATE BASE
- updateView("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, fromJson(?))", 0, "ascii text", "1");
+ updateView("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, from_json(?))", 0, "ascii text", "1");
assertRows(execute("SELECT bigintval FROM %s WHERE k = ? and asciival = ?", 0, "ascii text"), row(1L));
//Check the MV
diff --git a/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java b/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java
index 09d7e15..b58777e 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java
@@ -254,313 +254,313 @@
}
}
- // fromJson() can only be used when the receiver type is known
- assertInvalidMessage("fromJson() cannot be used in the selection clause", "SELECT fromJson(asciival) FROM %s", 0, 0);
+ // from_json() can only be used when the receiver type is known
+ assertInvalidMessage("from_json() cannot be used in the selection clause", "SELECT from_json(asciival) FROM %s", 0, 0);
String func1 = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (a int) CALLED ON NULL INPUT RETURNS text LANGUAGE java AS $$ return a.toString(); $$");
createFunctionOverload(func1, "int", "CREATE FUNCTION %s (a text) CALLED ON NULL INPUT RETURNS text LANGUAGE java AS $$ return new String(a); $$");
// ================ ascii ================
- updateView("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
+ updateView("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii \\\" text\"");
+ updateView("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"ascii \\\" text\"");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, "ascii \" text"));
- // test that we can use fromJson() in other valid places in queries
- assertRows(execute("SELECT asciival FROM %s WHERE k = fromJson(?)", "0"), row("ascii \" text"));
+ // test that we can use from_json() in other valid places in queries
+ assertRows(execute("SELECT asciival FROM %s WHERE k = from_json(?)", "0"), row("ascii \" text"));
//Check the MV
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii text"));
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii \" text"), row(0, null));
- updateView("UPDATE %s SET asciival = fromJson(?) WHERE k = fromJson(?)", "\"ascii \\\" text\"", "0");
+ updateView("UPDATE %s SET asciival = from_json(?) WHERE k = from_json(?)", "\"ascii \\\" text\"", "0");
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii \" text"), row(0, null));
- updateView("DELETE FROM %s WHERE k = fromJson(?)", "0");
+ updateView("DELETE FROM %s WHERE k = from_json(?)", "0");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0));
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii \" text"));
- updateView("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
+ updateView("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii text"), row(0, null));
// ================ bigint ================
- updateView("INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "123123123123");
+ updateView("INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "123123123123");
assertRows(execute("SELECT k, bigintval FROM %s WHERE k = ?", 0), row(0, 123123123123L));
assertRows(execute("SELECT k, asciival from mv_bigintval WHERE bigintval = ?", 123123123123L), row(0, "ascii text"));
// ================ blob ================
- updateView("INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "\"0x00000001\"");
+ updateView("INSERT INTO %s (k, blobval) VALUES (?, from_json(?))", 0, "\"0x00000001\"");
assertRows(execute("SELECT k, blobval FROM %s WHERE k = ?", 0), row(0, ByteBufferUtil.bytes(1)));
assertRows(execute("SELECT k, asciival from mv_blobval WHERE blobval = ?", ByteBufferUtil.bytes(1)), row(0, "ascii text"));
// ================ boolean ================
- updateView("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "true");
+ updateView("INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "true");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, true));
assertRows(execute("SELECT k, asciival from mv_booleanval WHERE booleanval = ?", true), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "false");
+ updateView("INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "false");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, false));
assertRows(execute("SELECT k, asciival from mv_booleanval WHERE booleanval = ?", true));
assertRows(execute("SELECT k, asciival from mv_booleanval WHERE booleanval = ?", false), row(0, "ascii text"));
// ================ date ================
- updateView("INSERT INTO %s (k, dateval) VALUES (?, fromJson(?))", 0, "\"1987-03-23\"");
+ updateView("INSERT INTO %s (k, dateval) VALUES (?, from_json(?))", 0, "\"1987-03-23\"");
assertRows(execute("SELECT k, dateval FROM %s WHERE k = ?", 0), row(0, SimpleDateSerializer.dateStringToDays("1987-03-23")));
- assertRows(execute("SELECT k, asciival from mv_dateval WHERE dateval = fromJson(?)", "\"1987-03-23\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_dateval WHERE dateval = from_json(?)", "\"1987-03-23\""), row(0, "ascii text"));
// ================ decimal ================
- updateView("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123.123123");
+ updateView("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "123123.123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123")));
- assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "123123.123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = from_json(?)", "123123.123123"), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123");
+ updateView("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123")));
- assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "123123.123123"));
- assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = from_json(?)", "123123.123123"));
+ assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = from_json(?)", "123123"), row(0, "ascii text"));
// accept strings for numbers that cannot be represented as doubles
- updateView("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"123123.123123\"");
+ updateView("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "\"123123.123123\"");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123")));
- updateView("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"-1.23E-12\"");
+ updateView("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "\"-1.23E-12\"");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("-1.23E-12")));
- assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "\"-1.23E-12\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = from_json(?)", "\"-1.23E-12\""), row(0, "ascii text"));
// ================ double ================
- updateView("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "123123.123123");
+ updateView("INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "123123.123123");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.123123d));
- assertRows(execute("SELECT k, asciival from mv_doubleval WHERE doubleval = fromJson(?)", "123123.123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_doubleval WHERE doubleval = from_json(?)", "123123.123123"), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "123123");
+ updateView("INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.0d));
- assertRows(execute("SELECT k, asciival from mv_doubleval WHERE doubleval = fromJson(?)", "123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_doubleval WHERE doubleval = from_json(?)", "123123"), row(0, "ascii text"));
// ================ float ================
- updateView("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "123123.123123");
+ updateView("INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "123123.123123");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.123123f));
- assertRows(execute("SELECT k, asciival from mv_floatval WHERE floatval = fromJson(?)", "123123.123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_floatval WHERE floatval = from_json(?)", "123123.123123"), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "123123");
+ updateView("INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.0f));
- assertRows(execute("SELECT k, asciival from mv_floatval WHERE floatval = fromJson(?)", "123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_floatval WHERE floatval = from_json(?)", "123123"), row(0, "ascii text"));
// ================ inet ================
- updateView("INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"127.0.0.1\"");
+ updateView("INSERT INTO %s (k, inetval) VALUES (?, from_json(?))", 0, "\"127.0.0.1\"");
assertRows(execute("SELECT k, inetval FROM %s WHERE k = ?", 0), row(0, InetAddress.getByName("127.0.0.1")));
- assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = fromJson(?)", "\"127.0.0.1\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = from_json(?)", "\"127.0.0.1\""), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"::1\"");
+ updateView("INSERT INTO %s (k, inetval) VALUES (?, from_json(?))", 0, "\"::1\"");
assertRows(execute("SELECT k, inetval FROM %s WHERE k = ?", 0), row(0, InetAddress.getByName("::1")));
- assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = fromJson(?)", "\"127.0.0.1\""));
- assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = fromJson(?)", "\"::1\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = from_json(?)", "\"127.0.0.1\""));
+ assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = from_json(?)", "\"::1\""), row(0, "ascii text"));
// ================ int ================
- updateView("INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "123123");
+ updateView("INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, intval FROM %s WHERE k = ?", 0), row(0, 123123));
- assertRows(execute("SELECT k, asciival from mv_intval WHERE intval = fromJson(?)", "123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_intval WHERE intval = from_json(?)", "123123"), row(0, "ascii text"));
// ================ text (varchar) ================
- updateView("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"some \\\" text\"");
+ updateView("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"some \\\" text\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "some \" text"));
- updateView("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"\\u2013\"");
+ updateView("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"\\u2013\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "\u2013"));
- assertRows(execute("SELECT k, asciival from mv_textval WHERE textval = fromJson(?)", "\"\\u2013\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_textval WHERE textval = from_json(?)", "\"\\u2013\""), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"abcd\"");
+ updateView("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"abcd\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "abcd"));
- assertRows(execute("SELECT k, asciival from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, "ascii text"));
// ================ time ================
- updateView("INSERT INTO %s (k, timeval) VALUES (?, fromJson(?))", 0, "\"07:35:07.000111222\"");
+ updateView("INSERT INTO %s (k, timeval) VALUES (?, from_json(?))", 0, "\"07:35:07.000111222\"");
assertRows(execute("SELECT k, timeval FROM %s WHERE k = ?", 0), row(0, TimeSerializer.timeStringToLong("07:35:07.000111222")));
- assertRows(execute("SELECT k, asciival from mv_timeval WHERE timeval = fromJson(?)", "\"07:35:07.000111222\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_timeval WHERE timeval = from_json(?)", "\"07:35:07.000111222\""), row(0, "ascii text"));
// ================ timestamp ================
- updateView("INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "123123123123");
+ updateView("INSERT INTO %s (k, timestampval) VALUES (?, from_json(?))", 0, "123123123123");
assertRows(execute("SELECT k, timestampval FROM %s WHERE k = ?", 0), row(0, new Date(123123123123L)));
- assertRows(execute("SELECT k, asciival from mv_timestampval WHERE timestampval = fromJson(?)", "123123123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_timestampval WHERE timestampval = from_json(?)", "123123123123"), row(0, "ascii text"));
- updateView("INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "\"2014-01-01\"");
+ updateView("INSERT INTO %s (k, timestampval) VALUES (?, from_json(?))", 0, "\"2014-01-01\"");
assertRows(execute("SELECT k, timestampval FROM %s WHERE k = ?", 0), row(0, new SimpleDateFormat("y-M-d").parse("2014-01-01")));
- assertRows(execute("SELECT k, asciival from mv_timestampval WHERE timestampval = fromJson(?)", "\"2014-01-01\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_timestampval WHERE timestampval = from_json(?)", "\"2014-01-01\""), row(0, "ascii text"));
// ================ timeuuid ================
- updateView("INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
+ updateView("INSERT INTO %s (k, timeuuidval) VALUES (?, from_json(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
assertRows(execute("SELECT k, timeuuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- updateView("INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
+ updateView("INSERT INTO %s (k, timeuuidval) VALUES (?, from_json(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
assertRows(execute("SELECT k, timeuuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- assertRows(execute("SELECT k, asciival from mv_timeuuidval WHERE timeuuidval = fromJson(?)", "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_timeuuidval WHERE timeuuidval = from_json(?)", "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\""), row(0, "ascii text"));
// ================ uuidval ================
- updateView("INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
+ updateView("INSERT INTO %s (k, uuidval) VALUES (?, from_json(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
assertRows(execute("SELECT k, uuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- updateView("INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
+ updateView("INSERT INTO %s (k, uuidval) VALUES (?, from_json(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
assertRows(execute("SELECT k, uuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- assertRows(execute("SELECT k, asciival from mv_uuidval WHERE uuidval = fromJson(?)", "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_uuidval WHERE uuidval = from_json(?)", "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\""), row(0, "ascii text"));
// ================ varint ================
- updateView("INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "123123123123");
+ updateView("INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "123123123123");
assertRows(execute("SELECT k, varintval FROM %s WHERE k = ?", 0), row(0, new BigInteger("123123123123")));
- assertRows(execute("SELECT k, asciival from mv_varintval WHERE varintval = fromJson(?)", "123123123123"), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_varintval WHERE varintval = from_json(?)", "123123123123"), row(0, "ascii text"));
// accept strings for numbers that cannot be represented as longs
- updateView("INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "\"1234567890123456789012345678901234567890\"");
+ updateView("INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "\"1234567890123456789012345678901234567890\"");
assertRows(execute("SELECT k, varintval FROM %s WHERE k = ?", 0), row(0, new BigInteger("1234567890123456789012345678901234567890")));
- assertRows(execute("SELECT k, asciival from mv_varintval WHERE varintval = fromJson(?)", "\"1234567890123456789012345678901234567890\""), row(0, "ascii text"));
+ assertRows(execute("SELECT k, asciival from mv_varintval WHERE varintval = from_json(?)", "\"1234567890123456789012345678901234567890\""), row(0, "ascii text"));
// ================ lists ================
- updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[1, 2, 3]");
+ updateView("INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1, 2, 3)));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(1, 2, 3)));
- updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[1]");
+ updateView("INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[1]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1)));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1)));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(1)));
- updateView("UPDATE %s SET listval = listval + fromJson(?) WHERE k = ?", "[2]", 0);
+ updateView("UPDATE %s SET listval = listval + from_json(?) WHERE k = ?", "[2]", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2)));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1, 2)));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(1, 2)));
- updateView("UPDATE %s SET listval = fromJson(?) + listval WHERE k = ?", "[0]", 0);
+ updateView("UPDATE %s SET listval = from_json(?) + listval WHERE k = ?", "[0]", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(0, 1, 2)));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(0, 1, 2)));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(0, 1, 2)));
- updateView("UPDATE %s SET listval[1] = fromJson(?) WHERE k = ?", "10", 0);
+ updateView("UPDATE %s SET listval[1] = from_json(?) WHERE k = ?", "10", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(0, 10, 2)));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(0, 10, 2)));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(0, 10, 2)));
updateView("DELETE listval[1] FROM %s WHERE k = ?", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(0, 2)));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(0, 2)));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(0, 2)));
- updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[]");
+ updateView("INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, null));
- assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, null));
+ assertRows(execute("SELECT k, listval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, null));
// frozen
- updateView("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[1, 2, 3]");
+ updateView("INSERT INTO %s (k, frozenlistval) VALUES (?, from_json(?))", 0, "[1, 2, 3]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
- assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1, 2, 3)));
- assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = fromJson(?)", "[1, 2, 3]"), row(0, "abcd"));
+ assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(1, 2, 3)));
+ assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = from_json(?)", "[1, 2, 3]"), row(0, "abcd"));
- updateView("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[3, 2, 1]");
+ updateView("INSERT INTO %s (k, frozenlistval) VALUES (?, from_json(?))", 0, "[3, 2, 1]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list(3, 2, 1)));
- assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = fromJson(?)", "[1, 2, 3]"));
- assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = fromJson(?)", "[3, 2, 1]"), row(0, "abcd"));
- assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(3, 2, 1)));
+ assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = from_json(?)", "[1, 2, 3]"));
+ assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = from_json(?)", "[3, 2, 1]"), row(0, "abcd"));
+ assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list(3, 2, 1)));
- updateView("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[]");
+ updateView("INSERT INTO %s (k, frozenlistval) VALUES (?, from_json(?))", 0, "[]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list()));
- assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list()));
+ assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, list()));
// ================ sets ================
- updateView("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))",
+ updateView("INSERT INTO %s (k, setval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, setval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
// duplicates are okay, just like in CQL
- updateView("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))",
+ updateView("INSERT INTO %s (k, setval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, setval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
- updateView("UPDATE %s SET setval = setval + fromJson(?) WHERE k = ?", "[\"6bddc89a-5644-0000-97fc-56847afe9799\"]", 0);
+ updateView("UPDATE %s SET setval = setval + from_json(?) WHERE k = ?", "[\"6bddc89a-5644-0000-97fc-56847afe9799\"]", 0);
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-0000-97fc-56847afe9799"), UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, setval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-0000-97fc-56847afe9799"), UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
- updateView("UPDATE %s SET setval = setval - fromJson(?) WHERE k = ?", "[\"6bddc89a-5644-0000-97fc-56847afe9799\"]", 0);
+ updateView("UPDATE %s SET setval = setval - from_json(?) WHERE k = ?", "[\"6bddc89a-5644-0000-97fc-56847afe9799\"]", 0);
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, setval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
- updateView("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))", 0, "[]");
+ updateView("INSERT INTO %s (k, setval) VALUES (?, from_json(?))", 0, "[]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0), row(0, null));
- assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, setval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, null));
// frozen
- updateView("INSERT INTO %s (k, frozensetval) VALUES (?, fromJson(?))",
+ updateView("INSERT INTO %s (k, frozensetval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, frozensetval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- assertRows(execute("SELECT k, frozensetval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, frozensetval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
- updateView("INSERT INTO %s (k, frozensetval) VALUES (?, fromJson(?))",
+ updateView("INSERT INTO %s (k, frozensetval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-0000-11e4-97fc-56847afe9799\", \"6bddc89a-5644-11e4-97fc-56847afe9798\"]");
assertRows(execute("SELECT k, frozensetval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-0000-11e4-97fc-56847afe9799"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"))))
);
- assertRows(execute("SELECT k, frozensetval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, frozensetval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-0000-11e4-97fc-56847afe9799"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798")))));
// ================ maps ================
- updateView("INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": 2}");
+ updateView("INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": 2}");
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
- assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, map("a", 1, "b", 2)));
+ assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = from_json(?)", "\"abcd\""), row(0, map("a", 1, "b", 2)));
updateView("UPDATE %s SET mapval[?] = ? WHERE k = ?", "c", 3, 0);
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0),
row(0, map("a", 1, "b", 2, "c", 3))
);
- assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, map("a", 1, "b", 2, "c", 3)));
updateView("UPDATE %s SET mapval[?] = ? WHERE k = ?", "b", 10, 0);
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0),
row(0, map("a", 1, "b", 10, "c", 3))
);
- assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, map("a", 1, "b", 10, "c", 3)));
updateView("DELETE mapval[?] FROM %s WHERE k = ?", "b", 0);
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0),
row(0, map("a", 1, "c", 3))
);
- assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, map("a", 1, "c", 3)));
- updateView("INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{}");
+ updateView("INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "{}");
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0), row(0, null));
- assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
+ assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = from_json(?)", "\"abcd\""),
row(0, null));
// frozen
- updateView("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": 2}");
+ updateView("INSERT INTO %s (k, frozenmapval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": 2}");
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
- assertRows(execute("SELECT k, textval FROM mv_frozenmapval WHERE frozenmapval = fromJson(?)", "{\"a\": 1, \"b\": 2}"), row(0, "abcd"));
+ assertRows(execute("SELECT k, textval FROM mv_frozenmapval WHERE frozenmapval = from_json(?)", "{\"a\": 1, \"b\": 2}"), row(0, "abcd"));
- updateView("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, "{\"b\": 2, \"a\": 3}");
+ updateView("INSERT INTO %s (k, frozenmapval) VALUES (?, from_json(?))", 0, "{\"b\": 2, \"a\": 3}");
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 3, "b", 2)));
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 3, "b", 2)));
// ================ tuples ================
- updateView("INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))", 0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
+ updateView("INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))", 0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, tupleval FROM %s WHERE k = ?", 0),
row(0, tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))
);
assertRows(execute("SELECT k, textval FROM mv_tupleval WHERE tupleval = ?", tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))),
row(0, "abcd"));
- updateView("INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))", 0, "[1, null, \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
+ updateView("INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))", 0, "[1, null, \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, tupleval FROM %s WHERE k = ?", 0),
row(0, tuple(1, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))
);
@@ -569,37 +569,37 @@
row(0, "abcd"));
// ================ UDTs ================
- updateView("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
+ updateView("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
- assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
+ assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = from_json(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
row(0, "abcd"));
// order of fields shouldn't matter
- updateView("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"a\": 1, \"c\": [\"foo\", \"bar\"]}");
+ updateView("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"a\": 1, \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
- assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
+ assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = from_json(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
row(0, "abcd"));
// test nulls
- updateView("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
+ updateView("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
- assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"));
- assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
+ assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = from_json(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"));
+ assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = from_json(?)", "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
row(0, "abcd"));
// test missing fields
- updateView("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}");
+ updateView("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), null)
);
- assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"));
- assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}"),
+ assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = from_json(?)", "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"));
+ assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = from_json(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}"),
row(0, "abcd"));
}
diff --git a/test/unit/org/apache/cassandra/cql3/ViewTest.java b/test/unit/org/apache/cassandra/cql3/ViewTest.java
index d82fd32..a2793fd 100644
--- a/test/unit/org/apache/cassandra/cql3/ViewTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java
@@ -232,12 +232,12 @@
createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE k IS NOT NULL AND intval IS NOT NULL PRIMARY KEY (intval, k)");
- updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, fromJson(?))", 0, 0, "[1, 2, 3]");
+ updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, from_json(?))", 0, 0, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
assertRows(executeView("SELECT k, listval from %s WHERE intval = ?", 0), row(0, list(1, 2, 3)));
updateView("INSERT INTO %s (k, intval) VALUES (?, ?)", 1, 1);
- updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 1, "[1, 2, 3]");
+ updateView("INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 1, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 1), row(1, list(1, 2, 3)));
assertRows(executeView("SELECT k, listval from %s WHERE intval = ?", 1), row(1, list(1, 2, 3)));
}
@@ -249,7 +249,7 @@
createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s WHERE k IS NOT NULL AND listval IS NOT NULL PRIMARY KEY (k, listval)");
- updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, fromJson(?))",
+ updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, from_json(?))",
0,
0,
"[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]");
@@ -261,7 +261,7 @@
row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
// update listval with the same value and it will be compared in view generator
- updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))",
+ updateView("INSERT INTO %s (k, listval) VALUES (?, from_json(?))",
0,
"[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]");
// verify result
diff --git a/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java
index 6efbdc0..7fe3b61 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/CastFctsTest.java
@@ -326,7 +326,7 @@
assertRows(execute("SELECT v FROM %s"), row(3));
// Cast of placeholder without type hint
- assertInvalidRequestMessage("Ambiguous call to function system.castAsInt",
+ assertInvalidRequestMessage("Ambiguous call to function system.cast_as_int",
"INSERT INTO %s (k, v) VALUES (1, CAST(? AS int))", 3.4f);
// Type hint of cast
@@ -363,7 +363,7 @@
assertRows(execute("SELECT v FROM %s"), row(3));
// Cast of placeholder without type hint
- assertInvalidRequestMessage("Ambiguous call to function system.castAsInt",
+ assertInvalidRequestMessage("Ambiguous call to function system.cast_as_int",
"UPDATE %s SET v = CAST(? AS int) WHERE k = 1", 3.4f);
// Type hint of cast
@@ -405,7 +405,7 @@
assertRows(execute("SELECT v FROM %s WHERE k = ?", 3), row(3));
// Cast of placeholder without type hint
- assertInvalidRequestMessage("Ambiguous call to function system.castAsInt",
+ assertInvalidRequestMessage("Ambiguous call to function system.cast_as_int",
"UPDATE %s SET v = ? WHERE k = CAST(? AS int)", 3, 3.4f);
// Type hint of cast
@@ -444,7 +444,7 @@
assertRows(execute("SELECT k FROM %s WHERE k = CAST((float) ? AS int)", 3.4f), row(3));
// Cast of placeholder without type hint
- assertInvalidRequestMessage("Ambiguous call to function system.castAsInt",
+ assertInvalidRequestMessage("Ambiguous call to function system.cast_as_int",
"SELECT k FROM %s WHERE k = CAST(? AS int)", 3.4f);
// Type hint of cast
@@ -483,7 +483,7 @@
assertEmpty(execute("SELECT * FROM %s WHERE k = ?", 3));
// Cast of placeholder without type hint
- assertInvalidRequestMessage("Ambiguous call to function system.castAsInt",
+ assertInvalidRequestMessage("Ambiguous call to function system.cast_as_int",
"DELETE FROM %s WHERE k = CAST(? AS int)", 3.4f);
// Type hint of cast
diff --git a/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java
new file mode 100644
index 0000000..0ad299e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/CollectionFctsTest.java
@@ -0,0 +1,595 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+/**
+ * Tests for the functions defined on {@link CollectionFcts}.
+ */
+public class CollectionFctsTest extends CQLTester
+{
+ private static final BigInteger bigint1 = new BigInteger("12345678901234567890");
+ private static final BigInteger bigint2 = new BigInteger("23456789012345678901");
+ private static final BigDecimal bigdecimal1 = new BigDecimal("1234567890.1234567890");
+ private static final BigDecimal bigdecimal2 = new BigDecimal("2345678901.2345678901");
+
+ @Test
+ public void testNotNumericCollection() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v uuid, l list<text>, s set<boolean>, fl frozen<list<text>>, fs frozen<set<boolean>>)");
+
+ // sum
+ assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+ "but found argument v of type uuid",
+ InvalidRequestException.class,
+ "SELECT collection_sum(v) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+ "but found argument l of type list<text>",
+ InvalidRequestException.class,
+ "SELECT collection_sum(l) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+ "but found argument s of type set<boolean>",
+ InvalidRequestException.class,
+ "SELECT collection_sum(s) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+ "but found argument fl of type frozen<list<text>>",
+ InvalidRequestException.class,
+ "SELECT collection_sum(fl) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_sum requires a numeric set/list argument, " +
+ "but found argument fs of type frozen<set<boolean>>",
+ InvalidRequestException.class,
+ "SELECT collection_sum(fs) FROM %s");
+
+ // avg
+ assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+ "but found argument v of type uuid",
+ InvalidRequestException.class,
+ "SELECT collection_avg(v) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+ "but found argument l of type list<text>",
+ InvalidRequestException.class,
+ "SELECT collection_avg(l) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+ "but found argument s of type set<boolean>",
+ InvalidRequestException.class,
+ "SELECT collection_avg(s) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+ "but found argument fl of type frozen<list<text>>",
+ InvalidRequestException.class,
+ "SELECT collection_avg(fl) FROM %s");
+ assertInvalidThrowMessage("Function system.collection_avg requires a numeric set/list argument, " +
+ "but found argument fs of type frozen<set<boolean>>",
+ InvalidRequestException.class,
+ "SELECT collection_avg(fs) FROM %s");
+ }
+
+ @Test
+ public void testTinyInt() throws Throwable
+ {
+ createTable(CQL3Type.Native.TINYINT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, (byte) 0, (byte) 0));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, (byte) 0, (byte) 0));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list((byte) 1, (byte) 2), set((byte) 1, (byte) 2),
+ list((byte) 1, (byte) 2), set((byte) 1, (byte) 2),
+ map((byte) 1, (byte) 2, (byte) 3, (byte) 4),
+ map((byte) 1, (byte) 2, (byte) 3, (byte) 4));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set((byte) 1, (byte) 3), set((byte) 1, (byte) 3)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list((byte) 2, (byte) 4), list((byte) 2, (byte) 4)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row((byte) 1, (byte) 1, (byte) 1, (byte) 1));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row((byte) 2, (byte) 2, (byte) 2, (byte) 2));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row((byte) 3, (byte) 3, (byte) 3, (byte) 3));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row((byte) 1, (byte) 1, (byte) 1, (byte) 1));
+ }
+
+ @Test
+ public void testSmallInt() throws Throwable
+ {
+ createTable(CQL3Type.Native.SMALLINT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, (short) 0, (short) 0));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, (short) 0, (short) 0));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list((short) 1, (short) 2), set((short) 1, (short) 2),
+ list((short) 1, (short) 2), set((short) 1, (short) 2),
+ map((short) 1, (short) 2, (short) 3, (short) 4),
+ map((short) 1, (short) 2, (short) 3, (short) 4));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set((short) 1, (short) 3), set((short) 1, (short) 3)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list((short) 2, (short) 4), list((short) 2, (short) 4)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row((short) 1, (short) 1, (short) 1, (short) 1));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row((short) 2, (short) 2, (short) 2, (short) 2));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row((short) 3, (short) 3, (short) 3, (short) 3));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row((short) 1, (short) 1, (short) 1, (short) 1));
+ }
+
+ @Test
+ public void testInt() throws Throwable
+ {
+ createTable(CQL3Type.Native.INT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, 0, 0));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, 0, 0));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(1, 2), set(1, 2), list(1, 2), set(1, 2), map(1, 2, 3, 4), map(1, 2, 3, 4));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(1, 3), set(1, 3)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(2, 4), list(2, 4)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(1, 1, 1, 1));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(2, 2, 2, 2));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(3, 3, 3, 3));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(1, 1, 1, 1));
+ }
+
+ @Test
+ public void testBigInt() throws Throwable
+ {
+ createTable(CQL3Type.Native.BIGINT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, 0L, 0L));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, 0L, 0L));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(1L, 2L), set(1L, 2L), list(1L, 2L), set(1L, 2L), map(1L, 2L, 3L, 4L), map(1L, 2L, 3L, 4L));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(1L, 3L), set(1L, 3L)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(2L, 4L), list(2L, 4L)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(1L, 1L, 1L, 1L));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(2L, 2L, 2L, 2L));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(3L, 3L, 3L, 3L));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(1L, 1L, 1L, 1L));
+ }
+
+ @Test
+ public void testFloat() throws Throwable
+ {
+ createTable(CQL3Type.Native.FLOAT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, 0f, 0f));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, 0f, 0f));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(1.23f, 2.34f), list(1.23f, 2.34f),
+ set(1.23f, 2.34f), set(1.23f, 2.34f),
+ map(1.23f, 2.34f, 3.45f, 4.56f), map(1.23f, 2.34f, 3.45f, 4.56f));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(1.23f, 3.45f), set(1.23f, 3.45f)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(2.34f, 4.56f), list(2.34f, 4.56f)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(1.23f, 1.23f, 1.23f, 1.23f));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(2.34f, 2.34f, 2.34f, 2.34f));
+
+ float sum = 1.23f + 2.34f;
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(sum, sum, sum, sum));
+
+ float avg = (1.23f + 2.34f) / 2;
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(avg, avg, avg, avg));
+ }
+
+ @Test
+ public void testDouble() throws Throwable
+ {
+ createTable(CQL3Type.Native.DOUBLE);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, 0d, 0d));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, 0d, 0d));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(1.23d, 2.34d), list(1.23d, 2.34d),
+ set(1.23d, 2.34d), set(1.23d, 2.34d),
+ map(1.23d, 2.34d, 3.45d, 4.56d), map(1.23d, 2.34d, 3.45d, 4.56d));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(1.23d, 3.45d), set(1.23d, 3.45d)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(2.34d, 4.56d), list(2.34d, 4.56d)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(1.23d, 1.23d, 1.23d, 1.23d));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(2.34d, 2.34d, 2.34d, 2.34d));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(3.57d, 3.57d, 3.57d, 3.57d));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(1.785d, 1.785d, 1.785d, 1.785d));
+ }
+
+ @Test
+ public void testVarInt() throws Throwable
+ {
+ createTable(CQL3Type.Native.VARINT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, BigInteger.ZERO, BigInteger.ZERO));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, BigInteger.ZERO, BigInteger.ZERO));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(bigint1, bigint2), list(bigint1, bigint2),
+ set(bigint1, bigint2), set(bigint1, bigint2),
+ map(bigint1, bigint2, bigint2, bigint1), map(bigint1, bigint2, bigint2, bigint1));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(bigint1, bigint2), set(bigint1, bigint2)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(bigint2, bigint1), list(bigint2, bigint1)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(bigint1, bigint1, bigint1, bigint1));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(bigint2, bigint2, bigint2, bigint2));
+
+ BigInteger sum = bigint1.add(bigint2);
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(sum, sum, sum, sum));
+
+ BigInteger avg = bigint1.add(bigint2).divide(BigInteger.valueOf(2));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(avg, avg, avg, avg));
+ }
+
+ @Test
+ public void testDecimal() throws Throwable
+ {
+ createTable(CQL3Type.Native.DECIMAL);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(null, null, null, null));
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(null, null, BigDecimal.ZERO, BigDecimal.ZERO));
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(null, null, BigDecimal.ZERO, BigDecimal.ZERO));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(bigdecimal1, bigdecimal2), list(bigdecimal1, bigdecimal2),
+ set(bigdecimal1, bigdecimal2), set(bigdecimal1, bigdecimal2),
+ map(bigdecimal1, bigdecimal2, bigdecimal2, bigdecimal1),
+ map(bigdecimal1, bigdecimal2, bigdecimal2, bigdecimal1));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(bigdecimal1, bigdecimal2), set(bigdecimal1, bigdecimal2)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(bigdecimal2, bigdecimal1), list(bigdecimal2, bigdecimal1)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(bigdecimal1, bigdecimal1, bigdecimal1, bigdecimal1));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(bigdecimal2, bigdecimal2, bigdecimal2, bigdecimal2));
+
+ BigDecimal sum = bigdecimal1.add(bigdecimal2);
+ assertRows(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"),
+ row(sum, sum, sum, sum));
+
+ BigDecimal avg = bigdecimal1.add(bigdecimal2).divide(BigDecimal.valueOf(2), RoundingMode.HALF_EVEN);
+ assertRows(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"),
+ row(avg, avg, avg, avg));
+ }
+
+ @Test
+ public void testAscii() throws Throwable
+ {
+ createTable(CQL3Type.Native.ASCII);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list("abc", "bcd"), set("abc", "bcd"),
+ list("abc", "bcd"), set("abc", "bcd"),
+ map("abc", "bcd", "cde", "def"), map("abc", "bcd", "cde", "def"));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set("abc", "cde"), set("abc", "cde")));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list("bcd", "def"), list("bcd", "def")));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row("abc", "abc", "abc", "abc"));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row("bcd", "bcd", "bcd", "bcd"));
+ }
+
+ @Test
+ public void testText() throws Throwable
+ {
+ createTable(CQL3Type.Native.TEXT);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list("ábc", "bcd"), set("ábc", "bcd"),
+ list("ábc", "bcd"), set("ábc", "bcd"),
+ map("ábc", "bcd", "cdé", "déf"), map("ábc", "bcd", "cdé", "déf"));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set("ábc", "cdé"), set("ábc", "cdé")));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list("déf", "bcd"), list("déf", "bcd")));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row("bcd", "bcd", "bcd", "bcd"));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row("ábc", "ábc", "ábc", "ábc"));
+ }
+
+ @Test
+ public void testBoolean() throws Throwable
+ {
+ createTable(CQL3Type.Native.BOOLEAN);
+
+ // empty collections
+ assertRows(execute("SELECT map_keys(m), map_values(m), map_keys(fm), map_values(fm) FROM %s"),
+ row(null, null, set(), list()));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m), " +
+ "collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(null, null, null, 0, 0, 0));
+
+ // not empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(true, false), set(true, false),
+ list(true, false), set(true, false),
+ map(true, false, false, true), map(true, false, false, true));
+
+ assertRows(execute("SELECT map_keys(m), map_keys(fm) FROM %s"),
+ row(set(true, false), set(true, false)));
+ assertRows(execute("SELECT map_values(m), map_values(fm) FROM %s"),
+ row(list(true, false), list(true, false)));
+ assertRows(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"),
+ row(2, 2, 2));
+ assertRows(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"),
+ row(false, false, false, false));
+ assertRows(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"),
+ row(true, true, true, true));
+ }
+
+ private void createTable(CQL3Type.Native type) throws Throwable
+ {
+ createTable(String.format("CREATE TABLE %%s (" +
+ " k int PRIMARY KEY, " +
+ " l list<%s>, " +
+ " s set<%<s>, " +
+ " m map<%<s, %<s>, " +
+ " fl frozen<list<%<s>>, " +
+ " fs frozen<set<%<s>>, " +
+ " fm frozen<map<%<s, %<s>>)", type));
+
+ // test functions with an empty table
+ assertEmpty(execute("SELECT map_keys(m), map_keys(fm), map_values(m), map_values(fm) FROM %s"));
+ assertEmpty(execute("SELECT collection_count(l), collection_count(s), collection_count(m) FROM %s"));
+ assertEmpty(execute("SELECT collection_count(fl), collection_count(fs), collection_count(fm) FROM %s"));
+ assertEmpty(execute("SELECT collection_min(l), collection_min(s), collection_min(fl), collection_min(fs) FROM %s"));
+ assertEmpty(execute("SELECT collection_max(l), collection_max(s), collection_max(fl), collection_max(fs) FROM %s"));
+
+ String errorMsg = "requires a numeric set/list argument";
+ if (type.getType() instanceof NumberType)
+ {
+ assertEmpty(execute("SELECT collection_sum(l), collection_sum(s), collection_sum(fl), collection_sum(fs) FROM %s"));
+ assertEmpty(execute("SELECT collection_avg(l), collection_avg(s), collection_avg(fl), collection_avg(fs) FROM %s"));
+ }
+ else
+ {
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_sum(l) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_avg(l) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_sum(s) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_avg(s) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_sum(fl) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_avg(fl) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_sum(fs) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_avg(fs) FROM %s");
+ }
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_sum(m) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_avg(m) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_sum(fm) FROM %s");
+ assertInvalidThrowMessage(errorMsg, InvalidRequestException.class, "SELECT collection_avg(fm) FROM %s");
+
+ // prepare empty collections
+ execute("INSERT INTO %s (k, l, fl, s, fs, m, fm) VALUES (1, ?, ?, ?, ?, ?, ?)",
+ list(), list(), set(), set(), map(), map());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java b/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java
new file mode 100644
index 0000000..3107d16
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/FunctionFactoryTest.java
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+
+public class FunctionFactoryTest extends CQLTester
+{
+ /**
+ * A function that just returns its only argument without any changes.
+ * Calls to this function will try to infer the type of its argument, if missing, from the function's receiver.
+ */
+ private static final FunctionFactory IDENTITY = new FunctionFactory("identity", FunctionParameter.anyType(true))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return new NativeScalarFunction(name.name, argTypes.get(0), argTypes.get(0))
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocol, List<ByteBuffer> parameters)
+ {
+ return parameters.get(0);
+ }
+ };
+ }
+ };
+
+ /**
+ * A function that returns the string representation of its only argument.
+ * Calls to this function won't try to infer the type of its argument, if missing, from the function's receiver.
+ */
+ private static final FunctionFactory TO_STRING = new FunctionFactory("tostring", FunctionParameter.anyType(false))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return new NativeScalarFunction(name.name, UTF8Type.instance, argTypes.get(0))
+ {
+ @Override
+ public ByteBuffer execute(ProtocolVersion protocol, List<ByteBuffer> parameters)
+ {
+ ByteBuffer value = parameters.get(0);
+ if (value == null)
+ return null;
+
+ return UTF8Type.instance.decompose(argTypes.get(0).compose(value).toString());
+ }
+ };
+ }
+ };
+
+ private static final UUID uuid = UUID.fromString("62c3e96f-55cd-493b-8c8e-5a18883a1698");
+ private static final TimeUUID timeUUID = TimeUUID.fromString("00346642-2d2f-11ed-a261-0242ac120002");
+ private static final BigInteger bigint = new BigInteger("12345678901234567890");
+ private static final BigDecimal bigdecimal = new BigDecimal("1234567890.1234567890");
+ private static final Date date = new Date();
+ private static final Duration duration = Duration.newInstance(1, 2, 3);
+ private static final InetAddress inet = new InetSocketAddress(0).getAddress();
+ private static final ByteBuffer blob = ByteBufferUtil.hexToBytes("ABCDEF");
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ NativeFunctions.instance.add(IDENTITY);
+ NativeFunctions.instance.add(TO_STRING);
+ }
+
+ @Test
+ public void testInvalidNumberOfArguments() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY)");
+ String msg = "Invalid number of arguments for function system.identity(any)";
+ assertInvalidMessage(msg, "SELECT identity() FROM %s");
+ assertInvalidMessage(msg, "SELECT identity(1, 2) FROM %s");
+ assertInvalidMessage(msg, "SELECT identity('1', '2', '3') FROM %s");
+ }
+
+ @Test
+ public void testUnknownFunction() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY)");
+ assertInvalidMessage("Unknown function 'unknown'", "SELECT unknown() FROM %s");
+ }
+
+ @Test
+ public void testSimpleTypes() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, " +
+ "tinyint tinyint, " +
+ "smallint smallint, " +
+ "int int, " +
+ "bigint bigint, " +
+ "float float, " +
+ "double double, " +
+ "varint varint, " +
+ "decimal decimal, " +
+ "text text, " +
+ "ascii ascii, " +
+ "boolean boolean, " +
+ "date date, " +
+ "timestamp timestamp, " +
+ "duration duration, " +
+ "uuid uuid, " +
+ "timeuuid timeuuid," +
+ "inet inet," +
+ "blob blob)");
+
+ // Test with empty table
+ String select = "SELECT " +
+ "identity(tinyint), identity(smallint), identity(int), " +
+ "identity(bigint), identity(float), identity(double), " +
+ "identity(varint), identity(decimal), identity(text), " +
+ "identity(ascii), identity(boolean), identity(date), " +
+ "identity(timestamp), identity(duration), identity(uuid), " +
+ "identity(timeuuid), identity(inet), identity(blob) " +
+ "FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs,
+ "system.identity(tinyint)", "system.identity(smallint)", "system.identity(int)",
+ "system.identity(bigint)", "system.identity(float)", "system.identity(double)",
+ "system.identity(varint)", "system.identity(decimal)", "system.identity(text)",
+ "system.identity(ascii)", "system.identity(boolean)", "system.identity(date)",
+ "system.identity(timestamp)", "system.identity(duration)", "system.identity(uuid)",
+ "system.identity(timeuuid)", "system.identity(inet)", "system.identity(blob)");
+ assertEmpty(rs);
+
+ // Test with not-empty table
+ Object[] row = row((byte) 1, (short) 1, 123, 123L, 1.23f, 1.23d, bigint, bigdecimal,
+ "ábc", "abc", true, 1, date, duration, uuid, timeUUID, inet, blob);
+ execute("INSERT INTO %s (k, tinyint, smallint, int, bigint, float, double, varint, decimal, " +
+ "text, ascii, boolean, date, timestamp, duration, uuid, timeuuid, inet, blob) " +
+ "VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", row);
+ assertRows(execute(select), row);
+
+ // Test with bind markers
+ execute("INSERT INTO %s (k, tinyint, smallint, int, bigint, float, double, varint, decimal, " +
+ "text, ascii, boolean, date, timestamp, duration, uuid, timeuuid, inet, blob) " +
+ "VALUES (1, " +
+ "identity(?), identity(?), identity(?), identity(?), identity(?), identity(?), " +
+ "identity(?), identity(?), identity(?), identity(?), identity(?), identity(?), " +
+ "identity(?), identity(?), identity(?), identity(?), identity(?), identity(?))", row);
+ assertRows(execute(select), row);
+
+ // Test literals
+ testLiteral("(tinyint) 1", (byte) 1);
+ testLiteral("(smallint) 1", (short) 1);
+ testLiteral(123);
+ testLiteral(1234567890123L);
+ testLiteral(1.23);
+ testLiteral(1234567.1234567D);
+ testLiteral(bigint);
+ testLiteral(bigdecimal);
+ testLiteral("'abc'", "abc");
+ testLiteral("'ábc'", "ábc");
+ testLiteral(true);
+ testLiteral(false);
+ testLiteral("(timestamp) '1970-01-01 00:00:00.000+0000'", new Date(0));
+ testLiteral("(time) '00:00:00.000000'", 0L);
+ testLiteral(duration);
+ testLiteral(uuid);
+ testLiteral(timeUUID);
+ testLiteral("(inet) '0.0.0.0'", inet);
+ testLiteral("0x" + ByteBufferUtil.bytesToHex(blob), blob);
+ }
+
+ @Test
+ public void testSets() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<int>, fs frozen<set<int>>)");
+
+ // Test with empty table
+ String select = "SELECT identity(s), identity(fs) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs, "system.identity(s)", "system.identity(fs)");
+ assertEmpty(rs);
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, s, fs) VALUES (1, {1, 2}, {1, 2})");
+ execute("INSERT INTO %s (k, s, fs) VALUES (2, {1, 2, 3}, {1, 2, 3})");
+ execute("INSERT INTO %s (k, s, fs) VALUES (3, {2, 1}, {2, 1})");
+ assertRows(execute(select),
+ row(set(1, 2), set(1, 2)),
+ row(set(1, 2, 3), set(1, 2, 3)),
+ row(set(2, 1), set(2, 1)));
+
+ // Test with bind markers
+ Object[] row = row(set(1, 2, 3), set(4, 5, 6));
+ execute("INSERT INTO %s (k, s, fs) VALUES (4, identity(?), identity(?))", row);
+ assertRows(execute("SELECT s, fs FROM %s WHERE k = 4"), row);
+
+ // Test literals
+ testLiteralFails("[]");
+ testLiteral("{1, 1234567890}", set(1, 1234567890));
+ testLiteral(String.format("{1, %s}", bigint), set(BigInteger.ONE, bigint));
+ testLiteral("{'abc'}", set("abc"));
+ testLiteral("{'ábc'}", set("ábc"));
+ testLiteral("{'abc', 'ábc'}", set("abc", "ábc"));
+ testLiteral("{'ábc', 'abc'}", set("ábc", "abc"));
+ testLiteral("{true}", set(true));
+ testLiteral("{false}", set(false));
+ testLiteral(String.format("{%s}", uuid), set(uuid));
+ testLiteral(String.format("{%s}", timeUUID), set(timeUUID));
+ testLiteral(String.format("{%s, %s}", uuid, timeUUID), set(uuid, timeUUID.asUUID()));
+ }
+
+ @Test
+ public void testLists() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<int>, fl frozen<list<int>>)");
+
+ // Test with empty table
+ String select = "SELECT identity(l), identity(fl) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs, "system.identity(l)", "system.identity(fl)");
+ assertEmpty(rs);
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, l, fl) VALUES (1, [1, 2], [1, 2])");
+ execute("INSERT INTO %s (k, l, fl) VALUES (2, [1, 2, 3], [1, 2, 3])");
+ execute("INSERT INTO %s (k, l, fl) VALUES (3, [2, 1], [2, 1])");
+ assertRows(execute(select),
+ row(list(1, 2), list(1, 2)),
+ row(list(1, 2, 3), list(1, 2, 3)),
+ row(list(2, 1), list(2, 1)));
+
+ // Test with bind markers
+ Object[] row = row(list(1, 2, 3), list(4, 5, 6));
+ execute("INSERT INTO %s (k, l, fl) VALUES (4, identity(?), identity(?))", row);
+ assertRows(execute("SELECT l, fl FROM %s WHERE k = 4"), row);
+
+ // Test literals
+ testLiteralFails("[]");
+ testLiteral("[1, 1234567890]", list(1, 1234567890));
+ testLiteral(String.format("[1, %s]", bigint), list(BigInteger.ONE, bigint));
+ testLiteral("['abc']", list("abc"));
+ testLiteral("['ábc']", list("ábc"));
+ testLiteral("['abc', 'ábc']", list("abc", "ábc"));
+ testLiteral("['ábc', 'abc']", list("ábc", "abc"));
+ testLiteral("[true]", list(true));
+ testLiteral("[false]", list(false));
+ testLiteral(String.format("[%s]", uuid), list(uuid));
+ testLiteral(String.format("[%s]", timeUUID), list(timeUUID));
+ testLiteral(String.format("[%s, %s]", uuid, timeUUID), list(uuid, timeUUID.asUUID()));
+ }
+
+ @Test
+ public void testMaps() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<int, int>, fm frozen<map<int, int>>)");
+
+ // Test with empty table
+ String select = "SELECT identity(m), identity(fm) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs, "system.identity(m)", "system.identity(fm)");
+ assertEmpty(rs);
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, m, fm) VALUES (1, {1:10, 2:20}, {1:10, 2:20})");
+ execute("INSERT INTO %s (k, m, fm) VALUES (2, {1:10, 2:20, 3:30}, {1:10, 2:20, 3:30})");
+ execute("INSERT INTO %s (k, m, fm) VALUES (3, {2:20, 1:10}, {2:20, 1:10})");
+ assertRows(execute(select),
+ row(map(1, 10, 2, 20), map(1, 10, 2, 20)),
+ row(map(1, 10, 2, 20, 3, 30), map(1, 10, 2, 20, 3, 30)),
+ row(map(1, 10, 2, 20), map(1, 10, 2, 20)));
+
+ // Test with bind markers
+ Object[] row = row(map(1, 10, 2, 20), map(3, 30, 4, 40));
+ execute("INSERT INTO %s (k, m, fm) VALUES (4, identity(?), identity(?))", row);
+ assertRows(execute("SELECT m, fm FROM %s WHERE k = 4"), row);
+
+ // Test literals
+ testLiteralFails("{}");
+ testLiteralFails("{1: 10, 2: 20}");
+ testLiteral("(map<int, int>) {1: 10, 2: 20}", map(1, 10, 2, 20));
+ testLiteral("(map<int, text>) {1: 'abc', 2: 'ábc'}", map(1, "abc", 2, "ábc"));
+ testLiteral("(map<text, int>) {'abc': 1, 'ábc': 2}", map("abc", 1, "ábc", 2));
+ }
+
+ @Test
+ public void testTuples() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<int, text, boolean>)");
+
+ // Test with empty table
+ String select = "SELECT identity(t) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs, "system.identity(t)");
+ assertEmpty(rs);
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, t) VALUES (1, (1, 'a', false))");
+ execute("INSERT INTO %s (k, t) VALUES (2, (2, 'b', true))");
+ execute("INSERT INTO %s (k, t) VALUES (3, (3, null, true))");
+ assertRows(execute(select),
+ row(tuple(1, "a", false)),
+ row(tuple(2, "b", true)),
+ row(tuple(3, null, true)));
+
+ // Test with bind markers
+ Object[] row = row(tuple(4, "d", false));
+ execute("INSERT INTO %s (k, t) VALUES (4, identity(?))", row);
+ assertRows(execute("SELECT t FROM %s WHERE k = 4"), row);
+
+ // Test literals
+ testLiteralFails("(1)");
+ testLiteral("(tuple<int>) (1)", tuple(1));
+ testLiteral("(1, 'a')", tuple(1, "a"));
+ testLiteral("(1, 'a', false)", tuple(1, "a", false));
+ }
+
+ @Test
+ public void testUDTs() throws Throwable
+ {
+ String udt = createType("CREATE TYPE %s (x int)");
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, u frozen<" + udt + ">, fu frozen<" + udt + ">)");
+
+ // Test with empty table
+ String select = "SELECT identity(u), identity(fu) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs, "system.identity(u)", "system.identity(fu)");
+ assertEmpty(rs);
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, u, fu) VALUES (1, {x: 2}, null)");
+ execute("INSERT INTO %s (k, u, fu) VALUES (2, {x: 4}, {x: 6})");
+ execute("INSERT INTO %s (k, u, fu) VALUES (4, null, {x: 8})");
+ assertRows(execute(select),
+ row(userType("x", 2), null),
+ row(userType("x", 4), userType("x", 6)),
+ row(null, userType("x", 8)));
+
+ // Test with bind markers
+ Object[] row = row(userType("x", 4), userType("x", 5));
+ execute("INSERT INTO %s (k, u, fu) VALUES (4, identity(?), identity(?))", row);
+ assertRows(execute("SELECT u, fu FROM %s WHERE k = 4"), row);
+
+ // Test literals
+ testLiteralFails("{}");
+ testLiteralFails("{x: 10}");
+ testLiteral('(' + udt + "){x: 10}", tuple(10));
+ }
+
+ @Test
+ public void testNestedCalls() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, t text)");
+
+ // Test function that infers parameter type from receiver type
+ execute("INSERT INTO %s (k, v) VALUES (1, identity(identity(2)))");
+ assertRows(execute("SELECT v FROM %s WHERE k = 1"), row(2));
+ execute("INSERT INTO %s (k, v) VALUES (1, identity(identity((int) ?)))", 3);
+ assertRows(execute("SELECT v FROM %s WHERE k = 1"), row(3));
+ assertRows(execute("SELECT identity(identity(v)) FROM %s WHERE k = 1"), row(3));
+
+ // Test function that does not infer parameter type from receiver type
+ execute("INSERT INTO %s (k, t) VALUES (1, tostring(tostring(4)))");
+ assertRows(execute("SELECT t FROM %s WHERE k = 1"), row("4"));
+ execute("INSERT INTO %s (k, t) VALUES (1, tostring(tostring((int) ?)))", 5);
+ assertRows(execute("SELECT tostring(tostring(t)) FROM %s WHERE k = 1"), row("5"));
+ }
+
+ private void testLiteral(Object literal) throws Throwable
+ {
+ testLiteral(literal, literal);
+ }
+
+ private void testLiteral(Object functionArgs, Object expectedResult) throws Throwable
+ {
+ assertRows(execute(String.format("SELECT %s(%s) FROM %%s LIMIT 1", IDENTITY.name(), functionArgs)),
+ row(expectedResult));
+ }
+
+ private void testLiteralFails(Object functionArgs) throws Throwable
+ {
+ assertInvalidMessage("Cannot infer type of argument " + functionArgs,
+ String.format("SELECT %s(%s) FROM %%s", IDENTITY.name(), functionArgs));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/MathFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/MathFctsTest.java
new file mode 100644
index 0000000..daefc0a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/MathFctsTest.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.FloatType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.NumberType;
+import org.apache.cassandra.db.marshal.ShortType;
+import org.apache.cassandra.transport.ProtocolVersion;
+
+import static org.junit.Assert.assertEquals;
+
+public class MathFctsTest
+{
+ @Test
+ public void testAbs()
+ {
+ assertAbsEquals(1, Int32Type.instance, 1);
+ assertAbsEquals(0, Int32Type.instance, 0);
+ assertAbsEquals(3, Int32Type.instance, -3);
+
+ assertAbsEquals((byte) 1, ByteType.instance, (byte) 1);
+ assertAbsEquals((byte) 0, ByteType.instance, (byte) 0);
+ assertAbsEquals((byte) 3, ByteType.instance, (byte) -3);
+
+ assertAbsEquals((short) 1, ShortType.instance, (short) 1);
+ assertAbsEquals((short) 0, ShortType.instance, (short) 0);
+ assertAbsEquals((short) 3, ShortType.instance, (short) -3);
+
+ assertAbsEquals(1F, FloatType.instance, 1F);
+ assertAbsEquals(1.5F, FloatType.instance, 1.5F);
+ assertAbsEquals(0F, FloatType.instance, 0F);
+ assertAbsEquals(3F, FloatType.instance, -3F);
+ assertAbsEquals(3.7F, FloatType.instance, -3.7F);
+
+ assertAbsEquals(1L, LongType.instance, 1L);
+ assertAbsEquals(0L, LongType.instance, 0L);
+ assertAbsEquals(3L, LongType.instance, -3L);
+
+ assertAbsEquals(1L, CounterColumnType.instance, 1L);
+ assertAbsEquals(0L, CounterColumnType.instance, 0L);
+ assertAbsEquals(3L, CounterColumnType.instance, -3L);
+
+ assertAbsEquals(1.0, DoubleType.instance, 1.0);
+ assertAbsEquals(1.5, DoubleType.instance, 1.5);
+ assertAbsEquals(0.0, DoubleType.instance, 0.0);
+ assertAbsEquals(3.0, DoubleType.instance, -3.0);
+ assertAbsEquals(3.7, DoubleType.instance, -3.7);
+
+ assertAbsEquals(BigInteger.ONE, IntegerType.instance, BigInteger.ONE);
+ assertAbsEquals(BigInteger.ZERO, IntegerType.instance, BigInteger.ZERO);
+ assertAbsEquals(BigInteger.valueOf(3), IntegerType.instance, BigInteger.valueOf(-3));
+
+ assertAbsEquals(BigDecimal.ONE, DecimalType.instance, BigDecimal.ONE);
+ assertAbsEquals(BigDecimal.valueOf(1.5), DecimalType.instance, BigDecimal.valueOf(1.5));
+ assertAbsEquals(BigDecimal.ZERO, DecimalType.instance, BigDecimal.ZERO);
+ assertAbsEquals(BigDecimal.valueOf(3), DecimalType.instance, BigDecimal.valueOf(-3));
+ assertAbsEquals(BigDecimal.valueOf(3.7), DecimalType.instance, BigDecimal.valueOf(-3.7));
+ }
+
+ private <T extends Number> void assertAbsEquals(T expected, NumberType<T> inputType, T inputValue)
+ {
+ assertFctEquals(MathFcts.absFct(inputType), expected, inputType, inputValue);
+ }
+
+ @Test
+ public void testExp()
+ {
+ assertExpEquals((int) Math.pow(Math.E, 5), Int32Type.instance, 5);
+ assertExpEquals((int) Math.E, Int32Type.instance, 1);
+ assertExpEquals(1, Int32Type.instance, 0);
+ assertExpEquals((int) Math.pow(Math.E, -1), Int32Type.instance, -1);
+
+ assertExpEquals((byte) Math.pow(Math.E, 5), ByteType.instance, (byte) 5);
+ assertExpEquals((byte) Math.E, ByteType.instance, (byte) 1);
+ assertExpEquals((byte) 1, ByteType.instance, (byte) 0);
+ assertExpEquals((byte) Math.pow(Math.E, -1), ByteType.instance, (byte) -1);
+
+ assertExpEquals((short) Math.pow(Math.E, 5), ShortType.instance, (short) 5);
+ assertExpEquals((short) Math.E, ShortType.instance, (short) 1);
+ assertExpEquals((short) 1, ShortType.instance, (short) 0);
+ assertExpEquals((short) Math.pow(Math.E, -1), ShortType.instance, (short) -1);
+
+ assertExpEquals((float) Math.pow(Math.E, 5), FloatType.instance, 5F);
+ assertExpEquals((float) Math.E, FloatType.instance, 1F);
+ assertExpEquals(1F, FloatType.instance, 0F);
+ assertExpEquals((float) Math.pow(Math.E, -1), FloatType.instance, -1F);
+ assertExpEquals((float) Math.pow(Math.E, -2.5), FloatType.instance, -2.5F);
+
+ assertExpEquals((long) Math.pow(Math.E, 5), LongType.instance, 5L);
+ assertExpEquals((long) Math.E, LongType.instance, 1L);
+ assertExpEquals(1L, LongType.instance, 0L);
+ assertExpEquals((long) Math.pow(Math.E, -1), LongType.instance, -1L);
+
+ assertExpEquals((long) Math.pow(Math.E, 5), CounterColumnType.instance, 5L);
+ assertExpEquals((long) Math.E, CounterColumnType.instance, 1L);
+ assertExpEquals(1L, CounterColumnType.instance, 0L);
+ assertExpEquals((long) Math.pow(Math.E, -1), CounterColumnType.instance, -1L);
+
+ assertExpEquals(BigInteger.valueOf((long) Math.pow(Math.E, 5)), IntegerType.instance, BigInteger.valueOf(5));
+ assertExpEquals(BigInteger.valueOf((long) Math.E), IntegerType.instance, BigInteger.valueOf(1));
+ assertExpEquals(BigInteger.valueOf(1), IntegerType.instance, BigInteger.valueOf(0));
+ assertExpEquals(BigInteger.valueOf((long) Math.pow(Math.E, -1)), IntegerType.instance, BigInteger.valueOf(-1));
+
+ assertExpEquals(new BigDecimal("148.41315910257660342111558004055"), DecimalType.instance, BigDecimal.valueOf(5));
+ assertExpEquals(new BigDecimal("2.7182818284590452353602874713527"), DecimalType.instance, BigDecimal.valueOf(1));
+ assertExpEquals(new BigDecimal("1"), DecimalType.instance, BigDecimal.valueOf(0));
+ assertExpEquals(new BigDecimal("0.36787944117144232159552377016146"), DecimalType.instance, BigDecimal.valueOf(-1));
+ }
+
+ private <T extends Number> void assertExpEquals(T expected, NumberType<T> inputType, T inputValue)
+ {
+ assertFctEquals(MathFcts.expFct(inputType), expected, inputType, inputValue);
+ }
+
+
+
+ @Test
+ public void testLog()
+ {
+ assertLogEquals((int) Math.log(5), Int32Type.instance, 5);
+ assertLogEquals(0, Int32Type.instance, (int) Math.E);
+ assertLogEquals(0, Int32Type.instance, 1);
+ assertLogEquals((int) Math.log(-1), Int32Type.instance, -1);
+
+ assertLogEquals((byte) Math.log(5), ByteType.instance, (byte) 5);
+ assertLogEquals((byte) 0, ByteType.instance, (byte) Math.E);
+ assertLogEquals((byte) 0, ByteType.instance, (byte) 1);
+ assertLogEquals((byte) Math.log(-1), ByteType.instance, (byte) -1);
+
+ assertLogEquals((short) Math.log(5), ShortType.instance, (short) 5);
+ assertLogEquals((short) 0, ShortType.instance, (short) Math.E);
+ assertLogEquals((short) 0, ShortType.instance, (short) 1);
+ assertLogEquals((short) Math.log(-1), ShortType.instance, (short) -1);
+
+ assertLogEquals((float) Math.log(5.5F), FloatType.instance, 5.5F, 0.0000001);
+ assertLogEquals(1F, FloatType.instance, (float) Math.E, 0.0000001);
+ assertLogEquals(0F, FloatType.instance, 1F, 0.0000001);
+ assertLogEquals((float) Math.log(-1F), FloatType.instance, -1F, 0.0000001);
+
+
+ assertLogEquals((long) Math.log(5), LongType.instance, 5L);
+ assertLogEquals(0L, LongType.instance, (long) Math.E);
+ assertLogEquals(0L, LongType.instance, 1L);
+ assertLogEquals((long) Math.log(-1), LongType.instance, -1L);
+
+ assertLogEquals((long) Math.log(5), CounterColumnType.instance, 5L);
+ assertLogEquals(0L, CounterColumnType.instance, (long) Math.E);
+ assertLogEquals(0L, CounterColumnType.instance, 1L);
+ assertLogEquals((long) Math.log(-1), CounterColumnType.instance, -1L);
+
+ assertLogEquals(BigInteger.valueOf((long) Math.log(5)), IntegerType.instance, BigInteger.valueOf(5));
+ assertLogEquals(BigInteger.valueOf(0), IntegerType.instance, BigInteger.valueOf((long) Math.E));
+ assertLogEquals(BigInteger.valueOf(0), IntegerType.instance, BigInteger.valueOf(1));
+
+ assertLogEquals(new BigDecimal("1.6094379124341003746007593332262"), DecimalType.instance, BigDecimal.valueOf(5));
+ assertLogEquals(new BigDecimal("0"), DecimalType.instance, BigDecimal.valueOf(1));
+ }
+
+ private <T extends Number> void assertLogEquals(T expected, NumberType<T> inputType, T inputValue)
+ {
+ assertFctEquals(MathFcts.logFct(inputType), expected, inputType, inputValue);
+ }
+
+ private <T extends Number> void assertLogEquals(T expected, NumberType<T> inputType, T inputValue, double delta)
+ {
+ assertFctEquals(MathFcts.logFct(inputType), expected, inputType, inputValue, delta);
+ }
+
+ @Test
+ public void testLog10()
+ {
+ assertLog10Equals((int) Math.log10(5), Int32Type.instance, 5);
+ assertLog10Equals(0, Int32Type.instance, (int) Math.E);
+ assertLog10Equals(0, Int32Type.instance, 1);
+ assertLog10Equals((int) Math.log10(-1), Int32Type.instance, -1);
+
+ assertLog10Equals((byte) Math.log10(5), ByteType.instance, (byte) 5);
+ assertLog10Equals((byte) 0, ByteType.instance, (byte) Math.E);
+ assertLog10Equals((byte) 0, ByteType.instance, (byte) 1);
+ assertLog10Equals((byte) Math.log10(-1), ByteType.instance, (byte) -1);
+
+ assertLog10Equals((short) Math.log10(5), ShortType.instance, (short) 5);
+ assertLog10Equals((short) 0, ShortType.instance, (short) Math.E);
+ assertLog10Equals((short) 0, ShortType.instance, (short) 1);
+ assertLog10Equals((short) Math.log10(-1), ShortType.instance, (short) -1);
+
+ assertLog10Equals((float) Math.log10(5.5F), FloatType.instance, 5.5F, 0.0000001);
+ assertLog10Equals(1F, FloatType.instance, 10F, 0.0000001);
+ assertLog10Equals(0F, FloatType.instance, 1F, 0.0000001);
+ assertLog10Equals((float) Math.log10(-1F), FloatType.instance, -1F, 0.0000001);
+
+
+ assertLog10Equals((long) Math.log10(5), LongType.instance, 5L);
+ assertLog10Equals(0L, LongType.instance, (long) Math.E);
+ assertLog10Equals(0L, LongType.instance, 1L);
+ assertLog10Equals((long) Math.log10(-1), LongType.instance, -1L);
+
+ assertLog10Equals((long) Math.log10(5), CounterColumnType.instance, 5L);
+ assertLog10Equals(0L, CounterColumnType.instance, (long) Math.E);
+ assertLog10Equals(0L, CounterColumnType.instance, 1L);
+ assertLog10Equals((long) Math.log10(-1), CounterColumnType.instance, -1L);
+
+ assertLog10Equals(BigInteger.valueOf((long) Math.log10(5)), IntegerType.instance, BigInteger.valueOf(5));
+ assertLog10Equals(BigInteger.valueOf(0), IntegerType.instance, BigInteger.valueOf((long) Math.E));
+ assertLog10Equals(BigInteger.valueOf(0), IntegerType.instance, BigInteger.valueOf(1));
+
+ assertLog10Equals(new BigDecimal("0.69897000433601880478626110527551"), DecimalType.instance, BigDecimal.valueOf(5));
+ assertLog10Equals(new BigDecimal("0"), DecimalType.instance, BigDecimal.valueOf(1));
+ }
+
+ private <T extends Number> void assertLog10Equals(T expected, NumberType<T> inputType, T inputValue)
+ {
+ assertFctEquals(MathFcts.log10Fct(inputType), expected, inputType, inputValue);
+ }
+
+ private <T extends Number> void assertLog10Equals(T expected, NumberType<T> inputType, T inputValue, double delta)
+ {
+ assertFctEquals(MathFcts.log10Fct(inputType), expected, inputType, inputValue, delta);
+ }
+
+ @Test
+ public void testRound()
+ {
+ assertRoundEquals(5, Int32Type.instance, 5);
+ assertRoundEquals(0, Int32Type.instance, 0);
+ assertRoundEquals(-1, Int32Type.instance, -1);
+
+ assertRoundEquals((byte) 5, ByteType.instance, (byte) 5);
+ assertRoundEquals((byte) 0, ByteType.instance, (byte) 0);
+ assertRoundEquals((byte) -1, ByteType.instance, (byte) -1);
+
+ assertRoundEquals((short) 5, ShortType.instance, (short) 5);
+ assertRoundEquals((short) 0, ShortType.instance, (short) 0);
+ assertRoundEquals((short) -1, ShortType.instance, (short) -1);
+
+ assertRoundEquals((float) Math.round(5.5F), FloatType.instance, 5.5F);
+ assertRoundEquals(1F, FloatType.instance, 1F);
+ assertRoundEquals(0F, FloatType.instance, 0F);
+ assertRoundEquals((float) Math.round(-1.5F), FloatType.instance, -1.5F);
+
+
+ assertRoundEquals(5L, LongType.instance, 5L);
+ assertRoundEquals(0L, LongType.instance, 0L);
+ assertRoundEquals(-1L, LongType.instance, -1L);
+
+ assertRoundEquals(5L, CounterColumnType.instance, 5L);
+ assertRoundEquals(0L, CounterColumnType.instance, 0L);
+ assertRoundEquals(-1L, CounterColumnType.instance, -1L);
+
+ assertRoundEquals(BigInteger.valueOf(5), IntegerType.instance, BigInteger.valueOf(5));
+ assertRoundEquals(BigInteger.valueOf(0), IntegerType.instance, BigInteger.valueOf(0));
+ assertRoundEquals(BigInteger.valueOf(-1), IntegerType.instance, BigInteger.valueOf(-1));
+
+ assertRoundEquals(new BigDecimal("6"), DecimalType.instance, BigDecimal.valueOf(5.5));
+ assertRoundEquals(new BigDecimal("1"), DecimalType.instance, BigDecimal.valueOf(1));
+ assertRoundEquals(new BigDecimal("0"), DecimalType.instance, BigDecimal.valueOf(0));
+ assertRoundEquals(new BigDecimal("-2"), DecimalType.instance, BigDecimal.valueOf(-1.5));
+ }
+
+ private <T extends Number> void assertRoundEquals(T expected, NumberType<T> inputType, T inputValue)
+ {
+ assertFctEquals(MathFcts.roundFct(inputType), expected, inputType, inputValue);
+ }
+
+ private static ByteBuffer executeFunction(Function function, ByteBuffer input)
+ {
+ List<ByteBuffer> params = Collections.singletonList(input);
+ return ((ScalarFunction) function).execute(ProtocolVersion.CURRENT, params);
+ }
+
+ private <T extends Number> void assertFctEquals(NativeFunction fct, T expected, NumberType<T> inputType, T inputValue)
+ {
+ ByteBuffer input = inputType.decompose(inputValue);
+ if (expected instanceof BigDecimal)
+ {
+ // This block is to deal with the edgecase where two BigDecimals' values are equal but not their scale.
+ BigDecimal bdExpected = (BigDecimal) expected;
+ BigDecimal bdInputValue = (BigDecimal) inputType.compose(executeFunction(fct, input));
+ assertEquals(bdExpected.compareTo(bdInputValue), 0);
+ }
+ else
+ {
+ assertEquals(expected, inputType.compose(executeFunction(fct, input)));
+ }
+ }
+
+ private <T extends Number> void assertFctEquals(NativeFunction fct, T expected, NumberType<T> inputType, T inputValue, double delta)
+ {
+ ByteBuffer input = inputType.decompose(inputValue);
+ T actual = inputType.compose(executeFunction(fct, input));
+ if (Double.isNaN(expected.doubleValue())) {
+ Assert.assertTrue(Double.isNaN(actual.doubleValue()));
+ } else
+ {
+ Assert.assertTrue(Math.abs(expected.doubleValue() - actual.doubleValue()) <= delta);
+ }
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/NativeFunctionsTest.java b/test/unit/org/apache/cassandra/cql3/functions/NativeFunctionsTest.java
new file mode 100644
index 0000000..3bf3543
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/NativeFunctionsTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import org.apache.cassandra.schema.SchemaConstants;
+import org.assertj.core.api.Assertions;
+
+public class NativeFunctionsTest
+{
+ /**
+ * Map associating old functions that don't satisfy the naming conventions adopted by CASSANDRA-18037 to their
+ * new names according to those conventions.
+ */
+ private static final Map<String, String> LEGACY_FUNCTION_NAMES = new HashMap<String, String>()
+ {
+ {
+ put("castAsAscii", "cast_as_ascii");
+ put("castAsBigint", "cast_as_bigint");
+ put("castAsDate", "cast_as_date");
+ put("castAsDecimal", "cast_as_decimal");
+ put("castAsDouble", "cast_as_double");
+ put("castAsFloat", "cast_as_float");
+ put("castAsInt", "cast_as_int");
+ put("castAsSmallint", "cast_as_smallint");
+ put("castAsText", "cast_as_text");
+ put("castAsTimestamp", "cast_as_timestamp");
+ put("castAsTinyint", "cast_as_tinyint");
+ put("castAsVarint", "cast_as_varint");
+ put("blobasascii", "blob_as_ascii");
+ put("blobasbigint", "blob_as_bigint");
+ put("blobasboolean", "blob_as_boolean");
+ put("blobascounter", "blob_as_counter");
+ put("blobasdate", "blob_as_date");
+ put("blobasdecimal", "blob_as_decimal");
+ put("blobasdouble", "blob_as_double");
+ put("blobasduration", "blob_as_duration");
+ put("blobasempty", "blob_as_empty");
+ put("blobasfloat", "blob_as_float");
+ put("blobasinet", "blob_as_inet");
+ put("blobasint", "blob_as_int");
+ put("blobassmallint", "blob_as_smallint");
+ put("blobastext", "blob_as_text");
+ put("blobastime", "blob_as_time");
+ put("blobastimestamp", "blob_as_timestamp");
+ put("blobastimeuuid", "blob_as_timeuuid");
+ put("blobastinyint", "blob_as_tinyint");
+ put("blobasuuid", "blob_as_uuid");
+ put("blobasvarchar", "blob_as_varchar");
+ put("blobasvarint", "blob_as_varint");
+ put("asciiasblob", "ascii_as_blob");
+ put("bigintasblob", "bigint_as_blob");
+ put("booleanasblob", "boolean_as_blob");
+ put("counterasblob", "counter_as_blob");
+ put("dateasblob", "date_as_blob");
+ put("decimalasblob", "decimal_as_blob");
+ put("doubleasblob", "double_as_blob");
+ put("durationasblob", "duration_as_blob");
+ put("emptyasblob", "empty_as_blob");
+ put("floatasblob", "float_as_blob");
+ put("inetasblob", "inet_as_blob");
+ put("intasblob", "int_as_blob");
+ put("smallintasblob", "smallint_as_blob");
+ put("textasblob", "text_as_blob");
+ put("timeasblob", "time_as_blob");
+ put("timestampasblob", "timestamp_as_blob");
+ put("timeuuidasblob", "timeuuid_as_blob");
+ put("tinyintasblob", "tinyint_as_blob");
+ put("uuidasblob", "uuid_as_blob");
+ put("varcharasblob", "varchar_as_blob");
+ put("varintasblob", "varint_as_blob");
+ put("countRows", "count_rows");
+ put("maxtimeuuid", "max_timeuuid");
+ put("mintimeuuid", "min_timeuuid");
+ put("currentdate", "current_date");
+ put("currenttime", "current_time");
+ put("currenttimestamp", "current_timestamp");
+ put("currenttimeuuid", "current_timeuuid");
+ put("todate", "to_date");
+ put("totimestamp", "to_timestamp");
+ put("tounixtimestamp", "to_unix_timestamp");
+ }
+ };
+
+ /**
+ * Map associating old functions factories that don't satisfy the naming conventions adopted by CASSANDRA-18037 to
+ * their new names according to those conventions.
+ */
+ private static final Map<String, String> LEGACY_FUNCTION_FACTORY_NAMES = new HashMap<String, String>()
+ {
+ {
+ put("tojson", "to_json");
+ put("fromjson", "from_json");
+ }
+ };
+
+ /**
+ * Verify that the old functions that don't satisfy the naming conventions adopted by CASSANDRA-18037 have a
+ * replacement function that satisfies those conventions.
+ */
+ @Test
+ public void testDeprectedFunctionNames()
+ {
+ NativeFunctions nativeFunctions = NativeFunctions.instance;
+ LEGACY_FUNCTION_NAMES.forEach((oldName, newName) -> {
+ Assertions.assertThat(nativeFunctions.getFunctions(FunctionName.nativeFunction(oldName))).isNotEmpty();
+ Assertions.assertThat(nativeFunctions.getFunctions(FunctionName.nativeFunction(newName))).isNotEmpty();
+ });
+
+ for (NativeFunction function : nativeFunctions.getFunctions())
+ {
+ String name = function.name.name;
+
+ if (satisfiesConventions(function.name))
+ continue;
+
+ Assertions.assertThat(LEGACY_FUNCTION_NAMES).containsKey(name);
+ FunctionName newName = FunctionName.nativeFunction(LEGACY_FUNCTION_NAMES.get(name));
+
+ Function newFunction = FunctionResolver.get(SchemaConstants.SYSTEM_KEYSPACE_NAME,
+ newName,
+ function.argTypes,
+ null,
+ null,
+ function.returnType);
+
+ Assertions.assertThat(newFunction).isNotNull();
+ Assertions.assertThat(function).isNotEqualTo(newFunction);
+ Assertions.assertThat(function).isEqualTo(((NativeFunction) newFunction).withLegacyName());
+ Assertions.assertThat(function.argTypes()).isEqualTo(newFunction.argTypes());
+ Assertions.assertThat(function.returnType()).isEqualTo(newFunction.returnType());
+ Assertions.assertThat(function.getClass()).isEqualTo(newFunction.getClass());
+ Assertions.assertThat(function.name().name.toLowerCase())
+ .isEqualTo(StringUtils.remove(newFunction.name().name, '_'));
+ }
+ }
+
+ /**
+ * Verify that the old functions function factories that don't satisfy the naming conventions adopted by
+ * CASSANDRA-18037 have a replacement function factory that satisfies those conventions.
+ */
+ @Test
+ public void testDeprectedFunctionFactoryNames()
+ {
+ NativeFunctions nativeFunctions = NativeFunctions.instance;
+ LEGACY_FUNCTION_FACTORY_NAMES.forEach((oldName, newName) -> {
+ Assertions.assertThat(nativeFunctions.getFactories(FunctionName.nativeFunction(oldName))).isNotEmpty();
+ Assertions.assertThat(nativeFunctions.getFactories(FunctionName.nativeFunction(newName))).isNotEmpty();
+ });
+
+ for (FunctionFactory factory : nativeFunctions.getFactories())
+ {
+ String name = factory.name.name;
+
+ if (satisfiesConventions(factory.name))
+ continue;
+
+ Assertions.assertThat(LEGACY_FUNCTION_FACTORY_NAMES).containsKey(name);
+ FunctionName newName = FunctionName.nativeFunction(LEGACY_FUNCTION_FACTORY_NAMES.get(name));
+ Collection<FunctionFactory> newFactories = NativeFunctions.instance.getFactories(newName);
+
+ Assertions.assertThat(newFactories).hasSize(1);
+ FunctionFactory newFactory = newFactories.iterator().next();
+
+ Assertions.assertThat(factory).isNotEqualTo(newFactory);
+ Assertions.assertThat(factory.name).isNotEqualTo(newFactory.name);
+ Assertions.assertThat(factory.parameters).isEqualTo(newFactory.parameters);
+ Assertions.assertThat(factory.getClass()).isEqualTo(newFactory.getClass());
+ Assertions.assertThat(factory.name().name.toLowerCase())
+ .isEqualTo(StringUtils.remove(newFactory.name().name, '_'));
+ }
+ }
+
+ private static boolean satisfiesConventions(FunctionName functionName)
+ {
+ String name = functionName.name;
+ return name.equals(name.toLowerCase()) &&
+ !LEGACY_FUNCTION_NAMES.containsKey(name);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java
index a124e60..b0cc1e0 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/TimeFctsTest.java
@@ -88,16 +88,6 @@
}
@Test
- public void testDateOf()
- {
-
- long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
- ByteBuffer input = ByteBuffer.wrap(atUnixMillisAsBytes(timeInMillis, 0));
- ByteBuffer output = executeFunction(TimeFcts.dateOfFct, input);
- assertEquals(Date.from(DATE_TIME.toInstant()), TimestampType.instance.compose(output));
- }
-
- @Test
public void testTimeUuidToTimestamp()
{
long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
@@ -107,15 +97,6 @@
}
@Test
- public void testUnixTimestampOfFct()
- {
- long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
- ByteBuffer input = ByteBuffer.wrap(atUnixMillisAsBytes(timeInMillis, 0));
- ByteBuffer output = executeFunction(TimeFcts.unixTimestampOfFct, input);
- assertEquals(timeInMillis, LongType.instance.compose(output).longValue());
- }
-
- @Test
public void testTimeUuidToUnixTimestamp()
{
long timeInMillis = DATE_TIME.toInstant().toEpochMilli();
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionTester.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionTester.java
new file mode 100644
index 0000000..252aac5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionTester.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+
+import static java.lang.String.format;
+
+/**
+ * {@link ColumnMaskTester} verifying that masks can be applied to columns in any position (partition key columns,
+ * clustering key columns, static columns and regular columns). The columns of any depending materialized views should
+ * be udpated accordingly.
+ */
+@RunWith(Parameterized.class)
+public abstract class ColumnMaskInAnyPositionTester extends ColumnMaskTester
+{
+ /** The column mask as expressed in CQL statements right after the {@code MASKED WITH} keywords. */
+ @Parameterized.Parameter
+ public String mask;
+
+ /** The type of the masked column */
+ @Parameterized.Parameter(1)
+ public String type;
+
+ /** The types of the tested masking function partial arguments. */
+ @Parameterized.Parameter(2)
+ public List<AbstractType<?>> argumentTypes;
+
+ /** The serialized values of the tested masking function partial arguments. */
+ @Parameterized.Parameter(3)
+ public List<ByteBuffer> argumentValues;
+
+ @Test
+ public void testCreateTableWithMaskedColumns() throws Throwable
+ {
+ // Nothing is masked
+ createTable("CREATE TABLE %s (k int, c int, r int, s int static, PRIMARY KEY(k, c))");
+ assertTableColumnsAreNotMasked("k", "c", "r", "s");
+
+ // Masked partition key
+ createTable(format("CREATE TABLE %%s (k %s MASKED WITH %s PRIMARY KEY, r int)", type, mask));
+ assertTableColumnsAreMasked("k");
+ assertTableColumnsAreNotMasked("r");
+
+ // Masked partition key component
+ createTable(format("CREATE TABLE %%s (k1 int, k2 %s MASKED WITH %s, r int, PRIMARY KEY(k1, k2))", type, mask));
+ assertTableColumnsAreMasked("k2");
+ assertTableColumnsAreNotMasked("k1", "r");
+
+ // Masked clustering key
+ createTable(format("CREATE TABLE %%s (k int, c %s MASKED WITH %s, r int, PRIMARY KEY (k, c))", type, mask));
+ assertTableColumnsAreMasked("c");
+ assertTableColumnsAreNotMasked("k", "r");
+
+ // Masked clustering key with reverse order
+ createTable(format("CREATE TABLE %%s (k int, c %s MASKED WITH %s, r int, PRIMARY KEY (k, c)) " +
+ "WITH CLUSTERING ORDER BY (c DESC)", type, mask));
+ assertTableColumnsAreMasked("c");
+ assertTableColumnsAreNotMasked("k", "r");
+
+ // Masked clustering key component
+ createTable(format("CREATE TABLE %%s (k int, c1 int, c2 %s MASKED WITH %s, r int, PRIMARY KEY (k, c1, c2))", type, mask));
+ assertTableColumnsAreMasked("c2");
+ assertTableColumnsAreNotMasked("k", "c1", "r");
+
+ // Masked regular column
+ createTable(format("CREATE TABLE %%s (k int PRIMARY KEY, r1 %s MASKED WITH %s, r2 int)", type, mask));
+ assertTableColumnsAreMasked("r1");
+ assertTableColumnsAreNotMasked("k", "r2");
+
+ // Masked static column
+ createTable(format("CREATE TABLE %%s (k int, c int, r int, s %s STATIC MASKED WITH %s, PRIMARY KEY (k, c))", type, mask));
+ assertTableColumnsAreMasked("s");
+ assertTableColumnsAreNotMasked("k", "c", "r");
+
+ // Multiple masked columns
+ createTable(format("CREATE TABLE %%s (" +
+ "k1 int, k2 %s MASKED WITH %s, " +
+ "c1 int, c2 %s MASKED WITH %s, " +
+ "r1 int, r2 %s MASKED WITH %s, " +
+ "s1 int static, s2 %s static MASKED WITH %s, " +
+ "PRIMARY KEY((k1, k2), c1, c2))",
+ type, mask, type, mask, type, mask, type, mask));
+ assertTableColumnsAreMasked("k2", "c2", "r2", "s2");
+ assertTableColumnsAreNotMasked("k1", "c1", "r1", "s1");
+ }
+
+ @Test
+ public void testCreateTableWithMaskedColumnsAndMaterializedView() throws Throwable
+ {
+ createTable(format("CREATE TABLE %%s (" +
+ "k1 int, k2 %s MASKED WITH %s, " +
+ "c1 int, c2 %s MASKED WITH %s, " +
+ "r1 int, r2 %s MASKED WITH %s, " +
+ "s1 int static, s2 %s static MASKED WITH %s, " +
+ "PRIMARY KEY((k1, k2), c1, c2))",
+ type, mask, type, mask, type, mask, type, mask));
+ createView("CREATE MATERIALIZED VIEW %s AS SELECT k1, k2, c1, c2, r1, r2 FROM %s " +
+ "WHERE k1 IS NOT NULL AND k2 IS NOT NULL " +
+ "AND c1 IS NOT NULL AND c2 IS NOT NULL " +
+ "AND r1 IS NOT NULL AND r2 IS NOT NULL " +
+ "PRIMARY KEY (r2, c2, c1, k2, k1)");
+
+ assertTableColumnsAreMasked("k2", "c2", "r2", "s2");
+ assertTableColumnsAreNotMasked("k1", "c1", "r1", "s1");
+
+ assertViewColumnsAreMasked("k2", "c2", "r2");
+ assertViewColumnsAreNotMasked("k1", "c1", "r1");
+ }
+
+ @Test
+ public void testAlterTableWithMaskedColumns() throws Throwable
+ {
+ // Create the table to be altered
+ createTable(format("CREATE TABLE %%s (k %s, c %<s, r1 %<s, r2 %<s MASKED WITH %s, r3 %s, s %<s static, " +
+ "PRIMARY KEY (k, c))", type, mask, type));
+ assertTableColumnsAreMasked("r2");
+ assertTableColumnsAreNotMasked("k", "c", "r1", "r3", "s");
+
+ // Add new masked column
+ alterTable(format("ALTER TABLE %%s ADD r4 %s MASKED WITH %s", type, mask));
+ assertTableColumnsAreMasked("r2", "r4");
+ assertTableColumnsAreNotMasked("k", "c", "r1", "r3", "s");
+
+ // Set mask for an existing but unmasked column
+ alterTable(format("ALTER TABLE %%s ALTER r1 MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("r1", "r2", "r4");
+
+ // Unmask a masked column
+ alterTable("ALTER TABLE %s ALTER r1 DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER r2 DROP MASKED");
+ assertTableColumnsAreMasked("r4");
+ assertTableColumnsAreNotMasked("r1", "r2", "r3");
+
+ // Mask and disable mask for primary key
+ alterTable(format("ALTER TABLE %%s ALTER k MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("k");
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ assertTableColumnsAreNotMasked("k");
+
+ // Mask and disable mask for clustering key
+ alterTable(format("ALTER TABLE %%s ALTER c MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("c");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ assertTableColumnsAreNotMasked("c");
+
+ // Mask and disable mask for static column
+ alterTable(format("ALTER TABLE %%s ALTER s MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("s");
+ alterTable("ALTER TABLE %s ALTER s DROP MASKED");
+ assertTableColumnsAreNotMasked("s");
+
+ // Add multiple masked columns within the same query
+ alterTable(format("ALTER TABLE %%s ADD (" +
+ "r5 %s MASKED WITH %s, " +
+ "r6 %s, " +
+ "r7 %s MASKED WITH %s, " +
+ "r8 %s)",
+ type, mask, type, type, mask, type));
+ assertTableColumnsAreMasked("r5", "r7");
+ assertTableColumnsAreNotMasked("r6", "r8");
+ }
+
+ @Test
+ public void testAlterTableWithMaskedColumnsAndMaterializedView() throws Throwable
+ {
+ createTable(format("CREATE TABLE %%s (" +
+ "k %s, c %<s, r1 %<s, r2 %<s, s1 %<s static, s2 %<s static, " +
+ "PRIMARY KEY(k, c))", type));
+ createView("CREATE MATERIALIZED VIEW %s AS SELECT k, c, r2 FROM %s " +
+ "WHERE k IS NOT NULL AND c IS NOT NULL " +
+ "AND r1 IS NOT NULL AND r2 IS NOT NULL " +
+ "PRIMARY KEY (r2, c, k)");
+
+ // Adding a column to the table doesn't have an effect on the view
+ alterTable(format("ALTER TABLE %%s ADD r3 %s MASKED WITH %s", type, mask));
+ assertTableColumnsAreMasked("r3");
+ assertTableColumnsAreNotMasked("k", "c", "r1", "r2", "s1", "s2");
+ assertViewColumnsAreNotMasked("k", "c", "r2");
+
+ // Masking a column that is not part of the view doesn't have an effect on the view
+ alterTable(format("ALTER TABLE %%s ALTER r1 MASKED WITH %s", mask));
+ alterTable(format("ALTER TABLE %%s ALTER s1 MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("r1", "r3", "s1");
+ assertTableColumnsAreNotMasked("k", "c", "r2", "s2");
+ assertViewColumnsAreNotMasked("k", "c", "r2");
+
+ // Masking a column that is part of the view should have an effect on the view
+ alterTable(format("ALTER TABLE %%s ALTER r2 MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("r1", "r2", "r3", "s1");
+ assertTableColumnsAreNotMasked("k", "c", "s2");
+ assertViewColumnsAreMasked("r2");
+ assertViewColumnsAreNotMasked("k", "c");
+
+ // Mask the rest of the columns
+ alterTable(format("ALTER TABLE %%s ALTER k MASKED WITH %s", mask));
+ alterTable(format("ALTER TABLE %%s ALTER c MASKED WITH %s", mask));
+ alterTable(format("ALTER TABLE %%s ALTER s2 MASKED WITH %s", mask));
+ assertTableColumnsAreMasked("k", "c", "r1", "r2", "r3", "s1", "s2");
+ assertViewColumnsAreMasked("k", "c", "r2");
+
+ // Unmask a column that is part of the view
+ alterTable("ALTER TABLE %s ALTER r2 DROP MASKED");
+ assertTableColumnsAreMasked("k", "c", "r1", "r3", "s1", "s2");
+ assertTableColumnsAreNotMasked("r2");
+ assertViewColumnsAreMasked("k", "c");
+ assertViewColumnsAreNotMasked("r2");
+
+ // Unmask the rest of the columns
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER r1 DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER r3 DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER s1 DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER s2 DROP MASKED");
+ assertTableColumnsAreNotMasked("k", "c", "r1", "r2", "r3", "s1", "s2");
+ assertViewColumnsAreNotMasked("k", "c", "r2");
+ }
+
+ private String functionName()
+ {
+ if (mask.equals("DEFAULT"))
+ return "mask_default";
+
+ return StringUtils.remove(StringUtils.substringBefore(mask, "("), KEYSPACE + ".");
+ }
+
+ private void assertTableColumnsAreMasked(String... columns) throws Throwable
+ {
+ for (String column : columns)
+ {
+ assertColumnIsMasked(currentTable(), column, functionName(), argumentTypes, argumentValues);
+ }
+ }
+
+ private void assertViewColumnsAreMasked(String... columns) throws Throwable
+ {
+ for (String column : columns)
+ {
+ assertColumnIsMasked(currentView(), column, functionName(), argumentTypes, argumentValues);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithDefaultTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithDefaultTest.java
new file mode 100644
index 0000000..1b9bf3a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithDefaultTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.runners.Parameterized;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * {@link ColumnMaskInAnyPositionTester} for {@link DefaultMaskingFunction}.
+ */
+public class ColumnMaskInAnyPositionWithDefaultTest extends ColumnMaskInAnyPositionTester
+{
+ @Parameterized.Parameters(name = "mask={0}, type={1}")
+ public static Collection<Object[]> options()
+ {
+ return Arrays.asList(new Object[][]{
+ { "DEFAULT", "int", emptyList(), emptyList() },
+ { "DEFAULT", "text", emptyList(), emptyList() },
+ { "DEFAULT", "frozen<list<uuid>>", emptyList(), emptyList() },
+ { "mask_default()", "int", emptyList(), emptyList() },
+ { "mask_default()", "text", emptyList(), emptyList() },
+ { "mask_default()", "frozen<list<uuid>>", emptyList(), emptyList() },
+ });
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithNullTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithNullTest.java
new file mode 100644
index 0000000..751e2ab
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithNullTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.runners.Parameterized;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * {@link ColumnMaskInAnyPositionTester} for {@link NullMaskingFunction}.
+ */
+public class ColumnMaskInAnyPositionWithNullTest extends ColumnMaskInAnyPositionTester
+{
+ @Parameterized.Parameters(name = "mask={0}, type={1}")
+ public static Collection<Object[]> options()
+ {
+ return Arrays.asList(new Object[][]{
+ { "mask_null()", "int", emptyList(), emptyList() },
+ { "mask_null()", "text", emptyList(), emptyList() },
+ { "mask_null()", "frozen<list<uuid>>", emptyList(), emptyList() },
+ });
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithPartialTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithPartialTest.java
new file mode 100644
index 0000000..cb0479a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithPartialTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+import static java.util.Arrays.asList;
+
+/**
+ * {@link ColumnMaskInAnyPositionTester} for {@link PartialMaskingFunction}.
+ */
+public class ColumnMaskInAnyPositionWithPartialTest extends ColumnMaskInAnyPositionTester
+{
+ @Parameterized.Parameters(name = "mask={0}, type={1}")
+ public static Collection<Object[]> options()
+ {
+ return Arrays.asList(new Object[][]{
+ { "mask_inner(1, 2)", "text",
+ asList(Int32Type.instance, Int32Type.instance),
+ asList(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)) },
+ { "mask_outer(1, 2)", "text",
+ asList(Int32Type.instance, Int32Type.instance),
+ asList(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)) },
+ { "mask_inner(1, 2, '#')", "text",
+ asList(Int32Type.instance, Int32Type.instance, UTF8Type.instance),
+ asList(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2), UTF8Type.instance.decompose("#")) },
+ { "mask_outer(1, 2, '#')", "text",
+ asList(Int32Type.instance, Int32Type.instance, UTF8Type.instance),
+ asList(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2), UTF8Type.instance.decompose("#")) }
+ });
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithReplaceTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithReplaceTest.java
new file mode 100644
index 0000000..8fc514d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithReplaceTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+/**
+ * {@link ColumnMaskInAnyPositionTester} for {@link ReplaceMaskingFunction}.
+ */
+public class ColumnMaskInAnyPositionWithReplaceTest extends ColumnMaskInAnyPositionTester
+{
+ @Parameterized.Parameters(name = "mask={0}, type={1}")
+ public static Collection<Object[]> options()
+ {
+ return Arrays.asList(new Object[][]{
+ { "mask_replace(0)", "int",
+ singletonList(Int32Type.instance),
+ singletonList(Int32Type.instance.decompose(0)) },
+ { "mask_replace('redacted')", "text",
+ singletonList(UTF8Type.instance),
+ singletonList(UTF8Type.instance.decompose("redacted")) },
+ { "mask_replace([])", "frozen<list<int>>",
+ singletonList(ListType.getInstance(Int32Type.instance, false)),
+ singletonList(ListType.getInstance(Int32Type.instance, false).decompose(emptyList())) },
+ });
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithUDFTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithUDFTest.java
new file mode 100644
index 0000000..09e5020
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithUDFTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Before;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * {@link ColumnMaskInAnyPositionTester} for user-defined functions.
+ */
+public class ColumnMaskInAnyPositionWithUDFTest extends ColumnMaskInAnyPositionTester
+{
+ private static final String TEXT_REPLACE_FUNCTION = KEYSPACE + ".mask_text_replace";
+ private static final String INT_REPLACE_FUNCTION = KEYSPACE + ".mask_int_replace";
+
+ @Before
+ public void setupSchema() throws Throwable
+ {
+ createFunction(KEYSPACE,
+ "text, text",
+ "CREATE FUNCTION IF NOT EXISTS " + TEXT_REPLACE_FUNCTION + " (column text, replacement text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return replacement;'");
+ createFunction(KEYSPACE,
+ "int, int",
+ "CREATE FUNCTION IF NOT EXISTS " + INT_REPLACE_FUNCTION + " (column int, replacement int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return replacement;'");
+ }
+
+ @Parameterized.Parameters(name = "mask={0}, type={1}")
+ public static Collection<Object[]> options()
+ {
+ return Arrays.asList(new Object[][]{
+ { TEXT_REPLACE_FUNCTION + "('redacted')", "text",
+ singletonList(UTF8Type.instance),
+ singletonList(UTF8Type.instance.decompose("redacted")) },
+ { INT_REPLACE_FUNCTION + "(0)", "int",
+ singletonList(Int32Type.instance),
+ singletonList(Int32Type.instance.decompose(0)) }
+ });
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskNativeTypesTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskNativeTypesTest.java
new file mode 100644
index 0000000..a151526
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskNativeTypesTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cql3.CQL3Type;
+
+import static java.lang.String.format;
+import static java.util.Collections.emptyList;
+
+/**
+ * {@link ColumnMaskTester} verifying that we can attach column masks to table columns with any native data type.
+ */
+@RunWith(Parameterized.class)
+public class ColumnMaskNativeTypesTest extends ColumnMaskTester
+{
+ /** The type of the column. */
+ @Parameterized.Parameter
+ public CQL3Type.Native type;
+
+ @Parameterized.Parameters(name = "type={0}")
+ public static Collection<Object[]> options()
+ {
+ List<Object[]> parameters = new ArrayList<>();
+ for (CQL3Type.Native type : CQL3Type.Native.values())
+ {
+ if (type != CQL3Type.Native.EMPTY)
+ parameters.add(new Object[]{ type });
+ }
+ return parameters;
+ }
+
+ @Test
+ public void testNativeDataTypes() throws Throwable
+ {
+ String def = format("%s MASKED WITH DEFAULT", type);
+ String keyDef = type == CQL3Type.Native.COUNTER || type == CQL3Type.Native.DURATION
+ ? "int MASKED WITH DEFAULT" : def;
+ String staticDef = format("%s STATIC MASKED WITH DEFAULT", type);
+
+ // Create table with masks
+ String table = createTable(format("CREATE TABLE %%s (k %s, c %<s, r %s, s %s, PRIMARY KEY(k, c))", keyDef, def, staticDef));
+ assertColumnIsMasked(table, "k", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "c", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "r", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "s", "mask_default", emptyList(), emptyList());
+
+ // Alter column masks
+ alterTable("ALTER TABLE %s ALTER k MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER c MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER r MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER s MASKED WITH mask_null()");
+ assertColumnIsMasked(table, "k", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "c", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "r", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "s", "mask_null", emptyList(), emptyList());
+
+ // Drop masks
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER r DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER s DROP MASKED");
+ assertTableColumnsAreNotMasked("k", "c", "r", "s");
+ }
+}
+
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryTester.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryTester.java
new file mode 100644
index 0000000..15a87fb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryTester.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.datastax.driver.core.ResultSet;
+
+import static java.lang.String.format;
+
+/**
+ * Test queries on columns that have attached a dynamic data masking function.
+ */
+@RunWith(Parameterized.class)
+public abstract class ColumnMaskQueryTester extends ColumnMaskTester
+{
+ @Parameterized.Parameter
+ public String order;
+
+ @Parameterized.Parameter(1)
+ public String mask;
+
+ @Parameterized.Parameter(2)
+ public String columnType;
+
+ @Parameterized.Parameter(3)
+ public Object columnValue;
+
+ @Parameterized.Parameter(4)
+ public Object maskedValue;
+
+ @Before
+ public void setupSchema() throws Throwable
+ {
+ createTable("CREATE TABLE %s (" +
+ format("k1 %s, k2 %<s MASKED WITH %s, ", columnType, mask) +
+ format("c1 %s, c2 %<s MASKED WITH %s, ", columnType, mask) +
+ format("r1 %s, r2 %<s MASKED WITH %s, ", columnType, mask) +
+ format("s1 %s static, s2 %<s static MASKED WITH %s, ", columnType, mask) +
+ format("PRIMARY KEY((k1, k2), c1, c2)) WITH CLUSTERING ORDER BY (c1 %s, c2 %<s)", order));
+
+ createView("CREATE MATERIALIZED VIEW %s AS SELECT k2, k1, c2, c1, r2, r1 FROM %s " +
+ "WHERE k1 IS NOT NULL AND k2 IS NOT NULL " +
+ "AND c1 IS NOT NULL AND c2 IS NOT NULL " +
+ "AND r1 IS NOT NULL AND r2 IS NOT NULL " +
+ "PRIMARY KEY ((c2, c1), k2, k1)");
+
+ executeNet("INSERT INTO %s(k1, k2, c1, c2, r1, r2, s1, s2) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+ columnValue, columnValue, columnValue, columnValue, columnValue, columnValue, columnValue, columnValue);
+ }
+
+ @Test
+ public void testSelectWithWilcard() throws Throwable
+ {
+ ResultSet rs = executeNet("SELECT * FROM %s");
+ assertColumnNames(rs, "k1", "k2", "c1", "c2", "s1", "s2", "r1", "r2");
+ assertRowsNet(rs, row(columnValue, maskedValue,
+ columnValue, maskedValue,
+ columnValue, maskedValue,
+ columnValue, maskedValue));
+
+ rs = executeNet(format("SELECT * FROM %s.%s", KEYSPACE, currentView()));
+ assertColumnNames(rs, "c2", "c1", "k2", "k1", "r1", "r2");
+ assertRowsNet(rs, row(maskedValue, columnValue,
+ maskedValue, columnValue,
+ columnValue, maskedValue));
+ }
+
+ @Test
+ public void testSelectWithAllColumnNames() throws Throwable
+ {
+ ResultSet rs = executeNet("SELECT c2, c1, k2, k1, r2, r1, s2, s1 FROM %s");
+ assertColumnNames(rs, "c2", "c1", "k2", "k1", "r2", "r1", "s2", "s1");
+ assertRowsNet(rs, row(maskedValue, columnValue,
+ maskedValue, columnValue,
+ maskedValue, columnValue,
+ maskedValue, columnValue));
+
+ rs = executeNet(format("SELECT c2, c1, k2, k1, r2, r1 FROM %s.%s", KEYSPACE, currentView()));
+ assertColumnNames(rs, "c2", "c1", "k2", "k1", "r2", "r1");
+ assertRowsNet(rs, row(maskedValue, columnValue,
+ maskedValue, columnValue,
+ maskedValue, columnValue));
+ }
+
+ @Test
+ public void testSelectOnlyMaskedColumns() throws Throwable
+ {
+ ResultSet rs = executeNet("SELECT k2, c2, s2, r2 FROM %s");
+ assertColumnNames(rs, "k2", "c2", "s2", "r2");
+ assertRowsNet(rs, row(maskedValue, maskedValue, maskedValue, maskedValue));
+
+ rs = executeNet(format("SELECT k2, c2, r2 FROM %s.%s", KEYSPACE, currentView()));
+ assertColumnNames(rs, "k2", "c2", "r2");
+ assertRowsNet(rs, row(maskedValue, maskedValue, maskedValue));
+ }
+
+ @Test
+ public void testSelectOnlyNotMaskedColumns() throws Throwable
+ {
+ ResultSet rs = executeNet("SELECT k1, c1, s1, r1 FROM %s");
+ assertColumnNames(rs, "k1", "c1", "s1", "r1");
+ assertRowsNet(rs, row(columnValue, columnValue, columnValue, columnValue));
+
+ rs = executeNet(format("SELECT k1, c1, r1 FROM %s.%s", KEYSPACE, currentView()));
+ assertColumnNames(rs, "k1", "c1", "r1");
+ assertRowsNet(rs, row(columnValue, columnValue, columnValue));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
new file mode 100644
index 0000000..edef2a8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link ColumnMaskQueryTester} for {@link DefaultMaskingFunction}.
+ */
+public class ColumnMaskQueryWithDefaultTest extends ColumnMaskQueryTester
+{
+ @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
+ public static Collection<Object[]> options()
+ {
+ List<Object[]> options = new ArrayList<>();
+ for (String order : Arrays.asList("ASC", "DESC"))
+ {
+ options.add(new Object[]{ order, "DEFAULT", "text", "abc", "****" });
+ options.add(new Object[]{ order, "DEFAULT", "int", 123, 0 });
+ options.add(new Object[]{ order, "mask_default()", "text", "abc", "****" });
+ options.add(new Object[]{ order, "mask_default()", "int", 123, 0, });
+ }
+ return options;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithNullTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithNullTest.java
new file mode 100644
index 0000000..46e40ba
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithNullTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link ColumnMaskQueryTester} for {@link NullMaskingFunction}.
+ */
+public class ColumnMaskQueryWithNullTest extends ColumnMaskQueryTester
+{
+ @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
+ public static Collection<Object[]> options()
+ {
+ List<Object[]> options = new ArrayList<>();
+ for (String order : Arrays.asList("ASC", "DESC"))
+ {
+ options.add(new Object[]{ order, "mask_null()", "text", "abc", null });
+ options.add(new Object[]{ order, "mask_null()", "int", 123, null });
+ }
+ return options;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithPartialTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithPartialTest.java
new file mode 100644
index 0000000..b1a4237
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithPartialTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link ColumnMaskQueryTester} for {@link PartialMaskingFunction}.
+ */
+public class ColumnMaskQueryWithPartialTest extends ColumnMaskQueryTester
+{
+ @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
+ public static Collection<Object[]> options()
+ {
+ List<Object[]> options = new ArrayList<>();
+ for (String order : Arrays.asList("ASC", "DESC"))
+ {
+ options.add(new Object[]{ order, "mask_inner(null, null)", "text", "abcdef", "******" });
+ options.add(new Object[]{ order, "mask_inner(1, 2)", "text", "abcdef", "a***ef" });
+ options.add(new Object[]{ order, "mask_inner(1, 2, '#')", "text", "abcdef", "a###ef" });
+ options.add(new Object[]{ order, "mask_outer(1, null)", "text", "abcdef", "*bcdef" });
+ options.add(new Object[]{ order, "mask_outer(1, 2)", "text", "abcdef", "*bcd**" });
+ options.add(new Object[]{ order, "mask_outer(1, 2, '#')", "text", "abcdef", "#bcd##", });
+ }
+ return options;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
new file mode 100644
index 0000000..9a3a570
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link ColumnMaskQueryTester} for {@link ReplaceMaskingFunction}.
+ */
+public class ColumnMaskQueryWithReplaceTest extends ColumnMaskQueryTester
+{
+ @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
+ public static Collection<Object[]> options()
+ {
+ List<Object[]> options = new ArrayList<>();
+ for (String order : Arrays.asList("ASC", "DESC"))
+ {
+ options.add(new Object[]{ order, "mask_replace(null)", "int", 123, null });
+ options.add(new Object[]{ order, "mask_replace('redacted')", "ascii", "abc", "redacted" });
+ options.add(new Object[]{ order, "mask_replace('redacted')", "text", "abc", "redacted" });
+ options.add(new Object[]{ order, "mask_replace(0)", "int", 123, 0 });
+ options.add(new Object[]{ order, "mask_replace(0)", "bigint", 123L, 0L });
+ options.add(new Object[]{ order, "mask_replace(0)", "varint", BigInteger.valueOf(123), BigInteger.ZERO });
+ }
+ return options;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithUDFTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithUDFTest.java
new file mode 100644
index 0000000..f6b90b7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithUDFTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.runners.Parameterized;
+
+/**
+ * {@link ColumnMaskQueryTester} for user-defined functions.
+ */
+public class ColumnMaskQueryWithUDFTest extends ColumnMaskQueryTester
+{
+ private static final String TEXT_REPLACE_FUNCTION = KEYSPACE + ".mask_text_replace";
+ private static final String INT_REPLACE_FUNCTION = KEYSPACE + ".mask_int_replace";
+
+ @Before
+ @Override
+ public void setupSchema() throws Throwable
+ {
+ createFunction(KEYSPACE,
+ "text, text",
+ "CREATE FUNCTION IF NOT EXISTS " + TEXT_REPLACE_FUNCTION + " (column text, replacement text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return replacement;'");
+ createFunction(KEYSPACE,
+ "int, int",
+ "CREATE FUNCTION IF NOT EXISTS " + INT_REPLACE_FUNCTION + " (column int, replacement int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return replacement;'");
+ super.setupSchema();
+ }
+
+ @Parameterized.Parameters(name = "order={0}, mask={1}, type={2}, value={3}")
+ public static Collection<Object[]> options()
+ {
+ List<Object[]> options = new ArrayList<>();
+ for (String order : Arrays.asList("ASC", "DESC"))
+ {
+ options.add(new Object[]{ order, TEXT_REPLACE_FUNCTION + "('redacted')", "text", "abc", "redacted" });
+ options.add(new Object[]{ order, TEXT_REPLACE_FUNCTION + "('secret')", "text", "abc", "secret" });
+ options.add(new Object[]{ order, INT_REPLACE_FUNCTION + "(0)", "int", 123, 0 });
+ options.add(new Object[]{ order, INT_REPLACE_FUNCTION + "(-1)", "int", 123, -1 });
+ }
+ return options;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
new file mode 100644
index 0000000..d7e10e2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
@@ -0,0 +1,580 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.functions.FunctionFactory;
+import org.apache.cassandra.cql3.functions.FunctionParameter;
+import org.apache.cassandra.cql3.functions.NativeFunction;
+import org.apache.cassandra.cql3.functions.NativeFunctions;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+import static java.lang.String.format;
+import static java.util.Collections.emptyList;
+import static org.apache.cassandra.cql3.functions.masking.ColumnMask.DISABLED_ERROR_MESSAGE;
+
+/**
+ * Tests schema altering queries ({@code CREATE TABLE}, {@code ALTER TABLE}, etc.) that attach/dettach dynamic data
+ * masking functions to column definitions.
+ */
+public class ColumnMaskTest extends ColumnMaskTester
+{
+ @Test
+ public void testCollections() throws Throwable
+ {
+ // Create table with masks
+ String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, " +
+ "s set<int> MASKED WITH DEFAULT, " +
+ "l list<int> MASKED WITH DEFAULT, " +
+ "m map<int, int> MASKED WITH DEFAULT, " +
+ "fs frozen<set<int>> MASKED WITH DEFAULT, " +
+ "fl frozen<list<int>> MASKED WITH DEFAULT, " +
+ "fm frozen<map<int, int>> MASKED WITH DEFAULT)");
+ assertColumnIsMasked(table, "s", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "l", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "m", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "fs", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "fl", "mask_default", emptyList(), emptyList());
+ assertColumnIsMasked(table, "fm", "mask_default", emptyList(), emptyList());
+
+ // Alter column masks
+ alterTable("ALTER TABLE %s ALTER s MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER l MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER m MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER fs MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER fl MASKED WITH mask_null()");
+ alterTable("ALTER TABLE %s ALTER fm MASKED WITH mask_null()");
+ assertColumnIsMasked(table, "s", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "l", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "m", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "fs", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "fl", "mask_null", emptyList(), emptyList());
+ assertColumnIsMasked(table, "fm", "mask_null", emptyList(), emptyList());
+
+ // Drop masks
+ alterTable("ALTER TABLE %s ALTER s DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER l DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER m DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER fs DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER fl DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER fm DROP MASKED");
+ assertTableColumnsAreNotMasked("s", "l", "m", "fs", "fl", "fm");
+ }
+
+ @Test
+ public void testUDTs() throws Throwable
+ {
+ String type = createType("CREATE TYPE %s (a1 varint, a2 varint, a3 varint);");
+
+ // Create table with mask
+ String table = createTable(format("CREATE TABLE %%s (k int PRIMARY KEY, v %s MASKED WITH DEFAULT)", type));
+ assertColumnIsMasked(table, "v", "mask_default", emptyList(), emptyList());
+
+ // Alter column mask
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_null()");
+ assertColumnIsMasked(table, "v", "mask_null", emptyList(), emptyList());
+
+ // Drop mask
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertTableColumnsAreNotMasked("v");
+ }
+
+ @Test
+ public void testAlterTableAddMaskingToNonExistingColumn() throws Throwable
+ {
+ String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+ execute("ALTER TABLE %s ALTER IF EXISTS unknown MASKED WITH DEFAULT");
+ assertInvalidMessage(format("Column with name 'unknown' doesn't exist on table '%s'", table),
+ formatQuery("ALTER TABLE %s ALTER unknown MASKED WITH DEFAULT"));
+ }
+
+ @Test
+ public void testAlterTableRemoveMaskingFromNonExistingColumn() throws Throwable
+ {
+ String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+ execute("ALTER TABLE %s ALTER IF EXISTS unknown DROP MASKED");
+ assertInvalidMessage(format("Column with name 'unknown' doesn't exist on table '%s'", table),
+ formatQuery("ALTER TABLE %s ALTER unknown DROP MASKED"));
+ }
+
+ @Test
+ public void testAlterTableRemoveMaskFromUnmaskedColumn() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+ execute("ALTER TABLE %s ALTER v DROP MASKED");
+ assertTableColumnsAreNotMasked("v");
+ }
+
+ @Test
+ public void testInvalidMaskingFunctionName() throws Throwable
+ {
+ // create table
+ createTableName();
+ assertInvalidMessage("Unable to find masking function for v, no declared function matches the signature mask_missing()",
+ formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v int MASKED WITH mask_missing())"));
+
+ // alter table
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+ assertInvalidMessage("Unable to find masking function for v, no declared function matches the signature mask_missing()",
+ "ALTER TABLE %s ALTER v MASKED WITH mask_missing()");
+
+ assertTableColumnsAreNotMasked("k", "v");
+ }
+
+ @Test
+ public void testInvalidMaskingFunctionArguments() throws Throwable
+ {
+ // create table
+ createTableName();
+ assertInvalidMessage("Invalid number of arguments for function system.mask_default(any)",
+ formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v int MASKED WITH mask_default(1))"));
+
+ // alter table
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+ assertInvalidMessage("Invalid number of arguments for function system.mask_default(any)",
+ "ALTER TABLE %s ALTER v MASKED WITH mask_default(1)");
+
+ assertTableColumnsAreNotMasked("k", "v");
+ }
+
+ @Test
+ public void testInvalidMaskingFunctionArgumentTypes() throws Throwable
+ {
+ // create table
+ createTableName();
+ assertInvalidMessage("Function system.mask_inner requires an argument of type int, but found argument 'a' of type ascii",
+ formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text MASKED WITH mask_inner('a', 'b'))"));
+
+ // alter table
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+ assertInvalidMessage("Function system.mask_inner requires an argument of type int, but found argument 'a' of type ascii",
+ "ALTER TABLE %s ALTER v MASKED WITH mask_inner('a', 'b')");
+ assertTableColumnsAreNotMasked("k", "v");
+ }
+
+ @Test
+ public void testColumnMaskingWithNotMaskingFunction() throws Throwable
+ {
+ // create table
+ createTableName();
+ assertInvalidMessage("Not-masking function tojson() cannot be used for masking table columns",
+ formatQuery("CREATE TABLE %s (k int PRIMARY KEY, v text MASKED WITH tojson())"));
+
+ // alter table
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+ assertInvalidMessage("Not-masking function tojson() cannot be used for masking table columns",
+ "ALTER TABLE %s ALTER v MASKED WITH tojson()");
+ assertTableColumnsAreNotMasked("k", "v");
+ }
+
+ @Test
+ @SuppressWarnings("resource")
+ public void testPreparedStatement() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v text MASKED WITH DEFAULT)");
+ execute("INSERT INTO %s (k, v) VALUES (0, 'sensitive')");
+
+ Session session = sessionNet();
+ PreparedStatement prepared = session.prepare(formatQuery("SELECT v FROM %s WHERE k = ?"));
+ BoundStatement bound = prepared.bind(0);
+ assertRowsNet(session.execute(bound), row("****"));
+
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertRowsNet(session.execute(bound), row("sensitive"));
+
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_replace('redacted')");
+ assertRowsNet(session.execute(bound), row("redacted"));
+ }
+
+ @Test
+ public void testViews() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, v text MASKED WITH mask_replace('redacted'), PRIMARY KEY (k, c))");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 'sensitive')");
+ String view = createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
+ "WHERE k IS NOT NULL AND c IS NOT NULL AND v IS NOT NULL " +
+ "PRIMARY KEY (v, k, c)");
+ waitForViewMutations();
+ assertRowsNet(executeNet(format("SELECT v FROM %s.%s", KEYSPACE, view)), row("redacted"));
+ assertRowsNet(executeNet(format("SELECT v FROM %s.%s WHERE v='sensitive'", KEYSPACE, view)), row("redacted"));
+ assertRowsNet(executeNet(format("SELECT v FROM %s.%s WHERE v='redacted'", KEYSPACE, view)));
+
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertRowsNet(executeNet(format("SELECT v FROM %s.%s", KEYSPACE, view)), row("sensitive"));
+ assertRowsNet(executeNet(format("SELECT v FROM %s.%s WHERE v='sensitive'", KEYSPACE, view)), row("sensitive"));
+ assertRowsNet(executeNet(format("SELECT v FROM %s.%s WHERE v='redacted'", KEYSPACE, view)));
+ }
+
+ @Test
+ @SuppressWarnings("resource")
+ public void testPreparedStatementOnView() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, v text MASKED WITH DEFAULT, PRIMARY KEY (k, c))");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 'sensitive')");
+ String view = createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
+ "WHERE k IS NOT NULL AND c IS NOT NULL AND v IS NOT NULL " +
+ "PRIMARY KEY (v, k, c)");
+ waitForViewMutations();
+
+ Session session = sessionNet();
+ PreparedStatement prepared = session.prepare(format("SELECT v FROM %s.%s WHERE v=?", KEYSPACE, view));
+ BoundStatement bound = prepared.bind("sensitive");
+ assertRowsNet(session.execute(bound), row("****"));
+
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertRowsNet(session.execute(bound), row("sensitive"));
+
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_replace('redacted')");
+ assertRowsNet(session.execute(bound), row("redacted"));
+ }
+
+ @Test
+ public void testGroupBy() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY (k, c))");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 'sensitive')");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 'sensitive')");
+ execute("INSERT INTO %s (k, c, v) VALUES (1, 0, 'sensitive')");
+ execute("INSERT INTO %s (k, c, v) VALUES (1, 1, 'sensitive')");
+
+ // without masks
+ String query = "SELECT * FROM %s GROUP BY k";
+ assertRowsNet(executeNet(query), row(1, 0, "sensitive"), row(0, 0, "sensitive"));
+
+ // with masked regular column
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_replace('redacted')");
+ assertRowsNet(executeNet(query), row(1, 0, "redacted"), row(0, 0, "redacted"));
+
+ // with masked clustering key
+ alterTable("ALTER TABLE %s ALTER c MASKED WITH mask_replace(-1)");
+ assertRowsNet(executeNet(query), row(1, -1, "redacted"), row(0, -1, "redacted"));
+
+ // with masked partition key
+ alterTable("ALTER TABLE %s ALTER k MASKED WITH mask_replace(-1)");
+ assertRowsNet(executeNet(query), row(-1, -1, "redacted"), row(-1, -1, "redacted"));
+
+ // again without masks
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertRowsNet(executeNet(query), row(1, 0, "sensitive"), row(0, 0, "sensitive"));
+ }
+
+ @Test
+ public void testPaging() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, v text, PRIMARY KEY (k, c))");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 'sensitive')");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 'sensitive')");
+ execute("INSERT INTO %s (k, c, v) VALUES (1, 0, 'sensitive')");
+
+ // without masks
+ assertRowsWithPaging("SELECT * FROM %s", row(1, 0, "sensitive"), row(0, 0, "sensitive"), row(0, 1, "sensitive"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 1", row(1, 0, "sensitive"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0", row(0, 0, "sensitive"), row(0, 1, "sensitive"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0 AND c = 1", row(0, 1, "sensitive"));
+
+ // with masked regular column
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_replace('redacted')");
+ assertRowsWithPaging("SELECT * FROM %s", row(1, 0, "redacted"), row(0, 0, "redacted"), row(0, 1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 1", row(1, 0, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0", row(0, 0, "redacted"), row(0, 1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0 AND c = 1", row(0, 1, "redacted"));
+
+ // with masked clustering key
+ alterTable("ALTER TABLE %s ALTER c MASKED WITH mask_replace(-1)");
+ assertRowsWithPaging("SELECT * FROM %s", row(1, -1, "redacted"), row(0, -1, "redacted"), row(0, -1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 1", row(1, -1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0", row(0, -1, "redacted"), row(0, -1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0 AND c = 1", row(0, -1, "redacted"));
+
+ // with masked partition key
+ alterTable("ALTER TABLE %s ALTER k MASKED WITH mask_replace(-1)");
+ assertRowsWithPaging("SELECT * FROM %s", row(-1, -1, "redacted"), row(-1, -1, "redacted"), row(-1, -1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 1", row(-1, -1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0", row(-1, -1, "redacted"), row(-1, -1, "redacted"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0 AND c = 1", row(-1, -1, "redacted"));
+
+ // again without masks
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertRowsWithPaging("SELECT * FROM %s", row(1, 0, "sensitive"), row(0, 0, "sensitive"), row(0, 1, "sensitive"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 1", row(1, 0, "sensitive"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0", row(0, 0, "sensitive"), row(0, 1, "sensitive"));
+ assertRowsWithPaging("SELECT * FROM %s WHERE k = 0 AND c = 1", row(0, 1, "sensitive"));
+ }
+
+ /**
+ * Tests that rows are always ordered according to the clear values of the columns, even for the post-ordering done
+ * for queries with {@code IN} restrictions and {@code ORDER BY} clauses.
+ */
+ @Test
+ public void testPostOrdering() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c))");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 0)");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 1)");
+ execute("INSERT INTO %s (k, c, v) VALUES (1, 0, 3)");
+ execute("INSERT INTO %s (k, c, v) VALUES (1, 1, 4)");
+ execute("INSERT INTO %s (k, c, v) VALUES (2, 0, 6)");
+ execute("INSERT INTO %s (k, c, v) VALUES (2, 1, 7)");
+ NativeFunctions.instance.add(NEGATIVE);
+
+ // Test ordering without masking, just for reference
+ assertRowsNet(executeNet("SELECT * FROM %s WHERE k IN (0, 1, 2)"),
+ row(0, 0, 0),
+ row(0, 1, 1),
+ row(1, 0, 3),
+ row(1, 1, 4),
+ row(2, 0, 6),
+ row(2, 1, 7));
+ assertRowsNet(executeNetWithoutPaging("SELECT * FROM %s WHERE k IN (0, 1, 2) ORDER BY c ASC"),
+ row(0, 0, 0),
+ row(1, 0, 3),
+ row(2, 0, 6),
+ row(0, 1, 1),
+ row(1, 1, 4),
+ row(2, 1, 7));
+ assertRowsNet(executeNetWithoutPaging("SELECT * FROM %s WHERE k IN (0, 1, 2) ORDER BY c DESC"),
+ row(0, 1, 1),
+ row(1, 1, 4),
+ row(2, 1, 7),
+ row(0, 0, 0),
+ row(1, 0, 3),
+ row(2, 0, 6));
+
+ // Test ordering with manually applied masking function, just for reference
+ assertRowsNet(executeNet("SELECT k, mask_negative(c), v FROM %s WHERE k IN (0, 1, 2)"),
+ row(0, -0, 0), // (0, 0, 0)
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -0, 3), // (1, 0, 3)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -0, 6), // (2, 0, 6)
+ row(2, -1, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT k, mask_negative(c), v FROM %s WHERE k IN (0, 1, 2) ORDER BY c ASC"),
+ row(0, -0, 0), // (0, 0, 0)
+ row(1, -0, 3), // (1, 0, 3)
+ row(2, -0, 6), // (2, 0, 6)
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -1, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT k, mask_negative(c), v FROM %s WHERE k IN (0, 1, 2) ORDER BY c DESC"),
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -1, 7), // (2, 1, 7)
+ row(0, -0, 0), // (0, 0, 0)
+ row(1, -0, 3), // (1, 0, 3)
+ row(2, -0, 6)); // (2, 0, 6)
+
+ alterTable("ALTER TABLE %s ALTER c MASKED WITH mask_negative()");
+
+ // Test ordering of wildcard queries with masked column
+ assertRowsNet(executeNet("SELECT * FROM %s WHERE k IN (0, 1, 2)"),
+ row(0, -0, 0), // (0, 0, 0)
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -0, 3), // (1, 0, 3)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -0, 6), // (2, 0, 6)
+ row(2, -1, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT * FROM %s WHERE k IN (0, 1, 2) ORDER BY c ASC"),
+ row(0, -0, 0), // (0, 0, 0)
+ row(1, -0, 3), // (1, 0, 3)
+ row(2, -0, 6), // (2, 0, 6)
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -1, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT * FROM %s WHERE k IN (0, 1, 2) ORDER BY c DESC"),
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -1, 7), // (2, 1, 7)
+ row(0, -0, 0), // (0, 0, 0)
+ row(1, -0, 3), // (1, 0, 3)
+ row(2, -0, 6)); // (2, 0, 6)
+
+ // Test ordering of column selection queries with masked column
+ assertRowsNet(executeNet("SELECT k, c, v FROM %s WHERE k IN (0, 1, 2)"),
+ row(0, -0, 0), // (0, 0, 0)
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -0, 3), // (1, 0, 3)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -0, 6), // (2, 0, 6)
+ row(2, -1, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT k, c, v FROM %s WHERE k IN (0, 1, 2) ORDER BY c ASC"),
+ row(0, -0, 0), // (0, 0, 0)
+ row(1, -0, 3), // (1, 0, 3)
+ row(2, -0, 6), // (2, 0, 6)
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -1, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT k, c, v FROM %s WHERE k IN (0, 1, 2) ORDER BY c DESC"),
+ row(0, -1, 1), // (0, 1, 1)
+ row(1, -1, 4), // (1, 1, 4)
+ row(2, -1, 7), // (2, 1, 7)
+ row(0, -0, 0), // (0, 0, 0)
+ row(1, -0, 3), // (1, 0, 3)
+ row(2, -0, 6)); // (2, 0, 6)
+
+ // Test ordering of column selection queries with masked column, without selecting the ordered column
+ assertRowsNet(executeNet("SELECT k, v FROM %s WHERE k IN (0, 1, 2)"),
+ row(0, 0), // (0, 0, 0)
+ row(0, 1), // (0, 1, 1)
+ row(1, 3), // (1, 0, 3)
+ row(1, 4), // (1, 1, 4)
+ row(2, 6), // (2, 0, 6)
+ row(2, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT k, v FROM %s WHERE k IN (0, 1, 2) ORDER BY c ASC"),
+ row(0, 0), // (0, 0, 0)
+ row(1, 3), // (1, 0, 3)
+ row(2, 6), // (2, 0, 6)
+ row(0, 1), // (0, 1, 1)
+ row(1, 4), // (1, 1, 4)
+ row(2, 7)); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT k, v FROM %s WHERE k IN (0, 1, 2) ORDER BY c DESC"),
+ row(0, 1), // (0, 1, 1)
+ row(1, 4), // (1, 1, 4)
+ row(2, 7), // (2, 1, 7)
+ row(0, 0), // (0, 0, 0)
+ row(1, 3), // (1, 0, 3)
+ row(2, 6)); // (2, 0, 6)
+
+ // Test ordering of wildcard JSON queries with masked column
+ assertRowsNet(executeNet("SELECT JSON * FROM %s WHERE k IN (0, 1, 2)"),
+ row("{\"k\": 0, \"c\": 0, \"v\": 0}"), // (0, 0, 0)
+ row("{\"k\": 0, \"c\": -1, \"v\": 1}"), // (0, 1, 1)
+ row("{\"k\": 1, \"c\": 0, \"v\": 3}"), // (1, 0, 3)
+ row("{\"k\": 1, \"c\": -1, \"v\": 4}"), // (1, 1, 4)
+ row("{\"k\": 2, \"c\": 0, \"v\": 6}"), // (2, 0, 6)
+ row("{\"k\": 2, \"c\": -1, \"v\": 7}")); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT JSON * FROM %s WHERE k IN (0, 1, 2) ORDER BY c ASC"),
+ row("{\"k\": 0, \"c\": 0, \"v\": 0}"), // (0, 0, 0)
+ row("{\"k\": 1, \"c\": 0, \"v\": 3}"), // (1, 0, 3)
+ row("{\"k\": 2, \"c\": 0, \"v\": 6}"), // (2, 0, 6)
+ row("{\"k\": 0, \"c\": -1, \"v\": 1}"), // (0, 1, 1)
+ row("{\"k\": 1, \"c\": -1, \"v\": 4}"), // (1, 1, 4)
+ row("{\"k\": 2, \"c\": -1, \"v\": 7}")); // (2, 1, 7)
+ assertRowsNet(executeNetWithoutPaging("SELECT JSON * FROM %s WHERE k IN (0, 1, 2) ORDER BY c DESC"),
+ row("{\"k\": 0, \"c\": -1, \"v\": 1}"), // (0, 1, 1)
+ row("{\"k\": 1, \"c\": -1, \"v\": 4}"), // (1, 1, 4)
+ row("{\"k\": 2, \"c\": -1, \"v\": 7}"), // (2, 1, 7)
+ row("{\"k\": 0, \"c\": 0, \"v\": 0}"), // (0, 0, 0)
+ row("{\"k\": 1, \"c\": 0, \"v\": 3}"), // (1, 0, 3)
+ row("{\"k\": 2, \"c\": 0, \"v\": 6}")); // (2, 0, 6)
+ }
+
+ @Test
+ public void testEnableFlag() throws Throwable
+ {
+ // verify that we cannot create tables with masked columns if DDM is disabled
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(false);
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "CREATE TABLE t (k int PRIMARY KEY, v text MASKED WITH DEFAULT)");
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "CREATE TABLE t (k int MASKED WITH DEFAULT PRIMARY KEY, v text)");
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "CREATE TABLE t (k int, c int MASKED WITH DEFAULT, v text, PRIMARY KEY(k, c))");
+
+ // verify that we cannot mask an existing column if DDM is disabled
+ createTable("CREATE TABLE %s (k int, c int, s int static, r int, PRIMARY KEY(k, c))");
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "ALTER TABLE %s ALTER k MASKED WITH DEFAULT");
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "ALTER TABLE %s ALTER c MASKED WITH DEFAULT");
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "ALTER TABLE %s ALTER s MASKED WITH DEFAULT");
+ assertInvalidThrowMessage(DISABLED_ERROR_MESSAGE,
+ InvalidRequestException.class,
+ "ALTER TABLE %s ALTER r MASKED WITH DEFAULT");
+
+ // enable DDM and add some masked data
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(true);
+ alterTable("ALTER TABLE %s ALTER k MASKED WITH DEFAULT");
+ alterTable("ALTER TABLE %s ALTER c MASKED WITH DEFAULT");
+ alterTable("ALTER TABLE %s ALTER s MASKED WITH DEFAULT");
+ alterTable("ALTER TABLE %s ALTER r MASKED WITH DEFAULT");
+ execute("INSERT INTO %s (k, c, s, r) VALUES (1, 2, 3, 4)");
+ assertRowsNet(executeNet("SELECT * FROM %s"), row(0, 0, 0, 0));
+
+ // verify that column masks are not applied if DDM is disabled
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(false);
+ assertRowsNet(executeNet("SELECT * FROM %s"), row(1, 2, 3, 4));
+
+ // verify that we can drop column masks even if DDM is disabled
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER s DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER r DROP MASKED");
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(true);
+ assertRowsNet(executeNet("SELECT * FROM %s"), row(1, 2, 3, 4));
+ }
+
+ private void assertRowsWithPaging(String query, Object[]... rows)
+ {
+ for (int pageSize : Arrays.asList(1, 2, 3, 4, 5, 100))
+ {
+ assertRowsNet(executeNetWithPaging(query, pageSize), rows);
+
+ for (int limit : Arrays.asList(1, 2, 3, 4, 5, 100))
+ {
+ assertRowsNet(executeNetWithPaging(query + " LIMIT " + limit, pageSize),
+ Arrays.copyOfRange(rows, 0, Math.min(limit, rows.length)));
+ }
+ }
+ }
+
+ private static final FunctionFactory NEGATIVE = new FunctionFactory("mask_negative", FunctionParameter.fixed(CQL3Type.Native.INT))
+ {
+ @Override
+ protected NativeFunction doGetOrCreateFunction(List<AbstractType<?>> argTypes, AbstractType<?> receiverType)
+ {
+ return new MaskingFunction(name, argTypes.get(0), argTypes.get(0))
+ {
+ @Override
+ public Masker masker(ByteBuffer... parameters)
+ {
+ return bb -> {
+ if (bb == null)
+ return null;
+
+ Integer value = Int32Type.instance.compose(bb) * -1;
+ return Int32Type.instance.decompose(value);
+ };
+ }
+ };
+ }
+ };
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java
new file mode 100644
index 0000000..93f8b00
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.functions.ScalarFunction;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspace;
+import org.apache.cassandra.schema.SchemaKeyspaceTables;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.UserFunctions;
+
+import static java.lang.String.format;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests table columns with attached dynamic data masking functions.
+ */
+public class ColumnMaskTester extends CQLTester
+{
+ protected static final String USERNAME = "ddm_user";
+ protected static final String PASSWORD = "ddm_password";
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ CQLTester.setUpClass();
+ requireAuthentication();
+ requireNetwork();
+ }
+
+ @Before
+ public void before() throws Throwable
+ {
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(true);
+ useSuperUser();
+ executeNet(format("CREATE USER IF NOT EXISTS %s WITH PASSWORD '%s'", USERNAME, PASSWORD));
+ executeNet(format("GRANT ALL ON KEYSPACE %s TO %s", KEYSPACE, USERNAME));
+ executeNet(format("REVOKE UNMASK ON KEYSPACE %s FROM %s", KEYSPACE, USERNAME));
+ useUser(USERNAME, PASSWORD);
+ }
+
+ protected void assertTableColumnsAreNotMasked(String... columns) throws Throwable
+ {
+ for (String column : columns)
+ {
+ assertColumnIsNotMasked(currentTable(), column);
+ }
+ }
+
+ protected void assertViewColumnsAreNotMasked(String... columns) throws Throwable
+ {
+ for (String column : columns)
+ {
+ assertColumnIsNotMasked(currentView(), column);
+ }
+ }
+
+ protected void assertColumnIsNotMasked(String table, String column) throws Throwable
+ {
+ ColumnMask mask = getColumnMask(table, column);
+ assertNull(format("Mask for column '%s'", column), mask);
+
+ assertRows(execute(format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ? AND column_name = ?",
+ SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.COLUMN_MASKS),
+ KEYSPACE, table, column));
+ }
+
+ protected void assertColumnIsMasked(String table,
+ String column,
+ String functionName,
+ List<AbstractType<?>> partialArgumentTypes,
+ List<ByteBuffer> partialArgumentValues) throws Throwable
+ {
+ KeyspaceMetadata keyspaceMetadata = Keyspace.open(KEYSPACE).getMetadata();
+ TableMetadata tableMetadata = keyspaceMetadata.getTableOrViewNullable(table);
+ assertNotNull(tableMetadata);
+ ColumnMetadata columnMetadata = tableMetadata.getColumn(ColumnIdentifier.getInterned(column, false));
+ assertNotNull(columnMetadata);
+ AbstractType<?> columnType = columnMetadata.type;
+
+ // Verify the column mask in the in-memory schema
+ ColumnMask mask = getColumnMask(table, column);
+ assertNotNull(mask);
+ assertThat(mask.partialArgumentTypes()).isEqualTo(columnType.isReversed() && functionName.equals("mask_replace")
+ ? Collections.singletonList(ReversedType.getInstance(partialArgumentTypes.get(0)))
+ : partialArgumentTypes);
+ assertThat(mask.partialArgumentValues()).isEqualTo(partialArgumentValues);
+
+ // Verify the function in the column mask
+ ScalarFunction function = mask.function;
+ assertNotNull(function);
+ assertThat(function.name().name).isEqualTo(functionName);
+ assertThat(function.argTypes().get(0).asCQL3Type()).isEqualTo(columnMetadata.type.asCQL3Type());
+ assertThat(function.argTypes().size()).isEqualTo(partialArgumentTypes.size() + 1);
+
+ // Retrieve the persisted column metadata
+ UntypedResultSet columnRows = execute("SELECT * FROM system_schema.columns " +
+ "WHERE keyspace_name = ? AND table_name = ? AND column_name = ?",
+ KEYSPACE, table, column);
+ ColumnMetadata persistedColumn = SchemaKeyspace.createColumnFromRow(columnRows.one(), keyspaceMetadata.types, UserFunctions.none());
+
+ // Verify the column mask in the persisted schema
+ ColumnMask savedMask = persistedColumn.getMask();
+ assertNotNull(savedMask);
+ assertThat(mask).isEqualTo(savedMask);
+ assertThat(mask.function.argTypes()).isEqualTo(savedMask.function.argTypes());
+ }
+
+ @Nullable
+ protected ColumnMask getColumnMask(String table, String column)
+ {
+ TableMetadata tableMetadata = Schema.instance.getTableMetadata(KEYSPACE, table);
+ assertNotNull(tableMetadata);
+ ColumnMetadata columnMetadata = tableMetadata.getColumn(ColumnIdentifier.getInterned(column, false));
+
+ if (columnMetadata == null)
+ fail(format("Unknown column '%s'", column));
+
+ return columnMetadata.getMask();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithTypeAlteringFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithTypeAlteringFunctionTest.java
new file mode 100644
index 0000000..47350a0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithTypeAlteringFunctionTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cql3.CQL3Type;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.CQL3Type.Native.ASCII;
+import static org.apache.cassandra.cql3.CQL3Type.Native.BIGINT;
+import static org.apache.cassandra.cql3.CQL3Type.Native.BLOB;
+import static org.apache.cassandra.cql3.CQL3Type.Native.INT;
+import static org.apache.cassandra.cql3.CQL3Type.Native.TEXT;
+import static org.apache.cassandra.cql3.CQL3Type.Native.VARINT;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * {@link ColumnMaskTester} for masks using functions that might return values with a type different to the type of the
+ * masked column. Those queries should fail.
+ */
+@RunWith(Parameterized.class)
+public class ColumnMaskWithTypeAlteringFunctionTest extends ColumnMaskTester
+{
+ /** The column mask as expressed in CQL statements right after the {@code MASKED WITH} keywords. */
+ @Parameterized.Parameter
+ public String mask;
+
+ /** The type of the masked column. */
+ @Parameterized.Parameter(1)
+ public CQL3Type.Native type;
+
+ /** The type returned by the tested masking function. */
+ @Parameterized.Parameter(2)
+ public CQL3Type returnedType;
+
+ private boolean shouldSucceed;
+ private String errorMessage;
+
+ @Parameterized.Parameters(name = "mask={0} type={1}")
+ public static Collection<Object[]> options()
+ {
+ return Arrays.asList(new Object[][]{
+ { "mask_default()", INT, INT },
+ { "mask_default()", TEXT, TEXT },
+ { "mask_null()", INT, INT },
+ { "mask_null()", TEXT, TEXT },
+ { "mask_hash()", INT, BLOB },
+ { "mask_hash('SHA-512')", INT, BLOB },
+ { "mask_inner(1,2)", TEXT, TEXT },
+ { "mask_outer(1,2)", TEXT, TEXT },
+ { "mask_replace(0)", INT, INT },
+ { "mask_replace(0)", BIGINT, BIGINT },
+ { "mask_replace(0)", VARINT, VARINT },
+ { "mask_replace('redacted')", ASCII, ASCII },
+ { "mask_replace('redacted')", TEXT, TEXT } });
+ }
+
+ @Before
+ public void setupExpectedResults()
+ {
+ shouldSucceed = returnedType == type;
+ errorMessage = shouldSucceed ? null : format("Masking function %s return type is %s.", mask, returnedType);
+ }
+
+ @Test
+ public void testTypeAlteringFunctionOnCreateTable()
+ {
+ testOnCreateTable("CREATE TABLE %s.%s (k int PRIMARY KEY, v %s MASKED WITH %s)");
+ testOnCreateTable("CREATE TABLE %s.%s (k %s MASKED WITH %s PRIMARY KEY, v int)");
+ testOnCreateTable("CREATE TABLE %s.%s (k1 int, k2 %s MASKED WITH %s, PRIMARY KEY((k1, k2)))");
+ testOnCreateTable("CREATE TABLE %s.%s (k int, c %s MASKED WITH %s, PRIMARY KEY(k, c)) WITH CLUSTERING ORDER BY (c ASC)");
+ testOnCreateTable("CREATE TABLE %s.%s (k int, c %s MASKED WITH %s, PRIMARY KEY(k, c)) WITH CLUSTERING ORDER BY (c DESC)");
+ testOnCreateTable("CREATE TABLE %s.%s (k int, c int, s %s STATIC MASKED WITH %s, PRIMARY KEY(k, c))");
+ }
+
+ private void testOnCreateTable(String query)
+ {
+ String formattedQuery = format(query, KEYSPACE, createTableName(), type, mask);
+ if (shouldSucceed)
+ createTable(formattedQuery);
+ else
+ assertThatThrownBy(() -> execute(formattedQuery)).hasMessageContaining(errorMessage);
+ }
+
+ @Test
+ public void testTypeAlteringFunctionOnMaskColumn()
+ {
+ testOnAlterColumn("CREATE TABLE %s.%s (k int PRIMARY KEY, v %s)",
+ "ALTER TABLE %s.%s ALTER v MASKED WITH %s");
+ testOnAlterColumn("CREATE TABLE %s.%s (k %s PRIMARY KEY, v int)",
+ "ALTER TABLE %s.%s ALTER k MASKED WITH %s");
+ testOnAlterColumn("CREATE TABLE %s.%s (k1 int, k2 %s, PRIMARY KEY((k1, k2)))",
+ "ALTER TABLE %s.%s ALTER k2 MASKED WITH %s");
+ testOnAlterColumn("CREATE TABLE %s.%s (k int, c %s, PRIMARY KEY(k, c)) WITH CLUSTERING ORDER BY (c ASC)",
+ "ALTER TABLE %s.%s ALTER c MASKED WITH %s");
+ testOnAlterColumn("CREATE TABLE %s.%s (k int, c %s, PRIMARY KEY(k, c)) WITH CLUSTERING ORDER BY (c DESC)",
+ "ALTER TABLE %s.%s ALTER c MASKED WITH %s");
+ testOnAlterColumn("CREATE TABLE %s.%s (k int, c int, s %s STATIC, PRIMARY KEY(k, c))",
+ "ALTER TABLE %s.%s ALTER s MASKED WITH %s");
+ }
+
+ private void testOnAlterColumn(String createQuery, String alterQuery)
+ {
+ String table = createTableName();
+ createTable(format(createQuery, KEYSPACE, table, type));
+
+ String formattedQuery = format(alterQuery, KEYSPACE, table, mask);
+ if (shouldSucceed)
+ alterTable(formattedQuery);
+ else
+ assertThatThrownBy(() -> execute(formattedQuery)).hasMessageContaining(errorMessage);
+ }
+
+ @Test
+ public void testTypeAlteringFunctionOnAddColumn()
+ {
+ String table = createTable(format("CREATE TABLE %%s (k int PRIMARY KEY, v %s)", type));
+ String query = format("ALTER TABLE %s.%s ADD n %s MASKED WITH %s", KEYSPACE, table, type, mask);
+ if (shouldSucceed)
+ alterTable(query);
+ else
+ assertThatThrownBy(() -> execute(query)).hasMessageContaining(errorMessage);
+ }
+}
+
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithUDFTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithUDFTest.java
new file mode 100644
index 0000000..3b26a56
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithUDFTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.junit.Test;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.assertj.core.api.Assertions;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertNull;
+
+/**
+ * {@link ColumnMaskTester} for column masks using a UDF.
+ */
+public class ColumnMaskWithUDFTest extends ColumnMaskTester
+{
+ @Test
+ @SuppressWarnings("resource")
+ public void testUDF() throws Throwable
+ {
+ // create a table masked with a UDF and with a materialized view
+ String function = createAddFunction();
+ createTable(format("CREATE TABLE %%s (k int, c int, v int MASKED WITH %s(7), PRIMARY KEY (k, c))", function));
+ String view = createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
+ "WHERE k IS NOT NULL AND c IS NOT NULL AND v IS NOT NULL " +
+ "PRIMARY KEY (v, k, c)");
+
+ // add some data
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 0, 10)");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 20)");
+ Object[][] clearRows = rows(row(0, 0, 10), row(0, 1, 20));
+ Object[][] maskedRows = rows(row(0, 0, 17), row(0, 1, 27));
+ waitForViewMutations();
+
+ Session session = sessionNet();
+
+ // query the table and verify that the column mask is applied
+ String tableQuery = "SELECT k, c, v FROM %s";
+ assertRowsNet(executeNet(tableQuery), maskedRows);
+ BoundStatement tablePrepared = session.prepare(formatQuery(tableQuery)).bind();
+ assertRowsNet(session.execute(tablePrepared), maskedRows);
+
+ // query the view and verify that the column mask is applied
+ String viewQuery = format("SELECT k, c, v FROM %s.%s", KEYSPACE, view);
+ BoundStatement viewPrepared = session.prepare(formatQuery(tableQuery)).bind();
+ assertRowsNet(executeNet(viewQuery), maskedRows);
+ assertRowsNet(session.execute(viewPrepared), maskedRows);
+
+ // drop the column mask and verify that the column is not masked anymore
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertRowsNet(executeNet(tableQuery), clearRows);
+ assertRowsNet(executeNet(viewQuery), clearRows);
+ assertRowsNet(session.execute(tablePrepared), clearRows);
+ assertRowsNet(session.execute(viewPrepared), clearRows);
+
+ // mask the column again, but this time with a different argument
+ alterTable(format("ALTER TABLE %%s ALTER v MASKED WITH %s(8)", function));
+ maskedRows = rows(row(0, 0, 18), row(0, 1, 28));
+ assertRowsNet(executeNet(tableQuery), maskedRows);
+ assertRowsNet(executeNet(viewQuery), maskedRows);
+ assertRowsNet(session.execute(tablePrepared), maskedRows);
+ assertRowsNet(session.execute(viewPrepared), maskedRows);
+ }
+
+ @Test
+ public void testUDFWithReversed() throws Throwable
+ {
+ // create a table masked with a UDF and with a materialized view
+ String function = createAddFunction();
+ createTable(format("CREATE TABLE %%s (k int, c int MASKED WITH %s(100), v int, PRIMARY KEY (k, c)) " +
+ "WITH CLUSTERING ORDER BY (c DESC)", function));
+ String view = createView("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " +
+ "WHERE k IS NOT NULL AND c IS NOT NULL AND v IS NOT NULL " +
+ "PRIMARY KEY (v, c, K) " +
+ "WITH CLUSTERING ORDER BY (c DESC, k ASC)");
+
+ // add some data
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 1, 0)");
+ execute("INSERT INTO %s (k, c, v) VALUES (0, 2, 0)");
+ Object[][] clearRows = rows(row(0, 2, 0), row(0, 1, 0));
+ Object[][] maskedRows = rows(row(0, 102, 0), row(0, 101, 0));
+ waitForViewMutations();
+
+ // query the table and verify that the column mask is applied
+ String tableQuery = "SELECT k, c, v FROM %s";
+ assertRowsNet(executeNet(tableQuery), maskedRows);
+
+ // query the view and verify that the column mask is applied
+ String viewQuery = format("SELECT k, c, v FROM %s.%s", KEYSPACE, view);
+ assertRowsNet(executeNet(viewQuery), maskedRows);
+
+ // drop the column mask and verify that the column is not masked anymore
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ assertRowsNet(executeNet(tableQuery), clearRows);
+ assertRowsNet(executeNet(viewQuery), clearRows);
+
+ // mask the column again, but this time with a different argument
+ alterTable(format("ALTER TABLE %%s ALTER c MASKED WITH %s(1000)", function));
+ maskedRows = rows(row(0, 1002, 0), row(0, 1001, 0));
+ assertRowsNet(executeNet(tableQuery), maskedRows);
+ assertRowsNet(executeNet(viewQuery), maskedRows);
+ }
+
+ /**
+ * Verifies that queries dropping a UDF that is used for masking columns are rejected.
+ */
+ @Test
+ public void testDropUDFWithDependentMasks() throws Throwable
+ {
+ String function = createAddFunction();
+ String mask = format("MASKED WITH %s(7)", function);
+ String table1 = createTable(format("CREATE TABLE %%s (k int %s PRIMARY KEY, v int %<s)", mask));
+ String table2 = createTable(format("CREATE TABLE %%s (k int %s, c int %<s, v int %<s, s int static %<s, PRIMARY KEY (k, c))", mask));
+
+ String dropFunctionQuery = format("DROP FUNCTION %s", function);
+
+ String message = format("Function '%s' is still referenced by column masks in tables %s, %s", function, table1, table2);
+ assertInvalidThrowMessage(message, InvalidRequestException.class, dropFunctionQuery);
+
+ alterTable(format("ALTER TABLE %s.%s ALTER k DROP MASKED", KEYSPACE, table1));
+ alterTable(format("ALTER TABLE %s.%s ALTER k DROP MASKED", KEYSPACE, table2));
+ assertInvalidThrowMessage(message, InvalidRequestException.class, dropFunctionQuery);
+
+ alterTable(format("ALTER TABLE %s.%s ALTER v DROP MASKED", KEYSPACE, table1));
+ alterTable(format("ALTER TABLE %s.%s ALTER v DROP MASKED", KEYSPACE, table2));
+ message = format("Function '%s' is still referenced by column masks in tables %s", function, table2);
+ assertInvalidThrowMessage(message, InvalidRequestException.class, dropFunctionQuery);
+
+ alterTable(format("ALTER TABLE %s.%s ALTER c DROP MASKED", KEYSPACE, table2));
+ assertInvalidThrowMessage(message, InvalidRequestException.class, dropFunctionQuery);
+
+ alterTable(format("ALTER TABLE %s.%s ALTER s DROP MASKED", KEYSPACE, table2));
+ schemaChange(dropFunctionQuery);
+ }
+
+ @Test
+ public void testMissingUDF() throws Throwable
+ {
+ assertMaskingFails("Unable to find masking function for v, no declared function matches the signature %s()",
+ "missing_udf");
+ }
+
+ @Test
+ public void testUDFWithNoArguments() throws Throwable
+ {
+ assertMaskingFails("Invalid number of arguments in call to function %s: 0 required but 1 provided",
+ createFunction(KEYSPACE,
+ "",
+ "CREATE FUNCTION %s () " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return 42;'"));
+ }
+
+ @Test
+ public void testUDFWithWrongArgumentCount() throws Throwable
+ {
+ assertMaskingFails("Invalid number of arguments in call to function %s: 2 required but 3 provided",
+ createAddFunction(),
+ "(1, 3)");
+ }
+
+ @Test
+ public void testUDFWithWrongArgumentType() throws Throwable
+ {
+ assertMaskingFails("Type error: org.apache.cassandra.db.marshal.Int32Type " +
+ "cannot be passed as argument 0 of function %s of type text",
+ createFunction(KEYSPACE,
+ "text",
+ "CREATE FUNCTION %s (a text) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return \"redacted\";'"));
+ }
+
+ @Test
+ public void testUDFWithWrongReturnType() throws Throwable
+ {
+ assertMaskingFails("Masking function %s() return type is text",
+ createFunction(KEYSPACE,
+ "int",
+ "CREATE FUNCTION %s (a int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS text " +
+ "LANGUAGE java " +
+ "AS 'return \"redacted\";'"));
+ }
+
+ @Test
+ public void testUDFInOtherKeyspace() throws Throwable
+ {
+ assertMaskingFails("Masking function %s() doesn't belong to the same keyspace as the table",
+ createFunction(KEYSPACE_PER_TEST,
+ "int",
+ "CREATE FUNCTION %s (input int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return Integer.valueOf(input);'"));
+ }
+
+ @Test
+ public void testUDA() throws Throwable
+ {
+ String fState = createAddFunction();
+ String fFinal = createIdentityFunction();
+ assertMaskingFails("Aggregate function %s() cannot be used for masking table columns",
+ createAggregate(KEYSPACE,
+ "int",
+ "CREATE AGGREGATE %s(int) " +
+ "SFUNC " + shortFunctionName(fState) + " " +
+ "STYPE int " +
+ "FINALFUNC " + shortFunctionName(fFinal) + " " +
+ "INITCOND 42"));
+ }
+
+ private void assertMaskingFails(String message, String function) throws Throwable
+ {
+ assertMaskingFails(message, function, "()");
+ }
+
+ private void assertMaskingFails(String message, String function, String arguments) throws Throwable
+ {
+ message = format(message, function);
+
+ // create table should fail
+ Assertions.assertThatThrownBy(() -> execute(format("CREATE TABLE %s.%s (k int PRIMARY KEY, v int MASKED WITH %s%s)",
+ KEYSPACE, createTableName(), function, arguments)))
+ .isInstanceOf(InvalidRequestException.class)
+ .hasMessageContaining(message);
+ assertNull(currentTableMetadata());
+
+ // alter table should fail
+ String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+ Assertions.assertThatThrownBy(() -> execute(format("ALTER TABLE %s.%s ALTER v MASKED WITH %s%s",
+ KEYSPACE, table, function, arguments)))
+ .isInstanceOf(InvalidRequestException.class)
+ .hasMessageContaining(message);
+ assertColumnIsNotMasked(table, "v");
+ }
+
+ private String createAddFunction() throws Throwable
+ {
+ return createFunction(KEYSPACE,
+ "int, int",
+ "CREATE FUNCTION %s (a int, b int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return Integer.valueOf(a) + Integer.valueOf(b);'");
+ }
+
+ private String createIdentityFunction() throws Throwable
+ {
+ return createFunction(KEYSPACE,
+ "int, int",
+ "CREATE FUNCTION %s (a int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS int " +
+ "LANGUAGE java " +
+ "AS 'return a;'");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunctionTest.java
new file mode 100644
index 0000000..19e4ab1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/DefaultMaskingFunctionTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.apache.cassandra.cql3.CQL3Type;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for {@link DefaultMaskingFunction}.
+ */
+public class DefaultMaskingFunctionTest extends MaskingFunctionTester
+{
+ @Override
+ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
+ {
+ assertRows(execute(format("SELECT mask_default(%s) FROM %%s", name)),
+ row(type.getType().getMaskedValue()));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java
new file mode 100644
index 0000000..10fe755
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/HashMaskingFunctionTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import com.datastax.driver.core.ColumnDefinitions;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.SchemaConstants;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for {@link HashMaskingFunction}.
+ */
+public class HashMaskingFunctionTest extends MaskingFunctionTester
+{
+ @BeforeClass
+ public static void beforeClass()
+ {
+ requireNetwork();
+ }
+
+ @Override
+ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
+ {
+ ByteBuffer serializedValue = serializedValue(type, value);
+
+ // with default algorithm
+ assertRows(execute(format("SELECT mask_hash(%s) FROM %%s", name)),
+ row(HashMaskingFunction.hash(HashMaskingFunction.messageDigest(HashMaskingFunction.DEFAULT_ALGORITHM),
+ serializedValue)));
+
+ // with null algorithm
+ assertRows(execute(format("SELECT mask_hash(%s, null) FROM %%s", name)),
+ row(HashMaskingFunction.hash(HashMaskingFunction.messageDigest(HashMaskingFunction.DEFAULT_ALGORITHM),
+ serializedValue)));
+
+ // with manually specified algorithm
+ assertRows(execute(format("SELECT mask_hash(%s, 'SHA-512') FROM %%s", name)),
+ row(HashMaskingFunction.hash(HashMaskingFunction.messageDigest("SHA-512"), serializedValue)));
+
+ // with not found ASCII algorithm
+ assertInvalidThrowMessage("Hash algorithm not found",
+ InvalidRequestException.class,
+ format("SELECT mask_hash(%s, 'unknown-algorithm') FROM %%s", name));
+
+ // with not found UTF-8 algorithm
+ assertInvalidThrowMessage("Hash algorithm not found",
+ InvalidRequestException.class,
+ format("SELECT mask_hash(%s, 'áéíóú') FROM %%s", name));
+
+ // test result set metadata, it should always be of type blob
+ ResultSet rs = executeNet(format("SELECT mask_hash(%s) FROM %%s", name));
+ ColumnDefinitions definitions = rs.getColumnDefinitions();
+ Assert.assertEquals(1, definitions.size());
+ Assert.assertEquals(DataType.blob(), definitions.getType(0));
+ Assert.assertEquals(format("%s.mask_hash(%s)", SchemaConstants.SYSTEM_KEYSPACE_NAME, name),
+ definitions.getName(0));
+ }
+
+ private ByteBuffer serializedValue(CQL3Type type, Object value)
+ {
+ if (isNullOrEmptyMultiCell(type, value))
+ return null;
+
+ return makeByteBuffer(value, type.getType());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java b/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
new file mode 100644
index 0000000..a45f2c8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Map;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.TupleType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.serializers.SimpleDateSerializer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static java.lang.String.format;
+
+/**
+ * Abstract class for testing a specific implementation of {@link MaskingFunction}.
+ * <p>
+ * It tests the application of the function as defined by {@link #testMaskingOnColumn(String, CQL3Type, Object)}
+ * on all CQL data types on all the possible positions allowed for that type (primary key, regular and static columns).
+ */
+public abstract class MaskingFunctionTester extends CQLTester
+{
+ /**
+ * Tests the native masking function for all CQL native data types.
+ */
+ @Test
+ public void testMaskingOnNative() throws Throwable
+ {
+ for (CQL3Type.Native type : CQL3Type.Native.values())
+ {
+ switch (type)
+ {
+ case EMPTY:
+ break;
+ case COUNTER:
+ testMaskingOnCounterColumn(0L, -1L, 1L);
+ break;
+ case TEXT:
+ case ASCII:
+ case VARCHAR:
+ testMaskingOnAllColumns(type, "confidential");
+ break;
+ case BOOLEAN:
+ testMaskingOnAllColumns(type, true, false);
+ break;
+ case TINYINT:
+ testMaskingOnAllColumns(type, (byte) 0, (byte) 2);
+ break;
+ case SMALLINT:
+ testMaskingOnAllColumns(type, (short) 0, (short) 2);
+ break;
+ case INT:
+ testMaskingOnAllColumns(type, 2, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ break;
+ case BIGINT:
+ testMaskingOnAllColumns(type, 2L, Long.MIN_VALUE, Long.MAX_VALUE);
+ break;
+ case FLOAT:
+ testMaskingOnAllColumns(type, 2.3f, Float.MIN_VALUE, Float.MAX_VALUE);
+ break;
+ case DOUBLE:
+ testMaskingOnAllColumns(type, 2.3d, Double.MIN_VALUE, Double.MAX_VALUE);
+ break;
+ case VARINT:
+ testMaskingOnAllColumns(type, BigInteger.valueOf(-1), BigInteger.valueOf(0), BigInteger.valueOf(1));
+ break;
+ case DECIMAL:
+ testMaskingOnAllColumns(type, BigDecimal.valueOf(2.3), BigDecimal.valueOf(0), BigDecimal.valueOf(-2.3));
+ break;
+ case DATE:
+ testMaskingOnAllColumns(type,
+ SimpleDateSerializer.timeInMillisToDay(2),
+ SimpleDateSerializer.timeInMillisToDay(Long.MAX_VALUE));
+ break;
+ case DURATION:
+ testMaskingOnNotKeyColumns(type, Duration.newInstance(1, 2, 3), Duration.newInstance(3, 2, 1));
+ break;
+ case TIME:
+ testMaskingOnAllColumns(CQL3Type.Native.TIME, 2L, (long) Integer.MAX_VALUE);
+ break;
+ case TIMESTAMP:
+ testMaskingOnAllColumns(CQL3Type.Native.TIMESTAMP, new Date(2), new Date(Integer.MAX_VALUE));
+ break;
+ case UUID:
+ testMaskingOnAllColumns(type, UUID.randomUUID());
+ break;
+ case TIMEUUID:
+ testMaskingOnAllColumns(type, TimeUUID.minAtUnixMillis(2), TimeUUID.minAtUnixMillis(Long.MAX_VALUE));
+ break;
+ case INET:
+ testMaskingOnAllColumns(type, new InetSocketAddress(0).getAddress());
+ break;
+ case BLOB:
+ testMaskingOnAllColumns(type, UTF8Type.instance.decompose("confidential"));
+ break;
+ default:
+ throw new AssertionError("Type " + type + " should be tested for masking functions");
+ }
+ }
+ }
+
+ /**
+ * Tests the native masking function for collections.
+ */
+ @Test
+ public void testMaskingOnCollection() throws Throwable
+ {
+ // set
+ Object[] values = new Object[]{ set(), set(1, 2, 3) };
+ testMaskingOnAllColumns(SetType.getInstance(Int32Type.instance, false).asCQL3Type(), values);
+ testMaskingOnNotKeyColumns(SetType.getInstance(Int32Type.instance, true).asCQL3Type(), values);
+
+ // list
+ values = new Object[]{ list(), list(1, 2, 3) };
+ testMaskingOnAllColumns(ListType.getInstance(Int32Type.instance, false).asCQL3Type(), values);
+ testMaskingOnNotKeyColumns(ListType.getInstance(Int32Type.instance, true).asCQL3Type(), values);
+
+ // map
+ values = new Object[]{ map(), map(1, 10, 2, 20, 3, 30) };
+ testMaskingOnAllColumns(MapType.getInstance(Int32Type.instance, Int32Type.instance, false).asCQL3Type(), values);
+ testMaskingOnNotKeyColumns(MapType.getInstance(Int32Type.instance, Int32Type.instance, true).asCQL3Type(), values);
+ }
+
+ /**
+ * Tests the native masking function for tuples.
+ */
+ @Test
+ public void testMaskingOnTuple() throws Throwable
+ {
+ testMaskingOnAllColumns(new TupleType(ImmutableList.of(Int32Type.instance, Int32Type.instance)).asCQL3Type(),
+ tuple(1, 10), tuple(2, 20));
+ }
+
+ /**
+ * Tests the native masking function for UDTs.
+ */
+ @Test
+ public void testMaskingOnUDT() throws Throwable
+ {
+ String name = createType("CREATE TYPE %s (a int, b text)");
+
+ KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(keyspace());
+ Assert.assertNotNull(ks);
+
+ UserType udt = ks.types.get(ByteBufferUtil.bytes(name)).orElseThrow(AssertionError::new);
+ Assert.assertNotNull(udt);
+
+ Object[] values = new Object[]{ userType("a", 1, "b", "Alice"), userType("a", 2, "b", "Bob") };
+ testMaskingOnNotKeyColumns(udt.asCQL3Type(), values);
+ testMaskingOnAllColumns(udt.freeze().asCQL3Type(), values);
+ }
+
+ /**
+ * Tests the native masking function for the specified column type and values on all possible types of column.
+ * That is, when the column is part of the primary key, or a regular column, or a static column.
+ *
+ * @param type the type of the tested column
+ * @param values the values of the tested column
+ */
+ private void testMaskingOnAllColumns(CQL3Type type, Object... values) throws Throwable
+ {
+ createTable(format("CREATE TABLE %%s (pk %s, ck %<s, s %<s static, v %<s, PRIMARY KEY (pk, ck))", type));
+
+ for (Object value : values)
+ {
+ // Test null values
+ execute("INSERT INTO %s(pk, ck) VALUES (?, ?)", value, value);
+ testMaskingOnColumn("s", type, null);
+ testMaskingOnColumn("v", type, null);
+
+ // Test not-null values
+ execute("INSERT INTO %s(pk, ck, s, v) VALUES (?, ?, ?, ?)", value, value, value, value);
+ testMaskingOnColumn("pk", type, value);
+ testMaskingOnColumn("ck", type, value);
+ testMaskingOnColumn("s", type, value);
+ testMaskingOnColumn("v", type, value);
+
+ // Cleanup
+ execute("DELETE FROM %s WHERE pk=?", value);
+ }
+ }
+
+ /**
+ * Tests the native masking function for the specified column type and values when the column isn't part of the
+ * primary key. That is, when the column is either a regular column or a static column.
+ *
+ * @param type the type of the tested column
+ * @param values the values of the tested column
+ */
+ private void testMaskingOnNotKeyColumns(CQL3Type type, Object... values) throws Throwable
+ {
+ createTable(format("CREATE TABLE %%s (pk int, ck int, s %s static, v %<s, PRIMARY KEY (pk, ck))", type));
+
+ // Test null values
+ execute("INSERT INTO %s(pk, ck) VALUES (0, 0)");
+ testMaskingOnColumn("s", type, null);
+ testMaskingOnColumn("v", type, null);
+
+ // Test not-null values
+ for (Object value : values)
+ {
+ execute("INSERT INTO %s(pk, ck, s, v) VALUES (0, 0, ?, ?)", value, value);
+ testMaskingOnColumn("s", type, value);
+ testMaskingOnColumn("v", type, value);
+ }
+ }
+
+ /**
+ * Tests the native masking function on the specified counter column values in all the positions where it can appear.
+ * That is, when the counter column is either a regular column or a static column.
+ *
+ * @param values the values of the tested counter column
+ */
+ private void testMaskingOnCounterColumn(Object... values) throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, s counter static, v counter, PRIMARY KEY (pk, ck))");
+ for (Object value : values)
+ {
+ execute("UPDATE %s SET v = v + ?, s = s + ? WHERE pk = 0 AND ck = 0", value, value);
+ testMaskingOnColumn("s", CQL3Type.Native.COUNTER, value);
+ testMaskingOnColumn("v", CQL3Type.Native.COUNTER, value);
+ execute("TRUNCATE %s");
+ }
+ }
+
+ /**
+ * Tests the native masking function for the specified column type and value.
+ * This assumes that the table is already created.
+ *
+ * @param name the name of the tested column
+ * @param type the type of the tested column
+ * @param value the value of the tested column
+ */
+ protected abstract void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable;
+
+ protected boolean isNullOrEmptyMultiCell(CQL3Type type, Object value)
+ {
+ if (value == null)
+ return true;
+
+ AbstractType<?> dataType = type.getType();
+ if (dataType.isMultiCell() && dataType.isCollection())
+ {
+ return (((CollectionType<?>) dataType).kind == CollectionType.Kind.MAP)
+ ? ((Map<?, ?>) value).isEmpty()
+ : ((Collection<?>) value).isEmpty();
+ }
+
+ return false;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/NullMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/NullMaskingFunctionTest.java
new file mode 100644
index 0000000..bdf73eb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/NullMaskingFunctionTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.apache.cassandra.cql3.CQL3Type;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for {@link NullMaskingFunction}.
+ */
+public class NullMaskingFunctionTest extends MaskingFunctionTester
+{
+ @Override
+ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
+ {
+ assertRows(execute(format("SELECT mask_null(%s) FROM %%s", name)),
+ row((Object) null));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java
new file mode 100644
index 0000000..25f75a6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/PartialMaskingFunctionTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.datastax.driver.core.ColumnDefinitions;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.db.marshal.StringType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.assertj.core.api.Assertions;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for {@link PartialMaskingFunction}.
+ */
+public class PartialMaskingFunctionTest extends MaskingFunctionTester
+{
+ @Override
+ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
+ {
+ testMaskingOnColumn(PartialMaskingFunction.Type.INNER, name, type, value);
+ testMaskingOnColumn(PartialMaskingFunction.Type.OUTER, name, type, value);
+ }
+
+ protected void testMaskingOnColumn(PartialMaskingFunction.Type masker, String name, CQL3Type type, Object value) throws Throwable
+ {
+ String functionName = SchemaConstants.SYSTEM_KEYSPACE_NAME + ".mask_" + masker.name().toLowerCase();
+
+ if (type.getType() instanceof StringType)
+ {
+ StringType stringType = (StringType) type.getType();
+ String stringValue = (String) value;
+
+ // ... with default padding
+ assertRows(execute(format("SELECT %s(%s, 1, 2) FROM %%s", functionName, name)),
+ row(masker.mask(stringValue, 1, 2, PartialMaskingFunction.DEFAULT_PADDING_CHAR)));
+
+ // ... with manually specified ASCII padding
+ assertRows(execute(format("SELECT %s(%s, 1, 2, '#') FROM %%s", functionName, name)),
+ row(masker.mask(stringValue, 1, 2, '#')));
+
+ // ... with manually specified UTF-8 padding
+ assertRows(execute(format("SELECT %s((text) %s, 1, 2, 'é') FROM %%s", functionName, name)),
+ row(masker.mask(stringValue, 1, 2, 'é')));
+
+ // ... with not single-character padding
+ assertInvalidThrowMessage("should be single-character",
+ InvalidRequestException.class,
+ format("SELECT %s(%s, 1, 2, 'too_long') FROM %%s", functionName, name));
+
+ // ... with null padding
+ assertRows(execute(format("SELECT %s(%s, 1, 2, null) FROM %%s", functionName, name)),
+ row(masker.mask(stringValue, 1, 2, PartialMaskingFunction.DEFAULT_PADDING_CHAR)));
+
+ // ... with null begin
+ assertRows(execute(format("SELECT %s(%s, null, 2) FROM %%s", functionName, name)),
+ row(masker.mask(stringValue, 0, 2, PartialMaskingFunction.DEFAULT_PADDING_CHAR)));
+
+ // ... with null end
+ assertRows(execute(format("SELECT %s(%s, 1, null) FROM %%s", functionName, name)),
+ row(masker.mask(stringValue, 1, 0, PartialMaskingFunction.DEFAULT_PADDING_CHAR)));
+
+ // test result set metadata, it should always be of type text, regardless of the type of the column
+ ResultSet rs = executeNet(format("SELECT %s(%s, 1, 2) FROM %%s", functionName, name));
+ ColumnDefinitions definitions = rs.getColumnDefinitions();
+ Assert.assertEquals(1, definitions.size());
+ Assert.assertEquals(driverDataType(stringType), definitions.getType(0));
+ Assert.assertEquals(format("%s(%s, 1, 2)", functionName, name), definitions.getName(0));
+ }
+ else
+ {
+ assertInvalidThrowMessage(format("Function %s requires an argument of type [text|varchar|ascii], " +
+ "but found argument %s of type %s",
+ functionName, name, type),
+ InvalidRequestException.class,
+ format("SELECT %s(%s, 1, 2) FROM %%s", functionName, name));
+ }
+ }
+
+ private static DataType driverDataType(StringType type)
+ {
+ switch ((CQL3Type.Native) type.asCQL3Type())
+ {
+ case ASCII:
+ return DataType.ascii();
+ case VARCHAR:
+ return DataType.varchar();
+ case TEXT:
+ return DataType.text();
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Test
+ public void testMasking()
+ {
+ // null value
+ testMasking(null, 0, 0, '*', null, null);
+ testMasking(null, 9, 9, '*', null, null);
+ testMasking(null, 0, 0, '#', null, null);
+ testMasking(null, 9, 9, '#', null, null);
+
+ // empty value
+ testMasking("", 0, 0, '*', "", "");
+ testMasking("", 9, 9, '*', "", "");
+ testMasking("", 0, 0, '#', "", "");
+ testMasking("", 9, 9, '#', "", "");
+
+ // single-char value
+ testMasking("a", 0, 0, '*', "*", "a");
+ testMasking("a", 1, 1, '*', "a", "*");
+ testMasking("a", 10, 10, '*', "a", "*");
+ testMasking("a", 0, 0, '#', "#", "a");
+ testMasking("a", 1, 1, '#', "a", "#");
+ testMasking("a", 10, 10, '#', "a", "#");
+
+ // regular value
+ testMasking("abcde", 0, 0, '*', "*****", "abcde");
+ testMasking("abcde", 0, 1, '*', "****e", "abcd*");
+ testMasking("abcde", 0, 2, '*', "***de", "abc**");
+ testMasking("abcde", 0, 3, '*', "**cde", "ab***");
+ testMasking("abcde", 0, 4, '*', "*bcde", "a****");
+ testMasking("abcde", 0, 5, '*', "abcde", "*****");
+ testMasking("abcde", 0, 6, '*', "abcde", "*****");
+ testMasking("abcde", 1, 0, '*', "a****", "*bcde");
+ testMasking("abcde", 2, 0, '*', "ab***", "**cde");
+ testMasking("abcde", 3, 0, '*', "abc**", "***de");
+ testMasking("abcde", 4, 0, '*', "abcd*", "****e");
+ testMasking("abcde", 5, 0, '*', "abcde", "*****");
+ testMasking("abcde", 6, 0, '*', "abcde", "*****");
+ testMasking("abcde", 1, 1, '*', "a***e", "*bcd*");
+ testMasking("abcde", 2, 2, '*', "ab*de", "**c**");
+ testMasking("abcde", 3, 3, '*', "abcde", "*****");
+ testMasking("abcde", 4, 4, '*', "abcde", "*****");
+
+ // special characters
+ testMasking("á#íòü", 0, 0, '*', "*****", "á#íòü");
+ testMasking("á#íòü", 1, 1, '*', "á***ü", "*#íò*");
+ testMasking("á#íòü", 5, 5, '*', "á#íòü", "*****");
+ }
+
+ private static void testMasking(String unmaskedValue,
+ int begin,
+ int end,
+ char padding,
+ String innerMaskedValue,
+ String outerMaskedValue)
+ {
+ Assertions.assertThat(PartialMaskingFunction.Type.INNER.mask(unmaskedValue, begin, end, padding))
+ .isIn(innerMaskedValue);
+ Assertions.assertThat(PartialMaskingFunction.Type.OUTER.mask(unmaskedValue, begin, end, padding))
+ .isIn(outerMaskedValue);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
new file mode 100644
index 0000000..c00125c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.math.BigInteger;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for {@link ReplaceMaskingFunction}.
+ */
+public class ReplaceMaskingFunctionTest extends MaskingFunctionTester
+{
+ @Override
+ protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
+ {
+ // null replacement argument
+ assertRows(execute(format("SELECT mask_replace(%s, ?) FROM %%s", name), (Object) null),
+ row((Object) null));
+
+ // not-null replacement argument
+ AbstractType<?> t = type.getType();
+ Object replacementValue = t.compose(t.getMaskedValue());
+ String query = format("SELECT mask_replace(%s, ?) FROM %%s", name);
+ assertRows(execute(query, replacementValue), row(replacementValue));
+ }
+
+ @Test
+ public void testReplaceWithDifferentType() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v varint)");
+ execute("INSERT INTO %s(k) VALUES (0)");
+ assertRows(execute("SELECT mask_replace(v, 1) FROM %s"), row(BigInteger.ONE));
+ assertRows(execute("SELECT mask_replace(v, (int) 1) FROM %s"), row(BigInteger.ONE));
+ assertRows(execute("SELECT mask_replace(v, (bigint) 1) FROM %s"), row(BigInteger.ONE));
+ assertRows(execute("SELECT mask_replace(v, (varint) 1) FROM %s"), row(BigInteger.ONE));
+ assertRows(execute("SELECT mask_replace(v, ?) FROM %s", 1), row(BigInteger.ONE));
+ assertRows(execute("SELECT mask_replace(v, ?) FROM %s", 1L), row(BigInteger.ONE));
+ assertRows(execute("SELECT mask_replace(v, ?) FROM %s", BigInteger.ONE), row(BigInteger.ONE));
+ assertInvalidThrowMessage("Type error: 1.2 cannot be passed as argument 1 of function system.mask_replace of type varint",
+ InvalidRequestException.class, "SELECT mask_replace(v, 1.2) FROM %s");
+ assertInvalidThrowMessage("Type error: 'secret' cannot be passed as argument 1 of function system.mask_replace of type varint",
+ InvalidRequestException.class, "SELECT mask_replace(v, 'secret') FROM %s");
+
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v varchar)");
+ execute("INSERT INTO %s(k) VALUES (0)");
+ assertRows(execute("SELECT mask_replace(v, 'secret') FROM %s"), row("secret"));
+ assertRows(execute("SELECT mask_replace(v, (ascii) 'secret') FROM %s"), row("secret"));
+ assertRows(execute("SELECT mask_replace(v, (text) 'secret') FROM %s"), row("secret"));
+ assertRows(execute("SELECT mask_replace(v, (varchar) 'secret') FROM %s"), row("secret"));
+ assertInvalidThrowMessage("Type error: 1 cannot be passed as argument 1 of function system.mask_replace of type text",
+ InvalidRequestException.class, "SELECT mask_replace(v, 1) FROM %s");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/SelectMaskedPermissionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/SelectMaskedPermissionTest.java
new file mode 100644
index 0000000..81606ff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/SelectMaskedPermissionTest.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.awaitility.core.ThrowingRunnable;
+
+import static java.lang.String.format;
+
+/**
+ * Tests the {@link org.apache.cassandra.auth.Permission#SELECT_MASKED} permission.
+ */
+public class SelectMaskedPermissionTest extends CQLTester
+{
+ private static final Object[] CLEAR_ROW = row(7, 7, 7, 7);
+
+ private static final String USER = "ddm_user"; // user that will have their permissions changed
+ private static final String PASSWORD = "ddm_password";
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(true);
+ DatabaseDescriptor.setPermissionsValidity(0);
+ DatabaseDescriptor.setRolesValidity(0);
+ CQLTester.setUpClass();
+ requireAuthentication();
+ requireNetwork();
+ }
+
+ @Before
+ public void before() throws Throwable
+ {
+ useSuperUser();
+
+ createTable("CREATE TABLE %s (k int, c int, s int static, v int, PRIMARY KEY (k, c))");
+ executeNet("INSERT INTO %s(k, c, s, v) VALUES (?, ?, ?, ?)", CLEAR_ROW);
+
+ executeNet(format("CREATE USER IF NOT EXISTS %s WITH PASSWORD '%s'", USER, PASSWORD));
+ executeNet(format("GRANT SELECT ON ALL KEYSPACES TO %s", USER));
+ }
+
+ @After
+ public void after() throws Throwable
+ {
+ useSuperUser();
+ executeNet("DROP USER IF EXISTS " + USER);
+ alterTable("ALTER TABLE %s ALTER k DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER c DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER s DROP MASKED");
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ }
+
+ @Test
+ public void testPartitionKeyColumn() throws Throwable
+ {
+ alterTable("ALTER TABLE %s ALTER k MASKED WITH DEFAULT");
+ Object[] maskedRow = row(0, 7, 7, 7);
+
+ // test queries with default permissions (no UNMASK nor SELECT_MASKED)
+ testPartitionKeyColumnWithDefaultPermissions(maskedRow);
+
+ // test queries with only SELECT_MASKED permission
+ executeNet(format("GRANT SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testPartitionKeyColumnWithOnlySelectMasked(maskedRow);
+
+ // test queries with only UNMASK permission (which includes SELECT_MASKED)
+ executeNet(format("REVOKE SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ executeNet(format("GRANT UNMASK ON ALL KEYSPACES TO %s", USER));
+ testPartitionKeyColumnWithUnmask();
+
+ // test queries with both UNMASK and SELECT_MASKED permissions
+ executeNet(format("GRANT UNMASK, SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testPartitionKeyColumnWithUnmask();
+
+ // test queries again without both UNMASK and SELECT_MASKED permissions
+ executeNet(format("REVOKE UNMASK, SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ testPartitionKeyColumnWithDefaultPermissions(maskedRow);
+ }
+
+ private void testPartitionKeyColumnWithDefaultPermissions(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE k = 7", "[k]");
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE k >= 7 ALLOW FILTERING", "[k]");
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", "[k]");
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE k = 7 AND v = 7 ALLOW FILTERING", "[k]");
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", "[k]");
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE token(k) >= token(7)", "[k]");
+ });
+ }
+
+ private void testPartitionKeyColumnWithOnlySelectMasked(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k >= 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND v = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) >= token(7)", maskedRow);
+ });
+ }
+
+ private void testPartitionKeyColumnWithUnmask() throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k >= 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND v = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) >= token(7)", CLEAR_ROW);
+ });
+ }
+
+ @Test
+ public void testClusteringKeyColumn() throws Throwable
+ {
+ alterTable("ALTER TABLE %s ALTER c MASKED WITH DEFAULT");
+ Object[] maskedRow = row(7, 0, 7, 7);
+
+ // test queries with default permissions (no UNMASK nor SELECT_MASKED)
+ testClusteringKeyColumnWithDefaultPermissions(maskedRow);
+
+ // test queries with only SELECT_MASKED permission
+ executeNet(format("GRANT SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testClusteringKeyColumnWithOnlySelectMasked(maskedRow);
+
+ // test queries with only UNMASK permission (which includes SELECT_MASKED)
+ executeNet(format("REVOKE SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ executeNet(format("GRANT UNMASK ON ALL KEYSPACES TO %s", USER));
+ testClusteringKeyColumnWithUnmask();
+
+ // test queries with both UNMASK and SELECT_MASKED permissions
+ executeNet(format("GRANT UNMASK, SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testClusteringKeyColumnWithUnmask();
+
+ // test queries again without both UNMASK and SELECT_MASKED permissions
+ executeNet(format("REVOKE UNMASK, SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ testClusteringKeyColumnWithDefaultPermissions(maskedRow);
+ }
+
+ private void testClusteringKeyColumnWithDefaultPermissions(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", "[c]");
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", "[c]");
+ });
+ }
+
+ private void testClusteringKeyColumnWithOnlySelectMasked(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", maskedRow);
+ });
+ }
+
+ private void testClusteringKeyColumnWithUnmask() throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", CLEAR_ROW);
+ });
+ }
+
+ @Test
+ public void testStaticColumn() throws Throwable
+ {
+ alterTable("ALTER TABLE %s ALTER s MASKED WITH DEFAULT");
+ Object[] maskedRow = row(7, 7, 0, 7);
+
+ // test queries with default permissions (no UNMASK nor SELECT_MASKED)
+ testStaticColumnWithDefaultPermissions(maskedRow);
+
+ // test queries with only SELECT_MASKED permission
+ executeNet(format("GRANT SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testStaticColumnWithOnlySelectMasked(maskedRow);
+
+ // test queries with only UNMASK permission (which includes SELECT_MASKED)
+ executeNet(format("REVOKE SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ executeNet(format("GRANT UNMASK ON ALL KEYSPACES TO %s", USER));
+ testStaticColumnWithUnmask();
+
+ // test queries with both UNMASK and SELECT_MASKED permissions
+ executeNet(format("GRANT UNMASK, SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testStaticColumnWithUnmask();
+
+ // test queries again without both UNMASK and SELECT_MASKED permissions
+ executeNet(format("REVOKE UNMASK, SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ testStaticColumnWithDefaultPermissions(maskedRow);
+ }
+
+ private void testStaticColumnWithDefaultPermissions(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", "[s]");
+ });
+ }
+
+ private void testStaticColumnWithOnlySelectMasked(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ });
+ }
+
+ private void testStaticColumnWithUnmask() throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", CLEAR_ROW);
+ });
+ }
+
+ @Test
+ public void testCollections() throws Throwable
+ {
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH DEFAULT");
+ Object[] maskedRow = row(7, 7, 7, 0);
+
+ // test queries with default permissions (no UNMASK nor SELECT_MASKED)
+ testCollectionsWithDefaultPermissions(maskedRow);
+
+ // test queries with only SELECT_MASKED permission
+ executeNet(format("GRANT SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testCollectionsWithOnlySelectMasked(maskedRow);
+
+ // test queries with only UNMASK permission (which includes SELECT_MASKED)
+ executeNet(format("REVOKE SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ executeNet(format("GRANT UNMASK ON ALL KEYSPACES TO %s", USER));
+ testCollectionsWithUnmask();
+
+ // test queries with both UNMASK and SELECT_MASKED permissions
+ executeNet(format("GRANT UNMASK, SELECT_MASKED ON ALL KEYSPACES TO %s", USER));
+ testCollectionsWithUnmask();
+
+ // test queries again without both UNMASK and SELECT_MASKED permissions
+ executeNet(format("REVOKE UNMASK, SELECT_MASKED ON ALL KEYSPACES FROM %s", USER));
+ testCollectionsWithDefaultPermissions(maskedRow);
+ }
+
+ private void testCollectionsWithDefaultPermissions(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertUnauthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", "[v]");
+ });
+ }
+
+ private void testCollectionsWithOnlySelectMasked(Object[] maskedRow) throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", maskedRow);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", maskedRow);
+ });
+ }
+
+ private void testCollectionsWithUnmask() throws Throwable
+ {
+ assertWithUser(() -> {
+ assertAuthorizedQuery("SELECT * FROM %s", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE token(k) = token(7)", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE k = 7 AND c = 7", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE s = 7 ALLOW FILTERING", CLEAR_ROW);
+ assertAuthorizedQuery("SELECT * FROM %s WHERE v = 7 ALLOW FILTERING", CLEAR_ROW);
+ });
+ }
+
+ private void assertAuthorizedQuery(String query, Object[]... rows) throws Throwable
+ {
+ assertRowsNet(executeNet(query), rows);
+ }
+
+ private void assertUnauthorizedQuery(String query, String unauthorizedColumns) throws Throwable
+ {
+ assertInvalidMessageNet(format("User %s has no UNMASK nor SELECT_MASKED permission on table %s.%s, " +
+ "cannot query masked columns %s",
+ USER, KEYSPACE, currentTable(), unauthorizedColumns), query);
+ }
+
+ private void assertWithUser(ThrowingRunnable assertion) throws Throwable
+ {
+ useUser(USER, PASSWORD);
+ assertion.run();
+ useSuperUser();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/functions/masking/UnmaskPermissionTest.java b/test/unit/org/apache/cassandra/cql3/functions/masking/UnmaskPermissionTest.java
new file mode 100644
index 0000000..5a1cdc2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/UnmaskPermissionTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.functions.masking;
+
+import java.util.Arrays;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+
+import static java.lang.String.format;
+
+/**
+ * Tests the {@link org.apache.cassandra.auth.Permission#UNMASK} permission.
+ * <p>
+ * The permission is tested for a regular user with the {@code UNMASK} permissions on different resources,
+ * while also verifying the absence of side effects on other ordinary users, superusers and internal queries.
+ */
+public class UnmaskPermissionTest extends CQLTester
+{
+ private static final String CREATE_KEYSPACE = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = " +
+ "{'class': 'SimpleStrategy', 'replication_factor': '1'}";
+ private static final String CREATE_TABLE = "CREATE TABLE IF NOT EXISTS %s.%s " +
+ "(k int, c int, v text MASKED WITH mask_replace('redacted'), " +
+ "PRIMARY KEY (k, c))";
+ private static final String INSERT = "INSERT INTO %s.%s (k, c, v) VALUES (?, ?, ?)";
+ private static final String SELECT_WILDCARD = "SELECT * FROM %s.%s";
+ private static final String SELECT_COLUMNS = "SELECT k, c, v FROM %s.%s";
+
+ private static final Object[] CLEAR_ROW = row(0, 0, "sensitive");
+ private static final Object[] MASKED_ROW = row(0, 0, "redacted");
+
+ private static final String KEYSPACE_1 = "mask_keyspace_1";
+ private static final String KEYSPACE_2 = "mask_keyspace_2";
+ private static final String TABLE_1 = "mask_table_1";
+ private static final String TABLE_2 = "mask_table_2";
+
+ private static final String USER = "ddm_user"; // user that will have their permissions changed
+ private static final String OTHER_USER = "ddm_ordinary_user"; // user that won't have their permissions altered
+ private static final String PASSWORD = "ddm_password";
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(true);
+ DatabaseDescriptor.setPermissionsValidity(0);
+ DatabaseDescriptor.setRolesValidity(0);
+ CQLTester.setUpClass();
+ requireAuthentication();
+ requireNetwork();
+ }
+
+ @Before
+ public void before() throws Throwable
+ {
+ useSuperUser();
+
+ schemaChange(format(CREATE_KEYSPACE, KEYSPACE_1));
+ schemaChange(format(CREATE_KEYSPACE, KEYSPACE_2));
+
+ createTable(format(CREATE_TABLE, KEYSPACE_1, TABLE_1));
+ createTable(format(CREATE_TABLE, KEYSPACE_1, TABLE_2));
+ createTable(format(CREATE_TABLE, KEYSPACE_2, TABLE_1));
+
+ execute(format(INSERT, KEYSPACE_1, TABLE_1), CLEAR_ROW);
+ execute(format(INSERT, KEYSPACE_1, TABLE_2), CLEAR_ROW);
+ execute(format(INSERT, KEYSPACE_2, TABLE_1), CLEAR_ROW);
+
+ for (String user : Arrays.asList(USER, OTHER_USER))
+ {
+ executeNet(format("CREATE USER IF NOT EXISTS %s WITH PASSWORD '%s'", user, PASSWORD));
+ executeNet(format("GRANT SELECT ON ALL KEYSPACES TO %s", user));
+ }
+ }
+
+ @After
+ public void after() throws Throwable
+ {
+ useSuperUser();
+ executeNet("DROP USER IF EXISTS " + USER);
+ }
+
+ @Test
+ public void testUnmaskDefaults() throws Throwable
+ {
+ // ordinary user without changed permissions should see masked data
+ useUser(OTHER_USER, PASSWORD);
+ assertMasked(KEYSPACE_1, TABLE_1);
+ assertMasked(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+
+ // super user should see unmasked data
+ useSuperUser();
+ assertClear(KEYSPACE_1, TABLE_1);
+ assertClear(KEYSPACE_1, TABLE_2);
+ assertClear(KEYSPACE_2, TABLE_1);
+
+ // internal user should see unmasked data
+ assertClearInternal(KEYSPACE_1, TABLE_1);
+ assertClearInternal(KEYSPACE_1, TABLE_2);
+ assertClearInternal(KEYSPACE_2, TABLE_1);
+ }
+
+ @Test
+ public void testUnmaskOnAllKeyspaces() throws Throwable
+ {
+ assertPermissions(format("GRANT UNMASK ON ALL KEYSPACES TO %s", USER), () -> {
+ assertClear(KEYSPACE_1, TABLE_1);
+ assertClear(KEYSPACE_1, TABLE_2);
+ assertClear(KEYSPACE_2, TABLE_1);
+ });
+
+ assertPermissions(format("REVOKE UNMASK ON ALL KEYSPACES FROM %s", USER), () -> {
+ assertMasked(KEYSPACE_1, TABLE_1);
+ assertMasked(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+ });
+ }
+
+ @Test
+ public void testUnmaskOnKeyspace() throws Throwable
+ {
+ assertPermissions(format("GRANT UNMASK ON KEYSPACE %s TO %s", KEYSPACE_1, USER), () -> {
+ assertClear(KEYSPACE_1, TABLE_1);
+ assertClear(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+ });
+
+ assertPermissions(format("REVOKE UNMASK ON KEYSPACE %s FROM %s", KEYSPACE_1, USER), () -> {
+ assertMasked(KEYSPACE_1, TABLE_1);
+ assertMasked(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+ });
+ }
+
+ @Test
+ public void testUnmaskOnTable() throws Throwable
+ {
+ assertPermissions(format("GRANT UNMASK ON TABLE %s.%s TO %s", KEYSPACE_1, TABLE_1, USER), () -> {
+ assertClear(KEYSPACE_1, TABLE_1);
+ assertMasked(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+ });
+
+ assertPermissions(format("REVOKE UNMASK ON TABLE %s.%s FROM %s", KEYSPACE_1, TABLE_1, USER), () -> {
+ assertMasked(KEYSPACE_1, TABLE_1);
+ assertMasked(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+ });
+ }
+
+ private void assertPermissions(String alterPermissionsQuery, ThrowingRunnable assertion) throws Throwable
+ {
+ // alter permissions as superuser
+ useSuperUser();
+ executeNet(alterPermissionsQuery);
+
+ // verify the tested user permissions
+ useUser(USER, PASSWORD);
+ assertion.run();
+
+ // the ordinary user without modified permissions should keep seeing masked data
+ useUser(OTHER_USER, PASSWORD);
+ assertMasked(KEYSPACE_1, TABLE_1);
+ assertMasked(KEYSPACE_1, TABLE_2);
+ assertMasked(KEYSPACE_2, TABLE_1);
+
+ // super user should keep seeing unmasked data
+ useSuperUser();
+ assertClear(KEYSPACE_1, TABLE_1);
+ assertClear(KEYSPACE_1, TABLE_2);
+ assertClear(KEYSPACE_2, TABLE_1);
+
+ // internal user should keep seeing unmasked data
+ assertClearInternal(KEYSPACE_1, TABLE_1);
+ assertClearInternal(KEYSPACE_1, TABLE_2);
+ assertClearInternal(KEYSPACE_2, TABLE_1);
+ }
+
+ private void assertMasked(String keyspace, String table) throws Throwable
+ {
+ assertRowsNet(executeNet(format(SELECT_WILDCARD, keyspace, table)), MASKED_ROW);
+ assertRowsNet(executeNet(format(SELECT_COLUMNS, keyspace, table)), MASKED_ROW);
+ }
+
+ private void assertClear(String keyspace, String table) throws Throwable
+ {
+ assertRowsNet(executeNet(format(SELECT_WILDCARD, keyspace, table)), CLEAR_ROW);
+ assertRowsNet(executeNet(format(SELECT_COLUMNS, keyspace, table)), CLEAR_ROW);
+ }
+
+ private void assertClearInternal(String keyspace, String table) throws Throwable
+ {
+ assertRows(execute(format(SELECT_WILDCARD, keyspace, table)), CLEAR_ROW);
+ assertRows(execute(format(SELECT_COLUMNS, keyspace, table)), CLEAR_ROW);
+ }
+
+ @FunctionalInterface
+ private interface ThrowingRunnable
+ {
+ void run() throws Throwable;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java b/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java
index d6de5ff..2339f5d 100644
--- a/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java
+++ b/test/unit/org/apache/cassandra/cql3/selection/SelectionColumnMappingTest.java
@@ -26,6 +26,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.schema.Schema;
@@ -54,6 +55,8 @@
@BeforeClass
public static void setUpClass() // overrides CQLTester.setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
prepareServer();
@@ -227,14 +230,14 @@
private void testFunction() throws Throwable
{
- // a function such as intasblob(<col>) is represented in ResultSet.Metadata
+ // a function such as int_as_blob(<col>) is represented in ResultSet.Metadata
// by a ColumnSpecification with the function name plus args and the type set
// to the function's return type
- ColumnSpecification fnSpec = columnSpecification("system.intasblob(v1)", BytesType.instance);
+ ColumnSpecification fnSpec = columnSpecification("system.int_as_blob(v1)", BytesType.instance);
SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
.addMapping(fnSpec, columnDefinition("v1"));
- verify(expected, "SELECT intasblob(v1) FROM %s");
+ verify(expected, "SELECT int_as_blob(v1) FROM %s");
}
private void testNoArgFunction() throws Throwable
@@ -326,7 +329,7 @@
SelectionColumnMapping expected = SelectionColumnMapping.newMapping()
.addMapping(fnSpec, columnDefinition("v1"));
- verify(expected, "SELECT intasblob(v1) AS fn_alias FROM %s");
+ verify(expected, "SELECT int_as_blob(v1) AS fn_alias FROM %s");
}
public void testNoArgumentFunction() throws Throwable
@@ -341,19 +344,19 @@
public void testNestedFunctionsWithArguments() throws Throwable
{
SelectionColumns expected = SelectionColumnMapping.newMapping()
- .addMapping(columnSpecification("system.blobasint(system.intasblob(v1))",
+ .addMapping(columnSpecification("system.blob_as_int(system.int_as_blob(v1))",
Int32Type.instance),
columnDefinition("v1"));
- verify(expected, "SELECT blobasint(intasblob(v1)) FROM %s");
+ verify(expected, "SELECT blob_as_int(int_as_blob(v1)) FROM %s");
}
public void testNestedFunctions() throws Throwable
{
SelectionColumns expected = SelectionColumnMapping.newMapping()
- .addMapping(columnSpecification("system.tounixtimestamp(system.now())",
+ .addMapping(columnSpecification("system.to_unix_timestamp(system.now())",
LongType.instance),
NULL_DEF);
- verify(expected, "SELECT tounixtimestamp(now()) FROM %s");
+ verify(expected, "SELECT to_unix_timestamp(now()) FROM %s");
}
public void testDuplicateFunctionsWithoutAliases() throws Throwable
@@ -361,11 +364,11 @@
// where duplicate functions are present, the ColumnSpecification list will
// contain an entry per-duplicate but the mappings will be deduplicated (i.e.
// a single mapping k/v pair regardless of the number of duplicates)
- ColumnSpecification spec = columnSpecification("system.intasblob(v1)", BytesType.instance);
+ ColumnSpecification spec = columnSpecification("system.int_as_blob(v1)", BytesType.instance);
SelectionColumns expected = SelectionColumnMapping.newMapping()
.addMapping(spec, columnDefinition("v1"))
.addMapping(spec, columnDefinition("v1"));
- verify(expected, "SELECT intasblob(v1), intasblob(v1) FROM %s");
+ verify(expected, "SELECT int_as_blob(v1), int_as_blob(v1) FROM %s");
}
public void testDuplicateFunctionsWithAliases() throws Throwable
@@ -378,7 +381,7 @@
columnDefinition("v1"))
.addMapping(columnSpecification("blob_2", BytesType.instance),
columnDefinition("v1"));
- verify(expected, "SELECT intasblob(v1) AS blob_1, intasblob(v1) AS blob_2 FROM %s");
+ verify(expected, "SELECT int_as_blob(v1) AS blob_1, int_as_blob(v1) AS blob_2 FROM %s");
}
public void testSelectDistinct() throws Throwable
@@ -534,8 +537,8 @@
" CREATE FUNCTION %s (a int, b int)" +
" RETURNS NULL ON NULL INPUT" +
" RETURNS int" +
- " LANGUAGE javascript" +
- " AS 'a + b'")).name;
+ " LANGUAGE java" +
+ " AS ' return a + b;'")).name;
String aFunc = createAggregate(KEYSPACE, "int, int",
" CREATE AGGREGATE %s (int)" +
@@ -547,15 +550,15 @@
" CREATE FUNCTION %s (a int)" +
" RETURNS NULL ON NULL INPUT" +
" RETURNS int" +
- " LANGUAGE javascript" +
- " AS 'a+1'");
+ " LANGUAGE java" +
+ " AS ' return a + 1;'");
String sqFunc = createFunction(KEYSPACE, "int",
" CREATE FUNCTION %s (a int)" +
" RETURNS NULL ON NULL INPUT" +
" RETURNS int" +
- " LANGUAGE javascript" +
- " AS 'a*a'");
+ " LANGUAGE java" +
+ " AS ' return a * a;'");
ColumnMetadata v1 = columnDefinition("v1");
SelectionColumns expected = SelectionColumnMapping.newMapping()
diff --git a/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java b/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java
index ee4dd35..530cb67 100644
--- a/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java
@@ -36,6 +36,7 @@
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
@@ -60,8 +61,10 @@
checkSerialization(table.getColumn(new ColumnIdentifier("c1", false)), table);
// Test WritetimeOrTTLSelector serialization
- checkSerialization(new Selectable.WritetimeOrTTL(table.getColumn(new ColumnIdentifier("v", false)), true), table);
- checkSerialization(new Selectable.WritetimeOrTTL(table.getColumn(new ColumnIdentifier("v", false)), false), table);
+ ColumnMetadata column = table.getColumn(new ColumnIdentifier("v", false));
+ checkSerialization(new Selectable.WritetimeOrTTL(column, column, Selectable.WritetimeOrTTL.Kind.WRITE_TIME), table);
+ checkSerialization(new Selectable.WritetimeOrTTL(column, column, Selectable.WritetimeOrTTL.Kind.TTL), table);
+ checkSerialization(new Selectable.WritetimeOrTTL(column, column, Selectable.WritetimeOrTTL.Kind.MAX_WRITE_TIME), table);
// Test ListSelector serialization
checkSerialization(new Selectable.WithList(asList(table.getColumn(new ColumnIdentifier("v", false)),
diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
index 52bb3c3..422fcd4 100644
--- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
+++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java
@@ -39,12 +39,14 @@
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.transport.ProtocolVersion;
import static java.lang.String.format;
import static org.apache.cassandra.schema.SchemaConstants.*;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -790,6 +792,56 @@
}
@Test
+ public void testDescribeTableWithColumnMasks() throws Throwable
+ {
+ requireNetwork();
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(true);
+
+ String table = createTable(KEYSPACE_PER_TEST,
+ "CREATE TABLE %s (" +
+ " pk1 text, " +
+ " pk2 int MASKED WITH DEFAULT, " +
+ " ck1 int, " +
+ " ck2 int MASKED WITH mask_default()," +
+ " s1 decimal static, " +
+ " s2 decimal static MASKED WITH mask_null(), " +
+ " v1 text, " +
+ " v2 text MASKED WITH mask_inner(1, null), " +
+ "PRIMARY KEY ((pk1, pk2), ck1, ck2 ))");
+
+ TableMetadata tableMetadata = Schema.instance.getTableMetadata(KEYSPACE_PER_TEST, table);
+ assertNotNull(tableMetadata);
+
+ String tableCreateStatement = "CREATE TABLE " + KEYSPACE_PER_TEST + "." + table + " (\n" +
+ " pk1 text,\n" +
+ " pk2 int MASKED WITH system.mask_default(),\n" +
+ " ck1 int,\n" +
+ " ck2 int MASKED WITH system.mask_default(),\n" +
+ " s1 decimal static,\n" +
+ " s2 decimal static MASKED WITH system.mask_null(),\n" +
+ " v1 text,\n" +
+ " v2 text MASKED WITH system.mask_inner(1, null),\n" +
+ " PRIMARY KEY ((pk1, pk2), ck1, ck2)\n" +
+ ") WITH ID = " + tableMetadata.id + "\n" +
+ " AND CLUSTERING ORDER BY (ck1 ASC, ck2 ASC)\n" +
+ " AND " + tableParametersCql();
+
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE_PER_TEST + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE_PER_TEST,
+ "table",
+ table,
+ tableCreateStatement));
+
+ // masks should be listed even if DDM is disabled
+ DatabaseDescriptor.setDynamicDataMaskingEnabled(false);
+ assertRowsNet(executeDescribeNet("DESCRIBE TABLE " + KEYSPACE_PER_TEST + "." + table + " WITH INTERNALS"),
+ row(KEYSPACE_PER_TEST,
+ "table",
+ table,
+ tableCreateStatement));
+ }
+
+ @Test
public void testUsingReservedInCreateType() throws Throwable
{
String type = createType(KEYSPACE_PER_TEST, "CREATE TYPE %s (\"token\" text, \"desc\" text);");
@@ -969,6 +1021,7 @@
private static String tableParametersCql()
{
return "additional_write_policy = '99p'\n" +
+ " AND allow_auto_snapshot = true\n" +
" AND bloom_filter_fp_chance = 0.01\n" +
" AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" +
" AND cdc = false\n" +
@@ -980,6 +1033,7 @@
" AND default_time_to_live = 0\n" +
" AND extensions = {}\n" +
" AND gc_grace_seconds = 864000\n" +
+ " AND incremental_backups = true\n" +
" AND max_index_interval = 2048\n" +
" AND memtable_flush_period_in_ms = 0\n" +
" AND min_index_interval = 128\n" +
@@ -990,6 +1044,7 @@
private static String mvParametersCql()
{
return "additional_write_policy = '99p'\n" +
+ " AND allow_auto_snapshot = true\n" +
" AND bloom_filter_fp_chance = 0.01\n" +
" AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" +
" AND cdc = false\n" +
@@ -1000,6 +1055,7 @@
" AND crc_check_chance = 1.0\n" +
" AND extensions = {}\n" +
" AND gc_grace_seconds = 864000\n" +
+ " AND incremental_backups = true\n" +
" AND max_index_interval = 2048\n" +
" AND memtable_flush_period_in_ms = 0\n" +
" AND min_index_interval = 128\n" +
@@ -1044,9 +1100,4 @@
executeNet(v, "USE " + useKs);
return v;
}
-
- private static Object[][] rows(Object[]... rows)
- {
- return rows;
- }
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
index fedb4e5..e6ef163 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
@@ -577,18 +577,6 @@
execute("ALTER TABLE %s ADD alist list<text>");
}
- /**
- * Migrated from cql_tests.py:TestCQL.collection_function_test()
- */
- @Test
- public void testFunctionsOnCollections() throws Throwable
- {
- createTable("CREATE TABLE %s (k int PRIMARY KEY, l set<int>)");
-
- assertInvalid("SELECT ttl(l) FROM %s WHERE k = 0");
- assertInvalid("SELECT writetime(l) FROM %s WHERE k = 0");
- }
-
@Test
public void testInRestrictionWithCollection() throws Throwable
{
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
index 5cbfb4c..c4cdca1 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
@@ -26,6 +26,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.db.marshal.*;
@@ -43,6 +44,7 @@
@BeforeClass
public static void setUpClass() // overrides CQLTester.setUpClass()
{
+ ServerTestUtils.daemonInitialization();
// Selecting partitioner for a table is not exposed on CREATE TABLE.
StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java
index 810ee5a..f97b4fa 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java
@@ -17,8 +17,8 @@
*/
package org.apache.cassandra.cql3.validation.entities;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.Json;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -28,6 +28,7 @@
import org.apache.cassandra.serializers.TimeSerializer;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JsonUtils;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -46,10 +47,11 @@
public class JsonTest extends CQLTester
{
- // This method will be ran instead of the CQLTester#setUpClass
+ // This method will be run instead of the CQLTester#setUpClass
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
if (ROW_CACHE_SIZE_IN_MIB > 0)
DatabaseDescriptor.setRowCacheSizeInMiB(ROW_CACHE_SIZE_IN_MIB);
@@ -80,8 +82,8 @@
row("{\"k1\": [\"" + uuid + "\", 2], \"c1\": [\"" + uuid + "\", 3], \"value\": 3}"),
row("{\"k1\": [\"" + uuid + "\", 2], \"c1\": [\"" + uuid + "\", 4], \"value\": 4}"));
- // SELECT toJson(column)
- assertRowsNet(executeNetWithPaging("SELECT toJson(k1), toJson(c1), toJson(value) FROM %s", pageSize),
+ // SELECT to_json(column)
+ assertRowsNet(executeNetWithPaging("SELECT to_json(k1), to_json(c1), to_json(value) FROM %s", pageSize),
row("[\"" + uuid + "\", 2]", "[\"" + uuid + "\", 1]", "1"),
row("[\"" + uuid + "\", 2]", "[\"" + uuid + "\", 2]", "2"),
row("[\"" + uuid + "\", 2]", "[\"" + uuid + "\", 3]", "3"),
@@ -110,8 +112,8 @@
row("{\"k1\": {\"1\": [\"" + uuid + "\", 1], \"2\": [\"" + uuid + "\", 2]}, \"c1\": [\"" + uuid + "\", 3], \"value\": 3}"),
row("{\"k1\": {\"1\": [\"" + uuid + "\", 1], \"2\": [\"" + uuid + "\", 2]}, \"c1\": [\"" + uuid + "\", 4], \"value\": 4}"));
- // SELECT toJson(column)
- assertRowsNet(executeNetWithPaging("SELECT toJson(k1), toJson(c1), toJson(value) FROM %s", pageSize),
+ // SELECT to_json(column)
+ assertRowsNet(executeNetWithPaging("SELECT to_json(k1), to_json(c1), to_json(value) FROM %s", pageSize),
row("{\"1\": [\"" + uuid + "\", 1], \"2\": [\"" + uuid + "\", 2]}", "[\"" + uuid + "\", 1]", "1"),
row("{\"1\": [\"" + uuid + "\", 1], \"2\": [\"" + uuid + "\", 2]}", "[\"" + uuid + "\", 2]", "2"),
row("{\"1\": [\"" + uuid + "\", 1], \"2\": [\"" + uuid + "\", 2]}", "[\"" + uuid + "\", 3]", "3"),
@@ -140,8 +142,8 @@
row("{\"k1\": [[[1, 2], 1], [[2, 3], 2]], \"c1\": [\"" + uuid + "\", 3], \"value\": 3}"),
row("{\"k1\": [[[1, 2], 1], [[2, 3], 2]], \"c1\": [\"" + uuid + "\", 4], \"value\": 4}"));
- // SELECT toJson(column)
- assertRowsNet(executeNetWithPaging("SELECT toJson(k1), toJson(c1), toJson(value) FROM %s", pageSize),
+ // SELECT to_json(column)
+ assertRowsNet(executeNetWithPaging("SELECT to_json(k1), to_json(c1), to_json(value) FROM %s", pageSize),
row("[[[1, 2], 1], [[2, 3], 2]]", "[\"" + uuid + "\", 1]", "1"),
row("[[[1, 2], 1], [[2, 3], 2]]", "[\"" + uuid + "\", 2]", "2"),
row("[[[1, 2], 1], [[2, 3], 2]]", "[\"" + uuid + "\", 3]", "3"),
@@ -170,8 +172,8 @@
row("{\"k1\": [[\"" + uuid + "\", 2], [\"" + uuid + "\", 3]], \"c1\": [\"" + uuid + "\", 3], \"value\": 3}"),
row("{\"k1\": [[\"" + uuid + "\", 2], [\"" + uuid + "\", 3]], \"c1\": [\"" + uuid + "\", 4], \"value\": 4}"));
- // SELECT toJson(column)
- assertRowsNet(executeNetWithPaging("SELECT toJson(k1), toJson(c1), toJson(value) FROM %s", pageSize),
+ // SELECT to_json(column)
+ assertRowsNet(executeNetWithPaging("SELECT to_json(k1), to_json(c1), to_json(value) FROM %s", pageSize),
row("[[\"" + uuid + "\", 2], [\"" + uuid + "\", 3]]", "[\"" + uuid + "\", 1]", "1"),
row("[[\"" + uuid + "\", 2], [\"" + uuid + "\", 3]]", "[\"" + uuid + "\", 2]", "2"),
row("[[\"" + uuid + "\", 2], [\"" + uuid + "\", 3]]", "[\"" + uuid + "\", 3]", "3"),
@@ -202,8 +204,8 @@
row("{\"k1\": {\"a\": 1, \"b\": 2, \"c\": [\"1\", \"2\"]}, \"c1\": [\"" + uuid + "\", 3], \"value\": 3}"),
row("{\"k1\": {\"a\": 1, \"b\": 2, \"c\": [\"1\", \"2\"]}, \"c1\": [\"" + uuid + "\", 4], \"value\": 4}"));
- // SELECT toJson(column)
- assertRowsNet(executeNetWithPaging("SELECT toJson(k1), toJson(c1), toJson(value) FROM %s", pageSize),
+ // SELECT to_json(column)
+ assertRowsNet(executeNetWithPaging("SELECT to_json(k1), to_json(c1), to_json(value) FROM %s", pageSize),
row("{\"a\": 1, \"b\": 2, \"c\": [\"1\", \"2\"]}", "[\"" + uuid + "\", 1]", "1"),
row("{\"a\": 1, \"b\": 2, \"c\": [\"1\", \"2\"]}", "[\"" + uuid + "\", 2]", "2"),
row("{\"a\": 1, \"b\": 2, \"c\": [\"1\", \"2\"]}", "[\"" + uuid + "\", 3]", "3"),
@@ -246,468 +248,468 @@
"udtval frozen<" + typeName + ">," +
"durationval duration)");
- // fromJson() can only be used when the receiver type is known
- assertInvalidMessage("fromJson() cannot be used in the selection clause", "SELECT fromJson(asciival) FROM %s", 0, 0);
+ // from_json() can only be used when the receiver type is known
+ assertInvalidMessage("from_json() cannot be used in the selection clause", "SELECT from_json(textval) FROM %s", 0, 0);
String func1 = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (a int) CALLED ON NULL INPUT RETURNS text LANGUAGE java AS $$ return a.toString(); $$");
createFunctionOverload(func1, "int", "CREATE FUNCTION %s (a text) CALLED ON NULL INPUT RETURNS text LANGUAGE java AS $$ return new String(a); $$");
assertInvalidMessage("Ambiguous call to function",
- "INSERT INTO %s (k, textval) VALUES (?, " + func1 + "(fromJson(?)))", 0, "123");
+ "INSERT INTO %s (k, textval) VALUES (?, " + func1 + "(from_json(?)))", 0, "123");
// fails JSON parsing
assertInvalidMessage("Could not decode JSON string '\u038E\u0394\u03B4\u03E0'",
- "INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\u038E\u0394\u03B4\u03E0");
+ "INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\u038E\u0394\u03B4\u03E0");
// handle nulls
- execute("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, null);
+ execute("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, null);
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, null));
- execute("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, null);
+ execute("INSERT INTO %s (k, frozenmapval) VALUES (?, from_json(?))", 0, null);
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, null));
- execute("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, null);
+ execute("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, null);
assertRows(execute("SELECT k, udtval FROM %s WHERE k = ?", 0), row(0, null));
// ================ ascii ================
- execute("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
+ execute("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, "ascii text"));
- execute("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii \\\" text\"");
+ execute("INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"ascii \\\" text\"");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, "ascii \" text"));
assertInvalidMessage("Invalid ASCII character in string literal",
- "INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"\\u1fff\\u2013\\u33B4\\u2014\"");
+ "INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "\"\\u1fff\\u2013\\u33B4\\u2014\"");
assertInvalidMessage("Expected an ascii string, but got a Integer",
- "INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, asciival) VALUES (?, from_json(?))", 0, "123");
- // test that we can use fromJson() in other valid places in queries
- assertRows(execute("SELECT asciival FROM %s WHERE k = fromJson(?)", "0"), row("ascii \" text"));
- execute("UPDATE %s SET asciival = fromJson(?) WHERE k = fromJson(?)", "\"ascii \\\" text\"", "0");
- execute("DELETE FROM %s WHERE k = fromJson(?)", "0");
+ // test that we can use from_json() in other valid places in queries
+ assertRows(execute("SELECT asciival FROM %s WHERE k = from_json(?)", "0"), row("ascii \" text"));
+ execute("UPDATE %s SET asciival = from_json(?) WHERE k = from_json(?)", "\"ascii \\\" text\"", "0");
+ execute("DELETE FROM %s WHERE k = from_json(?)", "0");
// ================ bigint ================
- execute("INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "123123123123");
+ execute("INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "123123123123");
assertRows(execute("SELECT k, bigintval FROM %s WHERE k = ?", 0), row(0, 123123123123L));
// strings are also accepted
- execute("INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "\"123123123123\"");
+ execute("INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "\"123123123123\"");
assertRows(execute("SELECT k, bigintval FROM %s WHERE k = ?", 0), row(0, 123123123123L));
// overflow (Long.MAX_VALUE + 1)
assertInvalidMessage("Expected a bigint value, but got a",
- "INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "9223372036854775808");
+ "INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "9223372036854775808");
assertInvalidMessage("Expected a bigint value, but got a",
- "INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "123.456");
+ "INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "123.456");
assertInvalidMessage("Unable to make long from",
- "INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "\"abc\"");
+ "INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "\"abc\"");
assertInvalidMessage("Expected a bigint value, but got a",
- "INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "[\"abc\"]");
+ "INSERT INTO %s (k, bigintval) VALUES (?, from_json(?))", 0, "[\"abc\"]");
// ================ blob ================
- execute("INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "\"0x00000001\"");
+ execute("INSERT INTO %s (k, blobval) VALUES (?, from_json(?))", 0, "\"0x00000001\"");
assertRows(execute("SELECT k, blobval FROM %s WHERE k = ?", 0), row(0, ByteBufferUtil.bytes(1)));
assertInvalidMessage("Value 'xyzz' is not a valid blob representation",
- "INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, blobval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("String representation of blob is missing 0x prefix: 123",
- "INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "\"123\"");
+ "INSERT INTO %s (k, blobval) VALUES (?, from_json(?))", 0, "\"123\"");
assertInvalidMessage("Value '0x123' is not a valid blob representation",
- "INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "\"0x123\"");
+ "INSERT INTO %s (k, blobval) VALUES (?, from_json(?))", 0, "\"0x123\"");
assertInvalidMessage("Value '123' is not a valid blob representation",
- "INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, blobval) VALUES (?, from_json(?))", 0, "123");
// ================ boolean ================
- execute("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "true");
+ execute("INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "true");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, true));
- execute("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "false");
+ execute("INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "false");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, false));
// strings are also accepted
- execute("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "\"false\"");
+ execute("INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "\"false\"");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, false));
assertInvalidMessage("Unable to make boolean from",
- "INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "\"abc\"");
+ "INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "\"abc\"");
assertInvalidMessage("Expected a boolean value, but got a Integer",
- "INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, booleanval) VALUES (?, from_json(?))", 0, "123");
// ================ date ================
- execute("INSERT INTO %s (k, dateval) VALUES (?, fromJson(?))", 0, "\"1987-03-23\"");
+ execute("INSERT INTO %s (k, dateval) VALUES (?, from_json(?))", 0, "\"1987-03-23\"");
assertRows(execute("SELECT k, dateval FROM %s WHERE k = ?", 0), row(0, SimpleDateSerializer.dateStringToDays("1987-03-23")));
assertInvalidMessage("Expected a string representation of a date",
- "INSERT INTO %s (k, dateval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, dateval) VALUES (?, from_json(?))", 0, "123");
assertInvalidMessage("Unable to coerce 'xyz' to a formatted date",
- "INSERT INTO %s (k, dateval) VALUES (?, fromJson(?))", 0, "\"xyz\"");
+ "INSERT INTO %s (k, dateval) VALUES (?, from_json(?))", 0, "\"xyz\"");
// ================ decimal ================
- execute("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123.123123");
+ execute("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "123123.123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123")));
- execute("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123");
+ execute("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123")));
// accept strings for numbers that cannot be represented as doubles
- execute("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"123123.123123\"");
+ execute("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "\"123123.123123\"");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123")));
- execute("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"-1.23E-12\"");
+ execute("INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "\"-1.23E-12\"");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("-1.23E-12")));
assertInvalidMessage("Value 'xyzz' is not a valid representation of a decimal value",
- "INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Value 'true' is not a valid representation of a decimal value",
- "INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, decimalval) VALUES (?, from_json(?))", 0, "true");
// ================ double ================
- execute("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "123123.123123");
+ execute("INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "123123.123123");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.123123d));
- execute("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "123123");
+ execute("INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.0d));
// strings are also accepted
- execute("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "\"123123\"");
+ execute("INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "\"123123\"");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.0d));
assertInvalidMessage("Unable to make double from",
- "INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Expected a double value, but got",
- "INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, doubleval) VALUES (?, from_json(?))", 0, "true");
// ================ float ================
- execute("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "123123.123123");
+ execute("INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "123123.123123");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.123123f));
- execute("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "123123");
+ execute("INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.0f));
// strings are also accepted
- execute("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "\"123123.0\"");
+ execute("INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "\"123123.0\"");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.0f));
assertInvalidMessage("Unable to make float from",
- "INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Expected a float value, but got a",
- "INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, floatval) VALUES (?, from_json(?))", 0, "true");
// ================ inet ================
- execute("INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"127.0.0.1\"");
+ execute("INSERT INTO %s (k, inetval) VALUES (?, from_json(?))", 0, "\"127.0.0.1\"");
assertRows(execute("SELECT k, inetval FROM %s WHERE k = ?", 0), row(0, InetAddress.getByName("127.0.0.1")));
- execute("INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"::1\"");
+ execute("INSERT INTO %s (k, inetval) VALUES (?, from_json(?))", 0, "\"::1\"");
assertRows(execute("SELECT k, inetval FROM %s WHERE k = ?", 0), row(0, InetAddress.getByName("::1")));
assertInvalidMessage("Unable to make inet address from 'xyzz'",
- "INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, inetval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Expected a string representation of an inet value, but got a Integer",
- "INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, inetval) VALUES (?, from_json(?))", 0, "123");
// ================ int ================
- execute("INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "123123");
+ execute("INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "123123");
assertRows(execute("SELECT k, intval FROM %s WHERE k = ?", 0), row(0, 123123));
// strings are also accepted
- execute("INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "\"123123\"");
+ execute("INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "\"123123\"");
assertRows(execute("SELECT k, intval FROM %s WHERE k = ?", 0), row(0, 123123));
// int overflow (2 ^ 32, or Integer.MAX_INT + 1)
assertInvalidMessage("Expected an int value, but got a",
- "INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "2147483648");
+ "INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "2147483648");
assertInvalidMessage("Expected an int value, but got a",
- "INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "123.456");
+ "INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "123.456");
assertInvalidMessage("Unable to make int from",
- "INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Expected an int value, but got a",
- "INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, intval) VALUES (?, from_json(?))", 0, "true");
// ================ smallint ================
- execute("INSERT INTO %s (k, smallintval) VALUES (?, fromJson(?))", 0, "32767");
+ execute("INSERT INTO %s (k, smallintval) VALUES (?, from_json(?))", 0, "32767");
assertRows(execute("SELECT k, smallintval FROM %s WHERE k = ?", 0), row(0, (short) 32767));
// strings are also accepted
- execute("INSERT INTO %s (k, smallintval) VALUES (?, fromJson(?))", 0, "\"32767\"");
+ execute("INSERT INTO %s (k, smallintval) VALUES (?, from_json(?))", 0, "\"32767\"");
assertRows(execute("SELECT k, smallintval FROM %s WHERE k = ?", 0), row(0, (short) 32767));
// smallint overflow (Short.MAX_VALUE + 1)
assertInvalidMessage("Unable to make short from",
- "INSERT INTO %s (k, smallintval) VALUES (?, fromJson(?))", 0, "32768");
+ "INSERT INTO %s (k, smallintval) VALUES (?, from_json(?))", 0, "32768");
assertInvalidMessage("Unable to make short from",
- "INSERT INTO %s (k, smallintval) VALUES (?, fromJson(?))", 0, "123.456");
+ "INSERT INTO %s (k, smallintval) VALUES (?, from_json(?))", 0, "123.456");
assertInvalidMessage("Unable to make short from",
- "INSERT INTO %s (k, smallintval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, smallintval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Expected a short value, but got a Boolean",
- "INSERT INTO %s (k, smallintval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, smallintval) VALUES (?, from_json(?))", 0, "true");
// ================ tinyint ================
- execute("INSERT INTO %s (k, tinyintval) VALUES (?, fromJson(?))", 0, "127");
+ execute("INSERT INTO %s (k, tinyintval) VALUES (?, from_json(?))", 0, "127");
assertRows(execute("SELECT k, tinyintval FROM %s WHERE k = ?", 0), row(0, (byte) 127));
// strings are also accepted
- execute("INSERT INTO %s (k, tinyintval) VALUES (?, fromJson(?))", 0, "\"127\"");
+ execute("INSERT INTO %s (k, tinyintval) VALUES (?, from_json(?))", 0, "\"127\"");
assertRows(execute("SELECT k, tinyintval FROM %s WHERE k = ?", 0), row(0, (byte) 127));
// tinyint overflow (Byte.MAX_VALUE + 1)
assertInvalidMessage("Unable to make byte from",
- "INSERT INTO %s (k, tinyintval) VALUES (?, fromJson(?))", 0, "128");
+ "INSERT INTO %s (k, tinyintval) VALUES (?, from_json(?))", 0, "128");
assertInvalidMessage("Unable to make byte from",
- "INSERT INTO %s (k, tinyintval) VALUES (?, fromJson(?))", 0, "123.456");
+ "INSERT INTO %s (k, tinyintval) VALUES (?, from_json(?))", 0, "123.456");
assertInvalidMessage("Unable to make byte from",
- "INSERT INTO %s (k, tinyintval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, tinyintval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Expected a byte value, but got a Boolean",
- "INSERT INTO %s (k, tinyintval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, tinyintval) VALUES (?, from_json(?))", 0, "true");
// ================ text (varchar) ================
- execute("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"\"");
+ execute("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, ""));
- execute("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"abcd\"");
+ execute("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"abcd\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "abcd"));
- execute("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"some \\\" text\"");
+ execute("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"some \\\" text\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "some \" text"));
- execute("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"\\u2013\"");
+ execute("INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "\"\\u2013\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "\u2013"));
assertInvalidMessage("Expected a UTF-8 string, but got a Integer",
- "INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, textval) VALUES (?, from_json(?))", 0, "123");
// ================ time ================
- execute("INSERT INTO %s (k, timeval) VALUES (?, fromJson(?))", 0, "\"07:35:07.000111222\"");
+ execute("INSERT INTO %s (k, timeval) VALUES (?, from_json(?))", 0, "\"07:35:07.000111222\"");
assertRows(execute("SELECT k, timeval FROM %s WHERE k = ?", 0), row(0, TimeSerializer.timeStringToLong("07:35:07.000111222")));
assertInvalidMessage("Expected a string representation of a time value",
- "INSERT INTO %s (k, timeval) VALUES (?, fromJson(?))", 0, "123456");
+ "INSERT INTO %s (k, timeval) VALUES (?, from_json(?))", 0, "123456");
assertInvalidMessage("Unable to coerce 'xyz' to a formatted time",
- "INSERT INTO %s (k, timeval) VALUES (?, fromJson(?))", 0, "\"xyz\"");
+ "INSERT INTO %s (k, timeval) VALUES (?, from_json(?))", 0, "\"xyz\"");
// ================ timestamp ================
- execute("INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "123123123123");
+ execute("INSERT INTO %s (k, timestampval) VALUES (?, from_json(?))", 0, "123123123123");
assertRows(execute("SELECT k, timestampval FROM %s WHERE k = ?", 0), row(0, new Date(123123123123L)));
- execute("INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "\"2014-01-01\"");
+ execute("INSERT INTO %s (k, timestampval) VALUES (?, from_json(?))", 0, "\"2014-01-01\"");
assertRows(execute("SELECT k, timestampval FROM %s WHERE k = ?", 0), row(0, new SimpleDateFormat("y-M-d").parse("2014-01-01")));
assertInvalidMessage("Expected a long or a datestring representation of a timestamp value, but got a Double",
- "INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "123.456");
+ "INSERT INTO %s (k, timestampval) VALUES (?, from_json(?))", 0, "123.456");
assertInvalidMessage("Unable to parse a date/time from 'abcd'",
- "INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "\"abcd\"");
+ "INSERT INTO %s (k, timestampval) VALUES (?, from_json(?))", 0, "\"abcd\"");
// ================ timeuuid ================
- execute("INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
+ execute("INSERT INTO %s (k, timeuuidval) VALUES (?, from_json(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
assertRows(execute("SELECT k, timeuuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- execute("INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
+ execute("INSERT INTO %s (k, timeuuidval) VALUES (?, from_json(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
assertRows(execute("SELECT k, timeuuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
assertInvalidMessage("TimeUUID supports only version 1 UUIDs",
- "INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"00000000-0000-0000-0000-000000000000\"");
+ "INSERT INTO %s (k, timeuuidval) VALUES (?, from_json(?))", 0, "\"00000000-0000-0000-0000-000000000000\"");
assertInvalidMessage("Expected a string representation of a timeuuid, but got a Integer",
- "INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, timeuuidval) VALUES (?, from_json(?))", 0, "123");
// ================ uuidval ================
- execute("INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
+ execute("INSERT INTO %s (k, uuidval) VALUES (?, from_json(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
assertRows(execute("SELECT k, uuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- execute("INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
+ execute("INSERT INTO %s (k, uuidval) VALUES (?, from_json(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
assertRows(execute("SELECT k, uuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
assertInvalidMessage("Unable to make UUID from",
- "INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"00000000-0000-0000-zzzz-000000000000\"");
+ "INSERT INTO %s (k, uuidval) VALUES (?, from_json(?))", 0, "\"00000000-0000-0000-zzzz-000000000000\"");
assertInvalidMessage("Expected a string representation of a uuid, but got a Integer",
- "INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, uuidval) VALUES (?, from_json(?))", 0, "123");
// ================ varint ================
- execute("INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "123123123123");
+ execute("INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "123123123123");
assertRows(execute("SELECT k, varintval FROM %s WHERE k = ?", 0), row(0, new BigInteger("123123123123")));
// accept strings for numbers that cannot be represented as longs
- execute("INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "\"1234567890123456789012345678901234567890\"");
+ execute("INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "\"1234567890123456789012345678901234567890\"");
assertRows(execute("SELECT k, varintval FROM %s WHERE k = ?", 0), row(0, new BigInteger("1234567890123456789012345678901234567890")));
assertInvalidMessage("Value '123123.123' is not a valid representation of a varint value",
- "INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "123123.123");
+ "INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "123123.123");
assertInvalidMessage("Value 'xyzz' is not a valid representation of a varint value",
- "INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "\"xyzz\"");
+ "INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "\"xyzz\"");
assertInvalidMessage("Value '' is not a valid representation of a varint value",
- "INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "\"\"");
+ "INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "\"\"");
assertInvalidMessage("Value 'true' is not a valid representation of a varint value",
- "INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "true");
+ "INSERT INTO %s (k, varintval) VALUES (?, from_json(?))", 0, "true");
// ================ lists ================
- execute("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[1, 2, 3]");
+ execute("INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
- execute("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[]");
+ execute("INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, null));
assertInvalidMessage("Expected a list, but got a Integer",
- "INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "123");
assertInvalidMessage("Unable to make int from",
- "INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[\"abc\"]");
+ "INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[\"abc\"]");
assertInvalidMessage("Invalid null element in list",
- "INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[null]");
+ "INSERT INTO %s (k, listval) VALUES (?, from_json(?))", 0, "[null]");
// frozen
- execute("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[1, 2, 3]");
+ execute("INSERT INTO %s (k, frozenlistval) VALUES (?, from_json(?))", 0, "[1, 2, 3]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
// ================ sets ================
- execute("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))",
+ execute("INSERT INTO %s (k, setval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
// duplicates are okay, just like in CQL
- execute("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))",
+ execute("INSERT INTO %s (k, setval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- execute("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))", 0, "[]");
+ execute("INSERT INTO %s (k, setval) VALUES (?, from_json(?))", 0, "[]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0), row(0, null));
assertInvalidMessage("Expected a list (representing a set), but got a Integer",
- "INSERT INTO %s (k, setval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, setval) VALUES (?, from_json(?))", 0, "123");
assertInvalidMessage("Unable to make UUID from",
- "INSERT INTO %s (k, setval) VALUES (?, fromJson(?))", 0, "[\"abc\"]");
+ "INSERT INTO %s (k, setval) VALUES (?, from_json(?))", 0, "[\"abc\"]");
assertInvalidMessage("Invalid null element in set",
- "INSERT INTO %s (k, setval) VALUES (?, fromJson(?))", 0, "[null]");
+ "INSERT INTO %s (k, setval) VALUES (?, from_json(?))", 0, "[null]");
// frozen
- execute("INSERT INTO %s (k, frozensetval) VALUES (?, fromJson(?))",
+ execute("INSERT INTO %s (k, frozensetval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, frozensetval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
- execute("INSERT INTO %s (k, frozensetval) VALUES (?, fromJson(?))",
+ execute("INSERT INTO %s (k, frozensetval) VALUES (?, from_json(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9799\", \"6bddc89a-5644-11e4-97fc-56847afe9798\"]");
assertRows(execute("SELECT k, frozensetval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
// ================ maps ================
- execute("INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": 2}");
+ execute("INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": 2}");
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
- execute("INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{}");
+ execute("INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "{}");
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0), row(0, null));
assertInvalidMessage("Expected a map, but got a Integer",
- "INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "123");
+ "INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "123");
assertInvalidMessage("Invalid ASCII character in string literal",
- "INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{\"\\u1fff\\u2013\\u33B4\\u2014\": 1}");
+ "INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "{\"\\u1fff\\u2013\\u33B4\\u2014\": 1}");
assertInvalidMessage("Invalid null value in map",
- "INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{\"a\": null}");
+ "INSERT INTO %s (k, mapval) VALUES (?, from_json(?))", 0, "{\"a\": null}");
// frozen
- execute("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": 2}");
+ execute("INSERT INTO %s (k, frozenmapval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": 2}");
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
- execute("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, "{\"b\": 2, \"a\": 1}");
+ execute("INSERT INTO %s (k, frozenmapval) VALUES (?, from_json(?))", 0, "{\"b\": 2, \"a\": 1}");
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
// ================ tuples ================
- execute("INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))", 0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
+ execute("INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))", 0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, tupleval FROM %s WHERE k = ?", 0),
row(0, tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))
);
- execute("INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))", 0, "[1, null, \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
+ execute("INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))", 0, "[1, null, \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, tupleval FROM %s WHERE k = ?", 0),
row(0, tuple(1, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))
);
assertInvalidMessage("Tuple contains extra items",
- "INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))",
+ "INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))",
0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\", 1, 2, 3]");
assertInvalidMessage("Tuple is missing items",
- "INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))",
+ "INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))",
0, "[1, \"foobar\"]");
assertInvalidMessage("Unable to make int from",
- "INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))",
+ "INSERT INTO %s (k, tupleval) VALUES (?, from_json(?))",
0, "[\"not an int\", \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
// ================ UDTs ================
- execute("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
+ execute("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
// ================ duration ================
- execute("INSERT INTO %s (k, durationval) VALUES (?, fromJson(?))", 0, "\"53us\"");
+ execute("INSERT INTO %s (k, durationval) VALUES (?, from_json(?))", 0, "\"53us\"");
assertRows(execute("SELECT k, durationval FROM %s WHERE k = ?", 0), row(0, Duration.newInstance(0, 0, 53000L)));
- execute("INSERT INTO %s (k, durationval) VALUES (?, fromJson(?))", 0, "\"P2W\"");
+ execute("INSERT INTO %s (k, durationval) VALUES (?, from_json(?))", 0, "\"P2W\"");
assertRows(execute("SELECT k, durationval FROM %s WHERE k = ?", 0), row(0, Duration.newInstance(0, 14, 0)));
assertInvalidMessage("Unable to convert 'xyz' to a duration",
- "INSERT INTO %s (k, durationval) VALUES (?, fromJson(?))", 0, "\"xyz\"");
+ "INSERT INTO %s (k, durationval) VALUES (?, from_json(?))", 0, "\"xyz\"");
// order of fields shouldn't matter
- execute("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"a\": 1, \"c\": [\"foo\", \"bar\"]}");
+ execute("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"a\": 1, \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
// test nulls
- execute("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
+ execute("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
// test missing fields
- execute("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}");
+ execute("INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), null)
);
- assertInvalidMessage("Unknown field", "INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"xxx\": 1}");
+ assertInvalidMessage("Unknown field", "INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"xxx\": 1}");
assertInvalidMessage("Unable to make int from",
- "INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": \"foobar\"}");
+ "INSERT INTO %s (k, udtval) VALUES (?, from_json(?))", 0, "{\"a\": \"foobar\"}");
}
@Test
@@ -745,212 +747,224 @@
"udtval frozen<" + typeName + ">," +
"durationval duration)");
- // toJson() can only be used in selections
- assertInvalidMessage("toJson() may only be used within the selection clause",
- "INSERT INTO %s (k, asciival) VALUES (?, toJson(?))", 0, 0);
- assertInvalidMessage("toJson() may only be used within the selection clause",
- "UPDATE %s SET asciival = toJson(?) WHERE k = ?", 0, 0);
- assertInvalidMessage("toJson() may only be used within the selection clause",
- "DELETE FROM %s WHERE k = fromJson(toJson(?))", 0);
+ // to_json() can be used out of the selection clause (with literals)
+ execute("INSERT INTO %s (k, textval) VALUES (?, to_json(1234))", 0);
+ assertRows(execute("SELECT textval FROM %s WHERE k = ?", 0), row("1234"));
+ assertRows(execute("SELECT textval FROM %s WHERE textval = to_json(1234) ALLOW FILTERING"), row("1234"));
+ execute("UPDATE %s SET textval = to_json(-1234) WHERE k = ?", 0);
+ assertRows(execute("SELECT textval FROM %s WHERE k = ?", 0), row("-1234"));
+ assertRows(execute("SELECT textval FROM %s WHERE textval = to_json(-1234) ALLOW FILTERING"), row("-1234"));
+ execute("DELETE FROM %s WHERE k = from_json(to_json(0))");
+ assertEmpty(execute("SELECT textval FROM %s WHERE k = ?", 0));
+
+ // to_json() can be used out of the selection clause (with markers)
+ execute("INSERT INTO %s (k, textval) VALUES (?, to_json((int) ?))", 0, 123123);
+ assertRows(execute("SELECT textval FROM %s WHERE k = ?", 0), row("123123"));
+ assertRows(execute("SELECT textval FROM %s WHERE textval = to_json((int) ?) ALLOW FILTERING", 123123), row("123123"));
+ execute("UPDATE %s SET textval = to_json((int) ?) WHERE k = ?", -123123, 0);
+ assertRows(execute("SELECT textval FROM %s WHERE k = ?", 0), row("-123123"));
+ assertRows(execute("SELECT textval FROM %s WHERE textval = to_json((int) ?) ALLOW FILTERING", -123123), row("-123123"));
+ execute("DELETE FROM %s WHERE k = from_json(to_json((int) ?))", 0);
+ assertEmpty(execute("SELECT textval FROM %s WHERE k = ?", 0));
// ================ ascii ================
execute("INSERT INTO %s (k, asciival) VALUES (?, ?)", 0, "ascii text");
- assertRows(execute("SELECT k, toJson(asciival) FROM %s WHERE k = ?", 0), row(0, "\"ascii text\""));
+ assertRows(execute("SELECT k, to_json(asciival) FROM %s WHERE k = ?", 0), row(0, "\"ascii text\""));
execute("INSERT INTO %s (k, asciival) VALUES (?, ?)", 0, "");
- assertRows(execute("SELECT k, toJson(asciival) FROM %s WHERE k = ?", 0), row(0, "\"\""));
+ assertRows(execute("SELECT k, to_json(asciival) FROM %s WHERE k = ?", 0), row(0, "\"\""));
// ================ bigint ================
execute("INSERT INTO %s (k, bigintval) VALUES (?, ?)", 0, 123123123123L);
- assertRows(execute("SELECT k, toJson(bigintval) FROM %s WHERE k = ?", 0), row(0, "123123123123"));
+ assertRows(execute("SELECT k, to_json(bigintval) FROM %s WHERE k = ?", 0), row(0, "123123123123"));
execute("INSERT INTO %s (k, bigintval) VALUES (?, ?)", 0, 0L);
- assertRows(execute("SELECT k, toJson(bigintval) FROM %s WHERE k = ?", 0), row(0, "0"));
+ assertRows(execute("SELECT k, to_json(bigintval) FROM %s WHERE k = ?", 0), row(0, "0"));
execute("INSERT INTO %s (k, bigintval) VALUES (?, ?)", 0, -123123123123L);
- assertRows(execute("SELECT k, toJson(bigintval) FROM %s WHERE k = ?", 0), row(0, "-123123123123"));
+ assertRows(execute("SELECT k, to_json(bigintval) FROM %s WHERE k = ?", 0), row(0, "-123123123123"));
// ================ blob ================
execute("INSERT INTO %s (k, blobval) VALUES (?, ?)", 0, ByteBufferUtil.bytes(1));
- assertRows(execute("SELECT k, toJson(blobval) FROM %s WHERE k = ?", 0), row(0, "\"0x00000001\""));
+ assertRows(execute("SELECT k, to_json(blobval) FROM %s WHERE k = ?", 0), row(0, "\"0x00000001\""));
execute("INSERT INTO %s (k, blobval) VALUES (?, ?)", 0, ByteBufferUtil.EMPTY_BYTE_BUFFER);
- assertRows(execute("SELECT k, toJson(blobval) FROM %s WHERE k = ?", 0), row(0, "\"0x\""));
+ assertRows(execute("SELECT k, to_json(blobval) FROM %s WHERE k = ?", 0), row(0, "\"0x\""));
// ================ boolean ================
execute("INSERT INTO %s (k, booleanval) VALUES (?, ?)", 0, true);
- assertRows(execute("SELECT k, toJson(booleanval) FROM %s WHERE k = ?", 0), row(0, "true"));
+ assertRows(execute("SELECT k, to_json(booleanval) FROM %s WHERE k = ?", 0), row(0, "true"));
execute("INSERT INTO %s (k, booleanval) VALUES (?, ?)", 0, false);
- assertRows(execute("SELECT k, toJson(booleanval) FROM %s WHERE k = ?", 0), row(0, "false"));
+ assertRows(execute("SELECT k, to_json(booleanval) FROM %s WHERE k = ?", 0), row(0, "false"));
// ================ date ================
execute("INSERT INTO %s (k, dateval) VALUES (?, ?)", 0, SimpleDateSerializer.dateStringToDays("1987-03-23"));
- assertRows(execute("SELECT k, toJson(dateval) FROM %s WHERE k = ?", 0), row(0, "\"1987-03-23\""));
+ assertRows(execute("SELECT k, to_json(dateval) FROM %s WHERE k = ?", 0), row(0, "\"1987-03-23\""));
// ================ decimal ================
execute("INSERT INTO %s (k, decimalval) VALUES (?, ?)", 0, new BigDecimal("123123.123123"));
- assertRows(execute("SELECT k, toJson(decimalval) FROM %s WHERE k = ?", 0), row(0, "123123.123123"));
+ assertRows(execute("SELECT k, to_json(decimalval) FROM %s WHERE k = ?", 0), row(0, "123123.123123"));
execute("INSERT INTO %s (k, decimalval) VALUES (?, ?)", 0, new BigDecimal("-1.23E-12"));
- assertRows(execute("SELECT k, toJson(decimalval) FROM %s WHERE k = ?", 0), row(0, "-1.23E-12"));
+ assertRows(execute("SELECT k, to_json(decimalval) FROM %s WHERE k = ?", 0), row(0, "-1.23E-12"));
// ================ double ================
execute("INSERT INTO %s (k, doubleval) VALUES (?, ?)", 0, 123123.123123d);
- assertRows(execute("SELECT k, toJson(doubleval) FROM %s WHERE k = ?", 0), row(0, "123123.123123"));
+ assertRows(execute("SELECT k, to_json(doubleval) FROM %s WHERE k = ?", 0), row(0, "123123.123123"));
execute("INSERT INTO %s (k, doubleval) VALUES (?, ?)", 0, 123123d);
- assertRows(execute("SELECT k, toJson(doubleval) FROM %s WHERE k = ?", 0), row(0, "123123.0"));
+ assertRows(execute("SELECT k, to_json(doubleval) FROM %s WHERE k = ?", 0), row(0, "123123.0"));
// ================ float ================
execute("INSERT INTO %s (k, floatval) VALUES (?, ?)", 0, 123.123f);
- assertRows(execute("SELECT k, toJson(floatval) FROM %s WHERE k = ?", 0), row(0, "123.123"));
+ assertRows(execute("SELECT k, to_json(floatval) FROM %s WHERE k = ?", 0), row(0, "123.123"));
execute("INSERT INTO %s (k, floatval) VALUES (?, ?)", 0, 123123f);
- assertRows(execute("SELECT k, toJson(floatval) FROM %s WHERE k = ?", 0), row(0, "123123.0"));
+ assertRows(execute("SELECT k, to_json(floatval) FROM %s WHERE k = ?", 0), row(0, "123123.0"));
// ================ inet ================
execute("INSERT INTO %s (k, inetval) VALUES (?, ?)", 0, InetAddress.getByName("127.0.0.1"));
- assertRows(execute("SELECT k, toJson(inetval) FROM %s WHERE k = ?", 0), row(0, "\"127.0.0.1\""));
+ assertRows(execute("SELECT k, to_json(inetval) FROM %s WHERE k = ?", 0), row(0, "\"127.0.0.1\""));
execute("INSERT INTO %s (k, inetval) VALUES (?, ?)", 0, InetAddress.getByName("::1"));
- assertRows(execute("SELECT k, toJson(inetval) FROM %s WHERE k = ?", 0), row(0, "\"0:0:0:0:0:0:0:1\""));
+ assertRows(execute("SELECT k, to_json(inetval) FROM %s WHERE k = ?", 0), row(0, "\"0:0:0:0:0:0:0:1\""));
// ================ int ================
execute("INSERT INTO %s (k, intval) VALUES (?, ?)", 0, 123123);
- assertRows(execute("SELECT k, toJson(intval) FROM %s WHERE k = ?", 0), row(0, "123123"));
+ assertRows(execute("SELECT k, to_json(intval) FROM %s WHERE k = ?", 0), row(0, "123123"));
execute("INSERT INTO %s (k, intval) VALUES (?, ?)", 0, 0);
- assertRows(execute("SELECT k, toJson(intval) FROM %s WHERE k = ?", 0), row(0, "0"));
+ assertRows(execute("SELECT k, to_json(intval) FROM %s WHERE k = ?", 0), row(0, "0"));
execute("INSERT INTO %s (k, intval) VALUES (?, ?)", 0, -123123);
- assertRows(execute("SELECT k, toJson(intval) FROM %s WHERE k = ?", 0), row(0, "-123123"));
+ assertRows(execute("SELECT k, to_json(intval) FROM %s WHERE k = ?", 0), row(0, "-123123"));
// ================ smallint ================
execute("INSERT INTO %s (k, smallintval) VALUES (?, ?)", 0, (short) 32767);
- assertRows(execute("SELECT k, toJson(smallintval) FROM %s WHERE k = ?", 0), row(0, "32767"));
+ assertRows(execute("SELECT k, to_json(smallintval) FROM %s WHERE k = ?", 0), row(0, "32767"));
execute("INSERT INTO %s (k, smallintval) VALUES (?, ?)", 0, (short) 0);
- assertRows(execute("SELECT k, toJson(smallintval) FROM %s WHERE k = ?", 0), row(0, "0"));
+ assertRows(execute("SELECT k, to_json(smallintval) FROM %s WHERE k = ?", 0), row(0, "0"));
execute("INSERT INTO %s (k, smallintval) VALUES (?, ?)", 0, (short) -32768);
- assertRows(execute("SELECT k, toJson(smallintval) FROM %s WHERE k = ?", 0), row(0, "-32768"));
+ assertRows(execute("SELECT k, to_json(smallintval) FROM %s WHERE k = ?", 0), row(0, "-32768"));
// ================ tinyint ================
execute("INSERT INTO %s (k, tinyintval) VALUES (?, ?)", 0, (byte) 127);
- assertRows(execute("SELECT k, toJson(tinyintval) FROM %s WHERE k = ?", 0), row(0, "127"));
+ assertRows(execute("SELECT k, to_json(tinyintval) FROM %s WHERE k = ?", 0), row(0, "127"));
execute("INSERT INTO %s (k, tinyintval) VALUES (?, ?)", 0, (byte) 0);
- assertRows(execute("SELECT k, toJson(tinyintval) FROM %s WHERE k = ?", 0), row(0, "0"));
+ assertRows(execute("SELECT k, to_json(tinyintval) FROM %s WHERE k = ?", 0), row(0, "0"));
execute("INSERT INTO %s (k, tinyintval) VALUES (?, ?)", 0, (byte) -128);
- assertRows(execute("SELECT k, toJson(tinyintval) FROM %s WHERE k = ?", 0), row(0, "-128"));
+ assertRows(execute("SELECT k, to_json(tinyintval) FROM %s WHERE k = ?", 0), row(0, "-128"));
// ================ text (varchar) ================
execute("INSERT INTO %s (k, textval) VALUES (?, ?)", 0, "");
- assertRows(execute("SELECT k, toJson(textval) FROM %s WHERE k = ?", 0), row(0, "\"\""));
+ assertRows(execute("SELECT k, to_json(textval) FROM %s WHERE k = ?", 0), row(0, "\"\""));
execute("INSERT INTO %s (k, textval) VALUES (?, ?)", 0, "abcd");
- assertRows(execute("SELECT k, toJson(textval) FROM %s WHERE k = ?", 0), row(0, "\"abcd\""));
+ assertRows(execute("SELECT k, to_json(textval) FROM %s WHERE k = ?", 0), row(0, "\"abcd\""));
execute("INSERT INTO %s (k, textval) VALUES (?, ?)", 0, "\u8422");
- assertRows(execute("SELECT k, toJson(textval) FROM %s WHERE k = ?", 0), row(0, "\"\u8422\""));
+ assertRows(execute("SELECT k, to_json(textval) FROM %s WHERE k = ?", 0), row(0, "\"\u8422\""));
execute("INSERT INTO %s (k, textval) VALUES (?, ?)", 0, "\u0000");
- assertRows(execute("SELECT k, toJson(textval) FROM %s WHERE k = ?", 0), row(0, "\"\\u0000\""));
+ assertRows(execute("SELECT k, to_json(textval) FROM %s WHERE k = ?", 0), row(0, "\"\\u0000\""));
// ================ time ================
execute("INSERT INTO %s (k, timeval) VALUES (?, ?)", 0, 123L);
- assertRows(execute("SELECT k, toJson(timeval) FROM %s WHERE k = ?", 0), row(0, "\"00:00:00.000000123\""));
+ assertRows(execute("SELECT k, to_json(timeval) FROM %s WHERE k = ?", 0), row(0, "\"00:00:00.000000123\""));
- execute("INSERT INTO %s (k, timeval) VALUES (?, fromJson(?))", 0, "\"07:35:07.000111222\"");
- assertRows(execute("SELECT k, toJson(timeval) FROM %s WHERE k = ?", 0), row(0, "\"07:35:07.000111222\""));
+ execute("INSERT INTO %s (k, timeval) VALUES (?, from_json(?))", 0, "\"07:35:07.000111222\"");
+ assertRows(execute("SELECT k, to_json(timeval) FROM %s WHERE k = ?", 0), row(0, "\"07:35:07.000111222\""));
// ================ timestamp ================
SimpleDateFormat sdf = new SimpleDateFormat("y-M-d");
sdf.setTimeZone(TimeZone.getTimeZone("UDT"));
execute("INSERT INTO %s (k, timestampval) VALUES (?, ?)", 0, sdf.parse("2014-01-01"));
- assertRows(execute("SELECT k, toJson(timestampval) FROM %s WHERE k = ?", 0), row(0, "\"2014-01-01 00:00:00.000Z\""));
+ assertRows(execute("SELECT k, to_json(timestampval) FROM %s WHERE k = ?", 0), row(0, "\"2014-01-01 00:00:00.000Z\""));
// ================ timeuuid ================
execute("INSERT INTO %s (k, timeuuidval) VALUES (?, ?)", 0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"));
- assertRows(execute("SELECT k, toJson(timeuuidval) FROM %s WHERE k = ?", 0), row(0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\""));
+ assertRows(execute("SELECT k, to_json(timeuuidval) FROM %s WHERE k = ?", 0), row(0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\""));
// ================ uuidval ================
execute("INSERT INTO %s (k, uuidval) VALUES (?, ?)", 0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"));
- assertRows(execute("SELECT k, toJson(uuidval) FROM %s WHERE k = ?", 0), row(0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\""));
+ assertRows(execute("SELECT k, to_json(uuidval) FROM %s WHERE k = ?", 0), row(0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\""));
// ================ varint ================
execute("INSERT INTO %s (k, varintval) VALUES (?, ?)", 0, new BigInteger("123123123123123123123"));
- assertRows(execute("SELECT k, toJson(varintval) FROM %s WHERE k = ?", 0), row(0, "123123123123123123123"));
+ assertRows(execute("SELECT k, to_json(varintval) FROM %s WHERE k = ?", 0), row(0, "123123123123123123123"));
// ================ lists ================
execute("INSERT INTO %s (k, listval) VALUES (?, ?)", 0, list(1, 2, 3));
- assertRows(execute("SELECT k, toJson(listval) FROM %s WHERE k = ?", 0), row(0, "[1, 2, 3]"));
+ assertRows(execute("SELECT k, to_json(listval) FROM %s WHERE k = ?", 0), row(0, "[1, 2, 3]"));
execute("INSERT INTO %s (k, listval) VALUES (?, ?)", 0, list());
- assertRows(execute("SELECT k, toJson(listval) FROM %s WHERE k = ?", 0), row(0, "null"));
+ assertRows(execute("SELECT k, to_json(listval) FROM %s WHERE k = ?", 0), row(0, "null"));
// frozen
execute("INSERT INTO %s (k, frozenlistval) VALUES (?, ?)", 0, list(1, 2, 3));
- assertRows(execute("SELECT k, toJson(frozenlistval) FROM %s WHERE k = ?", 0), row(0, "[1, 2, 3]"));
+ assertRows(execute("SELECT k, to_json(frozenlistval) FROM %s WHERE k = ?", 0), row(0, "[1, 2, 3]"));
// ================ sets ================
execute("INSERT INTO %s (k, setval) VALUES (?, ?)",
0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))));
- assertRows(execute("SELECT k, toJson(setval) FROM %s WHERE k = ?", 0),
+ assertRows(execute("SELECT k, to_json(setval) FROM %s WHERE k = ?", 0),
row(0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]")
);
execute("INSERT INTO %s (k, setval) VALUES (?, ?)", 0, set());
- assertRows(execute("SELECT k, toJson(setval) FROM %s WHERE k = ?", 0), row(0, "null"));
+ assertRows(execute("SELECT k, to_json(setval) FROM %s WHERE k = ?", 0), row(0, "null"));
// frozen
execute("INSERT INTO %s (k, frozensetval) VALUES (?, ?)",
0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))));
- assertRows(execute("SELECT k, toJson(frozensetval) FROM %s WHERE k = ?", 0),
+ assertRows(execute("SELECT k, to_json(frozensetval) FROM %s WHERE k = ?", 0),
row(0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]")
);
// ================ maps ================
execute("INSERT INTO %s (k, mapval) VALUES (?, ?)", 0, map("a", 1, "b", 2));
- assertRows(execute("SELECT k, toJson(mapval) FROM %s WHERE k = ?", 0), row(0, "{\"a\": 1, \"b\": 2}"));
+ assertRows(execute("SELECT k, to_json(mapval) FROM %s WHERE k = ?", 0), row(0, "{\"a\": 1, \"b\": 2}"));
execute("INSERT INTO %s (k, mapval) VALUES (?, ?)", 0, map());
- assertRows(execute("SELECT k, toJson(mapval) FROM %s WHERE k = ?", 0), row(0, "null"));
+ assertRows(execute("SELECT k, to_json(mapval) FROM %s WHERE k = ?", 0), row(0, "null"));
// frozen
execute("INSERT INTO %s (k, frozenmapval) VALUES (?, ?)", 0, map("a", 1, "b", 2));
- assertRows(execute("SELECT k, toJson(frozenmapval) FROM %s WHERE k = ?", 0), row(0, "{\"a\": 1, \"b\": 2}"));
+ assertRows(execute("SELECT k, to_json(frozenmapval) FROM %s WHERE k = ?", 0), row(0, "{\"a\": 1, \"b\": 2}"));
// ================ tuples ================
execute("INSERT INTO %s (k, tupleval) VALUES (?, ?)", 0, tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
- assertRows(execute("SELECT k, toJson(tupleval) FROM %s WHERE k = ?", 0),
+ assertRows(execute("SELECT k, to_json(tupleval) FROM %s WHERE k = ?", 0),
row(0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]")
);
execute("INSERT INTO %s (k, tupleval) VALUES (?, ?)", 0, tuple(1, "foobar", null));
- assertRows(execute("SELECT k, toJson(tupleval) FROM %s WHERE k = ?", 0),
+ assertRows(execute("SELECT k, to_json(tupleval) FROM %s WHERE k = ?", 0),
row(0, "[1, \"foobar\", null]")
);
// ================ UDTs ================
execute("INSERT INTO %s (k, udtval) VALUES (?, {a: ?, b: ?, c: ?})", 0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("foo", "bar"));
- assertRows(execute("SELECT k, toJson(udtval) FROM %s WHERE k = ?", 0),
+ assertRows(execute("SELECT k, to_json(udtval) FROM %s WHERE k = ?", 0),
row(0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"bar\", \"foo\"]}")
);
execute("INSERT INTO %s (k, udtval) VALUES (?, {a: ?, b: ?})", 0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"));
- assertRows(execute("SELECT k, toJson(udtval) FROM %s WHERE k = ?", 0),
+ assertRows(execute("SELECT k, to_json(udtval) FROM %s WHERE k = ?", 0),
row(0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": null}")
);
// ================ duration ================
execute("INSERT INTO %s (k, durationval) VALUES (?, 12µs)", 0);
- assertRows(execute("SELECT k, toJson(durationval) FROM %s WHERE k = ?", 0), row(0, "\"12us\""));
+ assertRows(execute("SELECT k, to_json(durationval) FROM %s WHERE k = ?", 0), row(0, "\"12us\""));
execute("INSERT INTO %s (k, durationval) VALUES (?, P1Y1M2DT10H5M)", 0);
- assertRows(execute("SELECT k, toJson(durationval) FROM %s WHERE k = ?", 0), row(0, "\"1y1mo2d10h5m\""));
+ assertRows(execute("SELECT k, to_json(durationval) FROM %s WHERE k = ?", 0), row(0, "\"1y1mo2d10h5m\""));
}
@Test
@@ -1024,8 +1038,8 @@
row("{\"foo\": 2}")
);
- assertRows(execute("SELECT JSON toJson(blobAsInt(intAsBlob(v))) FROM %s LIMIT 1"),
- row("{\"system.tojson(system.blobasint(system.intasblob(v)))\": \"0\"}")
+ assertRows(execute("SELECT JSON to_json(blob_as_int(int_as_blob(v))) FROM %s LIMIT 1"),
+ row("{\"system.to_json(system.blob_as_int(system.int_as_blob(v)))\": \"0\"}")
);
}
@@ -1268,11 +1282,11 @@
// map<set<text>, text> keys
String innerKey1 = "[\"0\", \"1\"]";
- String fullKey1 = String.format("{\"%s\": \"%s\"}", Json.quoteAsJsonString(innerKey1), "a");
- String stringKey1 = Json.quoteAsJsonString(fullKey1);
+ String fullKey1 = String.format("{\"%s\": \"%s\"}", JsonUtils.quoteAsJsonString(innerKey1), "a");
+ String stringKey1 = JsonUtils.quoteAsJsonString(fullKey1);
String innerKey2 = "[\"3\", \"4\"]";
- String fullKey2 = String.format("{\"%s\": \"%s\"}", Json.quoteAsJsonString(innerKey2), "b");
- String stringKey2 = Json.quoteAsJsonString(fullKey2);
+ String fullKey2 = String.format("{\"%s\": \"%s\"}", JsonUtils.quoteAsJsonString(innerKey2), "b");
+ String stringKey2 = JsonUtils.quoteAsJsonString(fullKey2);
execute("INSERT INTO %s JSON ?", "{\"k\": 0, \"nestedsetmap\": {\"" + stringKey1 + "\": true, \"" + stringKey2 + "\": false}}");
assertRows(execute("SELECT JSON k, nestedsetmap FROM %s"), row("{\"k\": 0, \"nestedsetmap\": {\"" + stringKey1 + "\": true, \"" + stringKey2 + "\": false}}"));
@@ -1357,11 +1371,11 @@
};
ExecutorService executor = Executors.newFixedThreadPool(numThreads);
- List<Future> futures = new ArrayList<>();
+ List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < numThreads; i++)
futures.add(executor.submit(worker));
- for (Future future : futures)
+ for (Future<?> future : futures)
future.get(30, TimeUnit.SECONDS);
executor.shutdown();
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
index 34b805d..77c611b 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
@@ -1184,39 +1184,39 @@
beforeAndAfterFlush(() -> {
// Test clustering columns restrictions
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) = (text_as_blob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) IN ((text_as_blob('')), (text_as_blob('1')));"),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("1")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) <= (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) < (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('') AND c < text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob('')) AND (c) < (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) > (text_as_blob('')) AND (c) < (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"));
});
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
@@ -1224,50 +1224,50 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) = (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) IN ((text_as_blob('')), (text_as_blob('1')));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) <= (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) < (text_as_blob('')) AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('') AND c < text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob('')) AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob('')) AND c < text_as_blob('') AND v = text_as_blob('1') ALLOW FILTERING;"));
// Test restrictions on non-primary key value
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('');"));
});
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
@@ -1275,7 +1275,7 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('');"),
row(bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER));
});
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java
index 13090a6..94baf82 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TimestampTest.java
@@ -17,11 +17,16 @@
*/
package org.apache.cassandra.cql3.validation.entities;
+import java.util.Arrays;
+import java.util.List;
+
import org.junit.Test;
import org.junit.Assert;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.utils.Pair;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -66,7 +71,7 @@
// wrap writetime(), ttl() in other functions (test for CASSANDRA-8451)
- res = getRows(execute("SELECT k, c, blobAsBigint(bigintAsBlob(writetime(c))), ttl(c) FROM %s"));
+ res = getRows(execute("SELECT k, c, blob_as_bigint(bigint_as_blob(writetime(c))), ttl(c) FROM %s"));
Assert.assertEquals(2, res.length);
for (Object[] r : res)
@@ -78,7 +83,7 @@
assertTrue(r[3] instanceof Integer || r[2] instanceof Long);
}
- res = getRows(execute("SELECT k, c, writetime(c), blobAsInt(intAsBlob(ttl(c))) FROM %s"));
+ res = getRows(execute("SELECT k, c, writetime(c), blob_as_int(int_as_blob(ttl(c))) FROM %s"));
Assert.assertEquals(2, res.length);
@@ -97,6 +102,84 @@
row(1, null, null));
}
+ private void setupSchemaForMaxTimestamp()
+ {
+ String myType = createType("CREATE TYPE %s (a int, b int)");
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, a text, " +
+ "l list<int>, fl frozen<list<int>>," +
+ "s set<int>, fs frozen<set<int>>," +
+ "m map<int, text>, fm frozen<map<int, text>>," +
+ "t " + myType + ", ft frozen<" + myType + ">)");
+ }
+
+ @Test
+ public void testCallMaxTimestampOnEmptyCollectionReturnsNull() throws Throwable
+ {
+ setupSchemaForMaxTimestamp();
+
+ execute("INSERT INTO %s (k) VALUES (1)");
+ Object[][] res = getRows(execute("SELECT maxwritetime(a), maxwritetime(l), maxwritetime(fl)," +
+ "maxwritetime(s), maxwritetime(fs), maxwritetime(m), maxwritetime(fm)," +
+ "maxwritetime(t), maxwritetime(ft) FROM %s WHERE k=1"));
+
+ assertEquals(1, res.length);
+ for (Object v : res[0])
+ {
+ assertNull("All the multi-cell data are empty (we did not insert), calling maxwritetime should return null",
+ v);
+ }
+ }
+
+ @Test
+ public void testMaxTimestamp() throws Throwable
+ {
+ setupSchemaForMaxTimestamp();
+
+ execute("INSERT INTO %s (k, a, l, fl, s, fs, m, fm, t, ft) VALUES " +
+ "(1, 'test', [1], [2], {1}, {2}, {1 : 'a'}, {2 : 'b'}, {a : 1, b : 1 }, {a : 2, b : 2}) USING TIMESTAMP 1");
+
+ // enumerate through all multi-cell types and make sure maxwritetime reflects the expected result
+ testMaxTimestampWithColumnUpdate(Arrays.asList(
+ Pair.create(1, "UPDATE %s USING TIMESTAMP 10 SET l = l + [10] WHERE k = 1"),
+ Pair.create(3, "UPDATE %s USING TIMESTAMP 11 SET s = s + {10} WHERE k = 1"),
+ Pair.create(5, "UPDATE %s USING TIMESTAMP 12 SET m = m + {10 : 'c'} WHERE k = 1"),
+ Pair.create(7, "UPDATE %s USING TIMESTAMP 13 SET t.a = 10 WHERE k = 1")
+ ));
+ }
+
+ private void testMaxTimestampWithColumnUpdate(List<Pair<Integer, String>> updateStatements) throws Throwable
+ {
+ for (Pair<Integer, String> update : updateStatements)
+ {
+ int fieldPos = update.left();
+ String statement = update.right();
+
+ // run the update statement and update the timestamp of the column
+ execute(statement);
+
+ Object[][] res = getRows(execute("SELECT maxwritetime(a), maxwritetime(l), maxwritetime(fl)," +
+ "maxwritetime(s), maxwritetime(fs), maxwritetime(m), maxwritetime(fm)," +
+ "maxwritetime(t), maxwritetime(ft) FROM %s WHERE k=1"));
+ Assert.assertEquals(1, res.length);
+ Assert.assertEquals("maxwritetime should work on both single cell and complex columns",
+ 9, res[0].length);
+ for (Object ts : res[0])
+ {
+ assertTrue(ts instanceof Long); // all the result fields are timestamps
+ }
+
+ long updatedTs = (long) res[0][fieldPos]; // maxwritetime the updated column
+
+ for (int i = 0; i < res[0].length; i++)
+ {
+ long ts = (long) res[0][i];
+ if (i != fieldPos)
+ assertTrue("The updated column should have a large maxwritetime since it is updated later",
+ ts < updatedTs);
+ }
+ }
+ }
+
/**
* Migrated from cql_tests.py:TestCQL.invalid_custom_timestamp_test()
*/
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TimeuuidTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TimeuuidTest.java
index 931451e..133ed4e 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TimeuuidTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TimeuuidTest.java
@@ -55,17 +55,18 @@
assertRowCount(execute("SELECT * FROM %s WHERE k = 0 AND t = ?", rows[0][1]), 1);
- assertInvalid("SELECT dateOf(k) FROM %s WHERE k = 0 AND t = ?", rows[0][1]);
+ assertInvalidMessage("k cannot be passed as argument 0 of function",
+ "SELECT min_timeuuid(k) FROM %s WHERE k = 0 AND t = ?", rows[0][1]);
for (int i = 0; i < 4; i++)
{
long timestamp = ((TimeUUID) rows[i][1]).unix(MILLISECONDS);
- assertRows(execute("SELECT dateOf(t), unixTimestampOf(t) FROM %s WHERE k = 0 AND t = ?", rows[i][1]),
+ assertRows(execute("SELECT to_timestamp(t), to_unix_timestamp(t) FROM %s WHERE k = 0 AND t = ?", rows[i][1]),
row(new Date(timestamp), timestamp));
}
- assertEmpty(execute("SELECT t FROM %s WHERE k = 0 AND t > maxTimeuuid(1234567) AND t < minTimeuuid('2012-11-07 18:18:22-0800')"));
- assertEmpty(execute("SELECT t FROM %s WHERE k = 0 AND t > maxTimeuuid(1564830182000) AND t < minTimeuuid('2012-11-07 18:18:22-0800')"));
+ assertEmpty(execute("SELECT t FROM %s WHERE k = 0 AND t > max_timeuuid(1234567) AND t < min_timeuuid('2012-11-07 18:18:22-0800')"));
+ assertEmpty(execute("SELECT t FROM %s WHERE k = 0 AND t > max_timeuuid(1564830182000) AND t < min_timeuuid('2012-11-07 18:18:22-0800')"));
}
/**
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
index 9f53db4..8e83fd2 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.utils.AbstractTypeGenerators.TypeSupport;
import org.quicktheories.core.Gen;
@@ -266,7 +267,7 @@
for (ByteBuffer value : testcase.uniqueRows)
{
map.put(value, count);
- ByteBuffer[] tupleBuffers = tupleType.split(value);
+ ByteBuffer[] tupleBuffers = tupleType.split(ByteBufferAccessor.instance, value);
// use cast to avoid warning
execute("INSERT INTO %s (id, value) VALUES (?, ?)", tuple((Object[]) tupleBuffers), count);
@@ -304,7 +305,7 @@
for (ByteBuffer value : testcase.uniqueRows)
{
map.put(value, count);
- ByteBuffer[] tupleBuffers = tupleType.split(value);
+ ByteBuffer[] tupleBuffers = tupleType.split(ByteBufferAccessor.instance, value);
// use cast to avoid warning
execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, tuple((Object[]) tupleBuffers), count);
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java
index ceb96b6..119a0fe 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TypeTest.java
@@ -49,12 +49,8 @@
{
createTable("CREATE TABLE %s (a int, b timestamp, c bigint, d varint, PRIMARY KEY (a, b, c, d))");
- execute("INSERT INTO %s (a, b, c, d) VALUES (0, toUnixTimestamp(now()), toTimestamp(now()), toTimestamp(now()))");
- UntypedResultSet results = execute("SELECT * FROM %s WHERE a=0 AND b <= toUnixTimestamp(now())");
- assertEquals(1, results.size());
-
- execute("INSERT INTO %s (a, b, c, d) VALUES (1, unixTimestampOf(now()), dateOf(now()), dateOf(now()))");
- results = execute("SELECT * FROM %s WHERE a=1 AND b <= toUnixTimestamp(now())");
+ execute("INSERT INTO %s (a, b, c, d) VALUES (0, to_unix_timestamp(now()), to_timestamp(now()), to_timestamp(now()))");
+ UntypedResultSet results = execute("SELECT * FROM %s WHERE a=0 AND b <= to_unix_timestamp(now())");
assertEquals(1, results.size());
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
index 4a2d71f..9681827 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java
@@ -28,9 +28,9 @@
import org.apache.cassandra.auth.*;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.functions.UserFunction;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.*;
-import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
import org.apache.cassandra.cql3.statements.BatchStatement;
import org.apache.cassandra.cql3.statements.ModificationStatement;
@@ -311,14 +311,14 @@
public void systemFunctionsRequireNoExplicitPrivileges() throws Throwable
{
// with terminal arguments, so evaluated at prepare time
- String cql = String.format("UPDATE %s SET v2 = 0 WHERE k = blobasint(intasblob(0)) and v1 = 0",
+ String cql = String.format("UPDATE %s SET v2 = 0 WHERE k = blob_as_int(int_as_blob(0)) and v1 = 0",
KEYSPACE + "." + currentTable());
getStatement(cql).authorize(clientState);
// with non-terminal arguments, so evaluated at execution
String functionName = createSimpleFunction();
grantExecuteOnFunction(functionName);
- cql = String.format("UPDATE %s SET v2 = 0 WHERE k = blobasint(intasblob(%s)) and v1 = 0",
+ cql = String.format("UPDATE %s SET v2 = 0 WHERE k = blob_as_int(int_as_blob(%s)) and v1 = 0",
KEYSPACE + "." + currentTable(),
functionCall(functionName));
getStatement(cql).authorize(clientState);
@@ -635,7 +635,7 @@
// It is here to avoid having to duplicate the functionality of CqlParser
// for transforming cql types into AbstractTypes
FunctionName fn = parseFunctionName(functionName);
- Collection<Function> functions = Schema.instance.getFunctions(fn);
+ Collection<UserFunction> functions = Schema.instance.getUserFunctions(fn);
assertEquals(String.format("Expected a single function definition for %s, but found %s",
functionName,
functions.size()),
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
index 47f1cbf..ad59b4c 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFJavaTest.java
@@ -657,7 +657,7 @@
"AS $$return " +
" udt.getString(\"txt\");$$;",
fName1replace, type));
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(fName1replace)).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(fName1replace)).size());
execute(String.format("CREATE OR REPLACE FUNCTION %s( udt %s ) " +
"CALLED ON NULL INPUT " +
"RETURNS int " +
@@ -665,7 +665,7 @@
"AS $$return " +
" Integer.valueOf(udt.getInt(\"i\"));$$;",
fName2replace, type));
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(fName2replace)).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(fName2replace)).size());
execute(String.format("CREATE OR REPLACE FUNCTION %s( udt %s ) " +
"CALLED ON NULL INPUT " +
"RETURNS double " +
@@ -673,7 +673,7 @@
"AS $$return " +
" Double.valueOf(udt.getDouble(\"added\"));$$;",
fName3replace, type));
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(fName3replace)).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(fName3replace)).size());
execute(String.format("CREATE OR REPLACE FUNCTION %s( udt %s ) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS %s " +
@@ -681,7 +681,7 @@
"AS $$return " +
" udt;$$;",
fName4replace, type, type));
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(fName4replace)).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(fName4replace)).size());
assertRows(execute("SELECT " + fName1replace + "(udt) FROM %s WHERE key = 2"),
row("two"));
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFPureScriptTupleCollectionTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFPureScriptTupleCollectionTest.java
deleted file mode 100644
index a1801b3..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFPureScriptTupleCollectionTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation.entities;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.junit.Test;
-
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.TupleType;
-import com.datastax.driver.core.TupleValue;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.transport.ProtocolVersion;
-
-public class UFPureScriptTupleCollectionTest extends CQLTester
-{
- // Just JavaScript UDFs to check how UDF - especially security/class-loading/sandboxing stuff -
- // behaves, if no Java UDF has been executed before.
-
- // Do not add any other test here!
- // See CASSANDRA-10141
-
- @Test
- public void testJavascriptTupleTypeCollection() throws Throwable
- {
- String tupleTypeDef = "tuple<double, list<double>, set<text>, map<int, boolean>>";
- createTable("CREATE TABLE %s (key int primary key, tup frozen<" + tupleTypeDef + ">)");
-
- String fTup1 = createFunction(KEYSPACE_PER_TEST, tupleTypeDef,
- "CREATE FUNCTION %s( tup " + tupleTypeDef + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS tuple<double, list<double>, set<text>, map<int, boolean>> " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " tup;$$;");
- String fTup2 = createFunction(KEYSPACE_PER_TEST, tupleTypeDef,
- "CREATE FUNCTION %s( tup " + tupleTypeDef + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " tup.getDouble(0);$$;");
- String fTup3 = createFunction(KEYSPACE_PER_TEST, tupleTypeDef,
- "CREATE FUNCTION %s( tup " + tupleTypeDef + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS list<double> " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " tup.getList(1, java.lang.Double.class);$$;");
- String fTup4 = createFunction(KEYSPACE_PER_TEST, tupleTypeDef,
- "CREATE FUNCTION %s( tup " + tupleTypeDef + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS set<text> " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " tup.getSet(2, java.lang.String.class);$$;");
- String fTup5 = createFunction(KEYSPACE_PER_TEST, tupleTypeDef,
- "CREATE FUNCTION %s( tup " + tupleTypeDef + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS map<int, boolean> " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " tup.getMap(3, java.lang.Integer.class, java.lang.Boolean.class);$$;");
-
- List<Double> list = Arrays.asList(1d, 2d, 3d);
- Set<String> set = new TreeSet<>(Arrays.asList("one", "three", "two"));
- Map<Integer, Boolean> map = new TreeMap<>();
- map.put(1, true);
- map.put(2, false);
- map.put(3, true);
-
- Object t = tuple(1d, list, set, map);
-
- execute("INSERT INTO %s (key, tup) VALUES (1, ?)", t);
-
- assertRows(execute("SELECT " + fTup1 + "(tup) FROM %s WHERE key = 1"),
- row(t));
- assertRows(execute("SELECT " + fTup2 + "(tup) FROM %s WHERE key = 1"),
- row(1d));
- assertRows(execute("SELECT " + fTup3 + "(tup) FROM %s WHERE key = 1"),
- row(list));
- assertRows(execute("SELECT " + fTup4 + "(tup) FROM %s WHERE key = 1"),
- row(set));
- assertRows(execute("SELECT " + fTup5 + "(tup) FROM %s WHERE key = 1"),
- row(map));
-
- // same test - but via native protocol
- // we use protocol V3 here to encode the expected version because the server
- // always serializes Collections using V3 - see CollectionSerializer's
- // serialize and deserialize methods.
- TupleType tType = tupleTypeOf(ProtocolVersion.V3,
- DataType.cdouble(),
- DataType.list(DataType.cdouble()),
- DataType.set(DataType.text()),
- DataType.map(DataType.cint(),
- DataType.cboolean()));
- TupleValue tup = tType.newValue(1d, list, set, map);
- for (ProtocolVersion version : PROTOCOL_VERSIONS)
- {
- assertRowsNet(version,
- executeNet(version, "SELECT " + fTup1 + "(tup) FROM %s WHERE key = 1"),
- row(tup));
- assertRowsNet(version,
- executeNet(version, "SELECT " + fTup2 + "(tup) FROM %s WHERE key = 1"),
- row(1d));
- assertRowsNet(version,
- executeNet(version, "SELECT " + fTup3 + "(tup) FROM %s WHERE key = 1"),
- row(list));
- assertRowsNet(version,
- executeNet(version, "SELECT " + fTup4 + "(tup) FROM %s WHERE key = 1"),
- row(set));
- assertRowsNet(version,
- executeNet(version, "SELECT " + fTup5 + "(tup) FROM %s WHERE key = 1"),
- row(map));
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
deleted file mode 100644
index fb33e6f..0000000
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.cql3.validation.entities;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.UUID;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.exceptions.FunctionExecutionException;
-import org.apache.cassandra.transport.ProtocolVersion;
-import org.apache.cassandra.utils.TimeUUID;
-
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-
-public class UFScriptTest extends CQLTester
-{
- // Just JavaScript UDFs to check how UDF - especially security/class-loading/sandboxing stuff -
- // behaves, if no Java UDF has been executed before.
-
- // Do not add any other test here - especially none using Java UDFs
-
- @Test
- public void testJavascriptSimpleCollections() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, lst list<double>, st set<text>, mp map<int, boolean>)");
-
- String fName1 = createFunction(KEYSPACE_PER_TEST, "list<double>",
- "CREATE FUNCTION %s( lst list<double> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS list<double> " +
- "LANGUAGE javascript\n" +
- "AS 'lst;';");
- String fName2 = createFunction(KEYSPACE_PER_TEST, "set<text>",
- "CREATE FUNCTION %s( st set<text> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS set<text> " +
- "LANGUAGE javascript\n" +
- "AS 'st;';");
- String fName3 = createFunction(KEYSPACE_PER_TEST, "map<int, boolean>",
- "CREATE FUNCTION %s( mp map<int, boolean> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS map<int, boolean> " +
- "LANGUAGE javascript\n" +
- "AS 'mp;';");
-
- List<Double> list = Arrays.asList(1d, 2d, 3d);
- Set<String> set = new TreeSet<>(Arrays.asList("one", "three", "two"));
- Map<Integer, Boolean> map = new TreeMap<>();
- map.put(1, true);
- map.put(2, false);
- map.put(3, true);
-
- execute("INSERT INTO %s (key, lst, st, mp) VALUES (1, ?, ?, ?)", list, set, map);
-
- assertRows(execute("SELECT lst, st, mp FROM %s WHERE key = 1"),
- row(list, set, map));
-
- assertRows(execute("SELECT " + fName1 + "(lst), " + fName2 + "(st), " + fName3 + "(mp) FROM %s WHERE key = 1"),
- row(list, set, map));
-
- for (ProtocolVersion version : PROTOCOL_VERSIONS)
- assertRowsNet(version,
- executeNet(version, "SELECT " + fName1 + "(lst), " + fName2 + "(st), " + fName3 + "(mp) FROM %s WHERE key = 1"),
- row(list, set, map));
- }
-
- @Test
- public void testJavascriptTupleType() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, tup frozen<tuple<double, text, int, boolean>>)");
-
- String fName = createFunction(KEYSPACE_PER_TEST, "tuple<double, text, int, boolean>",
- "CREATE FUNCTION %s( tup tuple<double, text, int, boolean> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS tuple<double, text, int, boolean> " +
- "LANGUAGE javascript\n" +
- "AS $$tup;$$;");
-
- Object t = tuple(1d, "foo", 2, true);
-
- execute("INSERT INTO %s (key, tup) VALUES (1, ?)", t);
-
- assertRows(execute("SELECT tup FROM %s WHERE key = 1"),
- row(t));
-
- assertRows(execute("SELECT " + fName + "(tup) FROM %s WHERE key = 1"),
- row(t));
- }
-
- @Test
- public void testJavascriptUserType() throws Throwable
- {
- String type = createType("CREATE TYPE %s (txt text, i int)");
-
- createTable("CREATE TABLE %s (key int primary key, udt frozen<" + type + ">)");
-
- String fUdt1 = createFunction(KEYSPACE, type,
- "CREATE FUNCTION %s( udt " + type + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " udt;$$;");
- String fUdt2 = createFunction(KEYSPACE, type,
- "CREATE FUNCTION %s( udt " + type + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " udt.getString(\"txt\");$$;");
- String fUdt3 = createFunction(KEYSPACE, type,
- "CREATE FUNCTION %s( udt " + type + " ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS int " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " udt.getInt(\"i\");$$;");
-
- execute("INSERT INTO %s (key, udt) VALUES (1, {txt: 'one', i:1})");
-
- UntypedResultSet rows = execute("SELECT " + fUdt1 + "(udt) FROM %s WHERE key = 1");
- Assert.assertEquals(1, rows.size());
- assertRows(execute("SELECT " + fUdt2 + "(udt) FROM %s WHERE key = 1"),
- row("one"));
- assertRows(execute("SELECT " + fUdt3 + "(udt) FROM %s WHERE key = 1"),
- row(1));
- }
-
- @Test
- public void testJavascriptUTCollections() throws Throwable
- {
- String type = createType("CREATE TYPE %s (txt text, i int)");
-
- createTable(String.format("CREATE TABLE %%s " +
- "(key int primary key, lst list<frozen<%s>>, st set<frozen<%s>>, mp map<int, frozen<%s>>)",
- type, type, type));
-
- String fName = createFunction(KEYSPACE, "list<frozen<" + type + ">>",
- "CREATE FUNCTION %s( lst list<frozen<" + type + ">> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " lst.get(1).getString(\"txt\");$$;");
- createFunctionOverload(fName, "set<frozen<" + type + ">>",
- "CREATE FUNCTION %s( st set<frozen<" + type + ">> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " st.iterator().next().getString(\"txt\");$$;");
- createFunctionOverload(fName, "map<int, frozen<" + type + ">>",
- "CREATE FUNCTION %s( mp map<int, frozen<" + type + ">> ) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS $$" +
- " mp.get(java.lang.Integer.valueOf(3)).getString(\"txt\");$$;");
-
- execute("INSERT INTO %s (key, lst, st, mp) values (1, " +
- // list<frozen<UDT>>
- "[ {txt: 'one', i:1}, {txt: 'three', i:1}, {txt: 'one', i:1} ] , " +
- // set<frozen<UDT>>
- "{ {txt: 'one', i:1}, {txt: 'three', i:3}, {txt: 'two', i:2} }, " +
- // map<int, frozen<UDT>>
- "{ 1: {txt: 'one', i:1}, 2: {txt: 'one', i:3}, 3: {txt: 'two', i:2} })");
-
- assertRows(execute("SELECT " + fName + "(lst) FROM %s WHERE key = 1"),
- row("three"));
- assertRows(execute("SELECT " + fName + "(st) FROM %s WHERE key = 1"),
- row("one"));
- assertRows(execute("SELECT " + fName + "(mp) FROM %s WHERE key = 1"),
- row("two"));
-
- String cqlSelect = "SELECT " + fName + "(lst), " + fName + "(st), " + fName + "(mp) FROM %s WHERE key = 1";
- assertRows(execute(cqlSelect),
- row("three", "one", "two"));
-
- // same test - but via native protocol
- for (ProtocolVersion version : PROTOCOL_VERSIONS)
- assertRowsNet(version,
- executeNet(version, cqlSelect),
- row("three", "one", "two"));
- }
-
- @Test
- public void testJavascriptFunction() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- String functionBody = '\n' +
- " Math.sin(val);\n";
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS '" + functionBody + "';");
-
- FunctionName fNameName = parseFunctionName(fName);
-
- assertRows(execute("SELECT language, body FROM system_schema.functions WHERE keyspace_name=? AND function_name=?",
- fNameName.keyspace, fNameName.name),
- row("javascript", functionBody));
-
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 2, 2d);
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 3, 3d);
- assertRows(execute("SELECT key, val, " + fName + "(val) FROM %s"),
- row(1, 1d, Math.sin(1d)),
- row(2, 2d, Math.sin(2d)),
- row(3, 3d, Math.sin(3d))
- );
- }
-
- @Test
- public void testJavascriptBadReturnType() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS '\"string\";';");
-
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
- // throws IRE with ClassCastException
- assertInvalidMessage("Invalid value for CQL type double", "SELECT key, val, " + fName + "(val) FROM %s");
- }
-
- @Test
- public void testJavascriptThrow() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS 'throw \"fool\";';");
-
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
- // throws IRE with ScriptException
- assertInvalidThrowMessage("fool", FunctionExecutionException.class,
- "SELECT key, val, " + fName + "(val) FROM %s");
- }
-
- @Test
- public void testScriptReturnTypeCasting() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
- execute("INSERT INTO %s (key, val) VALUES (?, ?)", 1, 1d);
-
- Object[][] variations = {
- new Object[] { "true", "boolean", true },
- new Object[] { "false", "boolean", false },
- new Object[] { "100", "tinyint", (byte)100 },
- new Object[] { "100.", "tinyint", (byte)100 },
- new Object[] { "100", "smallint", (short)100 },
- new Object[] { "100.", "smallint", (short)100 },
- new Object[] { "100", "int", 100 },
- new Object[] { "100.", "int", 100 },
- new Object[] { "100", "double", 100d },
- new Object[] { "100.", "double", 100d },
- new Object[] { "100", "bigint", 100L },
- new Object[] { "100.", "bigint", 100L },
- new Object[] { "100", "varint", BigInteger.valueOf(100L) },
- new Object[] { "100.", "varint", BigInteger.valueOf(100L) },
- new Object[] { "parseInt(\"100\");", "decimal", BigDecimal.valueOf(100d) },
- new Object[] { "100.", "decimal", BigDecimal.valueOf(100d) },
- };
-
- for (Object[] variation : variations)
- {
- Object functionBody = variation[0];
- Object returnType = variation[1];
- Object expectedResult = variation[2];
-
- String fName = createFunction(KEYSPACE, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " +returnType + ' ' +
- "LANGUAGE javascript " +
- "AS '" + functionBody + ";';");
- assertRows(execute("SELECT key, val, " + fName + "(val) FROM %s"),
- row(1, 1d, expectedResult));
- }
- }
-
- @Test
- public void testScriptParamReturnTypes() throws Throwable
- {
- UUID ruuid = UUID.randomUUID();
- TimeUUID tuuid = nextTimeUUID();
-
- createTable("CREATE TABLE %s (key int primary key, " +
- "tival tinyint, sival smallint, ival int, lval bigint, fval float, dval double, vval varint, ddval decimal, " +
- "timval time, dtval date, tsval timestamp, uval uuid, tuval timeuuid)");
- execute("INSERT INTO %s (key, tival, sival, ival, lval, fval, dval, vval, ddval, timval, dtval, tsval, uval, tuval) VALUES " +
- "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 1,
- (byte)1, (short)1, 1, 1L, 1f, 1d, BigInteger.valueOf(1L), BigDecimal.valueOf(1d), 1L, Integer.MAX_VALUE, new Date(1), ruuid, tuuid);
-
- Object[][] variations = {
- new Object[] { "tinyint", "tival", (byte)1, (byte)2 },
- new Object[] { "smallint", "sival", (short)1, (short)2 },
- new Object[] { "int", "ival", 1, 2 },
- new Object[] { "bigint", "lval", 1L, 2L },
- new Object[] { "float", "fval", 1f, 2f },
- new Object[] { "double", "dval", 1d, 2d },
- new Object[] { "varint", "vval", BigInteger.valueOf(1L), BigInteger.valueOf(2L) },
- new Object[] { "decimal", "ddval", BigDecimal.valueOf(1d), BigDecimal.valueOf(2d) },
- new Object[] { "time", "timval", 1L, 2L },
- };
-
- for (Object[] variation : variations)
- {
- Object type = variation[0];
- Object col = variation[1];
- Object expected1 = variation[2];
- Object expected2 = variation[3];
- String fName = createFunction(KEYSPACE, type.toString(),
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE javascript " +
- "AS 'val+1;';");
- assertRows(execute("SELECT key, " + col + ", " + fName + '(' + col + ") FROM %s"),
- row(1, expected1, expected2));
- }
-
- variations = new Object[][] {
- new Object[] { "timestamp","tsval", new Date(1), new Date(1) },
- new Object[] { "uuid", "uval", ruuid, ruuid },
- new Object[] { "timeuuid", "tuval", tuuid, tuuid },
- new Object[] { "date", "dtval", Integer.MAX_VALUE, Integer.MAX_VALUE },
- };
-
- for (Object[] variation : variations)
- {
- Object type = variation[0];
- Object col = variation[1];
- Object expected1 = variation[2];
- Object expected2 = variation[3];
- String fName = createFunction(KEYSPACE, type.toString(),
- "CREATE OR REPLACE FUNCTION %s(val " + type + ") " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS " + type + ' ' +
- "LANGUAGE javascript " +
- "AS 'val;';");
- assertRows(execute("SELECT key, " + col + ", " + fName + '(' + col + ") FROM %s"),
- row(1, expected1, expected2));
- }
- }
-
- @Test
- public void testJavascriptDisabled() throws Throwable
- {
- createTable("CREATE TABLE %s (key int primary key, val double)");
-
- DatabaseDescriptor.enableScriptedUserDefinedFunctions(false);
- try
- {
- assertInvalid("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS 'Math.sin(val);';");
- }
- finally
- {
- DatabaseDescriptor.enableScriptedUserDefinedFunctions(true);
- }
- }
-
- @Test
- public void testJavascriptCompileFailure() throws Throwable
- {
- assertInvalidMessage("Failed to compile function 'cql_test_keyspace.scrinv'",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS 'foo bar';");
- }
-
- @Test
- public void testScriptInvalidLanguage() throws Throwable
- {
- assertInvalidMessage("Invalid language 'artificial_intelligence' for function 'cql_test_keyspace.scrinv'",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE artificial_intelligence\n" +
- "AS 'question for 42?';");
- }
-}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
index 7422db43..fe1ec70 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
@@ -48,13 +48,13 @@
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
"LANGUAGE JAVA\n" +
- "AS 'System.getProperty(\"foo.bar.baz\"); return 0d;';");
+ "AS 'System.getProperty(\"foo.bar.baz\"); return 0d;';"); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
Assert.fail();
}
catch (FunctionExecutionException e)
{
- assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e);
+ assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
}
String[] cfnSources =
@@ -130,74 +130,6 @@
"LANGUAGE JAVA\n" +
"AS '" + typeAndSource[1] + "';");
}
-
- // JavaScript UDFs
-
- try
- {
- String fName = createFunction(KEYSPACE_PER_TEST, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS 'org.apache.cassandra.service.StorageService.instance.isShutdown(); 0;';");
- execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
- Assert.fail("Javascript security check failed");
- }
- catch (FunctionExecutionException e)
- {
- assertAccessControlException("", e);
- }
-
- String[] javascript =
- {
- "java.lang.management.ManagmentFactory.getThreadMXBean(); 0;",
- "new java.io.FileInputStream(\"/tmp/foo\"); 0;",
- "new java.io.FileOutputStream(\"/tmp/foo\"); 0;",
- "java.nio.file.FileSystems.getDefault().createFileExclusively(\"./foo_bar_baz\"); 0;",
- "java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")); 0;",
- "java.nio.channels.SocketChannel.open(); 0;",
- "new java.net.ServerSocket().bind(null); 0;",
- "var thread = new java.lang.Thread(); thread.start(); 0;",
- "java.lang.System.getProperty(\"foo.bar.baz\"); 0;",
- "java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); 0;",
- "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
- "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
- // TODO these (ugly) calls are still possible - these can consume CPU (as one could do with an evil loop, too)
-// "java.lang.Runtime.getRuntime().traceMethodCalls(true); 0;",
-// "java.lang.Runtime.getRuntime().gc(); 0;",
-// "java.lang.Runtime.getRuntime(); 0;",
- };
-
- for (String script : javascript)
- {
- try
- {
- String fName = createFunction(KEYSPACE_PER_TEST, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS '" + script + "';");
- execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
- Assert.fail("Javascript security check failed: " + script);
- }
- catch (FunctionExecutionException e)
- {
- assertAccessControlException(script, e);
- }
- }
-
- String script = "java.lang.Class.forName(\"java.lang.System\"); 0;";
- String fName = createFunction(KEYSPACE_PER_TEST, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE javascript\n" +
- "AS '" + script + "';");
- assertInvalidThrowMessage("Java reflection not supported when class filter is present",
- FunctionExecutionException.class,
- "SELECT " + fName + "(dval) FROM %s WHERE key=1");
}
private static void assertAccessControlException(String script, FunctionExecutionException e)
@@ -250,16 +182,6 @@
"AS 'long t=System.currentTimeMillis()+500; while (t>System.currentTimeMillis()) { }; return 0d;';");
assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
- // Javascript UDF
-
- fName = createFunction(KEYSPACE_PER_TEST, "double",
- "CREATE OR REPLACE FUNCTION %s(val double) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS double " +
- "LANGUAGE JAVASCRIPT\n" +
- "AS 'var t=java.lang.System.currentTimeMillis()+500; while (t>java.lang.System.currentTimeMillis()) { }; 0;';");
- assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
-
return;
}
catch (Error | RuntimeException e)
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
index e86b3df..1a9e6d2 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
@@ -87,17 +87,17 @@
}
@Test
- public void testSchemaChange() throws Throwable
+ public void testSchemaChange()
{
String f = createFunctionName(KEYSPACE);
String functionName = shortFunctionName(f);
registerFunction(f, "double, double");
- assertSchemaChange("CREATE OR REPLACE FUNCTION " + f + "(state double, val double) " +
- "RETURNS NULL ON NULL INPUT " +
+ assertSchemaChange("CREATE OR REPLACE FUNCTION " + f + "(state double, val double)" +
+ "CALLED ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS '\"string\";';",
+ "LANGUAGE java " +
+ "AS ' return Double.valueOf(Math.max(state.doubleValue(), val.doubleValue())); ';",
Change.CREATED,
Target.FUNCTION,
KEYSPACE, functionName,
@@ -108,22 +108,22 @@
assertSchemaChange("CREATE OR REPLACE FUNCTION " + f + "(state int, val int) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS int " +
- "LANGUAGE javascript " +
- "AS '\"string\";';",
+ "LANGUAGE java " +
+ "AS ' return Integer.valueOf(Math.max(state, val));';",
Change.CREATED,
Target.FUNCTION,
KEYSPACE, functionName,
"int", "int");
- assertSchemaChange("CREATE OR REPLACE FUNCTION " + f + "(state int, val int) " +
+ /*assertSchemaChange("CREATE OR REPLACE FUNCTION " + f + "(state int, val int) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS int " +
- "LANGUAGE javascript " +
- "AS '\"string1\";';",
+ "LANGUAGE java " +
+ "AS ' return Integer.valueOf(Math.max(state, val));';",
Change.UPDATED,
Target.FUNCTION,
KEYSPACE, functionName,
- "int", "int");
+ "int", "int");*/
assertSchemaChange("DROP FUNCTION " + f + "(double, double)",
Change.DROPPED, Target.FUNCTION,
@@ -137,8 +137,8 @@
assertSchemaChange("CREATE OR REPLACE FUNCTION " + fl + "(state list<tuple<int, int>>, val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS '\"string\";';",
+ "LANGUAGE java " +
+ "AS ' return val;';",
Change.CREATED, Target.FUNCTION,
KEYSPACE, shortFunctionName(fl),
"list<tuple<int, int>>", "double");
@@ -156,7 +156,7 @@
FunctionName fSinName = parseFunctionName(fSin);
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(fSin)).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(fSin)).size());
assertRows(execute("SELECT function_name, language FROM system_schema.functions WHERE keyspace_name=?", KEYSPACE_PER_TEST),
row(fSinName.name, "java"));
@@ -165,7 +165,7 @@
assertRows(execute("SELECT function_name, language FROM system_schema.functions WHERE keyspace_name=?", KEYSPACE_PER_TEST));
- Assert.assertEquals(0, Schema.instance.getFunctions(fSinName).size());
+ Assert.assertEquals(0, Schema.instance.getUserFunctions(fSinName).size());
}
@Test
@@ -182,7 +182,7 @@
FunctionName fSinName = parseFunctionName(fSin);
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(fSin)).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(fSin)).size());
// create a pairs of Select and Inserts. One statement in each pair uses the function so when we
// drop it those statements should be removed from the cache in QueryProcessor. The other statements
@@ -220,7 +220,7 @@
"LANGUAGE java " +
"AS 'return Double.valueOf(Math.sin(input));'");
- Assert.assertEquals(1, Schema.instance.getFunctions(fSinName).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(fSinName).size());
preparedSelect1= QueryProcessor.instance.prepare(
String.format("SELECT key, %s(d) FROM %s.%s", fSin, KEYSPACE, currentTable()),
@@ -333,9 +333,9 @@
"CREATE FUNCTION %s ( input double ) " +
"CALLED ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS 'input'");
- Assert.assertEquals(1, Schema.instance.getFunctions(parseFunctionName(function)).size());
+ "LANGUAGE java " +
+ "AS 'return Double.valueOf(Math.log(input.doubleValue()));'");
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(parseFunctionName(function)).size());
List<ResultMessage.Prepared> prepared = new ArrayList<>();
// prepare statements which use the function to provide a DelayedValue
@@ -438,7 +438,7 @@
execute("DROP FUNCTION IF EXISTS " + fSin);
// can't drop native functions
- assertInvalidMessage("System keyspace 'system' is not user-modifiable", "DROP FUNCTION totimestamp");
+ assertInvalidMessage("System keyspace 'system' is not user-modifiable", "DROP FUNCTION to_timestamp");
assertInvalidMessage("System keyspace 'system' is not user-modifiable", "DROP FUNCTION uuid");
// sin() no longer exists
@@ -665,7 +665,7 @@
@Test
public void testFunctionInSystemKS() throws Throwable
{
- execute("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".totimestamp(val timeuuid) " +
+ execute("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".to_timestamp(val timeuuid) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS timestamp " +
"LANGUAGE JAVA\n" +
@@ -679,7 +679,7 @@
"LANGUAGE JAVA\n" +
"AS 'return null;';");
assertInvalidMessage("System keyspace 'system' is not user-modifiable",
- "CREATE OR REPLACE FUNCTION system.totimestamp(val timeuuid) " +
+ "CREATE OR REPLACE FUNCTION system.to_timestamp(val timeuuid) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS timestamp " +
"LANGUAGE JAVA\n" +
@@ -696,7 +696,7 @@
"LANGUAGE JAVA\n" +
"AS 'return null;';");
assertInvalidMessage("System keyspace 'system' is not user-modifiable",
- "CREATE OR REPLACE FUNCTION totimestamp(val timeuuid) " +
+ "CREATE OR REPLACE FUNCTION to_timestamp(val timeuuid) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS timestamp " +
"LANGUAGE JAVA\n" +
@@ -769,7 +769,7 @@
FunctionName fNameName = parseFunctionName(fName);
- Assert.assertEquals(1, Schema.instance.getFunctions(fNameName).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(fNameName).size());
ResultMessage.Prepared prepared = QueryProcessor.instance.prepare(String.format("SELECT key, %s(udt) FROM %s.%s", fName, KEYSPACE, currentTable()),
ClientState.forInternalCalls());
@@ -786,18 +786,17 @@
Assert.assertNull(QueryProcessor.instance.getPrepared(prepared.statementId));
// function stays
- Assert.assertEquals(1, Schema.instance.getFunctions(fNameName).size());
+ Assert.assertEquals(1, Schema.instance.getUserFunctions(fNameName).size());
}
@Test
public void testDuplicateArgNames() throws Throwable
{
assertInvalidMessage("Duplicate argument names for given function",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double, val text) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript\n" +
- "AS '\"foo bar\";';");
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(input double, input int) " +
+ "CALLED ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE java AS 'return Math.max(input, input)';");
}
@Test
@@ -855,7 +854,7 @@
"AS 'throw new RuntimeException();';");
KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE_PER_TEST);
- UDFunction f = (UDFunction) ksm.functions.get(parseFunctionName(fName)).iterator().next();
+ UDFunction f = (UDFunction) ksm.userFunctions.get(parseFunctionName(fName)).iterator().next();
UDFunction broken = UDFunction.createBrokenFunction(f.name(),
f.argNames(),
@@ -865,7 +864,7 @@
"java",
f.body(),
new InvalidRequestException("foo bar is broken"));
- SchemaTestUtil.addOrUpdateKeyspace(ksm.withSwapped(ksm.functions.without(f.name(), f.argTypes()).with(broken)), false);
+ SchemaTestUtil.addOrUpdateKeyspace(ksm.withSwapped(ksm.userFunctions.without(f.name(), f.argTypes()).with(broken)), false);
assertInvalidThrowMessage("foo bar is broken", InvalidRequestException.class,
"SELECT key, " + fName + "(dval) FROM %s");
@@ -908,7 +907,7 @@
public void testEmptyString() throws Throwable
{
createTable("CREATE TABLE %s (key int primary key, sval text, aval ascii, bval blob, empty_int int)");
- execute("INSERT INTO %s (key, sval, aval, bval, empty_int) VALUES (?, ?, ?, ?, blobAsInt(0x))", 1, "", "", ByteBuffer.allocate(0));
+ execute("INSERT INTO %s (key, sval, aval, bval, empty_int) VALUES (?, ?, ?, ?, blob_as_int(0x))", 1, "", "", ByteBuffer.allocate(0));
String fNameSRC = createFunction(KEYSPACE_PER_TEST, "text",
"CREATE OR REPLACE FUNCTION %s(val text) " +
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
index 0b05e8f..0d99494 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
@@ -22,6 +22,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -32,6 +33,8 @@
@BeforeClass
public static void setUpClass() // overrides CQLTester.setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
// Selecting partitioner for a table is not exposed on CREATE TABLE.
StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
@@ -200,6 +203,56 @@
}
@Test
+ public void testNullsInIntUDT() throws Throwable
+ {
+ String myType = KEYSPACE + '.' + createType("CREATE TYPE %s (a int)");
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b frozen<" + myType + ">)");
+ execute("INSERT INTO %s (a, b) VALUES (1, ?)", userType("a", 1));
+
+ assertRows(execute("SELECT b.a FROM %s"), row(1));
+
+ flush();
+
+ schemaChange("ALTER TYPE " + myType + " ADD b int");
+ execute("INSERT INTO %s (a, b) VALUES (2, {a: 2, b: 2})");
+ execute("INSERT INTO %s (a, b) VALUES (3, {b: 3})");
+ execute("INSERT INTO %s (a, b) VALUES (4, {a: null, b: 4})");
+
+ beforeAndAfterFlush(() ->
+ assertRows(execute("SELECT b.a, b.b FROM %s"),
+ row(1, null),
+ row(2, 2),
+ row(null, 3),
+ row(null, 4))
+ );
+ }
+
+ @Test
+ public void testNullsInTextUDT() throws Throwable
+ {
+ String myType = KEYSPACE + '.' + createType("CREATE TYPE %s (a text)");
+ createTable("CREATE TABLE %s (a int PRIMARY KEY, b frozen<" + myType + ">)");
+ execute("INSERT INTO %s (a, b) VALUES (1, {a: ''})");
+
+ assertRows(execute("SELECT b.a FROM %s"), row(""));
+
+ flush();
+
+ schemaChange("ALTER TYPE " + myType + " ADD b text");
+ execute("INSERT INTO %s (a, b) VALUES (2, {a: '', b: ''})");
+ execute("INSERT INTO %s (a, b) VALUES (3, {b: ''})");
+ execute("INSERT INTO %s (a, b) VALUES (4, {a: null, b: ''})");
+
+ beforeAndAfterFlush(() ->
+ assertRows(execute("SELECT b.a, b.b FROM %s"),
+ row("", null),
+ row("", ""),
+ row(null, ""),
+ row(null, ""))
+ );
+ }
+
+ @Test
public void testAlterNonFrozenUDT() throws Throwable
{
String myType = KEYSPACE + '.' + createType("CREATE TYPE %s (a int, b text)");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java
index 5d3b134..b2da6e5 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/VirtualTableTest.java
@@ -38,6 +38,8 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.marshal.Int32Type;
@@ -56,7 +58,9 @@
import org.apache.cassandra.triggers.ITrigger;
+import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class VirtualTableTest extends CQLTester
@@ -66,6 +70,7 @@
private static final String VT2_NAME = "vt2";
private static final String VT3_NAME = "vt3";
private static final String VT4_NAME = "vt4";
+ private static final String VT5_NAME = "vt5";
// As long as we execute test queries using execute (and not executeNet) the virtual tables implementation
// do not need to be thread-safe. We choose to do it to avoid issues if the test framework was changed or somebody
@@ -204,6 +209,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
TableMetadata vt1Metadata = TableMetadata.builder(KS_NAME, VT1_NAME)
.kind(TableMetadata.Kind.VIRTUAL)
.addPartitionKeyColumn("pk", UTF8Type.instance)
@@ -343,7 +350,27 @@
};
- VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(vt1, vt2, vt3, vt4)));
+ VirtualTable vt5 = new AbstractVirtualTable(TableMetadata.builder(KS_NAME, VT5_NAME)
+ .kind(TableMetadata.Kind.VIRTUAL)
+ .addPartitionKeyColumn("pk", UTF8Type.instance)
+ .addClusteringColumn("c", UTF8Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", LongType.instance)
+ .build())
+ {
+ public DataSet data()
+ {
+ return new SimpleDataSet(metadata());
+ }
+
+ @Override
+ public boolean allowFilteringImplicitly()
+ {
+ return false;
+ }
+ };
+
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(vt1, vt2, vt3, vt4, vt5)));
CQLTester.setUpClass();
}
@@ -1025,6 +1052,46 @@
assertJMXFails(() -> mbean.getAutoCompactionStatus(KS_NAME, VT1_NAME));
}
+ @Test
+ public void testDisallowedFilteringOnRegularColumn() throws Throwable
+ {
+ try
+ {
+ executeNet(format("SELECT * FROM %s.%s WHERE v2 = 5", KS_NAME, VT5_NAME));
+ fail(format("should fail as %s.%s is not allowed to be filtered on implicitly.", KS_NAME, VT5_NAME));
+ }
+ catch (InvalidQueryException ex)
+ {
+ assertTrue(ex.getMessage().contains("Cannot execute this query as it might involve data filtering and thus may have unpredictable performance"));
+ }
+ }
+
+ @Test
+ public void testDisallowedFilteringOnClusteringColumn() throws Throwable
+ {
+ try
+ {
+ executeNet(format("SELECT * FROM %s.%s WHERE c = 'abc'", KS_NAME, VT5_NAME));
+ fail(format("should fail as %s.%s is not allowed to be filtered on implicitly.", KS_NAME, VT5_NAME));
+ }
+ catch (InvalidQueryException ex)
+ {
+ assertTrue(ex.getMessage().contains("Cannot execute this query as it might involve data filtering and thus may have unpredictable performance"));
+ }
+ }
+
+ @Test
+ public void testAllowedFilteringOnRegularColumn() throws Throwable
+ {
+ executeNet(format("SELECT * FROM %s.%s WHERE v2 = 5", KS_NAME, VT1_NAME));
+ }
+
+ @Test
+ public void testAllowedFilteringOnClusteringColumn() throws Throwable
+ {
+ executeNet(format("SELECT * FROM %s.%s WHERE c = 'abc'", KS_NAME, VT1_NAME));
+ }
+
@FunctionalInterface
private static interface ThrowingRunnable
{
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java
index cc6c663..571196d 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/WritetimeOrTTLTest.java
@@ -18,17 +18,27 @@
package org.apache.cassandra.cql3.validation.entities;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import static java.lang.String.format;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static java.lang.String.format;
+/**
+ * Tests for CQL's {@code WRITETIME}, {@code MAXWRITETIME} and {@code TTL} selection functions.
+ */
public class WritetimeOrTTLTest extends CQLTester
{
private static final long TIMESTAMP_1 = 1;
@@ -70,7 +80,42 @@
public void testList() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<int>)");
- assertInvalidMultiCellSelection("l", true);
+
+ // Null column
+ execute("INSERT INTO %s (k) VALUES (1) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("l", NO_TIMESTAMP, NO_TTL);
+
+ // Create empty
+ execute("INSERT INTO %s (k, l) VALUES (1, []) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("l", NO_TIMESTAMP, NO_TTL);
+
+ // Create with a single element without TTL
+ execute("INSERT INTO %s (k, l) VALUES (1, [1]) USING TIMESTAMP ?", TIMESTAMP_1);
+ assertWritetimeAndTTL("l", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+
+ // Add a new element to the list with a new timestamp and a TTL
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET l=l+[2] WHERE k=1", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("l", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+
+ assertInvalidListElementSelection("l[0]", "l");
+ assertInvalidListSliceSelection("l[..0]", "l");
+ assertInvalidListSliceSelection("l[0..]", "l");
+ assertInvalidListSliceSelection("l[1..1]", "l");
+ assertInvalidListSliceSelection("l[1..2]", "l");
+
+ // Read multiple rows to verify selector reset
+ execute("TRUNCATE TABLE %s");
+ execute("INSERT INTO %s (k, l) VALUES (1, [1, 2, 3]) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, l) VALUES (2, [1, 2]) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, l) VALUES (3, [1]) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, l) VALUES (4, []) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, l) VALUES (5, null) USING TIMESTAMP ?", TIMESTAMP_2);
+ assertRows("SELECT k, WRITETIME(l) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, timestamps(TIMESTAMP_1, TIMESTAMP_1, TIMESTAMP_1)),
+ row(2, timestamps(TIMESTAMP_2, TIMESTAMP_2)),
+ row(4, NO_TIMESTAMP),
+ row(3, timestamps(TIMESTAMP_1)));
}
@Test
@@ -86,6 +131,9 @@
execute("INSERT INTO %s (k, v) VALUES (1, []) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("v", TIMESTAMP_1, TTL_1);
+ // truncate, since previous columns would win on reconcilliation because of their TTL (CASSANDRA-14592)
+ execute("TRUNCATE TABLE %s");
+
// Update with a single element without TTL
execute("INSERT INTO %s (k, v) VALUES (1, [1]) USING TIMESTAMP ?", TIMESTAMP_1);
assertWritetimeAndTTL("v", TIMESTAMP_1, NO_TTL);
@@ -93,13 +141,119 @@
// Add a new element to the list with a new timestamp and a TTL
execute("INSERT INTO %s (k, v) VALUES (1, [1, 2, 3]) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_2, TTL_2);
assertWritetimeAndTTL("v", TIMESTAMP_2, TTL_2);
+
+ assertInvalidListElementSelection("v[1]", "v");
+ assertInvalidListSliceSelection("v[..0]", "v");
+ assertInvalidListSliceSelection("v[0..]", "v");
+ assertInvalidListSliceSelection("v[1..1]", "v");
+ assertInvalidListSliceSelection("v[1..2]", "v");
+
+ // Read multiple rows to verify selector reset
+ execute("TRUNCATE TABLE %s");
+ execute("INSERT INTO %s (k, v) VALUES (1, [1, 2, 3]) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, v) VALUES (2, [1, 2]) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, v) VALUES (3, [1]) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, v) VALUES (4, []) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, v) VALUES (5, null) USING TIMESTAMP ?", TIMESTAMP_2);
+ assertRows("SELECT k, WRITETIME(v) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, TIMESTAMP_2),
+ row(3, TIMESTAMP_1));
}
@Test
public void testSet() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<int>)");
- assertInvalidMultiCellSelection("s", true);
+
+ // Null column
+ execute("INSERT INTO %s (k) VALUES (1) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("s", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+
+ // Create empty
+ execute("INSERT INTO %s (k, s) VALUES (1, {}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("s", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+
+ // Update with a single element without TTL
+ execute("INSERT INTO %s (k, s) VALUES (1, {1}) USING TIMESTAMP ?", TIMESTAMP_1);
+ assertWritetimeAndTTL("s", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[..2]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[0..]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[1..]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[2..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[1..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[1..2]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[2..2]", NO_TIMESTAMP, NO_TTL);
+
+ // Add a new element to the set with a new timestamp and a TTL
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET s=s+{2} WHERE k=1", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[0..]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[1..]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[2..]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("s[3..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[0..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[0..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[1..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("s[1..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[1..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("s[2..2]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("s[2..3]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("s[3..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[3..4]", NO_TIMESTAMP, NO_TTL);
+
+ // Combine timestamp selection with other selections and orders
+ assertRows("SELECT k, WRITETIME(s[1]) FROM %s", row(1, TIMESTAMP_1));
+ assertRows("SELECT WRITETIME(s[1]), k FROM %s", row(TIMESTAMP_1, 1));
+ assertRows("SELECT WRITETIME(s[1]), WRITETIME(s[2]) FROM %s", row(TIMESTAMP_1, TIMESTAMP_2));
+ assertRows("SELECT WRITETIME(s[2]), WRITETIME(s[1]) FROM %s", row(TIMESTAMP_2, TIMESTAMP_1));
+
+ // Read multiple rows to verify selector reset
+ execute("TRUNCATE TABLE %s");
+ execute("INSERT INTO %s (k, s) VALUES (1, {1, 2, 3}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, s) VALUES (2, {1, 2}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, s) VALUES (3, {1}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, s) VALUES (4, {}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, s) VALUES (5, null) USING TIMESTAMP ?", TIMESTAMP_2);
+ assertRows("SELECT k, WRITETIME(s) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, timestamps(TIMESTAMP_1, TIMESTAMP_1, TIMESTAMP_1)),
+ row(2, timestamps(TIMESTAMP_2, TIMESTAMP_2)),
+ row(4, NO_TIMESTAMP),
+ row(3, timestamps(TIMESTAMP_1)));
+ assertRows("SELECT k, WRITETIME(s[1]) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, NO_TIMESTAMP),
+ row(3, TIMESTAMP_1));
}
@Test
@@ -110,25 +264,182 @@
// Null column
execute("INSERT INTO %s (k) VALUES (1) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("s", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
// Create empty
execute("INSERT INTO %s (k, s) VALUES (1, {}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("s", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+
+ // truncate, since previous columns would win on reconcilliation because of their TTL (CASSANDRA-14592)
+ execute("TRUNCATE TABLE %s");
// Update with a single element without TTL
execute("INSERT INTO %s (k, s) VALUES (1, {1}) USING TIMESTAMP ?", TIMESTAMP_1);
assertWritetimeAndTTL("s", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[..2]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[0..]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[1..]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[2..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[0..2]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[1..1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[1..2]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("s[2..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[2..3]", NO_TIMESTAMP, NO_TTL);
// Add a new element to the set with a new timestamp and a TTL
execute("INSERT INTO %s (k, s) VALUES (1, {1, 2}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_2, TTL_2);
assertWritetimeAndTTL("s", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[0..]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[1..]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[2..]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[3..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[0..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[0..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[0..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[1..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[1..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[1..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[2..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[2..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("s[3..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("s[3..4]", NO_TIMESTAMP, NO_TTL);
+
+ // Read multiple rows to verify selector reset
+ execute("TRUNCATE TABLE %s");
+ execute("INSERT INTO %s (k, s) VALUES (1, {1, 2, 3}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, s) VALUES (2, {1, 2}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, s) VALUES (3, {1}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, s) VALUES (4, {}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, s) VALUES (5, null) USING TIMESTAMP ?", TIMESTAMP_2);
+ assertRows("SELECT k, WRITETIME(s) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, TIMESTAMP_2),
+ row(3, TIMESTAMP_1));
+ assertRows("SELECT k, WRITETIME(s[1]) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, NO_TIMESTAMP),
+ row(3, TIMESTAMP_1));
}
@Test
public void testMap() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<int, int>)");
- assertInvalidMultiCellSelection("m", true);
+
+ // Null column
+ execute("INSERT INTO %s (k) VALUES (1) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("m", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+
+ // Create empty
+ execute("INSERT INTO %s (k, m) VALUES (1, {}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("m", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+
+ // Update with a single element without TTL
+ execute("INSERT INTO %s (k, m) VALUES (1, {1:10}) USING TIMESTAMP ?", TIMESTAMP_1);
+ assertWritetimeAndTTL("m", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[..2]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[0..]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[1..]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[2..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[0..2]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[1..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[1..2]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[2..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[2..3]", NO_TIMESTAMP, NO_TTL);
+
+ // Add a new element to the map with a new timestamp and a TTL
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET m=m+{2:20} WHERE k=1", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[0..]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[1..]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[2..]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("m[3..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[0..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[0..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[1..1]", timestamps(TIMESTAMP_1), ttls(NO_TTL));
+ assertWritetimeAndTTL("m[1..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[1..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(NO_TTL, TTL_2));
+ assertWritetimeAndTTL("m[2..2]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("m[2..3]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("m[3..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[3..4]", NO_TIMESTAMP, NO_TTL);
+
+ // Combine timestamp selection with other selections and orders
+ assertRows("SELECT k, WRITETIME(m[1]) FROM %s", row(1, TIMESTAMP_1));
+ assertRows("SELECT WRITETIME(m[1]), k FROM %s", row(TIMESTAMP_1, 1));
+ assertRows("SELECT WRITETIME(m[1]), WRITETIME(m[2]) FROM %s", row(TIMESTAMP_1, TIMESTAMP_2));
+ assertRows("SELECT WRITETIME(m[2]), WRITETIME(m[1]) FROM %s", row(TIMESTAMP_2, TIMESTAMP_1));
+
+ // Read multiple rows to verify selector reset
+ execute("TRUNCATE TABLE %s");
+ execute("INSERT INTO %s (k, m) VALUES (1, {1:10, 2:20, 3:30}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, m) VALUES (2, {1:10, 2:20}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, m) VALUES (3, {1:10}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, m) VALUES (4, {}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, m) VALUES (5, null) USING TIMESTAMP ?", TIMESTAMP_2);
+ assertRows("SELECT k, WRITETIME(m) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, timestamps(TIMESTAMP_1, TIMESTAMP_1, TIMESTAMP_1)),
+ row(2, timestamps(TIMESTAMP_2, TIMESTAMP_2)),
+ row(4, NO_TIMESTAMP),
+ row(3, timestamps(TIMESTAMP_1)));
+ assertRows("SELECT k, WRITETIME(m[1]) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, NO_TIMESTAMP),
+ row(3, TIMESTAMP_1));
}
@Test
@@ -139,18 +450,244 @@
// Null column
execute("INSERT INTO %s (k) VALUES (1) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("m", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
// Create empty
execute("INSERT INTO %s (k, m) VALUES (1, {}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("m", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+
+ // truncate, since previous columns would win on reconcilliation because of their TTL (CASSANDRA-14592)
+ execute("TRUNCATE TABLE %s");
// Create with a single element without TTL
execute("INSERT INTO %s (k, m) VALUES (1, {1:10}) USING TIMESTAMP ?", TIMESTAMP_1);
assertWritetimeAndTTL("m", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[..2]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[0..]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[1..]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[2..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[0..2]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[1..1]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[1..2]", TIMESTAMP_1, NO_TTL);
+ assertWritetimeAndTTL("m[2..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[2..3]", NO_TIMESTAMP, NO_TTL);
// Add a new element to the map with a new timestamp and a TTL
execute("INSERT INTO %s (k, m) VALUES (1, {1:10, 2:20}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_2, TTL_2);
assertWritetimeAndTTL("m", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[0..]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[1..]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[2..]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[3..]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[0..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[0..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[0..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[1..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[1..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[1..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[2..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[2..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("m[3..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("m[3..4]", NO_TIMESTAMP, NO_TTL);
+
+ // Read multiple rows to verify selector reset
+ execute("TRUNCATE TABLE %s");
+ execute("INSERT INTO %s (k, m) VALUES (1, {1:10, 2:20, 3:30}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, m) VALUES (2, {1:10, 2:20}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, m) VALUES (3, {1:10}) USING TIMESTAMP ?", TIMESTAMP_1);
+ execute("INSERT INTO %s (k, m) VALUES (4, {}) USING TIMESTAMP ?", TIMESTAMP_2);
+ execute("INSERT INTO %s (k, m) VALUES (5, null) USING TIMESTAMP ?", TIMESTAMP_2);
+ assertRows("SELECT k, WRITETIME(m) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, TIMESTAMP_2),
+ row(3, TIMESTAMP_1));
+ assertRows("SELECT k, WRITETIME(m[1]) FROM %s",
+ row(5, NO_TIMESTAMP),
+ row(1, TIMESTAMP_1),
+ row(2, TIMESTAMP_2),
+ row(4, NO_TIMESTAMP),
+ row(3, TIMESTAMP_1));
+ }
+
+ @Test
+ public void testNestedCollections() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v map<int,frozen<set<int>>>)");
+
+ // Null column
+ execute("INSERT INTO %s (k) VALUES (1) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][0]", NO_TIMESTAMP, NO_TTL);
+
+ execute("INSERT INTO %s (k, v) VALUES (1, {1:{1,2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET v=v+{2:{1, 2}} WHERE k=1", TIMESTAMP_2, TTL_2);
+
+ assertWritetimeAndTTL("v", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+
+ assertWritetimeAndTTL("v[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..1]", timestamps(TIMESTAMP_1), ttls(TTL_1));
+ assertWritetimeAndTTL("v[0..2]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("v[0..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("v[1..3]", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("v[2..3]", timestamps(TIMESTAMP_2), ttls(TTL_2));
+ assertWritetimeAndTTL("v[3..3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[1][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[2][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[2][1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[2][2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[2][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[3][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][0..1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][1..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][2..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[1][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1][0..1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][1..2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][2..3]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[2][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[2][0..1]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[2][1..2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[2][2..3]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[2][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[3][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][0..1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][1..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][2..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0..1][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..1][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[0..1][2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..2][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..2][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[0..2][2]", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("v[0..2][3]", NO_TIMESTAMP, NO_TTL);
+ }
+
+ @Test
+ public void testFrozenNestedCollections() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<map<int,frozen<set<int>>>>)");
+ execute("INSERT INTO %s (k, v) VALUES (1, {1:{1,2}, 2:{1,2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+
+ assertWritetimeAndTTL("v", TIMESTAMP_1, TTL_1);
+
+ assertWritetimeAndTTL("v[0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[1][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[2][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[2][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2][2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[3][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][3]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][0..1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][1..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][2..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[1][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1][0..1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][1..2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][2..3]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[2][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[2][0..1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2][1..2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2][2..3]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[3][0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][0..1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][1..2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][2..3]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[3][3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0..0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1..2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[2..3]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[3..4]", NO_TIMESTAMP, NO_TTL);
+
+ assertWritetimeAndTTL("v[0..0][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..0][1]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..1][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[0..1][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[0..1][2]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1..2][0]", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("v[1..2][1]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1..2][2]", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("v[1..2][3]", NO_TIMESTAMP, NO_TTL);
}
@Test
@@ -158,7 +695,58 @@
{
String type = createType("CREATE TYPE %s (f1 int, f2 int)");
createTable("CREATE TABLE %s (k int PRIMARY KEY, t " + type + ')');
- assertInvalidMultiCellSelection("t", false);
+
+ // Null column
+ execute("INSERT INTO %s (k) VALUES (0) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", NO_TIMESTAMP, NO_TTL);
+
+ // Both fields are empty
+ execute("INSERT INTO %s (k, t) VALUES (0, {f1:null, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=0", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1", "k=0", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=0", NO_TIMESTAMP, NO_TTL);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=0", row(0, NO_TIMESTAMP, NO_TIMESTAMP));
+
+ // Only the first field is set
+ execute("INSERT INTO %s (k, t) VALUES (1, {f1:1, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=1", timestamps(TIMESTAMP_1, NO_TIMESTAMP), ttls(TTL_1, NO_TTL));
+ assertWritetimeAndTTL("t.f1", "k=1", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=1", row(1, TIMESTAMP_1, NO_TIMESTAMP));
+ assertRows("SELECT k, WRITETIME(t.f2), WRITETIME(t.f1) FROM %s WHERE k=1", row(1, NO_TIMESTAMP, TIMESTAMP_1));
+
+ // Only the second field is set
+ execute("INSERT INTO %s (k, t) VALUES (2, {f1:null, f2:2}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=2", timestamps(NO_TIMESTAMP, TIMESTAMP_1), ttls(NO_TTL, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=2", TIMESTAMP_1, TTL_1);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=2", row(2, NO_TIMESTAMP, TIMESTAMP_1));
+ assertRows("SELECT k, WRITETIME(t.f2), WRITETIME(t.f1) FROM %s WHERE k=2", row(2, TIMESTAMP_1, NO_TIMESTAMP));
+
+ // Both fields are set
+ execute("INSERT INTO %s (k, t) VALUES (3, {f1:1, f2:2}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=3", timestamps(TIMESTAMP_1, TIMESTAMP_1), ttls(TTL_1, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=3", TIMESTAMP_1, TTL_1);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=3", row(3, TIMESTAMP_1, TIMESTAMP_1));
+
+ // Having only the first field set, update the second field
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET t.f2=2 WHERE k=1", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t", "k=1", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("t.f1", "k=1", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=1", TIMESTAMP_2, TTL_2);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=1", row(1, TIMESTAMP_1, TIMESTAMP_2));
+ assertRows("SELECT k, WRITETIME(t.f2), WRITETIME(t.f1) FROM %s WHERE k=1", row(1, TIMESTAMP_2, TIMESTAMP_1));
+
+ // Having only the second field set, update the second field
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET t.f1=1 WHERE k=2", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t", "k=2", timestamps(TIMESTAMP_2, TIMESTAMP_1), ttls(TTL_2, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=2", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t.f2", "k=2", TIMESTAMP_1, TTL_1);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=2", row(2, TIMESTAMP_2, TIMESTAMP_1));
+ assertRows("SELECT k, WRITETIME(t.f2), WRITETIME(t.f1) FROM %s WHERE k=2", row(2, TIMESTAMP_1, TIMESTAMP_2));
}
@Test
@@ -170,22 +758,343 @@
// Null column
execute("INSERT INTO %s (k) VALUES (0) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("t", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", NO_TIMESTAMP, NO_TTL);
// Both fields are empty
execute("INSERT INTO %s (k, t) VALUES (0, {f1:null, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("t", "k=0", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=0", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=0", NO_TIMESTAMP, NO_TTL);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=0", row(0, NO_TIMESTAMP, NO_TIMESTAMP));
// Only the first field is set
execute("INSERT INTO %s (k, t) VALUES (1, {f1:1, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("t", "k=1", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=1", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=1", row(1, TIMESTAMP_1, NO_TIMESTAMP));
+ assertRows("SELECT k, WRITETIME(t.f2), WRITETIME(t.f1) FROM %s WHERE k=1", row(1, NO_TIMESTAMP, TIMESTAMP_1));
// Only the second field is set
execute("INSERT INTO %s (k, t) VALUES (2, {f1:null, f2:2}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("t", "k=2", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=2", TIMESTAMP_1, TTL_1);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=2", row(2, NO_TIMESTAMP, TIMESTAMP_1));
+ assertRows("SELECT k, WRITETIME(t.f2), WRITETIME(t.f1) FROM %s WHERE k=2", row(2, TIMESTAMP_1, NO_TIMESTAMP));
// Both fields are set
execute("INSERT INTO %s (k, t) VALUES (3, {f1:1, f2:2}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
assertWritetimeAndTTL("t", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=3", TIMESTAMP_1, TTL_1);
+ assertRows("SELECT k, WRITETIME(t.f1), WRITETIME(t.f2) FROM %s WHERE k=3", row(3, TIMESTAMP_1, TIMESTAMP_1));
+ }
+
+ @Test
+ public void testNestedUDTs() throws Throwable
+ {
+ String nestedType = createType("CREATE TYPE %s (f1 int, f2 int)");
+ String type = createType(format("CREATE TYPE %%s (f1 frozen<%s>, f2 frozen<%<s>)", nestedType));
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, t " + type + ')');
+
+ // Both fields are empty
+ execute("INSERT INTO %s (k, t) VALUES (1, {f1:null, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, no nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (2, {f1:{f1:null,f2:null}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=2", timestamps(TIMESTAMP_1, NO_TIMESTAMP), ttls(TTL_1, NO_TTL));
+ assertWritetimeAndTTL("t.f1", "k=2", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=2", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, only the first nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (3, {f1:{f1:1,f2:null}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=2", timestamps(TIMESTAMP_1, NO_TIMESTAMP), ttls(TTL_1, NO_TTL));
+ assertWritetimeAndTTL("t.f1", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=3", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=3", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=3", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=3", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, only the second nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (4, {f1:{f1:null,f2:2}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=4", timestamps(TIMESTAMP_1, NO_TIMESTAMP), ttls(TTL_1, NO_TTL));
+ assertWritetimeAndTTL("t.f1", "k=4", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=4", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=4", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=4", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=4", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=4", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, both nested field are set
+ execute("INSERT INTO %s (k, t) VALUES (5, {f1:{f1:1,f2:2}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=5", timestamps(TIMESTAMP_1, NO_TIMESTAMP), ttls(TTL_1, NO_TTL));
+ assertWritetimeAndTTL("t.f1", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=5", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=5", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=5", NO_TIMESTAMP, NO_TTL);
+
+ // Only the second field is set, no nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (6, {f1:null, f2:{f1:null,f2:null}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=6", timestamps(NO_TIMESTAMP, TIMESTAMP_1), ttls(NO_TTL, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=6", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=6", NO_TIMESTAMP, NO_TTL);
+
+ // Only the second field is set, only the first nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (7, {f1:null, f2:{f1:1,f2:null}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=7", timestamps(NO_TIMESTAMP, TIMESTAMP_1), ttls(NO_TTL, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=7", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=7", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=7", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=7", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=7", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f2", "k=7", NO_TIMESTAMP, NO_TTL);
+
+ // Only the second field is set, only the second nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (8, {f1:null, f2:{f1:null,f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=8", timestamps(NO_TIMESTAMP, TIMESTAMP_1), ttls(NO_TTL, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=8", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=8", TIMESTAMP_1, TTL_1);
+
+ // Only the second field is set, both nested field are set
+ execute("INSERT INTO %s (k, t) VALUES (9, {f1:null, f2:{f1:1,f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=9", timestamps(NO_TIMESTAMP, TIMESTAMP_1), ttls(NO_TTL, TTL_1));
+ assertWritetimeAndTTL("t.f1", "k=9", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=9", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=9", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=9", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=9", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f2", "k=9", TIMESTAMP_1, TTL_1);
+
+ // Both fields are set, alternate fields are set
+ execute("INSERT INTO %s (k, t) VALUES (10, {f1:{f1:1}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET t.f2={f2:2} WHERE k=10", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t", "k=10", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("t.f1", "k=10", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=10", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=10", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=10", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t.f2.f1", "k=10", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=10", TIMESTAMP_2, TTL_2);
+
+ // Both fields are set, alternate fields are set
+ execute("INSERT INTO %s (k, t) VALUES (11, {f1:{f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET t.f2={f1:2} WHERE k=11", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t", "k=11", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("t.f1", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=11", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=11", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t.f2.f1", "k=11", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t.f2.f2", "k=11", NO_TIMESTAMP, NO_TTL);
+
+ // Both fields are set, all fields are set
+ execute("INSERT INTO %s (k, t) VALUES (12, {f1:{f1:1,f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ execute("UPDATE %s USING TIMESTAMP ? AND TTL ? SET t.f2={f1:1,f2:2} WHERE k=12", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t", "k=12", timestamps(TIMESTAMP_1, TIMESTAMP_2), ttls(TTL_1, TTL_2));
+ assertWritetimeAndTTL("t.f1", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=12", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t.f2.f1", "k=12", TIMESTAMP_2, TTL_2);
+ assertWritetimeAndTTL("t.f2.f2", "k=12", TIMESTAMP_2, TTL_2);
+ }
+
+ @Test
+ public void testFrozenNestedUDTs() throws Throwable
+ {
+ String nestedType = createType("CREATE TYPE %s (f1 int, f2 int)");
+ String type = createType(format("CREATE TYPE %%s (f1 frozen<%s>, f2 frozen<%<s>)", nestedType));
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, t frozen<" + type + ">)");
+
+ // Both fields are empty
+ execute("INSERT INTO %s (k, t) VALUES (1, {f1:null, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=1", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=1", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=1", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, no nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (2, {f1:{f1:null,f2:null}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=2", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=2", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=2", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=2", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, only the first nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (3, {f1:{f1:1,f2:null}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=3", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=3", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=3", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=3", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=3", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, only the second nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (4, {f1:{f1:null,f2:2}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=4", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=4", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=4", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=4", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=4", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=4", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=4", NO_TIMESTAMP, NO_TTL);
+
+ // Only the first field is set, both nested field are set
+ execute("INSERT INTO %s (k, t) VALUES (5, {f1:{f1:1,f2:2}, f2:null}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=5", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=5", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f1", "k=5", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=5", NO_TIMESTAMP, NO_TTL);
+
+ // Only the second field is set, no nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (6, {f1:null, f2:{f1:null,f2:null}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=6", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=6", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=6", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=6", NO_TIMESTAMP, NO_TTL);
+
+ // Only the second field is set, only the first nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (7, {f1:null, f2:{f1:1,f2:null}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=7", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=7", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=7", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=7", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=7", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=7", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f2", "k=7", NO_TIMESTAMP, NO_TTL);
+
+ // Only the second field is set, only the second nested field is set
+ execute("INSERT INTO %s (k, t) VALUES (8, {f1:null, f2:{f1:null,f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=8", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=8", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=8", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=8", TIMESTAMP_1, TTL_1);
+
+ // Only the second field is set, both nested field are set
+ execute("INSERT INTO %s (k, t) VALUES (9, {f1:null, f2:{f1:1,f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=9", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=9", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f1", "k=9", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=9", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=9", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=9", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f2", "k=9", TIMESTAMP_1, TTL_1);
+
+ // Both fields are set, alternate fields are set
+ execute("INSERT INTO %s (k, t) VALUES (10, {f1:{f1:1}, f2:{f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=10", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=10", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=10", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=10", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2", "k=10", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=10", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f2.f2", "k=10", TIMESTAMP_1, TTL_1);
+ // Both fields are set, alternate fields are set
+ execute("INSERT INTO %s (k, t) VALUES (11, {f1:{f2:2}, f2:{f1:1}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=11", NO_TIMESTAMP, NO_TTL);
+ assertWritetimeAndTTL("t.f1.f2", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=11", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f2", "k=11", NO_TIMESTAMP, NO_TTL);
+
+ // Both fields are set, all fields are set
+ execute("INSERT INTO %s (k, t) VALUES (12, {f1:{f1:1,f2:2},f2:{f1:1,f2:2}}) USING TIMESTAMP ? AND TTL ?", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f1", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f1.f2", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f1", "k=12", TIMESTAMP_1, TTL_1);
+ assertWritetimeAndTTL("t.f2.f2", "k=12", TIMESTAMP_1, TTL_1);
+ }
+
+ @Test
+ public void testFunctions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int, s set<int>, fs frozen<set<int>>)");
+ execute("INSERT INTO %s (k, v, s, fs) VALUES (0, 0, {1, 2, 3}, {1, 2, 3}) USING TIMESTAMP 1 AND TTL 1000");
+ execute("INSERT INTO %s (k, v, s, fs) VALUES (1, 1, {10, 20, 30}, {10, 20, 30}) USING TIMESTAMP 10 AND TTL 1000");
+ execute("UPDATE %s USING TIMESTAMP 2 AND TTL 2000 SET s = s + {2, 3} WHERE k = 0");
+ execute("UPDATE %s USING TIMESTAMP 20 AND TTL 2000 SET s = s + {20, 30} WHERE k = 1");
+
+ // Regular column
+ assertRows("SELECT min(v) FROM %s", row(0));
+ assertRows("SELECT max(v) FROM %s", row(1));
+ assertRows("SELECT writetime(v) FROM %s", row(10L), row(1L));
+ assertRows("SELECT min(writetime(v)) FROM %s", row(1L));
+ assertRows("SELECT max(writetime(v)) FROM %s", row(10L));
+ assertRows("SELECT min(maxwritetime(v)) FROM %s", row(1L));
+ assertRows("SELECT max(maxwritetime(v)) FROM %s", row(10L));
+
+ // Frozen collection
+ assertRows("SELECT min(fs) FROM %s", row(set(1, 2, 3)));
+ assertRows("SELECT max(fs) FROM %s", row(set(10, 20, 30)));
+ assertRows("SELECT writetime(fs) FROM %s", row(10L), row(1L));
+ assertRows("SELECT min(writetime(fs)) FROM %s", row(1L));
+ assertRows("SELECT max(writetime(fs)) FROM %s", row(10L));
+ assertRows("SELECT min(maxwritetime(fs)) FROM %s", row(1L));
+ assertRows("SELECT max(maxwritetime(fs)) FROM %s", row(10L));
+
+ // Multi-cell collection
+ assertRows("SELECT min(s) FROM %s", row(set(1, 2, 3)));
+ assertRows("SELECT max(s) FROM %s", row(set(10, 20, 30)));
+ assertRows("SELECT writetime(s) FROM %s", row(list(10L, 20L, 20L)), row(list(1L, 2L, 2L)));
+ assertRows("SELECT min(writetime(s)) FROM %s", row(list(1L, 2L, 2L)));
+ assertRows("SELECT max(writetime(s)) FROM %s", row(list(10L, 20L, 20L)));
+ assertRows("SELECT min(maxwritetime(s)) FROM %s", row(2L));
+ assertRows("SELECT max(maxwritetime(s)) FROM %s", row(20L));
+ }
+
+ private static List<Integer> ttls(Integer... a)
+ {
+ return Arrays.asList(a);
+ }
+
+ private static List<Long> timestamps(Long... a)
+ {
+ return Arrays.asList(a);
}
private void assertRows(String query, Object[]... rows) throws Throwable
@@ -198,23 +1107,34 @@
assertWritetimeAndTTL(column, null, timestamp, ttl);
}
+ private void assertWritetimeAndTTL(String column, List<Long> timestamps, List<Integer> ttls) throws Throwable
+ {
+ assertWritetimeAndTTL(column, null, timestamps, ttls);
+ }
+
private void assertWritetimeAndTTL(String column, String where, Long timestamp, Integer ttl)
throws Throwable
{
where = where == null ? "" : " WHERE " + where;
// Verify write time
- String writetimeQuery = String.format("SELECT WRITETIME(%s) FROM %%s %s", column, where);
- assertRows(writetimeQuery, row(timestamp));
+ assertRows(format("SELECT WRITETIME(%s) FROM %%s %s", column, where), row(timestamp));
+
+ // Verify max write time
+ assertRows(format("SELECT MAXWRITETIME(%s) FROM %%s %s", column, where), row(timestamp));
+
+ // Verify write time and max write time together
+ assertRows(format("SELECT WRITETIME(%s), MAXWRITETIME(%s) FROM %%s %s", column, column, where),
+ row(timestamp, timestamp));
// Verify ttl
- UntypedResultSet rs = execute(String.format("SELECT TTL(%s) FROM %%s %s", column, where));
+ UntypedResultSet rs = execute(format("SELECT TTL(%s) FROM %%s %s", column, where));
assertRowCount(rs, 1);
UntypedResultSet.Row row = rs.one();
- String ttlColumn = String.format("ttl(%s)", column);
+ String ttlColumn = format("ttl(%s)", column);
if (ttl == null)
{
- assertTTL(ttl, null);
+ assertFalse(row.has(ttlColumn));
}
else
{
@@ -222,6 +1142,45 @@
}
}
+ private void assertWritetimeAndTTL(String column, String where, List<Long> timestamps, List<Integer> ttls)
+ throws Throwable
+ {
+ where = where == null ? "" : " WHERE " + where;
+
+ // Verify write time
+ assertRows(format("SELECT WRITETIME(%s) FROM %%s %s", column, where), row(timestamps));
+
+ // Verify max write time
+ Long maxTimestamp = timestamps.stream().filter(Objects::nonNull).max(Long::compare).orElse(null);
+ assertRows(format("SELECT MAXWRITETIME(%s) FROM %%s %s", column, where), row(maxTimestamp));
+
+ // Verify write time and max write time together
+ assertRows(format("SELECT WRITETIME(%s), MAXWRITETIME(%s) FROM %%s %s", column, column, where),
+ row(timestamps, maxTimestamp));
+
+ // Verify ttl
+ UntypedResultSet rs = execute(format("SELECT TTL(%s) FROM %%s %s", column, where));
+ assertRowCount(rs, 1);
+ UntypedResultSet.Row row = rs.one();
+ String ttlColumn = format("ttl(%s)", column);
+ if (ttls == null)
+ {
+ assertFalse(row.has(ttlColumn));
+ }
+ else
+ {
+ List<Integer> actualTTLs = row.getList(ttlColumn, Int32Type.instance);
+ assertEquals(ttls.size(), actualTTLs.size());
+
+ for (int i = 0; i < actualTTLs.size(); i++)
+ {
+ Integer expectedTTL = ttls.get(i);
+ Integer actualTTL = actualTTLs.get(i);
+ assertTTL(expectedTTL, actualTTL);
+ }
+ }
+ }
+
/**
* Since the returned TTL is the remaining seconds since last update, it could be lower than the
* specified TTL depending on the test execution time, se we allow up to one-minute difference
@@ -242,23 +1201,42 @@
private void assertInvalidPrimaryKeySelection(String column) throws Throwable
{
- assertInvalidThrowMessage("Cannot use selection function writeTime on PRIMARY KEY part " + column,
+ assertInvalidThrowMessage("Cannot use selection function writetime on PRIMARY KEY part " + column,
InvalidRequestException.class,
- String.format("SELECT WRITETIME(%s) FROM %%s", column));
+ format("SELECT WRITETIME(%s) FROM %%s", column));
+ assertInvalidThrowMessage("Cannot use selection function maxwritetime on PRIMARY KEY part " + column,
+ InvalidRequestException.class,
+ format("SELECT MAXWRITETIME(%s) FROM %%s", column));
assertInvalidThrowMessage("Cannot use selection function ttl on PRIMARY KEY part " + column,
InvalidRequestException.class,
- String.format("SELECT TTL(%s) FROM %%s", column));
+ format("SELECT TTL(%s) FROM %%s", column));
}
- private void assertInvalidMultiCellSelection(String column, boolean isCollection) throws Throwable
+ private void assertInvalidListElementSelection(String column, String list) throws Throwable
{
- String message = format("Cannot use selection function %%s on non-frozen %s %s",
- isCollection ? "collection" : "UDT", column);
- assertInvalidThrowMessage(format(message, "writeTime"),
+ String message = format("Element selection is only allowed on sets and maps, but %s is a list", list);
+ assertInvalidThrowMessage(message,
InvalidRequestException.class,
- String.format("SELECT WRITETIME(%s) FROM %%s", column));
- assertInvalidThrowMessage(format(message, "ttl"),
+ format("SELECT WRITETIME(%s) FROM %%s", column));
+ assertInvalidThrowMessage(message,
InvalidRequestException.class,
- String.format("SELECT TTL(%s) FROM %%s", column));
+ format("SELECT MAXWRITETIME(%s) FROM %%s", column));
+ assertInvalidThrowMessage(message,
+ InvalidRequestException.class,
+ format("SELECT TTL(%s) FROM %%s", column));
+ }
+
+ private void assertInvalidListSliceSelection(String column, String list) throws Throwable
+ {
+ String message = format("Slice selection is only allowed on sets and maps, but %s is a list", list);
+ assertInvalidThrowMessage(message,
+ InvalidRequestException.class,
+ format("SELECT WRITETIME(%s) FROM %%s", column));
+ assertInvalidThrowMessage(message,
+ InvalidRequestException.class,
+ format("SELECT MAXWRITETIME(%s) FROM %%s", column));
+ assertInvalidThrowMessage(message,
+ InvalidRequestException.class,
+ format("SELECT TTL(%s) FROM %%s", column));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java
index 0d9e043..9092f94 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/OverflowTest.java
@@ -170,8 +170,8 @@
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, i varint, b blob)");
- execute("INSERT INTO %s (k, i, b) VALUES (0, blobAsVarint(bigintAsBlob(3)), textAsBlob('foobar'))");
- assertRows(execute("SELECT i, blobAsText(b) FROM %s WHERE k = 0"),
+ execute("INSERT INTO %s (k, i, b) VALUES (0, blob_as_varint(bigint_as_blob(3)), text_as_blob('foobar'))");
+ assertRows(execute("SELECT i, blob_as_text(b) FROM %s WHERE k = 0"),
row(BigInteger.valueOf(3), "foobar"));
}
@@ -230,7 +230,7 @@
createTable("CREATE TABLE %s (k int PRIMARY KEY, t timeuuid,)");
execute("INSERT INTO %s (k) VALUES (0)");
- Object[][] rows = getRows(execute("SELECT dateOf(t) FROM %s WHERE k=0"));
+ Object[][] rows = getRows(execute("SELECT to_timestamp(t) FROM %s WHERE k=0"));
assertNull(rows[0][0]);
}
@@ -288,9 +288,9 @@
createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
// A blob that is not 4 bytes should be rejected
- assertInvalid("INSERT INTO %s (k, v) VALUES (0, blobAsInt(0x01))");
+ assertInvalid("INSERT INTO %s (k, v) VALUES (0, blob_as_int(0x01))");
- execute("INSERT INTO %s (k, v) VALUES (0, blobAsInt(0x00000001))");
+ execute("INSERT INTO %s (k, v) VALUES (0, blob_as_int(0x00000001))");
assertRows(execute("select v from %s where k=0"), row(1));
}
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java
index ff0e546..a1223c1 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java
@@ -20,12 +20,16 @@
*/
package org.apache.cassandra.cql3.validation.miscellaneous;
+import java.util.Arrays;
+
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.metrics.ClearableHistogram;
import static org.junit.Assert.assertEquals;
@@ -50,6 +54,22 @@
numSSTablesIterated);
}
+ private void executeAndCheckRangeQuery(String query, int numSSTables, Object[]... rows) throws Throwable
+ {
+ logger.info("Executing query: {} with parameters: {}", query, Arrays.toString(rows));
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE_PER_TEST);
+
+ ((ClearableHistogram) cfs.metric.sstablesPerRangeReadHistogram.cf).clear(); // resets counts
+
+ assertRows(execute(query), rows);
+
+ long numSSTablesIterated = cfs.metric.sstablesPerRangeReadHistogram.cf.getSnapshot().getMax(); // max sstables read
+ assertEquals(String.format("Expected %d sstables iterated but got %d instead, with %d live sstables",
+ numSSTables, numSSTablesIterated, cfs.getLiveSSTables().size()),
+ numSSTables,
+ numSSTablesIterated);
+ }
+
@Override
protected String createTable(String query)
{
@@ -59,7 +79,7 @@
}
@Override
- protected UntypedResultSet execute(String query, Object... values) throws Throwable
+ protected UntypedResultSet execute(String query, Object... values)
{
return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
}
@@ -354,7 +374,7 @@
private void testDeletionOnIndexedSSTableDESC(boolean deleteWithRange) throws Throwable
{
// reduce the column index size so that columns get indexed during flush
- DatabaseDescriptor.setColumnIndexSize(1);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(1);
createTable("CREATE TABLE %s (id int, col int, val text, PRIMARY KEY (id, col)) WITH CLUSTERING ORDER BY (col DESC)");
@@ -402,7 +422,7 @@
private void testDeletionOnIndexedSSTableASC(boolean deleteWithRange) throws Throwable
{
// reduce the column index size so that columns get indexed during flush
- DatabaseDescriptor.setColumnIndexSize(1);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(1);
createTable("CREATE TABLE %s (id int, col int, val text, PRIMARY KEY (id, col)) WITH CLUSTERING ORDER BY (col ASC)");
@@ -431,13 +451,19 @@
}
flush();
+ // The code has to read the 1st and 3rd sstables to see that everything before the 2nd sstable is deleted, and
+ // overall has to read the 3 sstables
executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 1", 3, row(1, 1001, "1001"));
executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 2", 3, row(1, 1001, "1001"), row(1, 1002, "1002"));
executeAndCheck("SELECT * FROM %s WHERE id=1", 3, allRows);
- executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000 LIMIT 1", 2, row(1, 1001, "1001"));
+
+ // The 1st and 3rd sstables have data only up to 1000, so they will be skipped
+ executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000 LIMIT 1", 1, row(1, 1001, "1001"));
+ executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000", 1, allRows);
+
+ // The condition makes no difference to the code, and all 3 sstables have to read
executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 LIMIT 1", 3, row(1, 1001, "1001"));
- executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000", 2, allRows);
executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000", 3, allRows);
}
@@ -451,7 +477,7 @@
private void testDeletionOnOverlappingIndexedSSTable(boolean deleteWithRange) throws Throwable
{
// reduce the column index size so that columns get indexed during flush
- DatabaseDescriptor.setColumnIndexSize(1);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(1);
createTable("CREATE TABLE %s (id int, col int, val1 text, val2 text, PRIMARY KEY (id, col)) WITH CLUSTERING ORDER BY (col ASC)");
@@ -515,14 +541,30 @@
allRows[idx] = row(1, i, Integer.toString(i), Integer.toString(i));
}
- executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 1", 2, row(1, 1, "1", "1"));
- executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 2", 2, row(1, 1, "1", "1"), row(1, 2, "2", null));
+ // The 500th first rows are in the first sstable (and there is no partition deletion/static row), so the 'lower
+ // bound' optimization will kick in and we'll only read the 1st sstable.
+ executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 1", 1, row(1, 1, "1", "1"));
+ executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 2", 1, row(1, 1, "1", "1"), row(1, 2, "2", null));
+ // Getting everything obviously requires reading both sstables
executeAndCheck("SELECT * FROM %s WHERE id=1", 2, allRows);
+
+ // The 'lower bound' optimization don't help us because while the row to fetch is in the 1st sstable, the lower
+ // bound for the 2nd sstable is 501, which is lower than 1000.
executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000 LIMIT 1", 2, row(1, 1001, "1001", "1001"));
- executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 LIMIT 1", 2, row(1, 1, "1", "1"));
+
+ // Somewhat similar to the previous one: the row is in th 2nd sstable in this case, but as the lower bound for
+ // the first sstable is 1, this doesn't help.
executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 500 LIMIT 1", 2, row(1, 751, "751", "751"));
- executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 500 LIMIT 1", 2, row(1, 1, "1", "1"));
+
+ // The 'col <= ?' condition in both queries doesn't impact the read path, which can still make use of the lower
+ // bound optimization and read only the first sstable.
+ executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 LIMIT 1", 1, row(1, 1, "1", "1"));
+ executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 500 LIMIT 1", 1, row(1, 1, "1", "1"));
+
+ // Making sure the 'lower bound' optimization also work in reverse queries (in which it's more of a 'upper
+ // bound' optimization).
+ executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 ORDER BY col DESC LIMIT 1", 1, row(1, 2000, "2000", null));
}
@Test
@@ -996,7 +1038,7 @@
@Test
public void testCompactAndNonCompactTableWithCounter() throws Throwable
{
- for (String with : new String[]{"", " WITH COMPACT STORAGE"})
+ for (String with : new String[]{ "", " WITH COMPACT STORAGE" })
{
createTable("CREATE TABLE %s (pk int, c int, count counter, PRIMARY KEY(pk, c))" + with);
@@ -1038,7 +1080,7 @@
executeAndCheck("SELECT s, v FROM %s WHERE pk = 3 AND c = 3", 3, row(3, set(1)));
executeAndCheck("SELECT v FROM %s WHERE pk = 1 AND c = 1", 3, row(set(3)));
executeAndCheck("SELECT v FROM %s WHERE pk = 2 AND c = 1", 2, row(set(3)));
- executeAndCheck("SELECT v FROM %s WHERE pk = 3 AND c = 3", 3, row(set(1)));
+ executeAndCheck("SELECT v FROM %s WHERE pk = 3 AND c = 3", 1, row(set(1)));
executeAndCheck("SELECT s FROM %s WHERE pk = 1", 3, row((Integer) null));
executeAndCheck("SELECT s FROM %s WHERE pk = 2", 2, row(1), row(1));
executeAndCheck("SELECT DISTINCT s FROM %s WHERE pk = 2", 2, row(1));
@@ -1087,7 +1129,7 @@
@Test
public void testCompactAndNonCompactTableWithPartitionTombstones() throws Throwable
{
- for (Boolean compact : new Boolean[] {Boolean.FALSE, Boolean.TRUE})
+ for (Boolean compact : new Boolean[]{ Boolean.FALSE, Boolean.TRUE })
{
String with = compact ? " WITH COMPACT STORAGE" : "";
createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int)" + with);
@@ -1642,4 +1684,157 @@
executeAndCheck("SELECT v1 FROM %s WHERE pk = 5", 1);
executeAndCheck("SELECT v2 FROM %s WHERE pk = 5", 1);
}
-}
+
+ @Test
+ public void testSkippingBySliceInSinglePartitionReads() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))");
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 1, 1);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 2);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 3);
+ flush();
+ assertEquals(1, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 4);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 5);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 6);
+ flush();
+ assertEquals(2, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 7);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 8);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 9);
+ flush();
+ assertEquals(3, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 10);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 11);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 6, 12);
+ flush();
+ assertEquals(4, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ // point query - test whether sstables are skipped due to not covering the requested slice
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 0", 0);
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 1", 1, row(1, 1, 1));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 2", 2, row(1, 2, 4));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 3", 3, row(1, 3, 7));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 4", 3, row(1, 4, 10));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 5", 2, row(1, 5, 11));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 6", 1, row(1, 6, 12));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 7", 0);
+
+ // range query - test whether sstables are skipped due to not covering the requeste slice
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > -10 AND c <= 0", 0);
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > -10 AND c < 1", 0);
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > -10 AND c <= 1", 1, row(1, 1, 1));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > 1 AND c < 3", 2, row(1, 2, 4));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c >= 6", 1, row(1, 6, 12));
+ executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > 6", 0);
+ }
+
+ @Test
+ public void testSkippingBySliceInPartitionRangeReads() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))");
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 1, 1);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 2);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 3);
+ flush();
+ assertEquals(1, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 4);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 5);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 6);
+ flush();
+ assertEquals(2, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 7);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 8);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 9);
+ flush();
+ assertEquals(3, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 10);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 11);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 6, 12);
+ flush();
+ assertEquals(4, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ // point query - test whether sstables are skipped due to not covering the requested slice
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 0 ALLOW FILTERING", 0);
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 1 ALLOW FILTERING", 1, row(1, 1, 1));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 2 ALLOW FILTERING", 2, row(1, 2, 4));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 3 ALLOW FILTERING", 3, row(1, 3, 7));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 4 ALLOW FILTERING", 3, row(1, 4, 10));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 5 ALLOW FILTERING", 2, row(1, 5, 11));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 6 ALLOW FILTERING", 1, row(1, 6, 12));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", 0);
+
+ // range query - test whether sstables are skipped due to not covering the requeste slice
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > -10 AND c <= 0 ALLOW FILTERING", 0);
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > -10 AND c < 1 ALLOW FILTERING", 0);
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > -10 AND c <= 1 ALLOW FILTERING", 1, row(1, 1, 1));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > 1 AND c < 3 ALLOW FILTERING", 2, row(1, 2, 4));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c >= 6 ALLOW FILTERING", 1, row(1, 6, 12));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > 6 ALLOW FILTERING", 0);
+ }
+
+ @Test
+ public void testSkippingByKeyRangeInPartitionRangeReads() throws Throwable
+ {
+ DecoratedKey[] keys = new DecoratedKey[8];
+ for (int i = 0; i < keys.length; i++)
+ keys[i] = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(i));
+ Arrays.sort(keys);
+
+ int[] k = new int[keys.length];
+ String[] t = new String[keys.length];
+ for (int i = 0; i < keys.length; i++)
+ {
+ DecoratedKey key = keys[i];
+ k[i] = Int32Type.instance.compose(key.getKey());
+ t[i] = key.getToken().getTokenValue().toString();
+ }
+
+ createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))");
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[1], 1, 1);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[2], 2, 2);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[3], 3, 3);
+ flush();
+ assertEquals(1, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[2], 2, 4);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[3], 3, 5);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[4], 4, 6);
+ flush();
+ assertEquals(2, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[3], 3, 7);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[4], 4, 8);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[5], 5, 9);
+ flush();
+ assertEquals(3, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[4], 4, 10);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[5], 5, 11);
+ execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[6], 6, 12);
+ flush();
+ assertEquals(4, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size());
+
+ // range query - test whether sstables are skipped due to not covering the requested slice
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN (pk) <= " + t[0], 0);
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN (pk) < " + t[1], 0);
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[1] + " AND TOKEN (pk) < " + t[2], 1, row(k[1], 1, 1));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[2] + " AND TOKEN (pk) < " + t[3], 2, row(k[2], 2, 4));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[3] + " AND TOKEN (pk) < " + t[4], 3, row(k[3], 3, 7));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[3] + " AND TOKEN (pk) <= " + t[4], 4, row(k[3], 3, 7), row(k[4], 4, 10));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[4] + " AND TOKEN (pk) < " + t[5], 3, row(k[4], 4, 10));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[5] + " AND TOKEN (pk) < " + t[6], 2, row(k[5], 5, 11));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[6] + " AND TOKEN (pk) < " + t[7], 1, row(k[6], 6, 12));
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) > " + t[6], 0);
+ executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[7], 0);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
index 4b8e5e9..fb37f3f 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
@@ -241,6 +241,114 @@
}
@Test
+ public void testAggregateWithSets() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<int>, fs frozen<set<int>>)");
+
+ // Test with empty table
+ String select = "SELECT count(s), count(fs), min(s), min(fs), max(s), max(fs) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs,
+ "system.count(s)", "system.count(fs)",
+ "system.min(s)", "system.min(fs)",
+ "system.max(s)", "system.max(fs)");
+ assertRows(rs, row(0L, 0L, null, null, null, null));
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, s, fs) VALUES (1, {1, 2}, {1, 2})");
+ execute("INSERT INTO %s (k, s, fs) VALUES (2, {1, 2, 3}, {1, 2, 3})");
+ execute("INSERT INTO %s (k, s, fs) VALUES (3, {2, 1}, {2, 1})");
+ assertRows(execute(select), row(3L, 3L, set(1, 2), set(1, 2), set(1, 2, 3), set(1, 2, 3)));
+ }
+
+ @Test
+ public void testAggregateWithLists() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<int>, fl frozen<list<int>>)");
+
+ // Test with empty table
+ String select = "SELECT count(l), count(fl), min(l), min(fl), max(l), max(fl) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs,
+ "system.count(l)", "system.count(fl)",
+ "system.min(l)", "system.min(fl)",
+ "system.max(l)", "system.max(fl)");
+ assertRows(rs, row(0L, 0L, null, null, null, null));
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, l, fl) VALUES (1, [1, 2], [1, 2])");
+ execute("INSERT INTO %s (k, l, fl) VALUES (2, [1, 2, 3], [1, 2, 3])");
+ execute("INSERT INTO %s (k, l, fl) VALUES (3, [2, 1], [2, 1])");
+ assertRows(execute(select),
+ row(3L, 3L, list(1, 2), list(1, 2), list(2, 1), list(2, 1)));
+ }
+
+ @Test
+ public void testAggregateWithMaps() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<int, int>, fm frozen<map<int, int>>)");
+
+ // Test with empty table
+ String select = "SELECT count(m), count(fm), min(m), min(fm), max(m), max(fm) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs,
+ "system.count(m)", "system.count(fm)",
+ "system.min(m)", "system.min(fm)",
+ "system.max(m)", "system.max(fm)");
+ assertRows(rs, row(0L, 0L, null, null, null, null));
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, m, fm) VALUES (1, {1:10, 2:20}, {1:10, 2:20})");
+ execute("INSERT INTO %s (k, m, fm) VALUES (2, {1:10, 2:20, 3:30}, {1:10, 2:20, 3:30})");
+ execute("INSERT INTO %s (k, m, fm) VALUES (3, {2:20, 1:10}, {2:20, 1:10})");
+ assertRows(execute(select),
+ row(3L, 3L,
+ map(1, 10, 2, 20), map(1, 10, 2, 20),
+ map(1, 10, 2, 20, 3, 30), map(1, 10, 2, 20, 3, 30)));
+ }
+
+ @Test
+ public void testAggregateWithTuples() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<int, text, boolean>)");
+
+ // Test with empty table
+ String select = "SELECT count(t), min(t), max(t) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs, "system.count(t)", "system.min(t)", "system.max(t)");
+ assertRows(rs, row(0L, null, null));
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, t) VALUES (1, (1, 'a', false))");
+ execute("INSERT INTO %s (k, t) VALUES (2, (2, 'b', true))");
+ execute("INSERT INTO %s (k, t) VALUES (3, (3, null, true))");
+ assertRows(execute(select), row(3L, tuple(1, "a", false), tuple(3, null, true)));
+ }
+
+ @Test
+ public void testAggregateWithUDTs() throws Throwable
+ {
+ String udt = createType("CREATE TYPE %s (x int)");
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, u frozen<" + udt + ">, fu frozen<" + udt + ">)");
+
+ // Test with empty table
+ String select = "SELECT count(u), count(fu), min(u), min(fu), max(u), max(fu) FROM %s";
+ UntypedResultSet rs = execute(select);
+ assertColumnNames(rs,
+ "system.count(u)", "system.count(fu)",
+ "system.min(u)", "system.min(fu)",
+ "system.max(u)", "system.max(fu)");
+ assertRows(rs, row(0L, 0L, null, null, null, null));
+
+ // Test with not-empty table
+ execute("INSERT INTO %s (k, u, fu) VALUES (1, {x: 2}, null)");
+ execute("INSERT INTO %s (k, u, fu) VALUES (2, {x: 4}, {x: 6})");
+ execute("INSERT INTO %s (k, u, fu) VALUES (3, null, {x: 8})");
+ assertRows(execute(select),
+ row(2L, 2L, userType("x", 2), userType("x", 6), userType("x", 4), userType("x", 8)));
+ }
+
+ @Test
public void testAggregateWithUdtFields() throws Throwable
{
String myType = createType("CREATE TYPE %s (x int)");
@@ -259,8 +367,8 @@
assertRows(execute("SELECT count(b.x), max(b.x) as max, b.x, c.x as first FROM %s"),
row(3L, 8, 2, null));
- assertInvalidMessage("Invalid field selection: system.max(b) of type blob is not a user type",
- "SELECT max(b).x as max FROM %s");
+ assertRows(execute("SELECT count(b), min(b).x, max(b).x, count(c), min(c).x, max(c).x FROM %s"),
+ row(3L, 2, 8, 2L, 6, 12));
}
@Test
@@ -369,25 +477,25 @@
"LANGUAGE JAVA " +
"AS 'return Double.valueOf(Math.copySign(magnitude, sign));';");
- assertColumnNames(execute("SELECT max(a), max(toUnixTimestamp(b)) FROM %s"), "system.max(a)", "system.max(system.tounixtimestamp(b))");
- assertRows(execute("SELECT max(a), max(toUnixTimestamp(b)) FROM %s"), row(null, null));
- assertColumnNames(execute("SELECT max(a), toUnixTimestamp(max(b)) FROM %s"), "system.max(a)", "system.tounixtimestamp(system.max(b))");
- assertRows(execute("SELECT max(a), toUnixTimestamp(max(b)) FROM %s"), row(null, null));
+ assertColumnNames(execute("SELECT max(a), max(to_unix_timestamp(b)) FROM %s"), "system.max(a)", "system.max(system.to_unix_timestamp(b))");
+ assertRows(execute("SELECT max(a), max(to_unix_timestamp(b)) FROM %s"), row(null, null));
+ assertColumnNames(execute("SELECT max(a), to_unix_timestamp(max(b)) FROM %s"), "system.max(a)", "system.to_unix_timestamp(system.max(b))");
+ assertRows(execute("SELECT max(a), to_unix_timestamp(max(b)) FROM %s"), row(null, null));
assertColumnNames(execute("SELECT max(" + copySign + "(c, d)) FROM %s"), "system.max(" + copySign + "(c, d))");
assertRows(execute("SELECT max(" + copySign + "(c, d)) FROM %s"), row((Object) null));
- execute("INSERT INTO %s (a, b, c, d) VALUES (1, maxTimeuuid('2011-02-03 04:05:00+0000'), -1.2, 2.1)");
- execute("INSERT INTO %s (a, b, c, d) VALUES (2, maxTimeuuid('2011-02-03 04:06:00+0000'), 1.3, -3.4)");
- execute("INSERT INTO %s (a, b, c, d) VALUES (3, maxTimeuuid('2011-02-03 04:10:00+0000'), 1.4, 1.2)");
+ execute("INSERT INTO %s (a, b, c, d) VALUES (1, max_timeuuid('2011-02-03 04:05:00+0000'), -1.2, 2.1)");
+ execute("INSERT INTO %s (a, b, c, d) VALUES (2, max_timeuuid('2011-02-03 04:06:00+0000'), 1.3, -3.4)");
+ execute("INSERT INTO %s (a, b, c, d) VALUES (3, max_timeuuid('2011-02-03 04:10:00+0000'), 1.4, 1.2)");
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
format.setTimeZone(TimeZone.getTimeZone("GMT"));
Date date = format.parse("2011-02-03 04:10:00");
date = DateUtils.truncate(date, Calendar.MILLISECOND);
- assertRows(execute("SELECT max(a), max(toUnixTimestamp(b)) FROM %s"), row(3, date.getTime()));
- assertRows(execute("SELECT max(a), toUnixTimestamp(max(b)) FROM %s"), row(3, date.getTime()));
+ assertRows(execute("SELECT max(a), max(to_unix_timestamp(b)) FROM %s"), row(3, date.getTime()));
+ assertRows(execute("SELECT max(a), to_unix_timestamp(max(b)) FROM %s"), row(3, date.getTime()));
assertRows(execute("SELECT " + copySign + "(max(c), min(c)) FROM %s"), row(-1.4));
assertRows(execute("SELECT " + copySign + "(c, d) FROM %s"), row(1.2), row(-1.3), row(1.4));
@@ -404,16 +512,16 @@
"CREATE OR REPLACE FUNCTION %s(state double, val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS '\"string\";';");
+ "LANGUAGE java " +
+ "AS ' return state;';");
createFunctionOverload(f,
"double, double",
"CREATE OR REPLACE FUNCTION %s(state int, val int) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS int " +
- "LANGUAGE javascript " +
- "AS '\"string\";';");
+ "LANGUAGE java " +
+ "AS 'return state;';");
String a = createAggregateName(KEYSPACE);
String aggregateName = shortFunctionName(a);
@@ -456,8 +564,8 @@
"CREATE OR REPLACE FUNCTION %s(state double, val list<tuple<int, int>>) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS '\"string\";';");
+ "LANGUAGE java " +
+ "AS ' return state;';");
String a1 = createAggregateName(KEYSPACE);
registerAggregate(a1, "list<tuple<int, int>>");
@@ -479,16 +587,16 @@
"CREATE OR REPLACE FUNCTION %s(state double, val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS '\"string\";';");
+ "LANGUAGE java " +
+ "AS 'return state;';");
createFunctionOverload(f,
"double, double",
"CREATE OR REPLACE FUNCTION %s(state int, val int) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS int " +
- "LANGUAGE javascript " +
- "AS '\"string\";';");
+ "LANGUAGE java " +
+ "AS ' return state;';");
// DROP AGGREGATE must not succeed against a scalar
assertInvalidMessage("matches multiple function definitions", "DROP AGGREGATE " + f);
@@ -531,8 +639,8 @@
"CREATE OR REPLACE FUNCTION %s(state double, val double) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
- "LANGUAGE javascript " +
- "AS '\"string\";';");
+ "LANGUAGE java " +
+ "AS ' return state;';");
String a = createAggregate(KEYSPACE,
"double",
@@ -1031,81 +1139,6 @@
}
@Test
- public void testJavascriptAggregate() throws Throwable
- {
- createTable("CREATE TABLE %s (a int primary key, b int)");
- execute("INSERT INTO %s (a, b) VALUES (1, 1)");
- execute("INSERT INTO %s (a, b) VALUES (2, 2)");
- execute("INSERT INTO %s (a, b) VALUES (3, 3)");
-
- String fState = createFunction(KEYSPACE,
- "int, int",
- "CREATE FUNCTION %s(a int, b int) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS int " +
- "LANGUAGE javascript " +
- "AS 'a + b;'");
-
- String fFinal = createFunction(KEYSPACE,
- "int",
- "CREATE FUNCTION %s(a int) " +
- "RETURNS NULL ON NULL INPUT " +
- "RETURNS text " +
- "LANGUAGE javascript " +
- "AS '\"\"+a'");
-
- String a = createFunction(KEYSPACE,
- "int",
- "CREATE AGGREGATE %s(int) " +
- "SFUNC " + shortFunctionName(fState) + " " +
- "STYPE int " +
- "FINALFUNC " + shortFunctionName(fFinal) + " " +
- "INITCOND 42");
-
- // 42 + 1 + 2 + 3 = 48
- assertRows(execute("SELECT " + a + "(b) FROM %s"), row("48"));
-
- execute("DROP AGGREGATE " + a + "(int)");
-
- execute("DROP FUNCTION " + fFinal + "(int)");
- execute("DROP FUNCTION " + fState + "(int, int)");
-
- assertInvalidMessage("Unknown function", "SELECT " + a + "(b) FROM %s");
- }
-
- @Test
- public void testJavascriptAggregateSimple() throws Throwable
- {
- createTable("CREATE TABLE %s (a int primary key, b int)");
- execute("INSERT INTO %s (a, b) VALUES (1, 1)");
- execute("INSERT INTO %s (a, b) VALUES (2, 2)");
- execute("INSERT INTO %s (a, b) VALUES (3, 3)");
-
- String fState = createFunction(KEYSPACE,
- "int, int",
- "CREATE FUNCTION %s(a int, b int) " +
- "CALLED ON NULL INPUT " +
- "RETURNS int " +
- "LANGUAGE javascript " +
- "AS 'a + b;'");
-
- String a = createAggregate(KEYSPACE,
- "int, int",
- "CREATE AGGREGATE %s(int) " +
- "SFUNC " + shortFunctionName(fState) + " " +
- "STYPE int ");
-
- // 1 + 2 + 3 = 6
- assertRows(execute("SELECT " + a + "(b) FROM %s"), row(6));
-
- execute("DROP AGGREGATE " + a + "(int)");
-
- execute("DROP FUNCTION " + fState + "(int, int)");
-
- assertInvalidMessage("Unknown function", "SELECT " + a + "(b) FROM %s");
- }
-
- @Test
public void testFunctionDropPreparedStatement() throws Throwable
{
String otherKS = "cqltest_foo";
@@ -1120,8 +1153,8 @@
"CREATE FUNCTION %s(a int, b int) " +
"CALLED ON NULL INPUT " +
"RETURNS int " +
- "LANGUAGE javascript " +
- "AS 'a + b;'");
+ "LANGUAGE java " +
+ "AS ' return a + b;'");
String a = createAggregate(otherKS,
"int",
@@ -1163,8 +1196,8 @@
"CREATE FUNCTION %s(a int, b int) " +
"CALLED ON NULL INPUT " +
"RETURNS int " +
- "LANGUAGE javascript " +
- "AS 'a + b;'");
+ "LANGUAGE java " +
+ "AS ' return a + b;'");
String a = createAggregate(KEYSPACE,
"int",
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
index fddd59b..1881707 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
@@ -22,6 +22,7 @@
import org.junit.Assert;
import org.junit.Test;
+import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -29,7 +30,6 @@
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.memtable.SkipListMemtable;
import org.apache.cassandra.db.memtable.TestMemtable;
-import org.apache.cassandra.dht.OrderPreservingPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
@@ -268,9 +268,9 @@
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
metadata.updateHostId(UUID.randomUUID(), local);
- metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
+ metadata.updateNormalToken(Util.token("A"), local);
metadata.updateHostId(UUID.randomUUID(), remote);
- metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
+ metadata.updateNormalToken(Util.token("B"), remote);
// With two datacenters we should respect anything passed in as a manual override
String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy', 'replication_factor' : 1, '" + DATA_CENTER_REMOTE + "': 3}");
@@ -326,9 +326,9 @@
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
metadata.updateHostId(UUID.randomUUID(), local);
- metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
+ metadata.updateNormalToken(Util.token("A"), local);
metadata.updateHostId(UUID.randomUUID(), remote);
- metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
+ metadata.updateNormalToken(Util.token("B"), remote);
// Let's create a keyspace first with SimpleStrategy
String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2}");
@@ -363,9 +363,9 @@
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
metadata.updateHostId(UUID.randomUUID(), local);
- metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
+ metadata.updateNormalToken(Util.token("A"), local);
metadata.updateHostId(UUID.randomUUID(), remote);
- metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
+ metadata.updateNormalToken(Util.token("B"), remote);
DatabaseDescriptor.setDefaultKeyspaceRF(3);
@@ -560,6 +560,7 @@
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))");
assertSame(MemtableParams.DEFAULT.factory(), getCurrentColumnFamilyStore().metadata().params.memtable.factory());
assertSchemaOption("memtable", null);
+ Class<? extends Memtable> defaultClass = getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable().getClass();
testMemtableConfig("skiplist", SkipListMemtable.FACTORY, SkipListMemtable.class);
testMemtableConfig("test_fullname", TestMemtable.FACTORY, SkipListMemtable.class);
@@ -570,7 +571,7 @@
+ " WITH compression = {'class': 'LZ4Compressor'};");
assertSchemaOption("memtable", "test_shortname");
- testMemtableConfig("default", MemtableParams.DEFAULT.factory(), SkipListMemtable.class);
+ testMemtableConfig("default", MemtableParams.DEFAULT.factory(), defaultClass);
assertAlterTableThrowsException(ConfigurationException.class,
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit1Test.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit1Test.java
index 80bf51c..214d776 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit1Test.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit1Test.java
@@ -778,11 +778,11 @@
execute("INSERT INTO %s (pk, v) VALUES (?, ?)", bytes("foo123"), bytes("1"));
// Test restrictions on non-primary key value
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('');"));
execute("INSERT INTO %s (pk, v) VALUES (?, ?)", bytes("foo124"), EMPTY_BYTE_BUFFER);
- assertRows(execute("SELECT * FROM %s WHERE v = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE v = text_as_blob('');"),
row(bytes("foo124"), EMPTY_BYTE_BUFFER));
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java
index e399feb..1c4feb3 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageSplit2Test.java
@@ -1639,49 +1639,49 @@
beforeAndAfterFlush(() -> {
- assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk = textAsBlob('');");
- assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk IN (textAsBlob(''), textAsBlob('1'));");
+ assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk = text_as_blob('');");
+ assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk IN (text_as_blob(''), text_as_blob('1'));");
assertInvalidMessage("Key may not be empty",
"INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
EMPTY_BYTE_BUFFER, bytes("2"), bytes("2"));
// Test clustering columns restrictions
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) = (text_as_blob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) IN ((text_as_blob('')), (text_as_blob('1')));"),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) > (text_as_blob(''));"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob(''));"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) <= (text_as_blob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) < (text_as_blob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('') AND c < text_as_blob('');"));
});
if (options.contains("COMPACT"))
@@ -1696,60 +1696,60 @@
bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) = (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) IN ((text_as_blob('')), (text_as_blob('1')));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) > (text_as_blob(''));"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) <= (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) < (text_as_blob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('') AND c < text_as_blob('');"));
});
}
// Test restrictions on non-primary key value
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('') ALLOW FILTERING;"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER);
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('') ALLOW FILTERING;"),
row(bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER));
});
}
@@ -1764,76 +1764,76 @@
beforeAndAfterFlush(() -> {
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob('1'), textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) = (text_as_blob('1'), text_as_blob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) IN ((text_as_blob(''), text_as_blob('1')), (text_as_blob('1'), text_as_blob('1')));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 >= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 >= text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 <= textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 <= text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob('1'), textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) <= (text_as_blob('1'), text_as_blob(''));"));
});
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)",
bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4"));
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('') AND c2 = textAsBlob('1');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('') AND c2 = text_as_blob('1');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) = (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) IN ((text_as_blob(''), text_as_blob('1')), (text_as_blob('1'), text_as_blob('1')));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) >= (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) >= (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) <= (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) < (textAsBlob(''), textAsBlob('1'));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) < (text_as_blob(''), text_as_blob('1'));"));
});
}
@@ -1857,17 +1857,17 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')" + orderingClause));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('')" + orderingClause));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')" + orderingClause));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('')" + orderingClause));
});
assertInvalidMessage("Invalid empty or null value for column c",
@@ -1892,19 +1892,19 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 >= textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 >= text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
});
@@ -1914,19 +1914,19 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) IN ((text_as_blob(''), text_as_blob('1')), (text_as_blob('1'), text_as_blob('1')))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) >= (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) >= (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
index 68751cd..c2a8653 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
@@ -17,16 +17,6 @@
*/
package org.apache.cassandra.cql3.validation.operations;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.UUID;
-
-import org.junit.Assert;
-import org.junit.Test;
-
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
@@ -34,6 +24,7 @@
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.memtable.SkipListMemtable;
import org.apache.cassandra.db.memtable.TestMemtable;
+import org.apache.cassandra.db.memtable.TrieMemtable;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -43,25 +34,23 @@
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.MemtableParams;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.SchemaKeyspaceTables;
-import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.*;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.triggers.ITrigger;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.UUID;
import static java.lang.String.format;
-import static org.apache.cassandra.cql3.Duration.NANOS_PER_HOUR;
-import static org.apache.cassandra.cql3.Duration.NANOS_PER_MICRO;
-import static org.apache.cassandra.cql3.Duration.NANOS_PER_MILLI;
-import static org.apache.cassandra.cql3.Duration.NANOS_PER_MINUTE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.apache.cassandra.cql3.Duration.*;
+import static org.junit.Assert.*;
public class CreateTest extends CQLTester
{
@@ -613,14 +602,16 @@
{
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))");
assertSame(MemtableParams.DEFAULT.factory(), getCurrentColumnFamilyStore().metadata().params.memtable.factory());
+ Class<? extends Memtable> defaultClass = getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable().getClass();
assertSchemaOption("memtable", null);
testMemtableConfig("skiplist", SkipListMemtable.FACTORY, SkipListMemtable.class);
+ testMemtableConfig("trie", MemtableParams.get("trie").factory(), TrieMemtable.class);
testMemtableConfig("skiplist_remapped", SkipListMemtable.FACTORY, SkipListMemtable.class);
testMemtableConfig("test_fullname", TestMemtable.FACTORY, SkipListMemtable.class);
testMemtableConfig("test_shortname", SkipListMemtable.FACTORY, SkipListMemtable.class);
- testMemtableConfig("default", MemtableParams.DEFAULT.factory(), SkipListMemtable.class);
+ testMemtableConfig("default", MemtableParams.DEFAULT.factory(), defaultClass);
assertThrowsConfigurationException("The 'class_name' option must be specified.",
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
index 980b791..645becd 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/DeleteTest.java
@@ -1171,12 +1171,12 @@
createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY (pk, c))");
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');");
assertEmpty(execute("SELECT * FROM %s"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));");
assertEmpty(execute("SELECT * FROM %s"));
@@ -1184,27 +1184,27 @@
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('')");
assertRows(execute("SELECT * FROM %s"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('')");
assertEmpty(execute("SELECT * FROM %s"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('')");
assertEmpty(execute("SELECT * FROM %s"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("2"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')");
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('')");
assertRows(execute("SELECT * FROM %s"),
row(bytes("foo123"), bytes("1"), bytes("1")),
@@ -1217,12 +1217,12 @@
createTable("CREATE TABLE %s (pk blob, c1 blob, c2 blob, v blob, PRIMARY KEY (pk, c1, c2))");
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('');");
assertEmpty(execute("SELECT * FROM %s"));
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("1"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1');");
assertEmpty(execute("SELECT * FROM %s"));
@@ -1230,27 +1230,27 @@
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 > text_as_blob('')");
assertRows(execute("SELECT * FROM %s"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("0")));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 >= textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 >= text_as_blob('')");
assertEmpty(execute("SELECT * FROM %s"));
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 > text_as_blob('')");
assertEmpty(execute("SELECT * FROM %s"));
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"), bytes("1"));
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)", bytes("foo123"), bytes("1"), bytes("2"), bytes("3"));
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 <= textAsBlob('')");
- execute("DELETE FROM %s WHERE pk = textAsBlob('foo123') AND c1 < textAsBlob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 <= text_as_blob('')");
+ execute("DELETE FROM %s WHERE pk = text_as_blob('foo123') AND c1 < text_as_blob('')");
assertRows(execute("SELECT * FROM %s"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java
index 4e33ae3..f6e71ae 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertTest.java
@@ -18,6 +18,8 @@
package org.apache.cassandra.cql3.validation.operations;
+import java.nio.ByteBuffer;
+
import org.junit.Assert;
import org.junit.Test;
@@ -68,6 +70,15 @@
}
@Test
+ public void testEmptyTTL() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)");
+ execute("INSERT INTO %s (k, v) VALUES (0, 0) USING TTL ?", (Object) null);
+ execute("INSERT INTO %s (k, v) VALUES (1, 1) USING TTL ?", ByteBuffer.wrap(new byte[0]));
+ assertRows(execute("SELECT k, v, ttl(v) FROM %s"), row(1, 1, null), row(0, 0, null));
+ }
+
+ @Test
public void testInsertWithUnset() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, s text, i int)");
@@ -87,7 +98,7 @@
assertInvalidMessage("Invalid unset value for column k", "UPDATE %s SET i = 0 WHERE k = ?", unset());
assertInvalidMessage("Invalid unset value for column k", "DELETE FROM %s WHERE k = ?", unset());
- assertInvalidMessage("Invalid unset value for argument in call to function blobasint", "SELECT * FROM %s WHERE k = blobAsInt(?)", unset());
+ assertInvalidMessage("Invalid unset value for argument in call to function blob_as_int", "SELECT * FROM %s WHERE k = blob_as_int(?)", unset());
}
@Test
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionCollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionCollectionsTest.java
index c0a5139..fe4073c 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionCollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionCollectionsTest.java
@@ -29,6 +29,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
@@ -51,6 +52,8 @@
@Parameterized.Parameters(name = "{index}: clusterMinVersion={0}")
public static Collection<Object[]> data()
{
+ ServerTestUtils.daemonInitialization();
+
return InsertUpdateIfConditionTest.data();
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionStaticsTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionStaticsTest.java
index b9c9ce3..3270fc1 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionStaticsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionStaticsTest.java
@@ -27,6 +27,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.CQLTester;
/* InsertUpdateIfConditionCollectionsTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670)
@@ -47,6 +48,7 @@
@Parameterized.Parameters(name = "{index}: clusterMinVersion={0}")
public static Collection<Object[]> data()
{
+ ServerTestUtils.daemonInitialization();
return InsertUpdateIfConditionTest.data();
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
index 8429ec0..c617544 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
@@ -28,6 +28,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
@@ -61,6 +62,7 @@
@Parameterized.Parameters(name = "{index}: clusterMinVersion={0}")
public static Collection<Object[]> data()
{
+ ServerTestUtils.daemonInitialization();
return Arrays.asList(new Object[]{ "3.0", (Runnable) () -> {
Pair<Boolean, CassandraVersion> res = Gossiper.instance.isUpgradingFromVersionLowerThanC17653(new CassandraVersion("3.11"));
assertTrue(debugMsgCASSANDRA17653(res), res.left);
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java
index f46783b..0a0aecb 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java
@@ -2251,7 +2251,7 @@
row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L),
row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L));
- assertRows(execute("SELECT pk, floor(toTimestamp(time), 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(toTimestamp(time), 5m" + startingTime + ")"),
+ assertRows(execute("SELECT pk, floor(to_timestamp(time), 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(to_timestamp(time), 5m" + startingTime + ")"),
row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L),
row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L),
row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L),
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
index 2e419e1..347bc1d 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
@@ -23,6 +23,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
@@ -34,6 +35,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
index 8a3ae03..0cbe221 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectOrderByTest.java
@@ -73,17 +73,17 @@
beforeAndAfterFlush(() -> {
// order by the only column in the selection
- assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
row(0), row(1), row(2));
- assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
row(2), row(1), row(0));
// order by a column not in the selection
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c)) FROM %s WHERE a=? ORDER BY b ASC", 0),
row(0), row(1), row(2));
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c)) FROM %s WHERE a=? ORDER BY b DESC", 0),
row(2), row(1), row(0));
assertInvalid("SELECT * FROM %s WHERE a=? ORDER BY c ASC", 0);
@@ -109,7 +109,7 @@
assertRows(execute("SELECT c.a FROM %s WHERE a=? ORDER BY b DESC", 0),
row(2), row(1), row(0));
- assertRows(execute("SELECT blobAsInt(intAsBlob(c.a)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c.a)) FROM %s WHERE a=? ORDER BY b DESC", 0),
row(2), row(1), row(0));
});
dropTable("DROP TABLE %s");
@@ -198,39 +198,39 @@
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 2, 5);
beforeAndAfterFlush(() -> {
- assertInvalid("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY c ASC", 0);
- assertInvalid("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY c DESC", 0);
- assertInvalid("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b ASC, c DESC", 0);
- assertInvalid("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b DESC, c ASC", 0);
- assertInvalid("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY d ASC", 0);
+ assertInvalid("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY c ASC", 0);
+ assertInvalid("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY c DESC", 0);
+ assertInvalid("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b ASC, c DESC", 0);
+ assertInvalid("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b DESC, c ASC", 0);
+ assertInvalid("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY d ASC", 0);
// select and order by b
- assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
row(0), row(0), row(0), row(1), row(1), row(1));
- assertRows(execute("SELECT blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
row(1), row(1), row(1), row(0), row(0), row(0));
- assertRows(execute("SELECT b, blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ assertRows(execute("SELECT b, blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b ASC", 0),
row(0, 0), row(0, 0), row(0, 0), row(1, 1), row(1, 1), row(1, 1));
- assertRows(execute("SELECT b, blobAsInt(intAsBlob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ assertRows(execute("SELECT b, blob_as_int(int_as_blob(b)) FROM %s WHERE a=? ORDER BY b DESC", 0),
row(1, 1), row(1, 1), row(1, 1), row(0, 0), row(0, 0), row(0, 0));
// select c, order by b
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b ASC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c)) FROM %s WHERE a=? ORDER BY b ASC", 0),
row(0), row(1), row(2), row(0), row(1), row(2));
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c)) FROM %s WHERE a=? ORDER BY b DESC", 0),
row(2), row(1), row(0), row(2), row(1), row(0));
// select c, order by b, c
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b ASC, c ASC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c)) FROM %s WHERE a=? ORDER BY b ASC, c ASC", 0),
row(0), row(1), row(2), row(0), row(1), row(2));
- assertRows(execute("SELECT blobAsInt(intAsBlob(c)) FROM %s WHERE a=? ORDER BY b DESC, c DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(c)) FROM %s WHERE a=? ORDER BY b DESC, c DESC", 0),
row(2), row(1), row(0), row(2), row(1), row(0));
// select d, order by b, c
- assertRows(execute("SELECT blobAsInt(intAsBlob(d)) FROM %s WHERE a=? ORDER BY b ASC, c ASC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(d)) FROM %s WHERE a=? ORDER BY b ASC, c ASC", 0),
row(0), row(1), row(2), row(3), row(4), row(5));
- assertRows(execute("SELECT blobAsInt(intAsBlob(d)) FROM %s WHERE a=? ORDER BY b DESC, c DESC", 0),
+ assertRows(execute("SELECT blob_as_int(int_as_blob(d)) FROM %s WHERE a=? ORDER BY b DESC, c DESC", 0),
row(5), row(4), row(3), row(2), row(1), row(0));
});
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
index 17c7e87..ae316e0 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
@@ -541,8 +541,8 @@
assertInvalidMessage("Invalid unset value for argument in call to function token",
"SELECT * FROM %s WHERE token(k) >= token(?)", unset());
- assertInvalidMessage("Invalid unset value for argument in call to function blobasint",
- "SELECT * FROM %s WHERE k = blobAsInt(?)", unset());
+ assertInvalidMessage("Invalid unset value for argument in call to function blob_as_int",
+ "SELECT * FROM %s WHERE k = blob_as_int(?)", unset());
}
@Test
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
index 7bbaa85..8e3a756 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java
@@ -923,7 +923,7 @@
for (int i = 0; i < 5; i++)
execute("INSERT INTO %s (k, t) VALUES (?, now())", i);
- execute("SELECT dateOf(t) FROM %s");
+ execute("SELECT to_timestamp(t) FROM %s");
}
/**
@@ -959,7 +959,7 @@
assertTrue(ttl == 9 || ttl == 10);
// test aliasing a regular function
- rs = execute("SELECT intAsBlob(id) AS id_blob FROM %s WHERE id = 0");
+ rs = execute("SELECT int_as_blob(id) AS id_blob FROM %s WHERE id = 0");
assertEquals("id_blob", rs.metadata().get(0).name.toString());
assertEquals(ByteBuffer.wrap(new byte[4]), rs.one().getBlob(rs.metadata().get(0).name.toString()));
@@ -1110,7 +1110,7 @@
createTable("CREATE TABLE %s ( k int, v int, PRIMARY KEY (k, v))");
execute("INSERT INTO %s (k, v) VALUES (0, 0)");
-
+
flush();
assertRows(execute("SELECT v FROM %s WHERE k=0 AND v IN (1, 0)"),
@@ -1255,7 +1255,7 @@
execute("DELETE FROM %s WHERE a = 2 AND b = 2");
beforeAndAfterFlush(() -> {
-
+
// Checks filtering
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
"SELECT * FROM %s WHERE c = 4 AND d = 8");
@@ -2328,7 +2328,7 @@
// Test for CASSANDRA-11310 compatibility with 2i
createTable("CREATE TABLE %s (a text, b int, c text, d int, PRIMARY KEY (a, b, c));");
createIndex("CREATE INDEX ON %s(c)");
-
+
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", "a", 0, "b", 1);
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", "a", 1, "b", 2);
execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", "a", 2, "b", 3);
@@ -2513,49 +2513,49 @@
beforeAndAfterFlush(() -> {
- assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk = textAsBlob('');");
- assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk IN (textAsBlob(''), textAsBlob('1'));");
+ assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk = text_as_blob('');");
+ assertInvalidMessage("Key may not be empty", "SELECT * FROM %s WHERE pk IN (text_as_blob(''), text_as_blob('1'));");
assertInvalidMessage("Key may not be empty",
"INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
EMPTY_BYTE_BUFFER, bytes("2"), bytes("2"));
// Test clustering columns restrictions
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) = (text_as_blob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) IN ((text_as_blob('')), (text_as_blob('1')));"),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) > (text_as_blob(''));"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob(''));"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) <= (text_as_blob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) < (text_as_blob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('') AND c < text_as_blob('');"));
});
@@ -2563,59 +2563,59 @@
bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4"));
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c = text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) = (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) IN ((text_as_blob('')), (text_as_blob('1')));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) > (text_as_blob(''));"),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) >= (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob(''));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) <= (text_as_blob(''));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c) < (text_as_blob(''));"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND c < textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('') AND c < text_as_blob('');"));
});
// Test restrictions on non-primary key value
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('') ALLOW FILTERING;"));
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER);
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('') ALLOW FILTERING;"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND v = text_as_blob('') ALLOW FILTERING;"),
row(bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER));
});
}
@@ -2629,76 +2629,76 @@
beforeAndAfterFlush(() -> {
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 = textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 = text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob('1'), textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) = (text_as_blob('1'), text_as_blob(''));"));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 IN (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 IN (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) IN ((text_as_blob(''), text_as_blob('1')), (text_as_blob('1'), text_as_blob('1')));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 > textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 > text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 >= textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 >= text_as_blob('');"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 <= textAsBlob('');"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 <= text_as_blob('');"));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob('1'), textAsBlob(''));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) <= (text_as_blob('1'), text_as_blob(''));"));
});
execute("INSERT INTO %s (pk, c1, c2, v) VALUES (?, ?, ?, ?)",
bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4"));
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('') AND c2 = textAsBlob('1');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('') AND c2 = text_as_blob('1');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) = (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) = (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1');"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1');"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) IN ((text_as_blob(''), text_as_blob('1')), (text_as_blob('1'), text_as_blob('1')));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) >= (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) >= (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) <= (textAsBlob(''), textAsBlob('1'));"),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) <= (text_as_blob(''), text_as_blob('1'));"),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) < (textAsBlob(''), textAsBlob('1'));"));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) < (text_as_blob(''), text_as_blob('1'));"));
});
}
@@ -2722,17 +2722,17 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')" + orderingClause));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('')" + orderingClause));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')" + orderingClause));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('')" + orderingClause));
});
execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
@@ -2740,22 +2740,22 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c IN (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c >= text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
- assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('')" + orderingClause));
+ assertEmpty(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c < text_as_blob('')" + orderingClause));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c <= text_as_blob('')" + orderingClause),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("4")));
});
}
@@ -2775,19 +2775,19 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 > textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 > text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 = textAsBlob('1') AND c2 >= textAsBlob('')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 = text_as_blob('1') AND c2 >= text_as_blob('')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
});
@@ -2797,19 +2797,19 @@
beforeAndAfterFlush(() -> {
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c1 IN (textAsBlob(''), textAsBlob('1')) AND c2 = textAsBlob('1')" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND c1 IN (text_as_blob(''), text_as_blob('1')) AND c2 = text_as_blob('1')" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) IN ((textAsBlob(''), textAsBlob('1')), (textAsBlob('1'), textAsBlob('1')))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) IN ((text_as_blob(''), text_as_blob('1')), (text_as_blob('1'), text_as_blob('1')))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) > (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) > (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")));
- assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c1, c2) >= (textAsBlob(''), textAsBlob('1'))" + orderingClause),
+ assertRows(execute("SELECT * FROM %s WHERE pk = text_as_blob('foo123') AND (c1, c2) >= (text_as_blob(''), text_as_blob('1'))" + orderingClause),
row(bytes("foo123"), bytes("1"), bytes("2"), bytes("2")),
row(bytes("foo123"), bytes("1"), bytes("1"), bytes("1")),
row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"), bytes("4")));
@@ -3191,4 +3191,40 @@
execute("INSERT INTO %s (k1, k2) VALUES (uuid(), 'k2')");
assertRowCount(execute("SELECT system.token(k1, k2) FROM %s"), 1);
}
+
+ @Test
+ public void testQuotedMapTextData() throws Throwable
+ {
+ createTable("CREATE TABLE " + KEYSPACE + ".t1 (id int, data text, PRIMARY KEY (id))");
+ createTable("CREATE TABLE " + KEYSPACE + ".t2 (id int, data map<int, text>, PRIMARY KEY (id))");
+
+ execute("INSERT INTO " + KEYSPACE + ".t1 (id, data) VALUES (1, 'I''m newb')");
+ execute("INSERT INTO " + KEYSPACE + ".t2 (id, data) VALUES (1, {1:'I''m newb'})");
+
+ assertRows(execute("SELECT data FROM " + KEYSPACE + ".t1"), row("I'm newb"));
+ assertRows(execute("SELECT data FROM " + KEYSPACE + ".t2"), row( map(1, "I'm newb")));
+ }
+
+ @Test
+ public void testQuotedSimpleCollectionsData() throws Throwable
+ {
+ createTable("CREATE TABLE " + KEYSPACE + ".t3 (id int, set_data set<text>, list_data list<text>, tuple_data tuple<int, text>, PRIMARY KEY (id))");
+
+ execute("INSERT INTO " + KEYSPACE + ".t3 (id, set_data, list_data, tuple_data) values(1, {'I''m newb'}, ['I''m newb'], (1, 'I''m newb'))");
+
+ assertRows(execute("SELECT set_data FROM " + KEYSPACE + ".t3"), row(set("I'm newb")));
+ assertRows(execute("SELECT list_data FROM " + KEYSPACE + ".t3"), row(list("I'm newb")));
+ assertRows(execute("SELECT tuple_data FROM " + KEYSPACE + ".t3"), row(tuple(1, "I'm newb")));
+ }
+
+ @Test
+ public void testQuotedUDTData() throws Throwable
+ {
+ createType("CREATE TYPE " + KEYSPACE + ".random (data text)");
+ createTable("CREATE TABLE " + KEYSPACE + ".t4 (id int, udt_data frozen<random>, PRIMARY KEY (id))");
+
+ execute("INSERT INTO " + KEYSPACE + ".t4 (id, udt_data) values(1, {data: 'I''m newb'})");
+
+ assertRows(execute("SELECT udt_data FROM " + KEYSPACE + ".t4"), row(userType("random", "I'm newb")));
+ }
}
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java
index 9ef2520..7b66e31 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java
@@ -21,12 +21,11 @@
import java.io.IOException;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.io.util.File;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.apache.cassandra.Util;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Attributes;
@@ -36,7 +35,10 @@
import org.apache.cassandra.db.ExpirationDateOverflowHandling;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.rows.AbstractCell;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileInputStreamPlus;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.tools.StandaloneScrubber;
@@ -44,6 +46,7 @@
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -379,7 +382,7 @@
if (runScrub)
{
- cfs.scrub(true, false, true, reinsertOverflowedTTL, 1);
+ cfs.scrub(true, IScrubber.options().checkData().reinsertOverflowedTTLRows(reinsertOverflowedTTL).build(), 1);
if (reinsertOverflowedTTL)
{
@@ -402,9 +405,7 @@
}
if (runSStableScrub)
{
- System.setProperty(org.apache.cassandra.tools.Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
-
- try
+ try (WithProperties properties = new WithProperties().set(TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST, true))
{
ToolResult tool;
if (reinsertOverflowedTTL)
@@ -419,10 +420,6 @@
else
Assertions.assertThat(tool.getStdout()).contains("No valid partitions found while scrubbing");
}
- finally
- {
- System.clearProperty(org.apache.cassandra.tools.Util.ALLOW_TOOL_REINIT_FOR_TEST);
- }
}
try
@@ -475,4 +472,4 @@
else
return clustering ? COMPLEX_CLUSTERING : COMPLEX_NOCLUSTERING;
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/CellTest.java b/test/unit/org/apache/cassandra/db/CellTest.java
index 4100c54..260ce94 100644
--- a/test/unit/org/apache/cassandra/db/CellTest.java
+++ b/test/unit/org/apache/cassandra/db/CellTest.java
@@ -76,7 +76,8 @@
ColumnIdentifier.getInterned(name, false),
type,
ColumnMetadata.NO_POSITION,
- ColumnMetadata.Kind.REGULAR);
+ ColumnMetadata.Kind.REGULAR,
+ null);
}
@Test
diff --git a/test/unit/org/apache/cassandra/db/ClusteringBoundTest.java b/test/unit/org/apache/cassandra/db/ClusteringBoundTest.java
deleted file mode 100644
index 20fcc20..0000000
--- a/test/unit/org/apache/cassandra/db/ClusteringBoundTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ClusteringBoundTest
-{
- @Test
- public void arrayTopAndBottom()
- {
- Assert.assertTrue(ArrayClusteringBound.BOTTOM.isBottom());
- Assert.assertFalse(ArrayClusteringBound.BOTTOM.isTop());
- Assert.assertTrue(ArrayClusteringBound.TOP.isTop());
- Assert.assertFalse(ArrayClusteringBound.TOP.isBottom());
- }
-
- @Test
- public void bufferTopAndBottom()
- {
- Assert.assertTrue(BufferClusteringBound.BOTTOM.isBottom());
- Assert.assertFalse(BufferClusteringBound.BOTTOM.isTop());
- Assert.assertTrue(BufferClusteringBound.TOP.isTop());
- Assert.assertFalse(BufferClusteringBound.TOP.isBottom());
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java b/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java
index e97f067..3ba4ae6 100644
--- a/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/ClusteringHeapSizeTest.java
@@ -61,6 +61,22 @@
Assertions.assertThat(min).isLessThanOrEqualTo(max);
}
+ @Test
+ public void testSingletonClusteringHeapSize()
+ {
+ Clustering<?> clustering = this.clustering.accessor().factory().staticClustering();
+ Assertions.assertThat(clustering.unsharedHeapSize())
+ .isEqualTo(0);
+ Assertions.assertThat(clustering.unsharedHeapSizeExcludingData())
+ .isEqualTo(0);
+
+ clustering = this.clustering.accessor().factory().clustering();
+ Assertions.assertThat(clustering.unsharedHeapSize())
+ .isEqualTo(0);
+ Assertions.assertThat(clustering.unsharedHeapSizeExcludingData())
+ .isEqualTo(0);
+ }
+
@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> data() {
byte[] rawBytes = { 0, 1, 2, 3, 4, 5, 6 };
diff --git a/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java b/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java
new file mode 100644
index 0000000..a295b22
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ClusteringPrefixTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
+import org.apache.cassandra.utils.memory.MemtablePool;
+import org.apache.cassandra.utils.memory.NativeAllocator;
+import org.apache.cassandra.utils.memory.NativePool;
+import org.apache.cassandra.utils.memory.SlabAllocator;
+import org.apache.cassandra.utils.memory.SlabPool;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class ClusteringPrefixTest
+{
+ @Test
+ public void arrayTopAndBottom()
+ {
+ Assert.assertTrue(ArrayClusteringBound.BOTTOM.isBottom());
+ Assert.assertFalse(ArrayClusteringBound.BOTTOM.isTop());
+ Assert.assertTrue(ArrayClusteringBound.TOP.isTop());
+ Assert.assertFalse(ArrayClusteringBound.TOP.isBottom());
+ }
+
+ @Test
+ public void bufferTopAndBottom()
+ {
+ Assert.assertTrue(BufferClusteringBound.BOTTOM.isBottom());
+ Assert.assertFalse(BufferClusteringBound.BOTTOM.isTop());
+ Assert.assertTrue(BufferClusteringBound.TOP.isTop());
+ Assert.assertFalse(BufferClusteringBound.TOP.isBottom());
+ }
+
+ @Test
+ public void testRetainableArray()
+ {
+ testRetainable(ByteArrayAccessor.instance.factory(), x -> new byte[][] {x.getBytes(StandardCharsets.UTF_8)});
+ }
+
+ @Test
+ public void testRetainableOnHeap()
+ {
+ testRetainable(ByteBufferAccessor.instance.factory(), x -> new ByteBuffer[] {ByteBufferUtil.bytes(x)});
+ }
+
+ @Test
+ public void testRetainableOnHeapSliced()
+ {
+ for (int prepend = 0; prepend < 3; ++prepend)
+ {
+ for (int append = 0; append < 3; ++append)
+ {
+ testRetainable(ByteBufferAccessor.instance.factory(),
+ slicingAllocator(prepend, append));
+ }
+ }
+ }
+
+ private Function<String, ByteBuffer[]> slicingAllocator(int prepend, int append)
+ {
+ return x ->
+ {
+ ByteBuffer bytes = ByteBufferUtil.bytes(x);
+ ByteBuffer sliced = ByteBuffer.allocate(bytes.remaining() + prepend + append);
+ for (int i = 0; i < prepend; ++i)
+ sliced.put((byte) ThreadLocalRandom.current().nextInt());
+ sliced.put(bytes);
+ bytes.flip();
+ for (int i = 0; i < append; ++i)
+ sliced.put((byte) ThreadLocalRandom.current().nextInt());
+ sliced.position(prepend).limit(prepend + bytes.remaining());
+ return new ByteBuffer[]{ sliced.slice() };
+ };
+ }
+
+ @Test
+ public void testRetainableOffHeap()
+ {
+ testRetainable(ByteBufferAccessor.instance.factory(), x ->
+ {
+ ByteBuffer h = ByteBufferUtil.bytes(x);
+ ByteBuffer v = ByteBuffer.allocateDirect(h.remaining());
+ v.put(h);
+ v.flip();
+ return new ByteBuffer[] {v};
+ });
+ }
+
+ @Test
+ public void testRetainableOnHeapSlab() throws InterruptedException, TimeoutException
+ {
+ testRetainableSlab(true);
+ }
+
+ @Test
+ public void testRetainableOffHeapSlab() throws InterruptedException, TimeoutException
+ {
+ testRetainableSlab(false);
+ }
+
+ public void testRetainableSlab(boolean onHeap) throws InterruptedException, TimeoutException
+ {
+ MemtablePool pool = new SlabPool(1L << 24, onHeap ? 0 : 1L << 24, 1.0f, () -> ImmediateFuture.success(false));
+ SlabAllocator allocator = ((SlabAllocator) pool.newAllocator("test"));
+ assert !allocator.allocate(1).isDirect() == onHeap;
+ try
+ {
+ testRetainable(ByteBufferAccessor.instance.factory(), x ->
+ {
+ ByteBuffer h = ByteBufferUtil.bytes(x);
+ ByteBuffer v = allocator.allocate(h.remaining());
+ v.put(h);
+ v.flip();
+ return new ByteBuffer[] {v};
+ });
+ }
+ finally
+ {
+ pool.shutdownAndWait(10, TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ public void testRetainableNative() throws InterruptedException, TimeoutException
+ {
+ MemtablePool pool = new NativePool(1L << 24,1L << 24, 1.0f, () -> ImmediateFuture.success(false));
+ NativeAllocator allocator = (NativeAllocator) pool.newAllocator("test");
+ try
+ {
+ testRetainable(ByteBufferAccessor.instance.factory(),
+ x -> new ByteBuffer[] {ByteBufferUtil.bytes(x)},
+ x -> x.kind() == ClusteringPrefix.Kind.CLUSTERING
+ ? new NativeClustering(allocator, null, (Clustering<?>) x)
+ : x);
+ }
+ finally
+ {
+ pool.shutdownAndWait(10, TimeUnit.SECONDS);
+ }
+ }
+
+ public <V> void testRetainable(ValueAccessor.ObjectFactory<V> factory,
+ Function<String, V[]> allocator)
+ {
+ testRetainable(factory, allocator, null);
+ }
+
+ public <V> void testRetainable(ValueAccessor.ObjectFactory<V> factory,
+ Function<String, V[]> allocator,
+ Function<ClusteringPrefix<V>, ClusteringPrefix<V>> mapper)
+ {
+ ClusteringPrefix<V>[] clusterings = new ClusteringPrefix[]
+ {
+ factory.clustering(),
+ factory.staticClustering(),
+ factory.clustering(allocator.apply("test")),
+ factory.bound(ClusteringPrefix.Kind.INCL_START_BOUND, allocator.apply("testA")),
+ factory.bound(ClusteringPrefix.Kind.INCL_END_BOUND, allocator.apply("testB")),
+ factory.bound(ClusteringPrefix.Kind.EXCL_START_BOUND, allocator.apply("testC")),
+ factory.bound(ClusteringPrefix.Kind.EXCL_END_BOUND, allocator.apply("testD")),
+ factory.boundary(ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY, allocator.apply("testE")),
+ factory.boundary(ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY, allocator.apply("testF")),
+ };
+
+ if (mapper != null)
+ clusterings = Arrays.stream(clusterings)
+ .map(mapper)
+ .toArray(ClusteringPrefix[]::new);
+
+ testRetainable(clusterings);
+ }
+
+ public void testRetainable(ClusteringPrefix<?>[] clusterings)
+ {
+ for (ClusteringPrefix<?> clustering : clusterings)
+ {
+ ClusteringPrefix<?> retainable = clustering.retainable();
+ assertEquals(clustering, retainable);
+ assertClusteringIsRetainable(retainable);
+ }
+ }
+
+
+ public static void assertClusteringIsRetainable(ClusteringPrefix<?> clustering)
+ {
+ if (clustering instanceof AbstractArrayClusteringPrefix)
+ return; // has to be on-heap and minimized
+
+ assertTrue(clustering instanceof AbstractBufferClusteringPrefix);
+ AbstractBufferClusteringPrefix abcf = (AbstractBufferClusteringPrefix) clustering;
+ ByteBuffer[] buffers = abcf.getBufferArray();
+ for (ByteBuffer b : buffers)
+ {
+ assertFalse(b.isDirect());
+ assertTrue(b.hasArray());
+ assertEquals(b.capacity(), b.remaining());
+ assertEquals(0, b.arrayOffset());
+ assertEquals(b.capacity(), b.array().length);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 4d871fc..8855ceb 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -63,8 +63,10 @@
import org.apache.cassandra.index.transactions.UpdateTransaction;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.ScrubTest;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.ClearableHistogram;
@@ -200,7 +202,7 @@
}
@Test
- public void testDeleteStandardRowSticksAfterFlush() throws Throwable
+ public void testDeleteStandardRowSticksAfterFlush()
{
// test to make sure flushing after a delete doesn't resurrect delted cols.
String keyspaceName = KEYSPACE1;
@@ -258,7 +260,7 @@
}
@Test
- public void testClearEphemeralSnapshots() throws Throwable
+ public void testClearEphemeralSnapshots()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1);
@@ -347,9 +349,9 @@
KEYSPACE2,
CF_STANDARD1,
liveSSTable.descriptor.id,
- liveSSTable.descriptor.formatType);
+ liveSSTable.descriptor.version.format);
for (Component c : liveSSTable.getComponents())
- assertTrue("Cannot find backed-up file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists());
+ assertTrue("Cannot find backed-up file:" + desc.fileFor(c), desc.fileFor(c).exists());
}
}
@@ -578,8 +580,8 @@
Set<String> originalFiles = new HashSet<>();
Iterables.toList(cfs.concatWithIndexes()).stream()
- .flatMap(c -> c.getLiveSSTables().stream().map(t -> t.descriptor.filenameFor(Component.DATA)))
- .forEach(originalFiles::add);
+ .flatMap(c -> c.getLiveSSTables().stream().map(t -> t.descriptor.fileFor(Components.DATA)))
+ .forEach(e -> originalFiles.add(e.toString()));
assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(indexTableFile))).isTrue();
assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(baseTableFile))).isTrue();
}
@@ -667,9 +669,9 @@
assertEquals(1, ssTables.size());
SSTableReader ssTable = ssTables.iterator().next();
- String dataFileName = ssTable.descriptor.filenameFor(Component.DATA);
- String tmpDataFileName = ssTable.descriptor.tmpFilenameFor(Component.DATA);
- new File(dataFileName).tryMove(new File(tmpDataFileName));
+ File dataFile = ssTable.descriptor.fileFor(Components.DATA);
+ File tmpDataFile = ssTable.descriptor.tmpFileFor(Components.DATA);
+ dataFile.tryMove(tmpDataFile);
ssTable.selfRef().release();
@@ -701,6 +703,112 @@
{
@Override
+ public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup)
+ {
+ return 0;
+ }
+
+ @Override
+ public long partitionCount()
+ {
+ return 0;
+ }
+
+ @Override
+ public long getLiveDataSize()
+ {
+ return 0;
+ }
+
+ @Override
+ public void addMemoryUsageTo(MemoryUsage usage)
+ {
+ }
+
+ @Override
+ public void markExtraOnHeapUsed(long additionalSpace, Group opGroup)
+ {
+ }
+
+ @Override
+ public void markExtraOffHeapUsed(long additionalSpace, Group opGroup)
+ {
+ }
+
+ @Override
+ public FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPosition to)
+ {
+ return null;
+ }
+
+ @Override
+ public void switchOut(Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
+ {
+ }
+
+ @Override
+ public void discard()
+ {
+ }
+
+ @Override
+ public boolean accepts(Group opGroup, CommitLogPosition commitLogPosition)
+ {
+ return false;
+ }
+
+ @Override
+ public CommitLogPosition getApproximateCommitLogLowerBound()
+ {
+ return null;
+ }
+
+ @Override
+ public CommitLogPosition getCommitLogLowerBound()
+ {
+ return null;
+ }
+
+ @Override
+ public LastCommitLogPosition getFinalCommitLogUpperBound()
+ {
+ return null;
+ }
+
+ @Override
+ public boolean mayContainDataBefore(CommitLogPosition position)
+ {
+ return false;
+ }
+
+ @Override
+ public boolean isClean()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean shouldSwitch(FlushReason reason)
+ {
+ return false;
+ }
+
+ @Override
+ public void metadataUpdated()
+ {
+ }
+
+ @Override
+ public void localRangesUpdated()
+ {
+ }
+
+ @Override
+ public void performSnapshot(String snapshotName)
+ {
+ }
+
+ @Override
public UnfilteredRowIterator rowIterator(DecoratedKey key,
Slices slices,
ColumnFilter columnFilter,
@@ -716,113 +824,6 @@
{
return null;
}
-
- @Override
- public void switchOut(Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
- {
- }
-
- @Override
- public boolean shouldSwitch(FlushReason reason)
- {
- return false;
- }
-
- @Override
- public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup)
- {
- return 0;
- }
-
- @Override
- public void performSnapshot(String snapshotName)
- {
- }
-
- @Override
- public long partitionCount()
- {
- return 0;
- }
-
- @Override
- public void metadataUpdated()
- {
- }
-
- @Override
- public boolean mayContainDataBefore(CommitLogPosition position)
- {
- return false;
- }
-
- @Override
- public void markExtraOnHeapUsed(long additionalSpace, Group opGroup)
- {
- }
-
- @Override
- public void markExtraOffHeapUsed(long additionalSpace, Group opGroup)
- {
- }
-
- @Override
- public void localRangesUpdated()
- {
- }
-
- @Override
- public boolean isClean()
- {
- return false;
- }
-
- @Override
- public long getLiveDataSize()
- {
- return 0;
- }
-
- @Override
- public FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPosition to)
- {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public LastCommitLogPosition getFinalCommitLogUpperBound()
- {
- return null;
- }
-
- @Override
- public CommitLogPosition getCommitLogLowerBound()
- {
- return null;
- }
-
- @Override
- public CommitLogPosition getApproximateCommitLogLowerBound()
- {
- return null;
- }
-
- @Override
- public void discard()
- {
- }
-
- @Override
- public void addMemoryUsageTo(MemoryUsage usage)
- {
- }
-
- @Override
- public boolean accepts(Group opGroup, CommitLogPosition commitLogPosition)
- {
- return false;
- }
};
}
}
diff --git a/test/unit/org/apache/cassandra/db/ColumnsTest.java b/test/unit/org/apache/cassandra/db/ColumnsTest.java
index 37edcbc..4c8bcc0 100644
--- a/test/unit/org/apache/cassandra/db/ColumnsTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnsTest.java
@@ -496,7 +496,7 @@
private static ColumnMetadata def(String name, AbstractType<?> type, ColumnMetadata.Kind kind)
{
- return new ColumnMetadata(TABLE_METADATA, bytes(name), type, ColumnMetadata.NO_POSITION, kind);
+ return new ColumnMetadata(TABLE_METADATA, bytes(name), type, ColumnMetadata.NO_POSITION, kind, null);
}
private static TableMetadata mock(Columns columns)
diff --git a/test/unit/org/apache/cassandra/db/CorruptPrimaryIndexTest.java b/test/unit/org/apache/cassandra/db/CorruptPrimaryIndexTest.java
new file mode 100644
index 0000000..ba9f9fd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/CorruptPrimaryIndexTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.io.filesystem.ListenableFileSystem;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.util.File;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class CorruptPrimaryIndexTest extends CQLTester.InMemory
+{
+ protected ListenableFileSystem.PathFilter isCurrentTableIndexFile(String keyspace, String endsWith)
+ {
+ return path -> {
+ if (!path.getFileName().toString().endsWith(endsWith))
+ return false;
+ Descriptor desc = Descriptor.fromFile(new File(path));
+ if (!desc.ksname.equals(keyspace) && desc.cfname.equals(currentTable()))
+ return false;
+ return true;
+ };
+ }
+
+ @Test
+ public void bigPrimaryIndexDoesNotDetectDiskCorruption()
+ {
+ // Set listener early, before the file is opened; mmap access can not be listened to, so need to observe the open, which happens on flush
+ if (BigFormat.isSelected())
+ {
+ fs.onPostRead(isCurrentTableIndexFile(keyspace(), "Index.db"), (path, channel, position, dst, read) -> {
+ // Reading the Primary index for the test!
+ // format
+ // 2 bytes: length of bytes for PK
+ // 4 bytes: pk as an int32
+ // variable bytes (see org.apache.cassandra.io.sstable.format.big.RowIndexEntry.IndexSerializer.deserialize(org.apache.cassandra.io.util.FileDataInput))
+ assertThat(position).describedAs("Unexpected access, should start read from start of file").isEqualTo(0);
+
+ // simulate bit rot by having 1 byte change... but make sure it's the pk!
+ dst.put(2, Byte.MAX_VALUE);
+ });
+ }
+ else if (BtiFormat.isSelected())
+ {
+ fs.onPostRead(isCurrentTableIndexFile(keyspace(), "Partitions.db"), (path, channel, position, dst, read) -> {
+ // simulate bit rot by having 1 byte change...
+ // first read should be in the footer -- give it an invalid root position
+ // TODO: Change this to modify something in a more undetectable position in the file when checksumming is implemented
+ dst.put(2, Byte.MAX_VALUE);
+ });
+ }
+ else
+ throw Util.testMustBeImplementedForSSTableFormat();
+
+ createTable("CREATE TABLE %s (id int PRIMARY KEY, value int)");
+ execute("INSERT INTO %s (id, value) VALUES (?, ?)", 0, 0);
+ flush();
+
+ if (BigFormat.isSelected())
+ {
+ UntypedResultSet rs = execute("SELECT * FROM %s WHERE id=?", 0);
+ // this assert check is here to get the test to be green... if the format is fixed and this data loss is not
+ // happening anymore, then this check should be updated
+ assertThatThrownBy(() -> assertRows(rs, row(0, 0))).hasMessage("Got less rows than expected. Expected 1 but got 0");
+ }
+ else
+ {
+ assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE id=?", 0)).isInstanceOf(CorruptSSTableException.class);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index c701516..71a49cf 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -18,8 +18,11 @@
package org.apache.cassandra.db;
import java.io.IOException;
+import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.attribute.FileAttributeView;
+import java.nio.file.attribute.FileStoreAttributeView;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@@ -33,17 +36,17 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
-
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -52,7 +55,6 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.apache.cassandra.Util;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
// Our version of Sfl4j seems to be missing the ListAppender class.
@@ -63,6 +65,7 @@
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
+import org.apache.cassandra.Util;
import org.apache.cassandra.auth.AuthKeyspace;
import org.apache.cassandra.config.Config.DiskFailurePolicy;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -79,7 +82,7 @@
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.io.util.FileUtils;
@@ -110,15 +113,24 @@
public static final String TABLE_NAME = "FakeTable";
public static final String SNAPSHOT1 = "snapshot1";
public static final String SNAPSHOT2 = "snapshot2";
+ public static final String SNAPSHOT3 = "snapshot3";
public static final String LEGACY_SNAPSHOT_NAME = "42";
+
+
private static File tempDataDir;
private static final String KS = "ks";
private static String[] TABLES;
private static Set<TableMetadata> CFM;
private static Map<String, List<File>> sstablesByTableName;
- @Parameterized.Parameter(0)
+ private static final String MDCID = "test-DirectoriesTest-id";
+ private static AtomicInteger diyThreadId = new AtomicInteger(1);
+ private int myDiyId = -1;
+ private static Logger logger;
+ private ListAppender<ILoggingEvent> listAppender;
+
+ @Parameterized.Parameter
public SSTableId.Builder<? extends SSTableId> idBuilder;
@Parameterized.Parameter(1)
@@ -131,13 +143,6 @@
new Object[]{ UUIDBasedSSTableId.Builder.instance, Util.newUUIDGen() });
}
-
- private static final String MDCID = "test-DirectoriesTest-id";
- private static AtomicInteger diyThreadId = new AtomicInteger(1);
- private int myDiyId = -1;
- private static Logger logger;
- private ListAppender<ILoggingEvent> listAppender;
-
@BeforeClass
public static void beforeClass()
{
@@ -172,7 +177,7 @@
@AfterClass
public static void afterClass()
{
- FileUtils.deleteRecursive(tempDataDir);
+ tempDataDir.deleteRecursive();
}
@After
@@ -186,17 +191,7 @@
return new DataDirectory[] { new DataDirectory(location) };
}
- private static DataDirectory[] toDataDirectories(File[] locations)
- {
- DataDirectory[] dirs = new DataDirectory[locations.length];
- for (int i=0; i<locations.length; i++)
- {
- dirs[i] = new DataDirectory(locations[i]);
- }
- return dirs;
- }
-
- private void createTestFiles() throws IOException
+ private void createTestFiles()
{
for (TableMetadata cfm : CFM)
{
@@ -218,25 +213,27 @@
}
}
- class FakeSnapshot {
+ static class FakeSnapshot {
final TableMetadata table;
final String tag;
final File snapshotDir;
final SnapshotManifest manifest;
+ final boolean ephemeral;
- FakeSnapshot(TableMetadata table, String tag, File snapshotDir, SnapshotManifest manifest)
+ FakeSnapshot(TableMetadata table, String tag, File snapshotDir, SnapshotManifest manifest, boolean ephemeral)
{
this.table = table;
this.tag = tag;
this.snapshotDir = snapshotDir;
this.manifest = manifest;
+ this.ephemeral = ephemeral;
}
public TableSnapshot asTableSnapshot()
{
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
- return new TableSnapshot(table.keyspace, table.name, table.id.asUUID(), tag, createdAt, expiresAt, Collections.singleton(snapshotDir));
+ return new TableSnapshot(table.keyspace, table.name, table.id.asUUID(), tag, createdAt, expiresAt, Collections.singleton(snapshotDir), ephemeral);
}
}
@@ -248,39 +245,43 @@
.build();
}
- public FakeSnapshot createFakeSnapshot(TableMetadata table, String tag, boolean createManifest) throws IOException
+ public FakeSnapshot createFakeSnapshot(TableMetadata table, String tag, boolean createManifest, boolean ephemeral) throws IOException
{
File tableDir = cfDir(table);
tableDir.tryCreateDirectories();
File snapshotDir = new File(tableDir, Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + tag);
snapshotDir.tryCreateDirectories();
- Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), SSTableFormat.Type.BIG);
+ Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), DatabaseDescriptor.getSelectedSSTableFormat());
createFakeSSTable(sstableDesc);
SnapshotManifest manifest = null;
if (createManifest)
{
File manifestFile = Directories.getSnapshotManifestFile(snapshotDir);
- manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.filenameFor(Component.DATA)), new DurationSpec.IntSecondsBound("1m"), now());
+ manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.fileFor(Components.DATA).absolutePath()), new DurationSpec.IntSecondsBound("1m"), now(), ephemeral);
manifest.serializeToJsonFile(manifestFile);
}
+ else if (ephemeral)
+ {
+ Files.createFile(snapshotDir.toPath().resolve("ephemeral.snapshot"));
+ }
- return new FakeSnapshot(table, tag, snapshotDir, manifest);
+ return new FakeSnapshot(table, tag, snapshotDir, manifest, ephemeral);
}
private List<File> createFakeSSTable(File dir, String cf, int gen)
{
- Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), SSTableFormat.Type.BIG);
+ Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), DatabaseDescriptor.getSelectedSSTableFormat());
return createFakeSSTable(desc);
}
private List<File> createFakeSSTable(Descriptor desc)
{
List<File> components = new ArrayList<>(3);
- for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER })
+ for (Component c : DatabaseDescriptor.getSelectedSSTableFormat().uploadComponents())
{
- File f = new File(desc.filenameFor(c));
+ File f = desc.fileFor(c);
f.createFileIfNotExists();
components.add(f);
}
@@ -306,14 +307,14 @@
}
@Test
- public void testStandardDirs() throws IOException
+ public void testStandardDirs()
{
for (TableMetadata cfm : CFM)
{
Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
- Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG);
+ Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), DatabaseDescriptor.getSelectedSSTableFormat());
File snapshotDir = new File(cfDir(cfm), File.pathSeparator() + Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + LEGACY_SNAPSHOT_NAME);
assertEquals(snapshotDir.toCanonical(), Directories.getSnapshotDirectory(desc, LEGACY_SNAPSHOT_NAME));
@@ -333,22 +334,27 @@
assertThat(directories.listSnapshots()).isEmpty();
// Create snapshot with and without manifest
- FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true);
- FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false);
+ FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false);
+ FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false);
+ // ephemeral without manifst
+ FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true);
// Both snapshots should be present
Map<String, TableSnapshot> snapshots = directories.listSnapshots();
- assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2));
+ assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3));
assertThat(snapshots.get(SNAPSHOT1)).isEqualTo(snapshot1.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot());
+ assertThat(snapshots.get(SNAPSHOT3)).isEqualTo(snapshot3.asTableSnapshot());
// Now remove snapshot1
- FileUtils.deleteRecursive(snapshot1.snapshotDir);
+ snapshot1.snapshotDir.deleteRecursive();
- // Only snapshot 2 should be present
+ // Only snapshot 2 and 3 should be present
snapshots = directories.listSnapshots();
- assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2));
+ assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3));
assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot());
+ assertThat(snapshots.get(SNAPSHOT3)).isEqualTo(snapshot3.asTableSnapshot());
+ assertThat(snapshots.get(SNAPSHOT3).isEphemeral()).isTrue();
}
@Test
@@ -359,21 +365,23 @@
assertThat(directories.listSnapshotDirsByTag()).isEmpty();
// Create snapshot with and without manifest
- FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true);
- FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false);
+ FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false);
+ FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false);
+ FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true);
// Both snapshots should be present
Map<String, Set<File>> snapshotDirs = directories.listSnapshotDirsByTag();
- assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2));
+ assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3));
assertThat(snapshotDirs.get(SNAPSHOT1)).allMatch(snapshotDir -> snapshotDir.equals(snapshot1.snapshotDir));
assertThat(snapshotDirs.get(SNAPSHOT2)).allMatch(snapshotDir -> snapshotDir.equals(snapshot2.snapshotDir));
+ assertThat(snapshotDirs.get(SNAPSHOT3)).allMatch(snapshotDir -> snapshotDir.equals(snapshot3.snapshotDir));
// Now remove snapshot1
- FileUtils.deleteRecursive(snapshot1.snapshotDir);
+ snapshot1.snapshotDir.deleteRecursive();
- // Only snapshot 2 should be present
+ // Only snapshot 2 and 3 should be present
snapshotDirs = directories.listSnapshotDirsByTag();
- assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2));
+ assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3));
}
@Test
@@ -382,7 +390,7 @@
{
String tag = "test";
Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
- Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), SSTableFormat.Type.BIG);
+ Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, tag);
List<String> files = new LinkedList<>();
@@ -390,7 +398,7 @@
File manifestFile = directories.getSnapshotManifestFile(tag);
- SnapshotManifest manifest = new SnapshotManifest(files, new DurationSpec.IntSecondsBound("1m"), now());
+ SnapshotManifest manifest = new SnapshotManifest(files, new DurationSpec.IntSecondsBound("1m"), now(), false);
manifest.serializeToJsonFile(manifestFile);
Set<File> dirs = new HashSet<>();
@@ -429,8 +437,8 @@
{
assertEquals(cfDir(INDEX_CFM), dir);
}
- Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
- Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
+ Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
+ Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
// snapshot dir should be created under its parent's
File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, "test");
@@ -443,10 +451,10 @@
assertTrue(indexDirectories.snapshotExists("test"));
// check true snapshot size
- Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
- createFile(parentSnapshot.filenameFor(Component.DATA), 30);
- Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
- createFile(indexSnapshot.filenameFor(Component.DATA), 40);
+ Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
+ createFile(parentSnapshot.fileFor(Components.DATA), 30);
+ Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
+ createFile(indexSnapshot.fileFor(Components.DATA), 40);
assertEquals(30, parentDirectories.trueSnapshotsSize());
assertEquals(40, indexDirectories.trueSnapshotsSize());
@@ -463,16 +471,15 @@
assertEquals(parentBackupDirectory, indexBackupDirectory.parent());
}
- private File createFile(String fileName, int size)
+ private File createFile(File file, int size)
{
- File newFile = new File(fileName);
- try (FileOutputStreamPlus writer = new FileOutputStreamPlus(newFile);)
+ try (FileOutputStreamPlus writer = new FileOutputStreamPlus(file);)
{
writer.write(new byte[size]);
writer.flush();
}
catch (IOException ignore) {}
- return newFile;
+ return file;
}
@Test
@@ -525,7 +532,7 @@
}
@Test
- public void testTemporaryFile() throws IOException
+ public void testTemporaryFile()
{
for (TableMetadata cfm : CFM)
{
@@ -589,11 +596,10 @@
final Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
final String n = Long.toString(nanoTime());
- Callable<File> directoryGetter = new Callable<File>() {
- public File call() throws Exception {
- Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG);
- return Directories.getSnapshotDirectory(desc, n);
- }
+ Callable<File> directoryGetter = () ->
+ {
+ Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), DatabaseDescriptor.getSelectedSSTableFormat());
+ return Directories.getSnapshotDirectory(desc, n);
};
List<Future<File>> invoked = Executors.newFixedThreadPool(2).invokeAll(Arrays.asList(directoryGetter, directoryGetter));
for(Future<File> fut:invoked) {
@@ -757,7 +763,7 @@
Directories dirs = new Directories(cfm, paths);
for (DataDirectory dir : paths)
{
- Descriptor d = Descriptor.fromFilename(new File(dir.location, getNewFilename(cfm, false)).toString());
+ Descriptor d = Descriptor.fromFile(new File(dir.location, getNewFilename(cfm, false)));
String p = dirs.getDataDirectoryForFile(d).location.absolutePath() + File.pathSeparator();
assertTrue(p.startsWith(dir.location.absolutePath() + File.pathSeparator()));
}
@@ -786,9 +792,9 @@
for (TableMetadata tm : CFM)
{
Directories dirs = new Directories(tm, Sets.newHashSet(dd1, dd2));
- Descriptor desc = Descriptor.fromFilename(new File(ddir1.resolve(getNewFilename(tm, false))));
+ Descriptor desc = Descriptor.fromFile(new File(ddir1.resolve(getNewFilename(tm, false))));
assertEquals(new File(ddir1), dirs.getDataDirectoryForFile(desc).location);
- desc = Descriptor.fromFilename(new File(ddir2.resolve(getNewFilename(tm, false))));
+ desc = Descriptor.fromFile(new File(ddir2.resolve(getNewFilename(tm, false))));
assertEquals(new File(ddir2), dirs.getDataDirectoryForFile(desc).location);
}
}
@@ -836,9 +842,9 @@
for (TableMetadata tm : CFM)
{
Directories dirs = new Directories(tm, Sets.newHashSet(dd1, dd2));
- Descriptor desc = Descriptor.fromFilename(new File(ddir1.resolve(getNewFilename(tm, oldStyle))));
+ Descriptor desc = Descriptor.fromFile(new File(ddir1.resolve(getNewFilename(tm, oldStyle))));
assertEquals(new File(ddir1), dirs.getDataDirectoryForFile(desc).location);
- desc = Descriptor.fromFilename(new File(ddir2.resolve(getNewFilename(tm, oldStyle))));
+ desc = Descriptor.fromFile(new File(ddir2.resolve(getNewFilename(tm, oldStyle))));
assertEquals(new File(ddir2), dirs.getDataDirectoryForFile(desc).location);
}
}
@@ -889,31 +895,28 @@
assertFalse(iter.hasNext());
}
- private String getNewFilename(TableMetadata tm, boolean oldStyle)
+ @Test
+ public void testFreeCompactionSpace()
{
- return tm.keyspace + File.pathSeparator() + tm.name + (oldStyle ? "" : Component.separator + tm.id.toHexString()) + "/na-1-big-Data.db";
- }
-
- private List<Directories.DataDirectoryCandidate> getWriteableDirectories(DataDirectory[] dataDirectories, long writeSize)
- {
- // copied from Directories.getWriteableLocation(long)
- List<Directories.DataDirectoryCandidate> candidates = new ArrayList<>();
-
- long totalAvailable = 0L;
-
- for (DataDirectory dataDir : dataDirectories)
+ double oldMaxSpaceForCompactions = DatabaseDescriptor.getMaxSpaceForCompactionsPerDrive();
+ long oldFreeSpace = DatabaseDescriptor.getMinFreeSpacePerDriveInMebibytes();
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(100.0);
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(0);
+ FileStore fstore = new FakeFileStore();
+ try
{
- Directories.DataDirectoryCandidate candidate = new Directories.DataDirectoryCandidate(dataDir);
- // exclude directory if its total writeSize does not fit to data directory
- if (candidate.availableSpace < writeSize)
- continue;
- candidates.add(candidate);
- totalAvailable += candidate.availableSpace;
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(1);
+ assertEquals(100, Directories.getAvailableSpaceForCompactions(fstore));
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(.5);
+ assertEquals(50, Directories.getAvailableSpaceForCompactions(fstore));
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(0);
+ assertEquals(0, Directories.getAvailableSpaceForCompactions(fstore));
}
-
- Directories.sortWriteableCandidates(candidates, totalAvailable);
-
- return candidates;
+ finally
+ {
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(oldMaxSpaceForCompactions);
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(oldFreeSpace / FileUtils.ONE_MIB);
+ }
}
private int getDiyThreadId()
@@ -944,11 +947,10 @@
private List<ILoggingEvent> filterLogByDiyId(List<ILoggingEvent> log)
{
ArrayList<ILoggingEvent> filteredLog = new ArrayList<>();
- for (ILoggingEvent event : log)
+ for(ILoggingEvent event : log)
{
int mdcId = Integer.parseInt(event.getMDCPropertyMap().get(this.MDCID));
- if (mdcId == myDiyId)
- {
+ if(mdcId == myDiyId){
filteredLog.add(event);
}
}
@@ -960,101 +962,191 @@
int found=0;
for(ILoggingEvent e: log)
{
- System.err.println(e.getFormattedMessage());
if(e.getFormattedMessage().endsWith(expectedMessage))
- {
if (e.getLevel() == expectedLevel)
found++;
- }
}
assertEquals(expectedCount, found);
}
@Test
- public void testHasAvailableDiskSpace()
+ public void testHasAvailableSpace()
{
- DataDirectory[] dataDirectories = new DataDirectory[]
- {
- new DataDirectory(new File("/nearlyFullDir1"))
- {
- public long getAvailableSpace()
- {
- return 11L;
- }
- },
- new DataDirectory(new File("/uniformDir2"))
- {
- public long getAvailableSpace()
- {
- return 999L;
- }
- },
- new DataDirectory(new File("/veryFullDir"))
- {
- public long getAvailableSpace()
- {
- return 4L;
- }
- }
- };
+ double oldMaxSpaceForCompactions = DatabaseDescriptor.getMaxSpaceForCompactionsPerDrive();
+ long oldFreeSpace = DatabaseDescriptor.getMinFreeSpacePerDriveInMebibytes();
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(1.0);
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(0);
+ try
+ {
+ FakeFileStore fs1 = new FakeFileStore();
+ FakeFileStore fs2 = new FakeFileStore();
+ FakeFileStore fs3 = new FakeFileStore();
+ Map<FileStore, Long> writes = new HashMap<>();
- Directories d = new Directories( ((TableMetadata) CFM.toArray()[0]), dataDirectories);
+ fs1.usableSpace = 30;
+ fs2.usableSpace = 30;
+ fs3.usableSpace = 30;
- assertTrue(d.hasAvailableDiskSpace(1,2));
- assertTrue(d.hasAvailableDiskSpace(10,99));
- assertFalse(d.hasAvailableDiskSpace(10,1024));
- assertFalse(d.hasAvailableDiskSpace(1024,1024*1024));
+ writes.put(fs1, 20L);
+ writes.put(fs2, 20L);
+ writes.put(fs3, 20L);
+ assertTrue(Directories.hasDiskSpaceForCompactionsAndStreams(writes));
- List<ILoggingEvent> filteredLog = listAppender.list;
- //List<ILoggingEvent> filteredLog = filterLogByDiyId(listAppender.list);
- // Log messages can be out of order, even for the single thread. (e tui AsyncAppender?)
- // We can deal with it, it's sufficient to just check that all messages exist in the result
- assertEquals(23, filteredLog.size());
- String logMsgFormat = "DataDirectory %s has %d bytes available, checking if we can write %d bytes";
- String logMsg = String.format(logMsgFormat, "/nearlyFullDir1", 11, 2);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/uniformDir2", 999, 2);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/veryFullDir", 4, 2);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/nearlyFullDir1", 11, 2);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/uniformDir2", 999, 9);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/veryFullDir", 4, 9);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/nearlyFullDir1", 11, 102);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/uniformDir2", 999, 102);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/veryFullDir", 4, 102);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/nearlyFullDir1", 11, 1024);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/uniformDir2", 999, 1024);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/veryFullDir", 4, 1024);
- checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 1);
+ fs1.usableSpace = 19;
+ assertFalse(Directories.hasDiskSpaceForCompactionsAndStreams(writes));
- logMsgFormat = "DataDirectory %s can't be used for compaction. Only %s is available, but %s is the minimum write size.";
- logMsg = String.format(logMsgFormat, "/veryFullDir", "4 bytes", "9 bytes");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/nearlyFullDir1", "11 bytes", "102 bytes");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/veryFullDir", "4 bytes", "102 bytes");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/nearlyFullDir1", "11 bytes", "1 KiB");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/uniformDir2", "999 bytes", "1 KiB");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
- logMsg = String.format(logMsgFormat, "/veryFullDir", "4 bytes", "1 KiB");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
+ writes.put(fs2, 25L*1024*1024+9);
+ fs2.usableSpace = 20L*1024*1024-9;
+ writes.put(fs3, 999L*1024*1024*1024+9);
+ fs2.usableSpace = 20L*1024+99;
+ assertFalse(Directories.hasDiskSpaceForCompactionsAndStreams(writes));
- logMsgFormat = "Across %s there's only %s available, but %s is needed.";
- logMsg = String.format(logMsgFormat, "[/nearlyFullDir1,/uniformDir2,/veryFullDir]", "999 bytes", "1 KiB");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
- logMsg = String.format(logMsgFormat, "[/nearlyFullDir1,/uniformDir2,/veryFullDir]", "0 bytes", "1 MiB");
- checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
+ fs1.usableSpace = 30;
+ fs2.usableSpace = 30;
+ fs3.usableSpace = 30L*1024*1024*1024*1024;
+
+ writes.put(fs1, 20L);
+ writes.put(fs2, 20L);
+ writes.put(fs3, 30L*1024*1024*1024*1024+1);
+ assertFalse(Directories.hasDiskSpaceForCompactionsAndStreams(writes));
+
+ List<ILoggingEvent> filteredLog = filterLogByDiyId(listAppender.list);
+ // Log messages can be out of order, even for the single thread. (e tui AsyncAppender?)
+ // We can deal with it, it's sufficient to just check that all messages exist in the result
+ assertEquals(17, filteredLog.size());
+
+ String logMsg = "30 bytes available, checking if we can write 20 bytes";
+ checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 7);
+ logMsg = "19 bytes available, checking if we can write 20 bytes";
+ checkFormattedMessage(filteredLog, Level.DEBUG, logMsg, 2);
+
+
+ logMsg = "19 bytes available, but 20 bytes is needed";
+ checkFormattedMessage(filteredLog, Level.WARN, logMsg, 2);
+ logMsg = "has only 20.1 KiB available, but 25 MiB is needed";
+ checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
+ logMsg = "has only 30 bytes available, but 999 GiB is needed";
+ checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
+ logMsg = "has only 30 TiB available, but 30 TiB is needed";
+ checkFormattedMessage(filteredLog, Level.WARN, logMsg, 1);
+ }
+ finally
+ {
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(oldMaxSpaceForCompactions);
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(oldFreeSpace);
+ }
+ }
+
+ @Test
+ public void testHasAvailableSpaceSumming()
+ {
+ double oldMaxSpaceForCompactions = DatabaseDescriptor.getMaxSpaceForCompactionsPerDrive();
+ long oldFreeSpace = DatabaseDescriptor.getMinFreeSpacePerDriveInMebibytes();
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(1.0);
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(0);
+ try
+ {
+ FakeFileStore fs1 = new FakeFileStore();
+ FakeFileStore fs2 = new FakeFileStore();
+ Map<File, Long> expectedNewWriteSizes = new HashMap<>();
+ Map<File, Long> totalCompactionWriteRemaining = new HashMap<>();
+
+ fs1.usableSpace = 100;
+ fs2.usableSpace = 100;
+
+ File f1 = new File("f1");
+ File f2 = new File("f2");
+ File f3 = new File("f3");
+
+ expectedNewWriteSizes.put(f1, 20L);
+ expectedNewWriteSizes.put(f2, 20L);
+ expectedNewWriteSizes.put(f3, 20L);
+
+ totalCompactionWriteRemaining.put(f1, 20L);
+ totalCompactionWriteRemaining.put(f2, 20L);
+ totalCompactionWriteRemaining.put(f3, 20L);
+ Function<File, FileStore> filestoreMapper = (f) -> {
+ if (f == f1 || f == f2)
+ return fs1;
+ return fs2;
+ };
+ assertTrue(Directories.hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, filestoreMapper));
+ fs1.usableSpace = 79;
+ assertFalse(Directories.hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, filestoreMapper));
+ fs1.usableSpace = 81;
+ assertTrue(Directories.hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, filestoreMapper));
+
+ expectedNewWriteSizes.clear();
+ expectedNewWriteSizes.put(f1, 100L);
+ totalCompactionWriteRemaining.clear();
+ totalCompactionWriteRemaining.put(f2, 100L);
+ fs1.usableSpace = 150;
+
+ assertFalse(Directories.hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, filestoreMapper));
+ expectedNewWriteSizes.clear();
+ expectedNewWriteSizes.put(f1, 100L);
+ totalCompactionWriteRemaining.clear();
+ totalCompactionWriteRemaining.put(f3, 500L);
+ fs1.usableSpace = 150;
+ fs2.usableSpace = 400; // too little space for the ongoing compaction, but this filestore does not affect the new compaction so it should be allowed
+
+ assertTrue(Directories.hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, filestoreMapper));
+ }
+ finally
+ {
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(oldMaxSpaceForCompactions);
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(oldFreeSpace / FileUtils.ONE_MIB);
+ }
+ }
+
+ private String getNewFilename(TableMetadata tm, boolean oldStyle)
+ {
+ return tm.keyspace + File.pathSeparator() + tm.name + (oldStyle ? "" : Component.separator + tm.id.toHexString()) + "/na-1-big-Data.db";
+ }
+
+ private List<Directories.DataDirectoryCandidate> getWriteableDirectories(DataDirectory[] dataDirectories, long writeSize)
+ {
+ // copied from Directories.getWriteableLocation(long)
+ List<Directories.DataDirectoryCandidate> candidates = new ArrayList<>();
+
+ long totalAvailable = 0L;
+
+ for (DataDirectory dataDir : dataDirectories)
+ {
+ Directories.DataDirectoryCandidate candidate = new Directories.DataDirectoryCandidate(dataDir);
+ // exclude directory if its total writeSize does not fit to data directory
+ if (candidate.availableSpace < writeSize)
+ continue;
+ candidates.add(candidate);
+ totalAvailable += candidate.availableSpace;
+ }
+
+ Directories.sortWriteableCandidates(candidates, totalAvailable);
+
+ return candidates;
+ }
+
+ public static class FakeFileStore extends FileStore
+ {
+ public long usableSpace = 100;
+ public long getUsableSpace()
+ {
+ return usableSpace;
+ }
+ public String name() {return null;}
+ public String type() {return null;}
+ public boolean isReadOnly() {return false;}
+ public long getTotalSpace() {return 0;}
+ public long getUnallocatedSpace() {return 0;}
+ public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {return false;}
+ public boolean supportsFileAttributeView(String name) {return false;}
+ public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {return null;}
+ public Object getAttribute(String attribute) {return null;}
+
+ public String toString()
+ {
+ return "MockFileStore";
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java b/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
index cdf9a9a..f043c0b 100644
--- a/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
@@ -19,8 +19,11 @@
package org.apache.cassandra.db;
import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Before;
@@ -29,10 +32,16 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.BootStrapper;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertFalse;
@@ -45,6 +54,8 @@
private DiskBoundaryManager dbm;
private MockCFS mock;
private Directories dirs;
+ private List<Directories.DataDirectory> datadirs;
+ private List<File> tableDirs;
@Before
public void setup()
@@ -53,11 +64,13 @@
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
createTable("create table %s (id int primary key, x text)");
- dirs = new Directories(getCurrentColumnFamilyStore().metadata(), Lists.newArrayList(new Directories.DataDirectory(new File("/tmp/1")),
- new Directories.DataDirectory(new File("/tmp/2")),
- new Directories.DataDirectory(new File("/tmp/3"))));
+ datadirs = Lists.newArrayList(new Directories.DataDirectory(new File("/tmp/1")),
+ new Directories.DataDirectory(new File("/tmp/2")),
+ new Directories.DataDirectory(new File("/tmp/3")));
+ dirs = new Directories(getCurrentColumnFamilyStore().metadata(), datadirs);
mock = new MockCFS(getCurrentColumnFamilyStore(), dirs);
dbm = mock.diskBoundaryManager;
+ tableDirs = datadirs.stream().map(ddir -> mock.getDirectories().getLocationForDisk(ddir)).collect(Collectors.toList());
}
@Test
@@ -105,6 +118,76 @@
}
+ @Test
+ public void testGetDisksInBounds()
+ {
+ List<PartitionPosition> pps = new ArrayList<>();
+
+ pps.add(pp(100));
+ pps.add(pp(200));
+ pps.add(pp(Long.MAX_VALUE)); // last position is always the max token
+
+ DiskBoundaries diskBoundaries = new DiskBoundaries(mock, dirs.getWriteableLocations(), pps, 0, 0);
+
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(0)), diskBoundaries.getDisksInBounds(dk(10), dk(50)));
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(2)), diskBoundaries.getDisksInBounds(dk(250), dk(500)));
+ Assert.assertEquals(Lists.newArrayList(datadirs), diskBoundaries.getDisksInBounds(dk(0), dk(250)));
+ Assert.assertEquals(Lists.newArrayList(datadirs), diskBoundaries.getDisksInBounds(dk(0), dk(250)));
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(1), datadirs.get(2)), diskBoundaries.getDisksInBounds(dk(150), dk(250)));
+ Assert.assertEquals(Lists.newArrayList(datadirs), diskBoundaries.getDisksInBounds(null, dk(250)));
+
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(0)), diskBoundaries.getDisksInBounds(dk(0), dk(99)));
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(0)), diskBoundaries.getDisksInBounds(dk(0), dk(100))); // pp(100) is maxKeyBound, so dk(100) < pp(100)
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(0), datadirs.get(1)), diskBoundaries.getDisksInBounds(dk(100), dk(200)));
+ Assert.assertEquals(Lists.newArrayList(datadirs.get(1)), diskBoundaries.getDisksInBounds(dk(101), dk(101)));
+
+ }
+
+ @Test
+ public void testGetDataDirectoriesForFiles()
+ {
+ int gen = 1;
+ List<Murmur3Partitioner.LongToken> tokens = mock.getDiskBoundaries().positions.stream().map(t -> (Murmur3Partitioner.LongToken)t.getToken()).collect(Collectors.toList());
+ IPartitioner partitioner = Murmur3Partitioner.instance;
+
+ Murmur3Partitioner.LongToken sstableFirstDisk1 = (Murmur3Partitioner.LongToken) partitioner.midpoint(partitioner.getMinimumToken(), tokens.get(0));
+ Murmur3Partitioner.LongToken sstableEndDisk1 = (Murmur3Partitioner.LongToken) partitioner.midpoint(sstableFirstDisk1, tokens.get(0));
+ Murmur3Partitioner.LongToken sstableEndDisk2 = (Murmur3Partitioner.LongToken) partitioner.midpoint(tokens.get(0), tokens.get(1));
+ Murmur3Partitioner.LongToken sstableFirstDisk2 = (Murmur3Partitioner.LongToken) partitioner.midpoint(tokens.get(0), sstableEndDisk2);
+
+ SSTableReader containedDisk1 = MockSchema.sstable(gen++, (long)sstableFirstDisk1.getTokenValue(), (long)sstableEndDisk1.getTokenValue(), 0, mock);
+ SSTableReader startDisk1EndDisk2 = MockSchema.sstable(gen++, (long)sstableFirstDisk1.getTokenValue(), (long)sstableEndDisk2.getTokenValue(), 0, mock);
+ SSTableReader containedDisk2 = MockSchema.sstable(gen++, (long)sstableFirstDisk2.getTokenValue(), (long)sstableEndDisk2.getTokenValue(), 0, mock);
+
+ SSTableReader disk1Boundary = MockSchema.sstable(gen++, (long)sstableFirstDisk1.getTokenValue(), (long)tokens.get(0).getTokenValue(), 0, mock);
+ SSTableReader disk2Full = MockSchema.sstable(gen++, (long)tokens.get(0).increaseSlightly().getTokenValue(), (long)tokens.get(1).getTokenValue(), 0, mock);
+ SSTableReader disk3Full = MockSchema.sstable(gen++, (long)tokens.get(1).increaseSlightly().getTokenValue(), (long)partitioner.getMaximumToken().getTokenValue(), 0, mock);
+
+ Assert.assertEquals(tableDirs, mock.getDirectoriesForFiles(ImmutableSet.of()));
+ Assert.assertEquals(Lists.newArrayList(tableDirs.get(0)), mock.getDirectoriesForFiles(ImmutableSet.of(containedDisk1)));
+ Assert.assertEquals(Lists.newArrayList(tableDirs.get(0), tableDirs.get(1)), mock.getDirectoriesForFiles(ImmutableSet.of(containedDisk1, startDisk1EndDisk2)));
+ Assert.assertEquals(Lists.newArrayList(tableDirs.get(1)), mock.getDirectoriesForFiles(ImmutableSet.of(containedDisk2)));
+ Assert.assertEquals(Lists.newArrayList(tableDirs.get(0), tableDirs.get(1)), mock.getDirectoriesForFiles(ImmutableSet.of(containedDisk1, containedDisk2)));
+
+ Assert.assertEquals(Lists.newArrayList(tableDirs.get(0)), mock.getDirectoriesForFiles(ImmutableSet.of(disk1Boundary)));
+ Assert.assertEquals(Lists.newArrayList(tableDirs.get(1)), mock.getDirectoriesForFiles(ImmutableSet.of(disk2Full)));
+
+ Assert.assertEquals(tableDirs, mock.getDirectoriesForFiles(ImmutableSet.of(containedDisk1, disk3Full)));
+ }
+
+ private PartitionPosition pp(long t)
+ {
+ return t(t).maxKeyBound();
+ }
+ private Token t(long t)
+ {
+ return new Murmur3Partitioner.LongToken(t);
+ }
+ private DecoratedKey dk(long t)
+ {
+ return new BufferDecoratedKey(t(t), ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ }
+
private static void assertEquals(List<Directories.DataDirectory> dir1, Directories.DataDirectory[] dir2)
{
if (dir1.size() != dir2.length)
diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java
index ff843fa..9dc06f0 100644
--- a/test/unit/org/apache/cassandra/db/ImportTest.java
+++ b/test/unit/org/apache/cassandra/db/ImportTest.java
@@ -40,7 +40,7 @@
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.BootStrapper;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -236,7 +236,7 @@
sstable.selfRef().release();
for (File f : sstable.descriptor.directory.tryList())
{
- if (f.toString().contains(sstable.descriptor.baseFilename()))
+ if (f.toString().contains(sstable.descriptor.baseFile().toString()))
{
System.out.println("move " + f.toPath() + " to " + backupdir);
File moveFileTo = new File(backupdir, f.name());
@@ -314,8 +314,8 @@
getCurrentColumnFamilyStore().clearUnsafe();
- String filenameToCorrupt = sstableToCorrupt.descriptor.filenameFor(Component.STATS);
- try (FileChannel fileChannel = new File(filenameToCorrupt).newReadWriteChannel())
+ File fileToCorrupt = sstableToCorrupt.descriptor.fileFor(Components.STATS);
+ try (FileChannel fileChannel = fileToCorrupt.newReadWriteChannel())
{
fileChannel.position(0);
fileChannel.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
@@ -574,8 +574,8 @@
sstables.forEach(s -> s.selfRef().release());
// corrupt the sstable which is still in the data directory
SSTableReader sstableToCorrupt = sstables.iterator().next();
- String filenameToCorrupt = sstableToCorrupt.descriptor.filenameFor(Component.STATS);
- try (FileChannel fileChannel = new File(filenameToCorrupt).newReadWriteChannel())
+ File fileToCorrupt = sstableToCorrupt.descriptor.fileFor(Components.STATS);
+ try (FileChannel fileChannel = fileToCorrupt.newReadWriteChannel())
{
fileChannel.position(0);
fileChannel.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
@@ -615,7 +615,7 @@
assertEquals(20, rowCount);
assertEquals(expectedFiles, getCurrentColumnFamilyStore().getLiveSSTables());
for (SSTableReader sstable : expectedFiles)
- assertTrue(new File(sstable.descriptor.filenameFor(Component.DATA)).exists());
+ assertTrue(sstable.descriptor.fileFor(Components.DATA).exists());
getCurrentColumnFamilyStore().truncateBlocking();
LifecycleTransaction.waitForDeletions();
for (File f : sstableToCorrupt.descriptor.directory.tryList()) // clean up the corrupt files which truncate does not handle
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
deleted file mode 100644
index 475f1a1..0000000
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ /dev/null
@@ -1,467 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.ImmutableList;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.cache.AutoSavingCache;
-import org.apache.cassandra.cache.ICache;
-import org.apache.cassandra.cache.KeyCacheKey;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Refs;
-import org.hamcrest.Matchers;
-import org.mockito.Mockito;
-import org.mockito.internal.stubbing.answers.AnswersWithDelay;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-
-public class KeyCacheTest
-{
- private static final String KEYSPACE1 = "KeyCacheTest1";
- private static final String KEYSPACE2 = "KeyCacheTest2";
- private static final String COLUMN_FAMILY1 = "Standard1";
- private static final String COLUMN_FAMILY2 = "Standard2";
- private static final String COLUMN_FAMILY3 = "Standard3";
- private static final String COLUMN_FAMILY4 = "Standard4";
- private static final String COLUMN_FAMILY5 = "Standard5";
- private static final String COLUMN_FAMILY6 = "Standard6";
- private static final String COLUMN_FAMILY7 = "Standard7";
- private static final String COLUMN_FAMILY8 = "Standard8";
- private static final String COLUMN_FAMILY9 = "Standard9";
- private static final String COLUMN_FAMILY10 = "Standard10";
-
- private static final String COLUMN_FAMILY_K2_1 = "Standard1";
-
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY1),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY2),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY3),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY4),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY5),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY6),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY7),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY8),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY9),
- SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY10));
-
- SchemaLoader.createKeyspace(KEYSPACE2,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE2, COLUMN_FAMILY_K2_1));
-
- }
-
- @AfterClass
- public static void cleanup()
- {
- SchemaLoader.cleanupSavedCaches();
- }
-
- @Test
- public void testKeyCacheLoadShallowIndexEntry() throws Exception
- {
- DatabaseDescriptor.setColumnIndexCacheSize(0);
- testKeyCacheLoad(COLUMN_FAMILY2);
- }
-
- @Test
- public void testKeyCacheLoadIndexInfoOnHeap() throws Exception
- {
- DatabaseDescriptor.setColumnIndexCacheSize(8);
- testKeyCacheLoad(COLUMN_FAMILY5);
- }
-
- private void testKeyCacheLoad(String cf) throws Exception
- {
- CompactionManager.instance.disableAutoCompaction();
-
- ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
-
- // empty the cache
- CacheService.instance.invalidateKeyCache();
- assertKeyCacheSize(0, KEYSPACE1, cf);
-
- // insert data and force to disk
- SchemaLoader.insertData(KEYSPACE1, cf, 0, 100);
- Util.flush(store);
-
- // populate the cache
- readData(KEYSPACE1, cf, 0, 100);
- assertKeyCacheSize(100, KEYSPACE1, cf);
-
- // really? our caches don't implement the map interface? (hence no .addAll)
- Map<KeyCacheKey, RowIndexEntry> savedMap = new HashMap<>();
- Map<KeyCacheKey, RowIndexEntry.IndexInfoRetriever> savedInfoMap = new HashMap<>();
- for (Iterator<KeyCacheKey> iter = CacheService.instance.keyCache.keyIterator();
- iter.hasNext();)
- {
- KeyCacheKey k = iter.next();
- if (k.desc.ksname.equals(KEYSPACE1) && k.desc.cfname.equals(cf))
- {
- RowIndexEntry rie = CacheService.instance.keyCache.get(k);
- savedMap.put(k, rie);
- SSTableReader sstr = readerForKey(k);
- savedInfoMap.put(k, rie.openWithIndex(sstr.getIndexFile()));
- }
- }
-
- // force the cache to disk
- CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
-
- CacheService.instance.invalidateKeyCache();
- assertKeyCacheSize(0, KEYSPACE1, cf);
-
- CacheService.instance.keyCache.loadSaved();
- assertKeyCacheSize(savedMap.size(), KEYSPACE1, cf);
-
- // probably it's better to add equals/hashCode to RowIndexEntry...
- for (Map.Entry<KeyCacheKey, RowIndexEntry> entry : savedMap.entrySet())
- {
- RowIndexEntry expected = entry.getValue();
- RowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey());
- assertEquals(expected.position, actual.position);
- assertEquals(expected.columnsIndexCount(), actual.columnsIndexCount());
- for (int i = 0; i < expected.columnsIndexCount(); i++)
- {
- SSTableReader actualSstr = readerForKey(entry.getKey());
- try (RowIndexEntry.IndexInfoRetriever actualIir = actual.openWithIndex(actualSstr.getIndexFile()))
- {
- RowIndexEntry.IndexInfoRetriever expectedIir = savedInfoMap.get(entry.getKey());
- assertEquals(expectedIir.columnsIndex(i), actualIir.columnsIndex(i));
- }
- }
- if (expected.isIndexed())
- {
- assertEquals(expected.deletionTime(), actual.deletionTime());
- }
- }
-
- savedInfoMap.values().forEach(iir -> {
- try
- {
- if (iir != null)
- iir.close();
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- });
- }
-
- private static SSTableReader readerForKey(KeyCacheKey k)
- {
- return ColumnFamilyStore.getIfExists(k.desc.ksname, k.desc.cfname).getLiveSSTables()
- .stream()
- .filter(sstreader -> sstreader.descriptor.id == k.desc.id)
- .findFirst().get();
- }
-
- @Test
- public void testKeyCacheLoadWithLostTableShallowIndexEntry() throws Exception
- {
- DatabaseDescriptor.setColumnIndexCacheSize(0);
- testKeyCacheLoadWithLostTable(COLUMN_FAMILY3);
- }
-
- @Test
- public void testKeyCacheLoadWithLostTableIndexInfoOnHeap() throws Exception
- {
- DatabaseDescriptor.setColumnIndexCacheSize(8);
- testKeyCacheLoadWithLostTable(COLUMN_FAMILY6);
- }
-
- private void testKeyCacheLoadWithLostTable(String cf) throws Exception
- {
- CompactionManager.instance.disableAutoCompaction();
-
- ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
-
- // empty the cache
- CacheService.instance.invalidateKeyCache();
- assertKeyCacheSize(0, KEYSPACE1, cf);
-
- // insert data and force to disk
- SchemaLoader.insertData(KEYSPACE1, cf, 0, 100);
- Util.flush(store);
-
- Collection<SSTableReader> firstFlushTables = ImmutableList.copyOf(store.getLiveSSTables());
-
- // populate the cache
- readData(KEYSPACE1, cf, 0, 100);
- assertKeyCacheSize(100, KEYSPACE1, cf);
-
- // insert some new data and force to disk
- SchemaLoader.insertData(KEYSPACE1, cf, 100, 50);
- Util.flush(store);
-
- // check that it's fine
- readData(KEYSPACE1, cf, 100, 50);
- assertKeyCacheSize(150, KEYSPACE1, cf);
-
- // force the cache to disk
- CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
-
- CacheService.instance.invalidateKeyCache();
- assertKeyCacheSize(0, KEYSPACE1, cf);
-
- // check that the content is written correctly
- CacheService.instance.keyCache.loadSaved();
- assertKeyCacheSize(150, KEYSPACE1, cf);
-
- CacheService.instance.invalidateKeyCache();
- assertKeyCacheSize(0, KEYSPACE1, cf);
-
- // now remove the first sstable from the store to simulate losing the file
- store.markObsolete(firstFlushTables, OperationType.UNKNOWN);
-
- // check that reading now correctly skips over lost table and reads the rest (CASSANDRA-10219)
- CacheService.instance.keyCache.loadSaved();
- assertKeyCacheSize(50, KEYSPACE1, cf);
- }
-
- @Test
- public void testKeyCacheShallowIndexEntry() throws ExecutionException, InterruptedException
- {
- DatabaseDescriptor.setColumnIndexCacheSize(0);
- testKeyCache(COLUMN_FAMILY1);
- }
-
- @Test
- public void testKeyCacheIndexInfoOnHeap() throws ExecutionException, InterruptedException
- {
- DatabaseDescriptor.setColumnIndexCacheSize(8);
- testKeyCache(COLUMN_FAMILY4);
- }
-
- private void testKeyCache(String cf) throws ExecutionException, InterruptedException
- {
- CompactionManager.instance.disableAutoCompaction();
-
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cf);
-
- // just to make sure that everything is clean
- CacheService.instance.invalidateKeyCache();
-
- // KeyCache should start at size 0 if we're caching X% of zero data.
- assertKeyCacheSize(0, KEYSPACE1, cf);
-
- Mutation rm;
-
- // inserts
- new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("1").build().applyUnsafe();
- new RowUpdateBuilder(cfs.metadata(), 0, "key2").clustering("2").build().applyUnsafe();
-
- // to make sure we have SSTable
- Util.flush(cfs);
-
- // reads to cache key position
- Util.getAll(Util.cmd(cfs, "key1").build());
- Util.getAll(Util.cmd(cfs, "key2").build());
-
- assertKeyCacheSize(2, KEYSPACE1, cf);
-
- Set<SSTableReader> readers = cfs.getLiveSSTables();
- Refs<SSTableReader> refs = Refs.tryRef(readers);
- if (refs == null)
- throw new IllegalStateException();
-
- Util.compactAll(cfs, Integer.MAX_VALUE).get();
- boolean noEarlyOpen = DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMiB() < 0;
-
- // after compaction cache should have entries for new SSTables,
- // but since we have kept a reference to the old sstables,
- // if we had 2 keys in cache previously it should become 4
- assertKeyCacheSize(noEarlyOpen ? 2 : 4, KEYSPACE1, cf);
-
- refs.release();
-
- LifecycleTransaction.waitForDeletions();
-
- // after releasing the reference this should drop to 2
- assertKeyCacheSize(2, KEYSPACE1, cf);
-
- // re-read same keys to verify that key cache didn't grow further
- Util.getAll(Util.cmd(cfs, "key1").build());
- Util.getAll(Util.cmd(cfs, "key2").build());
-
- assertKeyCacheSize(noEarlyOpen ? 4 : 2, KEYSPACE1, cf);
- }
-
- @Test
- public void testKeyCacheLoadZeroCacheLoadTime() throws Exception
- {
- DatabaseDescriptor.setCacheLoadTimeout(0);
- String cf = COLUMN_FAMILY7;
-
- createAndInvalidateCache(Collections.singletonList(Pair.create(KEYSPACE1, cf)), 100);
-
- CacheService.instance.keyCache.loadSaved();
-
- // Here max time to load cache is zero which means no time left to load cache. So the keyCache size should
- // be zero after loadSaved().
- assertKeyCacheSize(0, KEYSPACE1, cf);
- }
-
- @Test
- public void testKeyCacheLoadTwoTablesTime() throws Exception
- {
- DatabaseDescriptor.setCacheLoadTimeout(60);
- String columnFamily1 = COLUMN_FAMILY8;
- String columnFamily2 = COLUMN_FAMILY_K2_1;
- int numberOfRows = 100;
- List<Pair<String, String>> tables = new ArrayList<>(2);
- tables.add(Pair.create(KEYSPACE1, columnFamily1));
- tables.add(Pair.create(KEYSPACE2, columnFamily2));
-
- createAndInvalidateCache(tables, numberOfRows);
-
- CacheService.instance.keyCache.loadSaved();
-
- // Here max time to load cache is negative which means no time left to load cache. So the keyCache size should
- // be zero after load.
- assertKeyCacheSize(numberOfRows, KEYSPACE1, columnFamily1);
- assertKeyCacheSize(numberOfRows, KEYSPACE2, columnFamily2);
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- @Test
- public void testKeyCacheLoadCacheLoadTimeExceedingLimit() throws Exception
- {
- DatabaseDescriptor.setCacheLoadTimeout(2);
- int delayMillis = 1000;
- int numberOfRows = 100;
-
- String cf = COLUMN_FAMILY9;
-
- createAndInvalidateCache(Collections.singletonList(Pair.create(KEYSPACE1, cf)), numberOfRows);
-
- // Testing cache load. Here using custom built AutoSavingCache instance as simulating delay is not possible with
- // 'CacheService.instance.keyCache'. 'AutoSavingCache.loadSaved()' is returning no.of entries loaded so we don't need
- // to instantiate ICache.class.
- CacheService.KeyCacheSerializer keyCacheSerializer = new CacheService.KeyCacheSerializer();
- CacheService.KeyCacheSerializer keyCacheSerializerSpy = Mockito.spy(keyCacheSerializer);
- AutoSavingCache autoSavingCache = new AutoSavingCache(mock(ICache.class),
- CacheService.CacheType.KEY_CACHE,
- keyCacheSerializerSpy);
-
- doAnswer(new AnswersWithDelay(delayMillis, answer -> keyCacheSerializer.deserialize(answer.getArgument(0),
- answer.getArgument(1)) ))
- .when(keyCacheSerializerSpy).deserialize(any(DataInputPlus.class), any(ColumnFamilyStore.class));
-
- long maxExpectedKeyCache = Math.min(numberOfRows,
- 1 + TimeUnit.SECONDS.toMillis(DatabaseDescriptor.getCacheLoadTimeout()) / delayMillis);
-
- long keysLoaded = autoSavingCache.loadSaved();
- assertThat(keysLoaded, Matchers.lessThanOrEqualTo(maxExpectedKeyCache));
- assertNotEquals(0, keysLoaded);
- Mockito.verify(keyCacheSerializerSpy, Mockito.times(1)).cleanupAfterDeserialize();
- }
-
- private void createAndInvalidateCache(List<Pair<String, String>> tables, int numberOfRows) throws ExecutionException, InterruptedException
- {
- CompactionManager.instance.disableAutoCompaction();
-
- // empty the cache
- CacheService.instance.invalidateKeyCache();
- assertEquals(0, CacheService.instance.keyCache.size());
-
- for(Pair<String, String> entry : tables)
- {
- String keyspace = entry.left();
- String cf = entry.right();
- ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(cf);
-
- // insert data and force to disk
- SchemaLoader.insertData(keyspace, cf, 0, numberOfRows);
- Util.flush(store);
- }
- for(Pair<String, String> entry : tables)
- {
- String keyspace = entry.left();
- String cf = entry.right();
- // populate the cache
- readData(keyspace, cf, 0, numberOfRows);
- assertKeyCacheSize(numberOfRows, keyspace, cf);
- }
-
- // force the cache to disk
- CacheService.instance.keyCache.submitWrite(CacheService.instance.keyCache.size()).get();
-
- CacheService.instance.invalidateKeyCache();
- assertEquals(0, CacheService.instance.keyCache.size());
- }
-
- private static void readData(String keyspace, String columnFamily, int startRow, int numberOfRows)
- {
- ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily);
- for (int i = 0; i < numberOfRows; i++)
- Util.getAll(Util.cmd(store, "key" + (i + startRow)).includeRow("col" + (i + startRow)).build());
- }
-
-
- private void assertKeyCacheSize(int expected, String keyspace, String columnFamily)
- {
- int size = 0;
- for (Iterator<KeyCacheKey> iter = CacheService.instance.keyCache.keyIterator();
- iter.hasNext();)
- {
- KeyCacheKey k = iter.next();
- if (k.desc.ksname.equals(keyspace) && k.desc.cfname.equals(columnFamily))
- size++;
- }
- assertEquals(expected, size);
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
index 2445fe3..2c5e793 100644
--- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
@@ -19,28 +19,36 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Collection;
-import org.apache.cassandra.schema.Schema;
-import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
import org.apache.cassandra.metrics.ClearableHistogram;
+import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.assertj.core.api.Assertions;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class KeyspaceTest extends CQLTester
{
@@ -53,7 +61,7 @@
}
@Override
- protected UntypedResultSet execute(String query, Object... values) throws Throwable
+ protected UntypedResultSet execute(String query, Object... values)
{
return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
}
@@ -407,8 +415,11 @@
// verify that we do indeed have multiple index entries
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- RowIndexEntry<?> indexEntry = sstable.getPosition(Util.dk("0"), SSTableReader.Operator.EQ);
- assert indexEntry.columnsIndexCount() > 2;
+ if (sstable instanceof BigTableReader)
+ {
+ AbstractRowIndexEntry indexEntry = ((BigTableReader) sstable).getRowIndexEntry(Util.dk("0"), SSTableReader.Operator.EQ);
+ assert indexEntry.blockCount() > 2;
+ }
validateSliceLarge(cfs);
}
diff --git a/test/unit/org/apache/cassandra/db/NativeCellTest.java b/test/unit/org/apache/cassandra/db/NativeCellTest.java
index c70bb3b..190bdce 100644
--- a/test/unit/org/apache/cassandra/db/NativeCellTest.java
+++ b/test/unit/org/apache/cassandra/db/NativeCellTest.java
@@ -118,7 +118,8 @@
ColumnIdentifier.getInterned(uuid.toString(), false),
isComplex ? new SetType<>(BytesType.instance, true) : BytesType.instance,
-1,
- ColumnMetadata.Kind.REGULAR);
+ ColumnMetadata.Kind.REGULAR,
+ null);
}
private static Cell<?> rndcell(ColumnMetadata col)
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandTest.java b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
index 43a7952..bf272b8 100644
--- a/test/unit/org/apache/cassandra/db/ReadCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
@@ -56,6 +56,7 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
@@ -232,7 +233,16 @@
assertEquals(2, Util.getAll(readCommand).size());
readCommand.abort();
- assertEquals(0, Util.getAll(readCommand).size());
+ boolean cancelled = false;
+ try
+ {
+ Util.getAll(readCommand);
+ }
+ catch (QueryCancelledException e)
+ {
+ cancelled = true;
+ }
+ assertTrue(cancelled);
}
@Test
@@ -263,7 +273,16 @@
assertEquals(2, partitions.get(0).rowCount());
readCommand.abort();
- assertEquals(0, Util.getAll(readCommand).size());
+ boolean cancelled = false;
+ try
+ {
+ Util.getAll(readCommand);
+ }
+ catch (QueryCancelledException e)
+ {
+ cancelled = true;
+ }
+ assertTrue(cancelled);
}
@Test
@@ -294,7 +313,16 @@
assertEquals(2, partitions.get(0).rowCount());
readCommand.abort();
- assertEquals(0, Util.getAll(readCommand).size());
+ boolean cancelled = false;
+ try
+ {
+ Util.getAll(readCommand);
+ }
+ catch (QueryCancelledException e)
+ {
+ cancelled = true;
+ }
+ assertTrue(cancelled);
}
@Test
diff --git a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
index bb272af..dc2eaae 100644
--- a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
+++ b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
@@ -31,6 +31,7 @@
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.ByteBufferUtil;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -261,6 +262,10 @@
}
}
}
+ else
+ {
+ while (rowIter.hasNext()) rowIter.next();
+ }
}
}
}
@@ -312,4 +317,4 @@
sstable.reloadSSTableMetadata();
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java
index 0bf8c5d..4b37e69 100644
--- a/test/unit/org/apache/cassandra/db/RowCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java
@@ -54,6 +54,7 @@
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT;
import static org.junit.Assert.*;
public class RowCacheTest
@@ -66,7 +67,7 @@
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
- System.setProperty("org.caffinitas.ohc.segmentCount", "16");
+ TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT.setInt(16);
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE_CACHED,
KeyspaceParams.simple(1),
diff --git a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
deleted file mode 100644
index a2fb57d..0000000
--- a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
+++ /dev/null
@@ -1,828 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import com.google.common.primitives.Ints;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.db.columniterator.AbstractSSTableIterator;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
-import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
-import org.apache.cassandra.db.rows.BTreeRow;
-import org.apache.cassandra.db.rows.BufferCell;
-import org.apache.cassandra.db.rows.ColumnData;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.RangeTombstoneMarker;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.SerializationHelper;
-import org.apache.cassandra.db.rows.Unfiltered;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.rows.UnfilteredSerializer;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.IndexInfo;
-import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
-import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.serializers.LongSerializer;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.btree.BTree;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class RowIndexEntryTest extends CQLTester
-{
- private static final List<AbstractType<?>> clusterTypes = Collections.singletonList(LongType.instance);
- private static final ClusteringComparator comp = new ClusteringComparator(clusterTypes);
-
- private static final byte[] dummy_100k = new byte[100000];
-
- private static Clustering<?> cn(long l)
- {
- return Util.clustering(comp, l);
- }
-
- @Test
- public void testC11206AgainstPreviousArray() throws Exception
- {
- DatabaseDescriptor.setColumnIndexCacheSize(99999);
- testC11206AgainstPrevious();
- }
-
- @Test
- public void testC11206AgainstPreviousShallow() throws Exception
- {
- DatabaseDescriptor.setColumnIndexCacheSize(0);
- testC11206AgainstPrevious();
- }
-
- private static void testC11206AgainstPrevious() throws Exception
- {
- // partition without IndexInfo
- try (DoubleSerializer doubleSerializer = new DoubleSerializer())
- {
- doubleSerializer.build(null, partitionKey(42L),
- Collections.singletonList(cn(42)),
- 0L);
- assertEquals(doubleSerializer.rieOldSerialized, doubleSerializer.rieNewSerialized);
- }
-
- // partition with multiple IndexInfo
- try (DoubleSerializer doubleSerializer = new DoubleSerializer())
- {
- doubleSerializer.build(null, partitionKey(42L),
- Arrays.asList(cn(42), cn(43), cn(44)),
- 0L);
- assertEquals(doubleSerializer.rieOldSerialized, doubleSerializer.rieNewSerialized);
- }
-
- // partition with multiple IndexInfo
- try (DoubleSerializer doubleSerializer = new DoubleSerializer())
- {
- doubleSerializer.build(null, partitionKey(42L),
- Arrays.asList(cn(42), cn(43), cn(44), cn(45), cn(46), cn(47), cn(48), cn(49), cn(50), cn(51)),
- 0L);
- assertEquals(doubleSerializer.rieOldSerialized, doubleSerializer.rieNewSerialized);
- }
- }
-
- private static DecoratedKey partitionKey(long l)
- {
- ByteBuffer key = LongSerializer.instance.serialize(l);
- Token token = Murmur3Partitioner.instance.getToken(key);
- return new BufferDecoratedKey(token, key);
- }
-
- private static class DoubleSerializer implements AutoCloseable
- {
- TableMetadata metadata =
- CreateTableStatement.parse("CREATE TABLE pipe.dev_null (pk bigint, ck bigint, val text, PRIMARY KEY(pk, ck))", "foo")
- .build();
-
- Version version = BigFormat.latestVersion;
-
- DeletionTime deletionInfo = new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds());
- LivenessInfo primaryKeyLivenessInfo = LivenessInfo.EMPTY;
- Row.Deletion deletion = Row.Deletion.LIVE;
-
- SerializationHeader header = new SerializationHeader(true, metadata, metadata.regularAndStaticColumns(), EncodingStats.NO_STATS);
-
- // create C-11206 + old serializer instances
- RowIndexEntry.IndexSerializer rieSerializer = new RowIndexEntry.Serializer(version, header);
- Pre_C_11206_RowIndexEntry.Serializer oldSerializer = new Pre_C_11206_RowIndexEntry.Serializer(metadata, version, header);
-
- @SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" })
- final DataOutputBuffer rieOutput = new DataOutputBuffer(1024);
- @SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" })
- final DataOutputBuffer oldOutput = new DataOutputBuffer(1024);
-
- final SequentialWriter dataWriterNew;
- final SequentialWriter dataWriterOld;
- final org.apache.cassandra.db.ColumnIndex columnIndex;
-
- RowIndexEntry rieNew;
- ByteBuffer rieNewSerialized;
- Pre_C_11206_RowIndexEntry rieOld;
- ByteBuffer rieOldSerialized;
-
- DoubleSerializer() throws IOException
- {
- SequentialWriterOption option = SequentialWriterOption.newBuilder().bufferSize(1024).build();
- File f = FileUtils.createTempFile("RowIndexEntryTest-", "db");
- dataWriterNew = new SequentialWriter(f, option);
- columnIndex = new org.apache.cassandra.db.ColumnIndex(header, dataWriterNew, version, Collections.emptyList(),
- rieSerializer.indexInfoSerializer());
-
- f = FileUtils.createTempFile("RowIndexEntryTest-", "db");
- dataWriterOld = new SequentialWriter(f, option);
- }
-
- public void close() throws Exception
- {
- dataWriterNew.close();
- dataWriterOld.close();
- }
-
- void build(Row staticRow, DecoratedKey partitionKey,
- Collection<Clustering<?>> clusterings, long startPosition) throws IOException
- {
-
- Iterator<Clustering<?>> clusteringIter = clusterings.iterator();
- columnIndex.buildRowIndex(makeRowIter(staticRow, partitionKey, clusteringIter, dataWriterNew));
- rieNew = RowIndexEntry.create(startPosition, 0L,
- deletionInfo, columnIndex.headerLength, columnIndex.columnIndexCount,
- columnIndex.indexInfoSerializedSize(),
- columnIndex.indexSamples(), columnIndex.offsets(),
- rieSerializer.indexInfoSerializer());
- rieSerializer.serialize(rieNew, rieOutput, columnIndex.buffer());
- rieNewSerialized = rieOutput.buffer().duplicate();
-
- Iterator<Clustering<?>> clusteringIter2 = clusterings.iterator();
- ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(makeRowIter(staticRow, partitionKey, clusteringIter2, dataWriterOld),
- dataWriterOld, header, Collections.emptySet(), BigFormat.latestVersion);
- rieOld = Pre_C_11206_RowIndexEntry.create(startPosition, deletionInfo, columnIndex);
- oldSerializer.serialize(rieOld, oldOutput);
- rieOldSerialized = oldOutput.buffer().duplicate();
- }
-
- private AbstractUnfilteredRowIterator makeRowIter(Row staticRow, DecoratedKey partitionKey,
- Iterator<Clustering<?>> clusteringIter, SequentialWriter dataWriter)
- {
- return new AbstractUnfilteredRowIterator(metadata, partitionKey, deletionInfo, metadata.regularAndStaticColumns(),
- staticRow, false, new EncodingStats(0, 0, 0))
- {
- protected Unfiltered computeNext()
- {
- if (!clusteringIter.hasNext())
- return endOfData();
- try
- {
- // write some fake bytes to the data file to force writing the IndexInfo object
- dataWriter.write(dummy_100k);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- return buildRow(clusteringIter.next());
- }
- };
- }
-
- private Unfiltered buildRow(Clustering<?> clustering)
- {
- BTree.Builder<ColumnData> builder = BTree.builder(ColumnData.comparator);
- builder.add(BufferCell.live(metadata.regularAndStaticColumns().iterator().next(),
- 1L,
- ByteBuffer.allocate(0)));
- return BTreeRow.create(clustering, primaryKeyLivenessInfo, deletion, builder.build());
- }
- }
-
- /**
- * Pre C-11206 code.
- */
- static final class ColumnIndex
- {
- final long partitionHeaderLength;
- final List<IndexInfo> columnsIndex;
-
- private static final ColumnIndex EMPTY = new ColumnIndex(-1, Collections.emptyList());
-
- private ColumnIndex(long partitionHeaderLength, List<IndexInfo> columnsIndex)
- {
- assert columnsIndex != null;
-
- this.partitionHeaderLength = partitionHeaderLength;
- this.columnsIndex = columnsIndex;
- }
-
- static ColumnIndex writeAndBuildIndex(UnfilteredRowIterator iterator,
- SequentialWriter output,
- SerializationHeader header,
- Collection<SSTableFlushObserver> observers,
- Version version) throws IOException
- {
- assert !iterator.isEmpty();
-
- Builder builder = new Builder(iterator, output, header, observers, version.correspondingMessagingVersion());
- return builder.build();
- }
-
- public static ColumnIndex nothing()
- {
- return EMPTY;
- }
-
- /**
- * Help to create an index for a column family based on size of columns,
- * and write said columns to disk.
- */
- private static class Builder
- {
- private final UnfilteredRowIterator iterator;
- private final SequentialWriter writer;
- private final SerializationHelper helper;
- private final SerializationHeader header;
- private final int version;
-
- private final List<IndexInfo> columnsIndex = new ArrayList<>();
- private final long initialPosition;
- private long headerLength = -1;
-
- private long startPosition = -1;
-
- private int written;
- private long previousRowStart;
-
- private ClusteringPrefix<?> firstClustering;
- private ClusteringPrefix<?> lastClustering;
-
- private DeletionTime openMarker;
-
- private final Collection<SSTableFlushObserver> observers;
-
- Builder(UnfilteredRowIterator iterator,
- SequentialWriter writer,
- SerializationHeader header,
- Collection<SSTableFlushObserver> observers,
- int version)
- {
- this.iterator = iterator;
- this.writer = writer;
- this.helper = new SerializationHelper(header);
- this.header = header;
- this.version = version;
- this.observers = observers == null ? Collections.emptyList() : observers;
- this.initialPosition = writer.position();
- }
-
- private void writePartitionHeader(UnfilteredRowIterator iterator) throws IOException
- {
- ByteBufferUtil.writeWithShortLength(iterator.partitionKey().getKey(), writer);
- DeletionTime.serializer.serialize(iterator.partitionLevelDeletion(), writer);
- if (header.hasStatic())
- UnfilteredSerializer.serializer.serializeStaticRow(iterator.staticRow(), helper, writer, version);
- }
-
- public ColumnIndex build() throws IOException
- {
- writePartitionHeader(iterator);
- this.headerLength = writer.position() - initialPosition;
-
- while (iterator.hasNext())
- add(iterator.next());
-
- return close();
- }
-
- private long currentPosition()
- {
- return writer.position() - initialPosition;
- }
-
- private void addIndexBlock()
- {
- IndexInfo cIndexInfo = new IndexInfo(firstClustering,
- lastClustering,
- startPosition,
- currentPosition() - startPosition,
- openMarker);
- columnsIndex.add(cIndexInfo);
- firstClustering = null;
- }
-
- private void add(Unfiltered unfiltered) throws IOException
- {
- long pos = currentPosition();
-
- if (firstClustering == null)
- {
- // Beginning of an index block. Remember the start and position
- firstClustering = unfiltered.clustering();
- startPosition = pos;
- }
-
- UnfilteredSerializer.serializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version);
-
- // notify observers about each new row
- if (!observers.isEmpty())
- observers.forEach((o) -> o.nextUnfilteredCluster(unfiltered));
-
- lastClustering = unfiltered.clustering();
- previousRowStart = pos;
- ++written;
-
- if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
- {
- RangeTombstoneMarker marker = (RangeTombstoneMarker)unfiltered;
- openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null;
- }
-
- // if we hit the column index size that we have to index after, go ahead and index it.
- if (currentPosition() - startPosition >= DatabaseDescriptor.getColumnIndexSize())
- addIndexBlock();
-
- }
-
- private ColumnIndex close() throws IOException
- {
- UnfilteredSerializer.serializer.writeEndOfPartition(writer);
-
- // It's possible we add no rows, just a top level deletion
- if (written == 0)
- return RowIndexEntryTest.ColumnIndex.EMPTY;
-
- // the last column may have fallen on an index boundary already. if not, index it explicitly.
- if (firstClustering != null)
- addIndexBlock();
-
- // we should always have at least one computed index block, but we only write it out if there is more than that.
- assert !columnsIndex.isEmpty() && headerLength >= 0;
- return new ColumnIndex(headerLength, columnsIndex);
- }
- }
- }
-
- @Test
- public void testSerializedSize() throws Throwable
- {
- String tableName = createTable("CREATE TABLE %s (a int, b text, c int, PRIMARY KEY(a, b))");
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
-
- Pre_C_11206_RowIndexEntry simple = new Pre_C_11206_RowIndexEntry(123);
-
- DataOutputBuffer buffer = new DataOutputBuffer();
- SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
- Pre_C_11206_RowIndexEntry.Serializer serializer = new Pre_C_11206_RowIndexEntry.Serializer(cfs.metadata(), BigFormat.latestVersion, header);
-
- serializer.serialize(simple, buffer);
-
- assertEquals(buffer.getLength(), serializer.serializedSize(simple));
-
- // write enough rows to ensure we get a few column index entries
- for (int i = 0; i <= DatabaseDescriptor.getColumnIndexSize() / 4; i++)
- execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, String.valueOf(i), i);
-
- ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build());
-
- File tempFile = FileUtils.createTempFile("row_index_entry_test", null);
- tempFile.deleteOnExit();
- SequentialWriter writer = new SequentialWriter(tempFile);
- ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(partition.unfilteredIterator(), writer, header, Collections.emptySet(), BigFormat.latestVersion);
- Pre_C_11206_RowIndexEntry withIndex = Pre_C_11206_RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex);
- IndexInfo.Serializer indexSerializer = IndexInfo.serializer(BigFormat.latestVersion, header);
-
- // sanity check
- assertTrue(columnIndex.columnsIndex.size() >= 3);
-
- buffer = new DataOutputBuffer();
- serializer.serialize(withIndex, buffer);
- assertEquals(buffer.getLength(), serializer.serializedSize(withIndex));
-
- // serialization check
-
- ByteBuffer bb = buffer.buffer();
- DataInputBuffer input = new DataInputBuffer(bb, false);
- serializationCheck(withIndex, indexSerializer, bb, input);
-
- // test with an output stream that doesn't support a file-pointer
- buffer = new DataOutputBuffer()
- {
- public boolean hasPosition()
- {
- return false;
- }
-
- public long position()
- {
- throw new UnsupportedOperationException();
- }
- };
- serializer.serialize(withIndex, buffer);
- bb = buffer.buffer();
- input = new DataInputBuffer(bb, false);
- serializationCheck(withIndex, indexSerializer, bb, input);
-
- //
-
- bb = buffer.buffer();
- input = new DataInputBuffer(bb, false);
- Pre_C_11206_RowIndexEntry.Serializer.skip(input, BigFormat.latestVersion);
- Assert.assertEquals(0, bb.remaining());
- }
-
- private static void serializationCheck(Pre_C_11206_RowIndexEntry withIndex, IndexInfo.Serializer indexSerializer, ByteBuffer bb, DataInputBuffer input) throws IOException
- {
- Assert.assertEquals(0xdeadbeef, input.readUnsignedVInt());
- Assert.assertEquals(withIndex.promotedSize(indexSerializer), input.readUnsignedVInt());
-
- Assert.assertEquals(withIndex.headerLength(), input.readUnsignedVInt());
- Assert.assertEquals(withIndex.deletionTime(), DeletionTime.serializer.deserialize(input));
- Assert.assertEquals(withIndex.columnsIndex().size(), input.readUnsignedVInt());
-
- int offset = bb.position();
- int[] offsets = new int[withIndex.columnsIndex().size()];
- for (int i = 0; i < withIndex.columnsIndex().size(); i++)
- {
- int pos = bb.position();
- offsets[i] = pos - offset;
- IndexInfo info = indexSerializer.deserialize(input);
- int end = bb.position();
-
- Assert.assertEquals(indexSerializer.serializedSize(info), end - pos);
-
- Assert.assertEquals(withIndex.columnsIndex().get(i).offset, info.offset);
- Assert.assertEquals(withIndex.columnsIndex().get(i).width, info.width);
- Assert.assertEquals(withIndex.columnsIndex().get(i).endOpenMarker, info.endOpenMarker);
- Assert.assertEquals(withIndex.columnsIndex().get(i).firstName, info.firstName);
- Assert.assertEquals(withIndex.columnsIndex().get(i).lastName, info.lastName);
- }
-
- for (int i = 0; i < withIndex.columnsIndex().size(); i++)
- Assert.assertEquals(offsets[i], input.readInt());
-
- Assert.assertEquals(0, bb.remaining());
- }
-
- static class Pre_C_11206_RowIndexEntry implements IMeasurableMemory
- {
- private static final long EMPTY_SIZE = ObjectSizes.measure(new Pre_C_11206_RowIndexEntry(0));
-
- public final long position;
-
- Pre_C_11206_RowIndexEntry(long position)
- {
- this.position = position;
- }
-
- protected int promotedSize(IndexInfo.Serializer idxSerializer)
- {
- return 0;
- }
-
- public static Pre_C_11206_RowIndexEntry create(long position, DeletionTime deletionTime, ColumnIndex index)
- {
- assert index != null;
- assert deletionTime != null;
-
- // we only consider the columns summary when determining whether to create an IndexedEntry,
- // since if there are insufficient columns to be worth indexing we're going to seek to
- // the beginning of the row anyway, so we might as well read the tombstone there as well.
- if (index.columnsIndex.size() > 1)
- return new Pre_C_11206_RowIndexEntry.IndexedEntry(position, deletionTime, index.partitionHeaderLength, index.columnsIndex);
- else
- return new Pre_C_11206_RowIndexEntry(position);
- }
-
- /**
- * @return true if this index entry contains the row-level tombstone and column summary. Otherwise,
- * caller should fetch these from the row header.
- */
- public boolean isIndexed()
- {
- return !columnsIndex().isEmpty();
- }
-
- public DeletionTime deletionTime()
- {
- throw new UnsupportedOperationException();
- }
-
- /**
- * The length of the row header (partition key, partition deletion and static row).
- * This value is only provided for indexed entries and this method will throw
- * {@code UnsupportedOperationException} if {@code !isIndexed()}.
- */
- public long headerLength()
- {
- throw new UnsupportedOperationException();
- }
-
- public List<IndexInfo> columnsIndex()
- {
- return Collections.emptyList();
- }
-
- public long unsharedHeapSize()
- {
- return EMPTY_SIZE;
- }
-
- public static class Serializer
- {
- private final IndexInfo.Serializer idxSerializer;
- private final Version version;
-
- Serializer(TableMetadata metadata, Version version, SerializationHeader header)
- {
- this.idxSerializer = IndexInfo.serializer(version, header);
- this.version = version;
- }
-
- public void serialize(Pre_C_11206_RowIndexEntry rie, DataOutputPlus out) throws IOException
- {
- out.writeUnsignedVInt(rie.position);
- out.writeUnsignedVInt(rie.promotedSize(idxSerializer));
-
- if (rie.isIndexed())
- {
- out.writeUnsignedVInt(rie.headerLength());
- DeletionTime.serializer.serialize(rie.deletionTime(), out);
- out.writeUnsignedVInt(rie.columnsIndex().size());
-
- // Calculate and write the offsets to the IndexInfo objects.
-
- int[] offsets = new int[rie.columnsIndex().size()];
-
- if (out.hasPosition())
- {
- // Out is usually a SequentialWriter, so using the file-pointer is fine to generate the offsets.
- // A DataOutputBuffer also works.
- long start = out.position();
- int i = 0;
- for (IndexInfo info : rie.columnsIndex())
- {
- offsets[i] = i == 0 ? 0 : (int)(out.position() - start);
- i++;
- idxSerializer.serialize(info, out);
- }
- }
- else
- {
- // Not sure this branch will ever be needed, but if it is called, it has to calculate the
- // serialized sizes instead of simply using the file-pointer.
- int i = 0;
- int offset = 0;
- for (IndexInfo info : rie.columnsIndex())
- {
- offsets[i++] = offset;
- idxSerializer.serialize(info, out);
- offset += idxSerializer.serializedSize(info);
- }
- }
-
- for (int off : offsets)
- out.writeInt(off);
- }
- }
-
- public Pre_C_11206_RowIndexEntry deserialize(DataInputPlus in) throws IOException
- {
- long position = in.readUnsignedVInt();
-
- int size = (int)in.readUnsignedVInt();
- if (size > 0)
- {
- long headerLength = in.readUnsignedVInt();
- DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
- int entries = (int)in.readUnsignedVInt();
- List<IndexInfo> columnsIndex = new ArrayList<>(entries);
- for (int i = 0; i < entries; i++)
- columnsIndex.add(idxSerializer.deserialize(in));
-
- in.skipBytesFully(entries * TypeSizes.sizeof(0));
-
- return new Pre_C_11206_RowIndexEntry.IndexedEntry(position, deletionTime, headerLength, columnsIndex);
- }
- else
- {
- return new Pre_C_11206_RowIndexEntry(position);
- }
- }
-
- // Reads only the data 'position' of the index entry and returns it. Note that this left 'in' in the middle
- // of reading an entry, so this is only useful if you know what you are doing and in most case 'deserialize'
- // should be used instead.
- static long readPosition(DataInputPlus in, Version version) throws IOException
- {
- return in.readUnsignedVInt();
- }
-
- public static void skip(DataInputPlus in, Version version) throws IOException
- {
- readPosition(in, version);
- skipPromotedIndex(in, version);
- }
-
- private static void skipPromotedIndex(DataInputPlus in, Version version) throws IOException
- {
- int size = (int)in.readUnsignedVInt();
- if (size <= 0)
- return;
-
- in.skipBytesFully(size);
- }
-
- public int serializedSize(Pre_C_11206_RowIndexEntry rie)
- {
- int indexedSize = 0;
- if (rie.isIndexed())
- {
- List<IndexInfo> index = rie.columnsIndex();
-
- indexedSize += TypeSizes.sizeofUnsignedVInt(rie.headerLength());
- indexedSize += DeletionTime.serializer.serializedSize(rie.deletionTime());
- indexedSize += TypeSizes.sizeofUnsignedVInt(index.size());
-
- for (IndexInfo info : index)
- indexedSize += idxSerializer.serializedSize(info);
-
- indexedSize += index.size() * TypeSizes.sizeof(0);
- }
-
- return TypeSizes.sizeofUnsignedVInt(rie.position) + TypeSizes.sizeofUnsignedVInt(indexedSize) + indexedSize;
- }
- }
-
- /**
- * An entry in the row index for a row whose columns are indexed.
- */
- private static final class IndexedEntry extends Pre_C_11206_RowIndexEntry
- {
- private final DeletionTime deletionTime;
-
- // The offset in the file when the index entry end
- private final long headerLength;
- private final List<IndexInfo> columnsIndex;
- private static final long BASE_SIZE =
- ObjectSizes.measure(new IndexedEntry(0, DeletionTime.LIVE, 0, Arrays.asList(null, null)))
- + ObjectSizes.measure(new ArrayList<>(1));
-
- private IndexedEntry(long position, DeletionTime deletionTime, long headerLength, List<IndexInfo> columnsIndex)
- {
- super(position);
- assert deletionTime != null;
- assert columnsIndex != null && columnsIndex.size() > 1;
- this.deletionTime = deletionTime;
- this.headerLength = headerLength;
- this.columnsIndex = columnsIndex;
- }
-
- @Override
- public DeletionTime deletionTime()
- {
- return deletionTime;
- }
-
- @Override
- public long headerLength()
- {
- return headerLength;
- }
-
- @Override
- public List<IndexInfo> columnsIndex()
- {
- return columnsIndex;
- }
-
- @Override
- protected int promotedSize(IndexInfo.Serializer idxSerializer)
- {
- long size = TypeSizes.sizeofUnsignedVInt(headerLength)
- + DeletionTime.serializer.serializedSize(deletionTime)
- + TypeSizes.sizeofUnsignedVInt(columnsIndex.size()); // number of entries
- for (IndexInfo info : columnsIndex)
- size += idxSerializer.serializedSize(info);
-
- size += columnsIndex.size() * TypeSizes.sizeof(0);
-
- return Ints.checkedCast(size);
- }
-
- @Override
- public long unsharedHeapSize()
- {
- long entrySize = 0;
- for (IndexInfo idx : columnsIndex)
- entrySize += idx.unsharedHeapSize();
-
- return BASE_SIZE
- + entrySize
- + deletionTime.unsharedHeapSize()
- + ObjectSizes.sizeOfReferenceArray(columnsIndex.size());
- }
- }
- }
-
- @Test
- public void testIndexFor() throws IOException
- {
- DeletionTime deletionInfo = new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds());
-
- List<IndexInfo> indexes = new ArrayList<>();
- indexes.add(new IndexInfo(cn(0L), cn(5L), 0, 0, deletionInfo));
- indexes.add(new IndexInfo(cn(10L), cn(15L), 0, 0, deletionInfo));
- indexes.add(new IndexInfo(cn(20L), cn(25L), 0, 0, deletionInfo));
-
- RowIndexEntry rie = new RowIndexEntry(0L)
- {
- public IndexInfoRetriever openWithIndex(FileHandle indexFile)
- {
- return new IndexInfoRetriever()
- {
- public IndexInfo columnsIndex(int index)
- {
- return indexes.get(index);
- }
-
- public void close()
- {
- }
- };
- }
-
- public int columnsIndexCount()
- {
- return indexes.size();
- }
- };
-
- AbstractSSTableIterator.IndexState indexState = new AbstractSSTableIterator.IndexState(
- null, comp, rie, false, null
- );
-
- assertEquals(0, indexState.indexFor(cn(-1L), -1));
- assertEquals(0, indexState.indexFor(cn(5L), -1));
- assertEquals(1, indexState.indexFor(cn(12L), -1));
- assertEquals(2, indexState.indexFor(cn(17L), -1));
- assertEquals(3, indexState.indexFor(cn(100L), -1));
- assertEquals(3, indexState.indexFor(cn(100L), 0));
- assertEquals(3, indexState.indexFor(cn(100L), 1));
- assertEquals(3, indexState.indexFor(cn(100L), 2));
- assertEquals(3, indexState.indexFor(cn(100L), 3));
-
- indexState = new AbstractSSTableIterator.IndexState(
- null, comp, rie, true, null
- );
-
- assertEquals(-1, indexState.indexFor(cn(-1L), -1));
- assertEquals(0, indexState.indexFor(cn(5L), 3));
- assertEquals(0, indexState.indexFor(cn(5L), 2));
- assertEquals(1, indexState.indexFor(cn(17L), 3));
- assertEquals(2, indexState.indexFor(cn(100L), 3));
- assertEquals(2, indexState.indexFor(cn(100L), 4));
- assertEquals(1, indexState.indexFor(cn(12L), 3));
- assertEquals(1, indexState.indexFor(cn(12L), 2));
- assertEquals(1, indexState.indexFor(cn(100L), 1));
- assertEquals(2, indexState.indexFor(cn(100L), 2));
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
index 63ef861..41244ca 100644
--- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
+++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
@@ -22,11 +22,11 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;
-import org.apache.cassandra.io.util.FileReader;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.cassandra.*;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
@@ -37,9 +37,7 @@
import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
+import org.apache.cassandra.utils.JsonUtils;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
@@ -297,6 +295,7 @@
assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata()),
containsString("CLUSTERING ORDER BY (cl1 ASC)\n" +
" AND additional_write_policy = 'ALWAYS'\n" +
+ " AND allow_auto_snapshot = true\n" +
" AND bloom_filter_fp_chance = 1.0\n" +
" AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" +
" AND cdc = false\n" +
@@ -308,6 +307,7 @@
" AND default_time_to_live = 4\n" +
" AND extensions = {'ext1': 0x76616c31}\n" +
" AND gc_grace_seconds = 5\n" +
+ " AND incremental_backups = true\n" +
" AND max_index_interval = 7\n" +
" AND memtable_flush_period_in_ms = 8\n" +
" AND min_index_interval = 6\n" +
@@ -446,9 +446,10 @@
assertThat(schema, containsString("CREATE INDEX IF NOT EXISTS " + tableName + "_reg2_idx ON " + keyspace() + '.' + tableName + " (reg2);"));
- JSONObject manifest = (JSONObject) new JSONParser().parse(new FileReader(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT)));
- JSONArray files = (JSONArray) manifest.get("files");
+ JsonNode manifest = JsonUtils.JSON_OBJECT_MAPPER.readTree(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).toJavaIOFile());
+ JsonNode files = manifest.get("files");
// two files, the second is index
+ Assert.assertTrue(files.isArray());
Assert.assertEquals(2, files.size());
}
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
deleted file mode 100644
index dfe9165..0000000
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ /dev/null
@@ -1,771 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.IOError;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.cassandra.io.util.File;
-import org.apache.commons.lang3.StringUtils;
-
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.compaction.Scrubber;
-import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.partitions.Partition;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.RequestExecutionException;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.io.sstable.SSTableTxnWriter;
-import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.TimeUUID;
-
-import static org.apache.cassandra.SchemaLoader.counterCFMD;
-import static org.apache.cassandra.SchemaLoader.createKeyspace;
-import static org.apache.cassandra.SchemaLoader.getCompressionParameters;
-import static org.apache.cassandra.SchemaLoader.loadSchema;
-import static org.apache.cassandra.SchemaLoader.standardCFMD;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeTrue;
-
-public class ScrubTest
-{
- public static final String INVALID_LEGACY_SSTABLE_ROOT_PROP = "invalid-legacy-sstable-root";
-
- public static final String CF = "Standard1";
- public static final String COUNTER_CF = "Counter1";
- public static final String CF_UUID = "UUIDKeys";
- public static final String CF_INDEX1 = "Indexed1";
- public static final String CF_INDEX2 = "Indexed2";
- public static final String CF_INDEX1_BYTEORDERED = "Indexed1_ordered";
- public static final String CF_INDEX2_BYTEORDERED = "Indexed2_ordered";
- public static final String COL_INDEX = "birthdate";
- public static final String COL_NON_INDEX = "notbirthdate";
-
- public static final Integer COMPRESSION_CHUNK_LENGTH = 4096;
-
- private static final AtomicInteger seq = new AtomicInteger();
-
- String ksName;
- Keyspace keyspace;
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- loadSchema();
- }
-
- @Before
- public void setup()
- {
- ksName = "scrub_test_" + seq.incrementAndGet();
- createKeyspace(ksName,
- KeyspaceParams.simple(1),
- standardCFMD(ksName, CF),
- counterCFMD(ksName, COUNTER_CF).compression(getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
- standardCFMD(ksName, CF_UUID, 0, UUIDType.instance),
- SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1, true),
- SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2, true),
- SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance),
- SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance));
- keyspace = Keyspace.open(ksName);
-
- CompactionManager.instance.disableAutoCompaction();
- System.setProperty(org.apache.cassandra.tools.Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
- }
-
- @AfterClass
- public static void clearClassEnv()
- {
- System.clearProperty(org.apache.cassandra.tools.Util.ALLOW_TOOL_REINIT_FOR_TEST);
- }
-
- @Test
- public void testScrubOnePartition() throws ExecutionException, InterruptedException
- {
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- CompactionManager.instance.performScrub(cfs, false, true, false, 2);
-
- // check data is still there
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testScrubLastBrokenPartition() throws ExecutionException, InterruptedException, IOException
- {
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ksName, CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- Set<SSTableReader> liveSSTables = cfs.getLiveSSTables();
- assertThat(liveSSTables).hasSize(1);
- String fileName = liveSSTables.iterator().next().getFilename();
- Files.write(Paths.get(fileName), new byte[10], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
- ChunkCache.instance.invalidateFile(fileName);
-
- CompactionManager.instance.performScrub(cfs, true, true, false, 2);
-
- // check data is still there
- assertOrderedAll(cfs, 0);
- }
-
- @Test
- public void testScrubCorruptedCounterPartition() throws IOException, WriteTimeoutException
- {
- // When compression is enabled, for testing corrupted chunks we need enough partitions to cover
- // at least 3 chunks of size COMPRESSION_CHUNK_LENGTH
- int numPartitions = 1000;
-
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
- cfs.truncateBlocking();
- fillCounterCF(cfs, numPartitions);
-
- assertOrderedAll(cfs, numPartitions);
-
- assertEquals(1, cfs.getLiveSSTables().size());
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- //make sure to override at most 1 chunk when compression is enabled
- overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
-
- // with skipCorrupted == false, the scrub is expected to fail
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true))
- {
- scrubber.scrub();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (IOError err) {
- assertTrue(err.getCause() instanceof CorruptSSTableException);
- }
-
- // with skipCorrupted == true, the corrupt rows will be skipped
- Scrubber.ScrubResult scrubResult;
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, true))
- {
- scrubResult = scrubber.scrubWithResult();
- }
-
- assertNotNull(scrubResult);
-
- boolean compression = Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"));
- assertEquals(0, scrubResult.emptyPartitions);
- if (compression)
- {
- assertEquals(numPartitions, scrubResult.badPartitions + scrubResult.goodPartitions);
- //because we only corrupted 1 chunk and we chose enough partitions to cover at least 3 chunks
- assertTrue(scrubResult.goodPartitions >= scrubResult.badPartitions * 2);
- }
- else
- {
- assertEquals(1, scrubResult.badPartitions);
- assertEquals(numPartitions-1, scrubResult.goodPartitions);
- }
- assertEquals(1, cfs.getLiveSSTables().size());
-
- assertOrderedAll(cfs, scrubResult.goodPartitions);
- }
-
- @Test
- public void testScrubCorruptedRowInSmallFile() throws IOException, WriteTimeoutException
- {
- // cannot test this with compression
- assumeTrue(!Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")));
-
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
-
- fillCounterCF(cfs, 2);
-
- assertOrderedAll(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- // overwrite one row with garbage
- overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
-
- // with skipCorrupted == false, the scrub is expected to fail
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true))
- {
- // with skipCorrupted == true, the corrupt row will be skipped
- scrubber.scrub();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (IOError err) {
- assertTrue(err.getCause() instanceof CorruptSSTableException);
- }
-
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, true))
- {
- // with skipCorrupted == true, the corrupt row will be skipped
- scrubber.scrub();
- }
-
- assertEquals(1, cfs.getLiveSSTables().size());
- // verify that we can read all of the rows, and there is now one less row
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testScrubOneRowWithCorruptedKey() throws IOException, ExecutionException, InterruptedException, ConfigurationException
- {
- // cannot test this with compression
- assumeTrue(!Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")));
-
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 4);
- assertOrderedAll(cfs, 4);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- overrideWithGarbage(sstable, 0, 2);
-
- CompactionManager.instance.performScrub(cfs, false, true, 2);
-
- // check data is still there
- assertOrderedAll(cfs, 4);
- }
-
- @Test
- public void testScrubCorruptedCounterRowNoEarlyOpen() throws IOException, WriteTimeoutException
- {
- boolean oldDisabledVal = SSTableRewriter.disableEarlyOpeningForTests;
- try
- {
- SSTableRewriter.disableEarlyOpeningForTests = true;
- testScrubCorruptedCounterPartition();
- }
- finally
- {
- SSTableRewriter.disableEarlyOpeningForTests = oldDisabledVal;
- }
- }
-
- @Test
- public void testScrubMultiRow() throws ExecutionException, InterruptedException
- {
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 10);
- assertOrderedAll(cfs, 10);
-
- CompactionManager.instance.performScrub(cfs, false, true, 2);
-
- // check data is still there
- assertOrderedAll(cfs, 10);
- }
-
- @Test
- public void testScrubNoIndex() throws ExecutionException, InterruptedException, ConfigurationException
- {
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 10);
- assertOrderedAll(cfs, 10);
-
- for (SSTableReader sstable : cfs.getLiveSSTables())
- assertTrue(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)).tryDelete());
-
- CompactionManager.instance.performScrub(cfs, false, true, 2);
-
- // check data is still there
- assertOrderedAll(cfs, 10);
- }
-
- @Test
- public void testScrubOutOfOrder()
- {
- // This test assumes ByteOrderPartitioner to create out-of-order SSTable
- IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner();
- DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner());
-
- // Create out-of-order SSTable
- File tempDir = FileUtils.createTempFile("ScrubTest.testScrubOutOfOrder", "").parent();
- // create ks/cf directory
- File tempDataDir = new File(tempDir, String.join(File.pathSeparator(), ksName, CF));
- assertTrue(tempDataDir.tryCreateDirectories());
- try
- {
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
- Descriptor desc = cfs.newSSTableDescriptor(tempDataDir);
-
- try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
- SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, keys.size(), cfs.metadata, txn)))
- {
- for (String k : keys)
- {
- PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), Util.dk(k))
- .newRow("someName").add("val", "someValue")
- .build();
-
- writer.append(update.unfilteredIterator());
- }
- writer.finish(false);
- }
-
- try
- {
- SSTableReader.open(desc, cfs.metadata);
- fail("SSTR validation should have caught the out-of-order rows");
- }
- catch (CorruptSSTableException ise)
- { /* this is expected */ }
-
- // open without validation for scrubbing
- Set<Component> components = new HashSet<>();
- if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
- components.add(Component.COMPRESSION_INFO);
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- components.add(Component.FILTER);
- components.add(Component.STATS);
- components.add(Component.SUMMARY);
- components.add(Component.TOC);
-
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- if (sstable.last.compareTo(sstable.first) < 0)
- sstable.last = sstable.first;
-
- try (LifecycleTransaction scrubTxn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true))
- {
- scrubber.scrub();
- }
- LifecycleTransaction.waitForDeletions();
- cfs.loadNewSSTables();
- assertOrderedAll(cfs, 7);
- }
- finally
- {
- FileUtils.deleteRecursive(tempDataDir);
- // reset partitioner
- DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner);
- }
- }
-
- static void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException
- {
- boolean compression = Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"));
- long startPosition, endPosition;
-
- if (compression)
- { // overwrite with garbage the compression chunks from key1 to key2
- CompressionMetadata compData = CompressionMetadata.create(sstable.getFilename());
-
- CompressionMetadata.Chunk chunk1 = compData.chunkFor(
- sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ).position);
- CompressionMetadata.Chunk chunk2 = compData.chunkFor(
- sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ).position);
-
- startPosition = Math.min(chunk1.offset, chunk2.offset);
- endPosition = Math.max(chunk1.offset + chunk1.length, chunk2.offset + chunk2.length);
-
- compData.close();
- }
- else
- { // overwrite with garbage from key1 to key2
- long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ).position;
- long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ).position;
- startPosition = Math.min(row0Start, row1Start);
- endPosition = Math.max(row0Start, row1Start);
- }
-
- overrideWithGarbage(sstable, startPosition, endPosition);
- }
-
- private static void overrideWithGarbage(SSTableReader sstable, long startPosition, long endPosition) throws IOException
- {
- try (FileChannel fileChannel = new File(sstable.getFilename()).newReadWriteChannel())
- {
- fileChannel.position(startPosition);
- fileChannel.write(ByteBufferUtil.bytes(StringUtils.repeat('z', (int) (endPosition - startPosition))));
- }
- if (ChunkCache.instance != null)
- ChunkCache.instance.invalidateFile(sstable.getFilename());
- }
-
- static void assertOrderedAll(ColumnFamilyStore cfs, int expectedSize)
- {
- assertOrdered(Util.cmd(cfs).build(), expectedSize);
- }
-
- private static void assertOrdered(ReadCommand cmd, int expectedSize)
- {
- int size = 0;
- DecoratedKey prev = null;
- for (Partition partition : Util.getAllUnfiltered(cmd))
- {
- DecoratedKey current = partition.partitionKey();
- assertTrue("key " + current + " does not sort after previous key " + prev, prev == null || prev.compareTo(current) < 0);
- prev = current;
- ++size;
- }
- assertEquals(expectedSize, size);
- }
-
- protected static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
- {
- for (int i = 0; i < partitionsPerSSTable; i++)
- {
- PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
- .newRow("r1").add("val", "1")
- .newRow("r1").add("val", "1")
- .build();
-
- new Mutation(update).applyUnsafe();
- }
-
- Util.flush(cfs);
- }
-
- public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long ... values)
- {
- assertEquals(0, values.length % 2);
- for (int i = 0; i < values.length; i +=2)
- {
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), String.valueOf(i));
- if (composite)
- {
- builder.newRow("c" + i)
- .add(COL_INDEX, values[i])
- .add(COL_NON_INDEX, values[i + 1]);
- }
- else
- {
- builder.newRow()
- .add(COL_INDEX, values[i])
- .add(COL_NON_INDEX, values[i + 1]);
- }
- new Mutation(builder.build()).applyUnsafe();
- }
-
- Util.flush(cfs);
- }
-
- protected static void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException
- {
- for (int i = 0; i < partitionsPerSSTable; i++)
- {
- PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
- .newRow("r1").add("val", 100L)
- .build();
- new CounterMutation(new Mutation(update), ConsistencyLevel.ONE).apply();
- }
-
- Util.flush(cfs);
- }
-
- @Test
- public void testScrubColumnValidation() throws InterruptedException, RequestExecutionException, ExecutionException
- {
- QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_static_columns (a bigint, b timeuuid, c boolean static, d text, PRIMARY KEY (a, b))", ksName), ConsistencyLevel.ONE);
-
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_static_columns");
-
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_static_columns (a, b, c, d) VALUES (123, c3db07e8-b602-11e3-bc6b-e0b9a54a6d93, true, 'foobar')", ksName));
- Util.flush(cfs);
- CompactionManager.instance.performScrub(cfs, false, true, 2);
-
- QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_scrub_validation (a text primary key, b int)", ksName), ConsistencyLevel.ONE);
- ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore("test_scrub_validation");
-
- new Mutation(UpdateBuilder.create(cfs2.metadata(), "key").newRow().add("b", Int32Type.instance.decompose(1)).build()).apply();
- Util.flush(cfs2);
-
- CompactionManager.instance.performScrub(cfs2, false, false, 2);
- }
-
- /**
- * For CASSANDRA-6892 too, check that for a compact table with one cluster column, we can insert whatever
- * we want as value for the clustering column, including something that would conflict with a CQL column definition.
- */
- @Test
- public void testValidationCompactStorage() throws Exception
- {
- QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_dynamic_columns (a int, b text, c text, PRIMARY KEY (a, b)) WITH COMPACT STORAGE", ksName), ConsistencyLevel.ONE);
-
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_dynamic_columns");
-
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'a', 'foo')", ksName));
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'b', 'bar')", ksName));
- QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'c', 'boo')", ksName));
- Util.flush(cfs);
- CompactionManager.instance.performScrub(cfs, true, true, 2);
-
- // Scrub is silent, but it will remove broken records. So reading everything back to make sure nothing to "scrubbed away"
- UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".test_compact_dynamic_columns", ksName));
- assertNotNull(rs);
- assertEquals(3, rs.size());
-
- Iterator<UntypedResultSet.Row> iter = rs.iterator();
- assertEquals("foo", iter.next().getString("c"));
- assertEquals("bar", iter.next().getString("c"));
- assertEquals("boo", iter.next().getString("c"));
- }
-
- @Test /* CASSANDRA-5174 */
- public void testScrubKeysIndex_preserveOrder() throws IOException, ExecutionException, InterruptedException
- {
- //If the partitioner preserves the order then SecondaryIndex uses BytesType comparator,
- // otherwise it uses LocalByPartitionerType
- testScrubIndex(CF_INDEX1_BYTEORDERED, COL_INDEX, false, true);
- }
-
- @Test /* CASSANDRA-5174 */
- public void testScrubCompositeIndex_preserveOrder() throws IOException, ExecutionException, InterruptedException
- {
- testScrubIndex(CF_INDEX2_BYTEORDERED, COL_INDEX, true, true);
- }
-
- @Test /* CASSANDRA-5174 */
- public void testScrubKeysIndex() throws IOException, ExecutionException, InterruptedException
- {
- testScrubIndex(CF_INDEX1, COL_INDEX, false, true);
- }
-
- @Test /* CASSANDRA-5174 */
- public void testScrubCompositeIndex() throws IOException, ExecutionException, InterruptedException
- {
- testScrubIndex(CF_INDEX2, COL_INDEX, true, true);
- }
-
- @Test /* CASSANDRA-5174 */
- public void testFailScrubKeysIndex() throws IOException, ExecutionException, InterruptedException
- {
- testScrubIndex(CF_INDEX1, COL_INDEX, false, false);
- }
-
- @Test /* CASSANDRA-5174 */
- public void testFailScrubCompositeIndex() throws IOException, ExecutionException, InterruptedException
- {
- testScrubIndex(CF_INDEX2, COL_INDEX, true, false);
- }
-
- @Test /* CASSANDRA-5174 */
- public void testScrubTwice() throws IOException, ExecutionException, InterruptedException
- {
- testScrubIndex(CF_INDEX2, COL_INDEX, true, true, true);
- }
-
- @SuppressWarnings("SameParameterValue")
- private void testScrubIndex(String cfName, String colName, boolean composite, boolean ... scrubs)
- throws IOException, ExecutionException, InterruptedException
- {
- CompactionManager.instance.disableAutoCompaction();
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
-
- int numRows = 1000;
- long[] colValues = new long [numRows * 2]; // each row has two columns
- for (int i = 0; i < colValues.length; i+=2)
- {
- colValues[i] = (i % 4 == 0 ? 1L : 2L); // index column
- colValues[i+1] = 3L; //other column
- }
- fillIndexCF(cfs, composite, colValues);
-
- // check index
-
- assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2);
-
- // scrub index
- Set<ColumnFamilyStore> indexCfss = cfs.indexManager.getAllIndexColumnFamilyStores();
- assertEquals(1, indexCfss.size());
- for(ColumnFamilyStore indexCfs : indexCfss)
- {
- for (int i = 0; i < scrubs.length; i++)
- {
- boolean failure = !scrubs[i];
- if (failure)
- { //make sure the next scrub fails
- overrideWithGarbage(indexCfs.getLiveSSTables().iterator().next(), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes(2L));
- }
- CompactionManager.AllSSTableOpStatus result = indexCfs.scrub(false, false, false, true, false,0);
- assertEquals(failure ?
- CompactionManager.AllSSTableOpStatus.ABORTED :
- CompactionManager.AllSSTableOpStatus.SUCCESSFUL,
- result);
- }
- }
-
-
- // check index is still working
- assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2);
- }
-
- private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, TableMetadataRef metadata, LifecycleTransaction txn)
- {
- SerializationHeader header = new SerializationHeader(true, metadata.get(), metadata.get().regularAndStaticColumns(), EncodingStats.NO_STATS);
- MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(0);
- return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, null, false, metadata, collector, header, txn), txn);
- }
-
- private static class TestMultiWriter extends SimpleSSTableMultiWriter
- {
- TestMultiWriter(SSTableWriter writer, LifecycleNewTracker lifecycleNewTracker)
- {
- super(writer, lifecycleNewTracker);
- }
- }
-
- /**
- * Test writer that allows to write out of order SSTable.
- */
- private static class TestWriter extends BigTableWriter
- {
- TestWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, TableMetadataRef metadata,
- MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn)
- {
- super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, Collections.emptySet(), txn);
- }
-
- @Override
- protected long beforeAppend(DecoratedKey decoratedKey)
- {
- return dataFile.position();
- }
- }
-
- /**
- * Tests with invalid sstables (containing duplicate entries in 2.0 and 3.0 storage format),
- * that were caused by upgrading from 2.x with duplicate range tombstones.
- *
- * See CASSANDRA-12144 for details.
- */
- @Test
- public void testFilterOutDuplicates() throws Exception
- {
- IPartitioner oldPart = DatabaseDescriptor.getPartitioner();
- try
- {
- DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
- QueryProcessor.process(String.format("CREATE TABLE \"%s\".cf_with_duplicates_3_0 (a int, b int, c int, PRIMARY KEY (a, b))", ksName), ConsistencyLevel.ONE);
-
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("cf_with_duplicates_3_0");
-
- Path legacySSTableRoot = Paths.get(System.getProperty(INVALID_LEGACY_SSTABLE_ROOT_PROP),
- "Keyspace1",
- "cf_with_duplicates_3_0");
-
- for (String filename : new String[]{ "mb-3-big-CompressionInfo.db",
- "mb-3-big-Digest.crc32",
- "mb-3-big-Index.db",
- "mb-3-big-Summary.db",
- "mb-3-big-Data.db",
- "mb-3-big-Filter.db",
- "mb-3-big-Statistics.db",
- "mb-3-big-TOC.txt" })
- {
- Files.copy(Paths.get(legacySSTableRoot.toString(), filename), cfs.getDirectories().getDirectoryForNewSSTables().toPath().resolve(filename));
- }
-
- cfs.loadNewSSTables();
-
- cfs.scrub(true, true, false, false, false, 1);
-
- UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".cf_with_duplicates_3_0", ksName));
- assertNotNull(rs);
- assertEquals(1, rs.size());
-
- QueryProcessor.executeInternal(String.format("DELETE FROM \"%s\".cf_with_duplicates_3_0 WHERE a=1 AND b =2", ksName));
- rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".cf_with_duplicates_3_0", ksName));
- assertNotNull(rs);
- assertEquals(0, rs.size());
- }
- finally
- {
- DatabaseDescriptor.setPartitionerUnsafe(oldPart);
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/ScrubToolTest.java b/test/unit/org/apache/cassandra/db/ScrubToolTest.java
deleted file mode 100644
index 280810c..0000000
--- a/test/unit/org/apache/cassandra/db/ScrubToolTest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.db;
-
-import java.io.IOError;
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.tools.StandaloneScrubber;
-import org.apache.cassandra.tools.ToolRunner;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.assertj.core.api.Assertions;
-
-import static org.apache.cassandra.SchemaLoader.counterCFMD;
-import static org.apache.cassandra.SchemaLoader.createKeyspace;
-import static org.apache.cassandra.SchemaLoader.getCompressionParameters;
-import static org.apache.cassandra.SchemaLoader.loadSchema;
-import static org.apache.cassandra.SchemaLoader.standardCFMD;
-import static org.apache.cassandra.db.ScrubTest.CF_INDEX1;
-import static org.apache.cassandra.db.ScrubTest.CF_INDEX1_BYTEORDERED;
-import static org.apache.cassandra.db.ScrubTest.CF_INDEX2;
-import static org.apache.cassandra.db.ScrubTest.CF_INDEX2_BYTEORDERED;
-import static org.apache.cassandra.db.ScrubTest.CF_UUID;
-import static org.apache.cassandra.db.ScrubTest.COMPRESSION_CHUNK_LENGTH;
-import static org.apache.cassandra.db.ScrubTest.COUNTER_CF;
-import static org.apache.cassandra.db.ScrubTest.assertOrderedAll;
-import static org.apache.cassandra.db.ScrubTest.fillCF;
-import static org.apache.cassandra.db.ScrubTest.fillCounterCF;
-import static org.apache.cassandra.db.ScrubTest.overrideWithGarbage;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class ScrubToolTest
-{
- private static final String CF = "scrub_tool_test";
- private static final AtomicInteger seq = new AtomicInteger();
-
- String ksName;
- Keyspace keyspace;
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- loadSchema();
- }
-
- @Before
- public void setup()
- {
- ksName = "scrub_test_" + seq.incrementAndGet();
- createKeyspace(ksName,
- KeyspaceParams.simple(1),
- standardCFMD(ksName, CF),
- counterCFMD(ksName, COUNTER_CF).compression(getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
- standardCFMD(ksName, CF_UUID, 0, UUIDType.instance),
- SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1, true),
- SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2, true),
- SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance),
- SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance));
- keyspace = Keyspace.open(ksName);
-
- CompactionManager.instance.disableAutoCompaction();
- System.setProperty(org.apache.cassandra.tools.Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
- }
-
- @Test
- public void testScrubOnePartitionWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
- Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
- tool.assertOnCleanExit();
-
- // check data is still there
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testSkipScrubCorruptedCounterPartitionWithTool() throws IOException, WriteTimeoutException
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
- int numPartitions = 1000;
-
- fillCounterCF(cfs, numPartitions);
- assertOrderedAll(cfs, numPartitions);
- assertEquals(1, cfs.getLiveSSTables().size());
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
-
- // with skipCorrupted == true, the corrupt rows will be skipped
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-s", ksName, COUNTER_CF);
- Assertions.assertThat(tool.getStdout()).contains("0 empty");
- Assertions.assertThat(tool.getStdout()).contains("partitions that were skipped");
- tool.assertOnCleanExit();
-
- assertEquals(1, cfs.getLiveSSTables().size());
- }
-
- @Test
- public void testNoSkipScrubCorruptedCounterPartitionWithTool() throws IOException, WriteTimeoutException
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
- int numPartitions = 1000;
-
- fillCounterCF(cfs, numPartitions);
- assertOrderedAll(cfs, numPartitions);
- assertEquals(1, cfs.getLiveSSTables().size());
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
-
- // with skipCorrupted == false, the scrub is expected to fail
- try
- {
- ToolRunner.invokeClass(StandaloneScrubber.class, ksName, COUNTER_CF);
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (IOError err) {
- assertTrue(err.getCause() instanceof CorruptSSTableException);
- }
- }
-
- @Test
- public void testNoCheckScrubMultiPartitionWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- // insert data and verify we get it back w/ range query
- fillCF(cfs, 10);
- assertOrderedAll(cfs, 10);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-n", ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
- Assertions.assertThat(tool.getStdout()).contains("10 partitions in new sstable and 0 empty");
- tool.assertOnCleanExit();
-
- // check data is still there
- assertOrderedAll(cfs, 10);
- }
-
- @Test
- public void testHeaderFixValidateOnlyWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate_only", ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Not continuing with scrub, since '--header-fix validate-only' was specified.");
- tool.assertOnCleanExit();
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testHeaderFixValidateWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate", ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
- Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
- tool.assertOnCleanExit();
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testHeaderFixFixOnlyWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix-only", ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Not continuing with scrub, since '--header-fix fix-only' was specified.");
- tool.assertOnCleanExit();
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testHeaderFixWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix", ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
- Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
- tool.assertOnCleanExit();
- assertOrderedAll(cfs, 1);
- }
-
- @Test
- public void testHeaderFixNoChecksWithTool()
- {
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- fillCF(cfs, 1);
- assertOrderedAll(cfs, 1);
-
- ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "off", ksName, CF);
- Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
- Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
- tool.assertOnCleanExit();
- assertOrderedAll(cfs, 1);
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
index 8f7e24b..d77e1fe 100644
--- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
@@ -18,7 +18,16 @@
package org.apache.cassandra.db;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
import com.google.common.io.Files;
+import org.junit.Assert;
+import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -38,23 +47,12 @@
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
-import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.concurrent.Callable;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-
-import org.apache.cassandra.io.util.File;
public class SerializationHeaderTest
{
@@ -68,9 +66,8 @@
@Test
public void testWrittenAsDifferentKind() throws Exception
{
+ SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
final String tableName = "testWrittenAsDifferentKind";
-// final String schemaCqlWithStatic = String.format("CREATE TABLE %s (k int, c int, v int static, PRIMARY KEY(k, c))", tableName);
-// final String schemaCqlWithRegular = String.format("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))", tableName);
ColumnIdentifier v = ColumnIdentifier.getInterned("v", false);
TableMetadata schemaWithStatic = TableMetadata.builder(KEYSPACE, tableName)
.addPartitionKeyColumn("k", Int32Type.instance)
@@ -87,16 +84,26 @@
schemaWithStatic = schemaWithStatic.unbuild().recordColumnDrop(columnRegular, 0L).build();
schemaWithRegular = schemaWithRegular.unbuild().recordColumnDrop(columnStatic, 0L).build();
+ SSTableReader readerWithStatic = null;
+ SSTableReader readerWithRegular = null;
Supplier<SequenceBasedSSTableId> id = Util.newSeqGen();
File dir = new File(Files.createTempDir());
try
{
BiFunction<TableMetadata, Function<ByteBuffer, Clustering<?>>, Callable<Descriptor>> writer = (schema, clusteringFunction) -> () -> {
- Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.keyspace, schema.name, id.get(), SSTableFormat.Type.BIG);
+ Descriptor descriptor = new Descriptor(format.getLatestVersion(), dir, schema.keyspace, schema.name, id.get());
SerializationHeader header = SerializationHeader.makeWithoutStats(schema);
try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
- SSTableWriter sstableWriter = BigTableWriter.create(TableMetadataRef.forOfflineTools(schema), descriptor, 1, 0L, null, false, 0, header, Collections.emptyList(), txn))
+ SSTableWriter sstableWriter = descriptor.getFormat().getWriterFactory()
+ .builder(descriptor)
+ .setTableMetadataRef(TableMetadataRef.forOfflineTools(schema))
+ .setKeyCount(1)
+ .setSerializationHeader(header)
+ .setFlushObservers(Collections.emptyList())
+ .setMetadataCollector(new MetadataCollector(schema.comparator))
+ .addDefaultComponents()
+ .build(txn, null))
{
ColumnMetadata cd = schema.getColumn(v);
for (int i = 0 ; i < 5 ; ++i) {
@@ -114,8 +121,8 @@
Descriptor sstableWithRegular = writer.apply(schemaWithRegular, BufferClustering::new).call();
Descriptor sstableWithStatic = writer.apply(schemaWithStatic, value -> Clustering.STATIC_CLUSTERING).call();
- SSTableReader readerWithStatic = SSTableReader.openNoValidation(sstableWithStatic, TableMetadataRef.forOfflineTools(schemaWithRegular));
- SSTableReader readerWithRegular = SSTableReader.openNoValidation(sstableWithRegular, TableMetadataRef.forOfflineTools(schemaWithStatic));
+ readerWithStatic = SSTableReader.openNoValidation(null, sstableWithStatic, TableMetadataRef.forOfflineTools(schemaWithRegular));
+ readerWithRegular = SSTableReader.openNoValidation(null, sstableWithRegular, TableMetadataRef.forOfflineTools(schemaWithStatic));
try (ISSTableScanner partitions = readerWithStatic.getScanner()) {
for (int i = 0 ; i < 5 ; ++i)
@@ -141,6 +148,10 @@
}
finally
{
+ if (readerWithStatic != null)
+ readerWithStatic.selfRef().close();
+ if (readerWithRegular != null)
+ readerWithRegular.selfRef().close();
FileUtils.deleteRecursive(dir);
}
}
diff --git a/test/unit/org/apache/cassandra/db/SnapshotTest.java b/test/unit/org/apache/cassandra/db/SnapshotTest.java
index aa726ec..fdfdd6b 100644
--- a/test/unit/org/apache/cassandra/db/SnapshotTest.java
+++ b/test/unit/org/apache/cassandra/db/SnapshotTest.java
@@ -24,7 +24,7 @@
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
@@ -38,7 +38,7 @@
getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables())
{
- File toc = new File(sstable.descriptor.filenameFor(Component.TOC));
+ File toc = sstable.descriptor.fileFor(Components.TOC);
Files.write(toc.toPath(), new byte[0], StandardOpenOption.TRUNCATE_EXISTING);
}
getCurrentColumnFamilyStore().snapshot("hello");
diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
index b543679..2a1561b 100644
--- a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
+++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
@@ -20,6 +20,8 @@
import java.net.InetAddress;
import java.nio.ByteBuffer;
+import java.util.Date;
+import java.util.Map;
import java.util.UUID;
import com.google.common.collect.ImmutableMap;
@@ -32,6 +34,7 @@
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.Range;
@@ -42,6 +45,8 @@
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import static org.apache.cassandra.utils.FBUtilities.now;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.junit.Assert.assertEquals;
@@ -264,4 +269,55 @@
}
assertEquals(1, rowCount);
}
+
+ @Test
+ public void testMigrateCompactionHistory() throws Throwable
+ {
+ String table = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.COMPACTION_HISTORY);
+ String insert = String.format("INSERT INTO %s ("
+ + "id, "
+ + "bytes_in, "
+ + "bytes_out, "
+ + "columnfamily_name, "
+ + "compacted_at, "
+ + "keyspace_name, "
+ + "rows_merged) "
+ + " values ( ?, ?, ?, ?, ?, ?, ? )",
+ table);
+ TimeUUID compactionId = TimeUUID.Generator.atUnixMillis(currentTimeMillis());
+ Date compactAt = Date.from(now());
+ Map<Integer, Long> rowsMerged = ImmutableMap.of(6, 1L);
+ execute(insert,
+ compactionId,
+ 10L,
+ 5L,
+ "table",
+ compactAt,
+ "keyspace",
+ rowsMerged);
+ SystemKeyspaceMigrator41.migrateCompactionHistory();
+
+ int rowCount = 0;
+ for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s where keyspace_name = 'keyspace' and columnfamily_name = 'table' allow filtering", table)))
+ {
+ rowCount++;
+ assertEquals(compactionId, row.getTimeUUID("id"));
+ assertEquals(10L, row.getLong("bytes_in"));
+ assertEquals(5L, row.getLong("bytes_out"));
+ assertEquals("table", row.getString("columnfamily_name"));
+ assertEquals(compactAt, row.getTimestamp("compacted_at"));
+ assertEquals("keyspace", row.getString("keyspace_name"));
+ assertEquals(rowsMerged, row.getMap("rows_merged", Int32Type.instance, LongType.instance));
+ assertEquals(ImmutableMap.of(), row.getMap("compaction_properties", UTF8Type.instance, UTF8Type.instance));
+ }
+ assertEquals(1, rowCount);
+
+ //Test nulls/missing don't prevent the row from propagating
+ execute(String.format("TRUNCATE %s", table));
+
+ execute(String.format("INSERT INTO %s (id) VALUES (?)", table), compactionId);
+ SystemKeyspaceMigrator41.migrateCompactionHistory();
+
+ assertEquals(1, execute(String.format("SELECT * FROM %s", table)).size());
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
index bf59b62..0bade4b 100644
--- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
@@ -33,6 +33,7 @@
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspace;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraVersion;
@@ -105,14 +106,14 @@
// First, check that in the absence of any previous installed version, we don't create snapshots
for (ColumnFamilyStore cfs : Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStores())
cfs.clearUnsafe();
- Keyspace.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
SystemKeyspace.snapshotOnVersionChange();
assertDeleted();
// now setup system.local as if we're upgrading from a previous version
setupReleaseVersion(getOlderVersionString());
- Keyspace.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
assertDeleted();
// Compare versions again & verify that snapshots were created for all tables in the system ks
@@ -125,7 +126,7 @@
// clear out the snapshots & set the previous recorded version equal to the latest, we shouldn't
// see any new snapshots created this time.
- Keyspace.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
setupReleaseVersion(FBUtilities.getReleaseVersionString());
SystemKeyspace.snapshotOnVersionChange();
@@ -134,7 +135,7 @@
// 10 files expected.
assertDeleted();
- Keyspace.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
}
@Test
diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/db/VerifyTest.java
deleted file mode 100644
index a58837a..0000000
--- a/test/unit/org/apache/cassandra/db/VerifyTest.java
+++ /dev/null
@@ -1,814 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.db;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.zip.CRC32;
-import java.util.zip.CheckedInputStream;
-
-import com.google.common.base.Charsets;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.batchlog.Batch;
-import org.apache.cassandra.batchlog.BatchlogManager;
-import org.apache.cassandra.cache.ChunkCache;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.Verifier;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.dht.ByteOrderedPartitioner;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-import static org.apache.cassandra.SchemaLoader.counterCFMD;
-import static org.apache.cassandra.SchemaLoader.createKeyspace;
-import static org.apache.cassandra.SchemaLoader.loadSchema;
-import static org.apache.cassandra.SchemaLoader.standardCFMD;
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test for {@link Verifier}.
- *
- * Note: the complete coverage is composed of:
- * - {@link org.apache.cassandra.tools.StandaloneVerifierOnSSTablesTest}
- * - {@link org.apache.cassandra.tools.StandaloneVerifierTest}
- * - {@link VerifyTest}
- */
-public class VerifyTest
-{
- public static final String KEYSPACE = "Keyspace1";
- public static final String CF = "Standard1";
- public static final String CF2 = "Standard2";
- public static final String CF3 = "Standard3";
- public static final String CF4 = "Standard4";
- public static final String COUNTER_CF = "Counter1";
- public static final String COUNTER_CF2 = "Counter2";
- public static final String COUNTER_CF3 = "Counter3";
- public static final String COUNTER_CF4 = "Counter4";
- public static final String CORRUPT_CF = "Corrupt1";
- public static final String CORRUPT_CF2 = "Corrupt2";
- public static final String CORRUPTCOUNTER_CF = "CounterCorrupt1";
- public static final String CORRUPTCOUNTER_CF2 = "CounterCorrupt2";
-
- public static final String CF_UUID = "UUIDKeys";
- public static final String BF_ALWAYS_PRESENT = "BfAlwaysPresent";
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- CompressionParams compressionParameters = CompressionParams.snappy(32768);
-
- loadSchema();
- createKeyspace(KEYSPACE,
- KeyspaceParams.simple(1),
- standardCFMD(KEYSPACE, CF).compression(compressionParameters),
- standardCFMD(KEYSPACE, CF2).compression(compressionParameters),
- standardCFMD(KEYSPACE, CF3),
- standardCFMD(KEYSPACE, CF4),
- standardCFMD(KEYSPACE, CORRUPT_CF),
- standardCFMD(KEYSPACE, CORRUPT_CF2),
- counterCFMD(KEYSPACE, COUNTER_CF).compression(compressionParameters),
- counterCFMD(KEYSPACE, COUNTER_CF2).compression(compressionParameters),
- counterCFMD(KEYSPACE, COUNTER_CF3),
- counterCFMD(KEYSPACE, COUNTER_CF4),
- counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF),
- counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF2),
- standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance),
- standardCFMD(KEYSPACE, BF_ALWAYS_PRESENT).bloomFilterFpChance(1.0));
- }
-
-
- @Test
- public void testVerifyCorrect()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
-
- fillCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testVerifyCounterCorrect()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
-
- fillCounterCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testExtendedVerifyCorrect()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);
-
- fillCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testExtendedVerifyCounterCorrect()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF2);
-
- fillCounterCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testVerifyCorrectUncompressed()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3);
-
- fillCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testVerifyCounterCorrectUncompressed()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF3);
-
- fillCounterCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testExtendedVerifyCorrectUncompressed()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF4);
-
- fillCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
- @Test
- public void testExtendedVerifyCounterCorrectUncompressed()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF4);
-
- fillCounterCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Unexpected CorruptSSTableException");
- }
- }
-
-
- @Test
- public void testVerifyIncorrectDigest() throws IOException, WriteTimeoutException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF);
-
- fillCF(cfs, 2);
-
- Util.getAll(Util.cmd(cfs).build());
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
-
- try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST))))
- {
- Long correctChecksum = file.readLong();
-
- writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
- }
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (CorruptSSTableException err) {}
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(false).build()))
- {
- verifier.verify();
- fail("Expected a RuntimeException to be thrown");
- }
- catch (RuntimeException err) {}
- }
-
-
- @Test
- public void testVerifyCorruptRowCorrectDigest() throws IOException, WriteTimeoutException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
-
- fillCF(cfs, 2);
-
- Util.getAll(Util.cmd(cfs).build());
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- // overwrite one row with garbage
- long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
- long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
- long startPosition = row0Start < row1Start ? row0Start : row1Start;
- long endPosition = row0Start < row1Start ? row1Start : row0Start;
-
- FileChannel file = new File(sstable.getFilename()).newReadWriteChannel();
- file.position(startPosition);
- file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
- file.close();
- if (ChunkCache.instance != null)
- ChunkCache.instance.invalidateFile(sstable.getFilename());
-
- // Update the Digest to have the right Checksum
- writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.filenameFor(Component.DIGEST));
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- // First a simple verify checking digest, which should succeed
- try
- {
- verifier.verify();
- }
- catch (CorruptSSTableException err)
- {
- fail("Simple verify should have succeeded as digest matched");
- }
- }
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
- {
- // Now try extended verify
- try
- {
- verifier.verify();
-
- }
- catch (CorruptSSTableException err)
- {
- return;
- }
- fail("Expected a CorruptSSTableException to be thrown");
- }
- }
-
- @Test
- public void testVerifyBrokenSSTableMetadata() throws IOException, WriteTimeoutException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
- cfs.truncateBlocking();
- fillCF(cfs, 2);
-
- Util.getAll(Util.cmd(cfs).build());
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
- String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS);
- FileChannel file = new File(filenameToCorrupt).newReadWriteChannel();
- file.position(0);
- file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
- file.close();
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (CorruptSSTableException err)
- {}
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(false).build()))
- {
- verifier.verify();
- fail("Expected a RuntimeException to be thrown");
- }
- catch (CorruptSSTableException err) { fail("wrong exception thrown"); }
- catch (RuntimeException err)
- {}
- }
-
- @Test
- public void testVerifyMutateRepairStatus() throws IOException, WriteTimeoutException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
- cfs.truncateBlocking();
- fillCF(cfs, 2);
-
- Util.getAll(Util.cmd(cfs).build());
-
- // make the sstable repaired:
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), sstable.getPendingRepair(), sstable.isTransient());
- sstable.reloadSSTableMetadata();
-
- // break the sstable:
- Long correctChecksum;
- try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST))))
- {
- correctChecksum = file.readLong();
- }
- writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (CorruptSSTableException err)
- {}
-
- assertTrue(sstable.isRepaired());
-
- // now the repair status should be changed:
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (CorruptSSTableException err)
- {}
- assertFalse(sstable.isRepaired());
- }
-
- @Test(expected = RuntimeException.class)
- public void testOutOfRangeTokens() throws IOException
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
- fillCF(cfs, 100);
- TokenMetadata tmd = StorageService.instance.getTokenMetadata();
- byte[] tk1 = new byte[1], tk2 = new byte[1];
- tk1[0] = 2;
- tk2[0] = 1;
- tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
- tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkOwnsTokens(true).extendedVerification(true).build()))
- {
- verifier.verify();
- }
- finally
- {
- StorageService.instance.getTokenMetadata().clearUnsafe();
- }
-
- }
-
- @Test
- public void testMutateRepair() throws IOException, ExecutionException, InterruptedException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
-
- fillCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, sstable.getPendingRepair(), sstable.isTransient());
- sstable.reloadSSTableMetadata();
- cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
- assertTrue(sstable.isRepaired());
- cfs.forceMajorCompaction();
-
- sstable = cfs.getLiveSSTables().iterator().next();
- Long correctChecksum;
- try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST))))
- {
- correctChecksum = file.readLong();
- }
- writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true).build()))
- {
- verifier.verify();
- fail("should be corrupt");
- }
- catch (CorruptSSTableException e)
- {}
- assertFalse(sstable.isRepaired());
- }
-
- @Test
- public void testVerifyIndex() throws IOException
- {
- testBrokenComponentHelper(Component.PRIMARY_INDEX);
- }
- @Test
- public void testVerifyBf() throws IOException
- {
- testBrokenComponentHelper(Component.FILTER);
- }
-
- @Test
- public void testVerifyIndexSummary() throws IOException
- {
- testBrokenComponentHelper(Component.SUMMARY);
- }
-
- private void testBrokenComponentHelper(Component componentToBreak) throws IOException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
-
- fillCF(cfs, 2);
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().build()))
- {
- verifier.verify(); //still not corrupt, should pass
- }
- String filenameToCorrupt = sstable.descriptor.filenameFor(componentToBreak);
- try(FileChannel fileChannel = new File(filenameToCorrupt).newReadWriteChannel())
- {
- fileChannel.truncate(3);
- }
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("should throw exception");
- }
- catch(CorruptSSTableException e)
- {
- //expected
- }
- }
-
- @Test
- public void testQuick() throws IOException
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF);
-
- fillCF(cfs, 2);
-
- Util.getAll(Util.cmd(cfs).build());
-
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
-
-
- try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST))))
- {
- Long correctChecksum = file.readLong();
-
- writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST));
- }
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("Expected a CorruptSSTableException to be thrown");
- }
- catch (CorruptSSTableException err) {}
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).quick(true).build())) // with quick = true we don't verify the digest
- {
- verifier.verify();
- }
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build()))
- {
- verifier.verify();
- fail("Expected a RuntimeException to be thrown");
- }
- catch (CorruptSSTableException err) {}
- }
-
- @Test
- public void testRangeOwnHelper()
- {
- List<Range<Token>> normalized = new ArrayList<>();
- normalized.add(r(Long.MIN_VALUE, Long.MIN_VALUE + 1));
- normalized.add(r(Long.MIN_VALUE + 5, Long.MIN_VALUE + 6));
- normalized.add(r(Long.MIN_VALUE + 10, Long.MIN_VALUE + 11));
- normalized.add(r(0,10));
- normalized.add(r(10,11));
- normalized.add(r(20,25));
- normalized.add(r(26,200));
-
- Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
-
- roh.validate(dk(1));
- roh.validate(dk(10));
- roh.validate(dk(11));
- roh.validate(dk(21));
- roh.validate(dk(25));
- boolean gotException = false;
- try
- {
- roh.validate(dk(26));
- }
- catch (Throwable t)
- {
- gotException = true;
- }
- assertTrue(gotException);
- }
-
- @Test(expected = AssertionError.class)
- public void testRangeOwnHelperBadToken()
- {
- List<Range<Token>> normalized = new ArrayList<>();
- normalized.add(r(0,10));
- Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
- roh.validate(dk(1));
- // call with smaller token to get exception
- roh.validate(dk(0));
- }
-
-
- @Test
- public void testRangeOwnHelperNormalize()
- {
- List<Range<Token>> normalized = Range.normalize(Collections.singletonList(r(0,0)));
- Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
- roh.validate(dk(Long.MIN_VALUE));
- roh.validate(dk(0));
- roh.validate(dk(Long.MAX_VALUE));
- }
-
- @Test
- public void testRangeOwnHelperNormalizeWrap()
- {
- List<Range<Token>> normalized = Range.normalize(Collections.singletonList(r(Long.MAX_VALUE - 1000,Long.MIN_VALUE + 1000)));
- Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized);
- roh.validate(dk(Long.MIN_VALUE));
- roh.validate(dk(Long.MAX_VALUE));
- boolean gotException = false;
- try
- {
- roh.validate(dk(26));
- }
- catch (Throwable t)
- {
- gotException = true;
- }
- assertTrue(gotException);
- }
-
- @Test
- public void testEmptyRanges()
- {
- new Verifier.RangeOwnHelper(Collections.emptyList()).validate(dk(1));
- }
-
- @Test
- public void testVerifyLocalPartitioner() throws UnknownHostException
- {
- TokenMetadata tmd = StorageService.instance.getTokenMetadata();
- byte[] tk1 = new byte[1], tk2 = new byte[1];
- tk1[0] = 2;
- tk2[0] = 1;
- tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
- tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
- // write some bogus to a localpartitioner table
- Batch bogus = Batch.createLocal(nextTimeUUID(), 0, Collections.emptyList());
- BatchlogManager.store(bogus);
- ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("batches");
- Util.flush(cfs);
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
-
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkOwnsTokens(true).build()))
- {
- verifier.verify();
- }
- }
- }
-
- @Test
- public void testNoFilterFile()
- {
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(BF_ALWAYS_PRESENT);
- fillCF(cfs, 100);
- assertEquals(1.0, cfs.metadata().params.bloomFilterFpChance, 0.0);
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- File f = new File(sstable.descriptor.filenameFor(Component.FILTER));
- assertFalse(f.exists());
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().build()))
- {
- verifier.verify();
- }
- }
- }
-
-
-
- private DecoratedKey dk(long l)
- {
- return new BufferDecoratedKey(t(l), ByteBufferUtil.EMPTY_BYTE_BUFFER);
- }
-
- private Range<Token> r(long s, long e)
- {
- return new Range<>(t(s), t(e));
- }
-
- private Token t(long t)
- {
- return new Murmur3Partitioner.LongToken(t);
- }
-
-
- protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
- {
- for (int i = 0; i < partitionsPerSSTable; i++)
- {
- UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
- .newRow("c1").add("val", "1")
- .newRow("c2").add("val", "2")
- .apply();
- }
-
- Util.flush(cfs);
- }
-
- protected void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException
- {
- for (int i = 0; i < partitionsPerSSTable; i++)
- {
- UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
- .newRow("c1").add("val", 100L)
- .apply();
- }
-
- Util.flush(cfs);
- }
-
- protected long simpleFullChecksum(String filename) throws IOException
- {
- try (FileInputStreamPlus inputStream = new FileInputStreamPlus(filename))
- {
- CRC32 checksum = new CRC32();
- CheckedInputStream cinStream = new CheckedInputStream(inputStream, checksum);
- byte[] b = new byte[128];
- while (cinStream.read(b) >= 0) {
- }
- return cinStream.getChecksum().getValue();
- }
- }
-
- public static void writeChecksum(long checksum, String filePath)
- {
- File outFile = new File(filePath);
- BufferedWriter out = null;
- try
- {
- out = Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8);
- out.write(String.valueOf(checksum));
- out.flush();
- out.close();
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, outFile);
- }
- finally
- {
- FileUtils.closeQuietly(out);
- }
-
- }
-
-}
diff --git a/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java b/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java
index 13fb0df..4363d81 100644
--- a/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java
+++ b/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -327,6 +328,6 @@
Selector.Factory factory = selectable.newSelectorFactory(table, null, new ArrayList<>(), VariableSpecifications.empty());
Selector selector = factory.newInstance(QueryOptions.DEFAULT);
- return GroupMaker.newSelectorGroupMaker(table.comparator, reversed.length, selector);
+ return GroupMaker.newSelectorGroupMaker(table.comparator, reversed.length, selector, Collections.singletonList(column));
}
}
diff --git a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java b/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java
index 9f9b7bb..169318d 100644
--- a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java
@@ -34,9 +34,10 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
import org.apache.cassandra.schema.KeyspaceParams;
public class SSTableReverseIteratorTest
@@ -85,9 +86,12 @@
Util.flush(tbl);
SSTableReader sstable = Iterables.getOnlyElement(tbl.getLiveSSTables());
DecoratedKey dk = tbl.getPartitioner().decorateKey(Int32Type.instance.decompose(key));
- RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ);
- Assert.assertTrue(indexEntry.isIndexed());
- Assert.assertTrue(indexEntry.columnsIndexCount() > 2);
+ if (sstable instanceof BigTableReader)
+ {
+ RowIndexEntry indexEntry = ((BigTableReader) sstable).getRowIndexEntry(dk, SSTableReader.Operator.EQ);
+ Assert.assertTrue(indexEntry.isIndexed());
+ Assert.assertTrue(indexEntry.blockCount() > 2);
+ }
// drop v1 so the first 2 index blocks only contain empty unfiltereds
QueryProcessor.executeInternal(String.format("ALTER TABLE %s.%s DROP v1", KEYSPACE, table));
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java
index aadd2fd..3dc4e5f 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogFailurePolicyTest.java
@@ -32,13 +32,15 @@
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.KillerForTests;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_STOP_ON_ERRORS;
+
public class CommitLogFailurePolicyTest
{
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
SchemaLoader.prepareServer();
- System.setProperty("cassandra.commitlog.stop_on_errors", "true");
+ COMMITLOG_STOP_ON_ERRORS.setBoolean(true);
}
@Test
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
index 3789b51..5f6f235 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java
@@ -27,6 +27,7 @@
import java.util.concurrent.TimeUnit;
import com.google.monitoring.runtime.instrumentation.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileReader;
import org.junit.Assert;
@@ -51,6 +52,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
DatabaseDescriptor.setCDCEnabled(true);
DatabaseDescriptor.setCDCTotalSpaceInMiB(1024);
CQLTester.setUpClass();
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
index 17bbece..d53c1d4 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
@@ -1,23 +1,25 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.cassandra.db.commitlog;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.io.util.File;
import java.io.ByteArrayOutputStream;
@@ -41,6 +43,7 @@
import com.google.common.io.Files;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
+
import org.junit.*;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -54,6 +57,7 @@
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.MemtableParams;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.SchemaTestUtil;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -85,6 +89,9 @@
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.vint.VIntCoding;
+import static java.lang.String.format;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMITLOG_IGNORE_REPLAY_ERRORS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.COMMIT_LOG_REPLAY_LIST;
import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@ -107,6 +114,11 @@
protected static final String STANDARD1 = "Standard1";
private static final String STANDARD2 = "Standard2";
private static final String CUSTOM1 = "Custom1";
+ private static final String KEYSPACE1_REPLAY = "CommitLogTestReplay1";
+ private static final String KEYSPACE1_REPLAY_TABLE1 = "CommitLogTestReplay1Table1";
+ private static final String KEYSPACE1_REPLAY_TABLE2 = "CommitLogTestReplay1Table2";
+ private static final String KEYSPACE2_REPLAY = "CommitLogTestReplay2";
+ private static final String KEYSPACE2_REPLAY_TABLE2 = "CommitLogTestReplay2Table2";
private static JVMStabilityInspector.Killer oldKiller;
private static KillerForTests testKiller;
@@ -121,14 +133,14 @@
public static Collection<Object[]> generateData() throws Exception
{
return Arrays.asList(new Object[][]
- {
- { null, EncryptionContextGenerator.createDisabledContext()}, // No compression, no encryption
- { null, newEncryptionContext() }, // Encryption
- { new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
- { new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- { new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()},
- { new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext()}
- });
+ {
+ { null, EncryptionContextGenerator.createDisabledContext() }, // No compression, no encryption
+ { null, newEncryptionContext() }, // Encryption
+ { new ParameterizedClass(LZ4Compressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
+ { new ParameterizedClass(SnappyCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
+ { new ParameterizedClass(DeflateCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() },
+ { new ParameterizedClass(ZstdCompressor.class.getName(), Collections.emptyMap()), EncryptionContextGenerator.createDisabledContext() }
+ });
}
private static EncryptionContext newEncryptionContext() throws Exception
@@ -152,12 +164,12 @@
MemtableParams skipListMemtable = MemtableParams.get("skiplist");
TableMetadata.Builder custom =
- TableMetadata.builder(KEYSPACE1, CUSTOM1)
- .addPartitionKeyColumn("k", IntegerType.instance)
- .addClusteringColumn("c1", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false))
- .addClusteringColumn("c2", SetType.getInstance(UTF8Type.instance, false))
- .addStaticColumn("s", IntegerType.instance)
- .memtable(skipListMemtable);
+ TableMetadata.builder(KEYSPACE1, CUSTOM1)
+ .addPartitionKeyColumn("k", IntegerType.instance)
+ .addClusteringColumn("c1", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false))
+ .addClusteringColumn("c2", SetType.getInstance(UTF8Type.instance, false))
+ .addStaticColumn("s", IntegerType.instance)
+ .memtable(skipListMemtable);
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
@@ -168,6 +180,14 @@
KeyspaceParams.simpleTransient(1),
SchemaLoader.standardCFMD(KEYSPACE2, STANDARD1, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable),
SchemaLoader.standardCFMD(KEYSPACE2, STANDARD2, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable));
+ SchemaLoader.createKeyspace(KEYSPACE1_REPLAY,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable),
+ SchemaLoader.standardCFMD(KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE2, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable));
+ SchemaLoader.createKeyspace(KEYSPACE2_REPLAY,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE2_REPLAY, KEYSPACE2_REPLAY_TABLE2, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable));
+
CompactionManager.instance.disableAutoCompaction();
testKiller = new KillerForTests();
@@ -181,6 +201,10 @@
@AfterClass
public static void afterClass()
{
+ SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE1, true);
+ SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE2, true);
+ SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE1_REPLAY, true);
+ SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE2_REPLAY, true);
JVMStabilityInspector.replaceKiller(oldKiller);
}
@@ -194,7 +218,7 @@
public void afterTest()
{
CommitLogSegmentReader.setAllowSkipSyncMarkerCrc(false);
- System.clearProperty("cassandra.replayList");
+ COMMIT_LOG_REPLAY_LIST.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
testKiller.reset();
}
@@ -315,11 +339,9 @@
@Test
public void testRecoveryWithGarbageLog_ignoredByProperty() throws Exception
{
- try {
- System.setProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY, "true");
+ try (WithProperties properties = new WithProperties().set(COMMITLOG_IGNORE_REPLAY_ERRORS, "true"))
+ {
testRecoveryWithGarbageLog();
- } finally {
- System.clearProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY);
}
}
@@ -382,10 +404,10 @@
ColumnFamilyStore cfs2 = ks.getColumnFamilyStore(STANDARD2);
// Roughly 32 MiB mutation
- Mutation rm = new RowUpdateBuilder(cfs1.metadata(), 0, "k")
- .clustering("bytes")
- .add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/4) - 1))
- .build();
+ Mutation rm = new RowUpdateBuilder(cfs1.metadata(), 0, "k")
+ .clustering("bytes")
+ .add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize() / 4) - 1))
+ .build();
// Adding it twice (won't change segment)
CommitLog.instance.add(rm);
@@ -412,7 +434,7 @@
Collection<CommitLogSegment> segments = CommitLog.instance.segmentManager.getActiveSegments();
- assertEquals(String.format("Expected 3 segments but got %d (%s)", segments.size(), getDirtyCFIds(segments)),
+ assertEquals(format("Expected 3 segments but got %d (%s)", segments.size(), getDirtyCFIds(segments)),
3,
segments.size());
@@ -425,7 +447,7 @@
segments = CommitLog.instance.segmentManager.getActiveSegments();
// Assert we still have both our segment
- assertEquals(String.format("Expected 1 segment but got %d (%s)", segments.size(), getDirtyCFIds(segments)),
+ assertEquals(format("Expected 1 segment but got %d (%s)", segments.size(), getDirtyCFIds(segments)),
1,
segments.size());
}
@@ -503,6 +525,7 @@
}
throw new AssertionError("mutation larger than limit was accepted");
}
+
@Test
public void testExceedRecordLimitWithMultiplePartitions() throws Exception
{
@@ -536,12 +559,12 @@
String message = exception.getMessage();
long mutationSize = mutation.serializedSize(MessagingService.current_version) + ENTRY_OVERHEAD_SIZE;
- final String expectedMessagePrefix = String.format("Rejected an oversized mutation (%d/%d) for keyspace: %s.",
- mutationSize,
- DatabaseDescriptor.getMaxMutationSize(),
- KEYSPACE1);
+ final String expectedMessagePrefix = format("Rejected an oversized mutation (%d/%d) for keyspace: %s.",
+ mutationSize,
+ DatabaseDescriptor.getMaxMutationSize(),
+ KEYSPACE1);
assertTrue(message.startsWith(expectedMessagePrefix));
- assertTrue(message.contains(String.format("%s.%s and 1 more.", STANDARD1, key)));
+ assertTrue(message.contains(format("%s.%s and 1 more.", STANDARD1, key)));
}
}
@@ -713,7 +736,7 @@
.add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4))
.build();
- for (int i = 0 ; i < 5 ; i++)
+ for (int i = 0; i < 5; i++)
CommitLog.instance.add(m2);
assertEquals(2, CommitLog.instance.segmentManager.getActiveSegments().size());
@@ -791,6 +814,124 @@
}
@Test
+ public void testReplayListProperty() throws Throwable
+ {
+ // only keyspace
+ assertReplay(2, COMMIT_LOG_REPLAY_LIST, KEYSPACE1_REPLAY);
+
+ // only keyspaces
+ assertReplay(3, COMMIT_LOG_REPLAY_LIST, format("%s,%s", KEYSPACE1_REPLAY, KEYSPACE2_REPLAY));
+
+ // only table with keyspace
+ assertReplay(1, COMMIT_LOG_REPLAY_LIST, format("%s.%s", KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1));
+
+ // mix of keyspace and tables with keyspaces
+ assertReplay(2, COMMIT_LOG_REPLAY_LIST, format("%s.%s,%s", KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1, KEYSPACE2_REPLAY));
+
+ // only tables with keyspaces
+ assertReplay(2, COMMIT_LOG_REPLAY_LIST, format("%s.%s,%s.%s", KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1, KEYSPACE2_REPLAY, KEYSPACE2_REPLAY_TABLE2));
+
+ // mix of keyspace and tables with keyspaces within same keyspace.
+ assertReplay(2, COMMIT_LOG_REPLAY_LIST, format("%s.%s,%s", KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1, KEYSPACE1_REPLAY));
+
+ // test for wrong formats
+
+ String invalidFormat = format("%s.%s.%s", KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1, KEYSPACE1_REPLAY);
+
+ try
+ {
+ assertReplay(2,
+ COMMIT_LOG_REPLAY_LIST, invalidFormat);
+ fail(format("replay should fail on -D%s=%s as it is in invalid format",
+ COMMIT_LOG_REPLAY_LIST.getKey(), invalidFormat));
+ }
+ catch (IllegalArgumentException ex)
+ {
+ assertEquals(format("%s property contains an item which is not " +
+ "in format 'keyspace' or 'keyspace.table' but it is '%s'",
+ COMMIT_LOG_REPLAY_LIST.getKey(), invalidFormat),
+ ex.getMessage());
+ }
+
+ String invalidFormat2 = format("%s.%s,%s.", KEYSPACE1_REPLAY, KEYSPACE1_REPLAY_TABLE1, KEYSPACE1_REPLAY);
+
+ try
+ {
+ assertReplay(2,
+ COMMIT_LOG_REPLAY_LIST, invalidFormat2);
+ fail(format("replay should fail on -D%s=%s as it is in invalid format",
+ COMMIT_LOG_REPLAY_LIST.getKey(), invalidFormat2));
+ }
+ catch (IllegalArgumentException ex)
+ {
+ assertEquals(format("Invalid pair: '%s.'", KEYSPACE1_REPLAY), ex.getMessage());
+ }
+ }
+
+ private static class ReplayListPropertyReplayer extends CommitLogReplayer
+ {
+ private final ReplayFilter replayFilter;
+
+ ReplayListPropertyReplayer(CommitLog commitLog,
+ CommitLogPosition globalPosition,
+ Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted,
+ ReplayFilter replayFilter)
+ {
+ super(commitLog, globalPosition, cfPersisted, replayFilter);
+ this.replayFilter = replayFilter;
+ }
+
+ public int count = 0;
+
+ @Override
+ public void handleMutation(Mutation m, int size, int entryLocation, CommitLogDescriptor desc)
+ {
+ count += Iterables.size(replayFilter.filter(m));
+ super.handleMutation(m, size, entryLocation, desc);
+ }
+ }
+
+ private void assertReplay(int expectedReplayedMutations, CassandraRelevantProperties property, String propertyValue) throws Throwable
+ {
+ try (WithProperties properties = new WithProperties().set(property, propertyValue))
+ {
+ CommitLog.instance.resetUnsafe(true);
+
+ ColumnFamilyStore ks1tb1 = Keyspace.open(KEYSPACE1_REPLAY).getColumnFamilyStore(KEYSPACE1_REPLAY_TABLE1);
+ ColumnFamilyStore ks1tb2 = Keyspace.open(KEYSPACE1_REPLAY).getColumnFamilyStore(KEYSPACE1_REPLAY_TABLE2);
+ ColumnFamilyStore ks2tb2 = Keyspace.open(KEYSPACE2_REPLAY).getColumnFamilyStore(KEYSPACE2_REPLAY_TABLE2);
+
+ Mutation mutation1 = new RowUpdateBuilder(ks1tb1.metadata(), 0, "key1")
+ .clustering("c1").add("val", ByteBuffer.allocate(100)).build();
+
+ Mutation mutation2 = new RowUpdateBuilder(ks1tb2.metadata(), 0, "key2")
+ .clustering("c2").add("val", ByteBuffer.allocate(100)).build();
+
+ Mutation mutation3 = new RowUpdateBuilder(ks2tb2.metadata(), 0, "key3")
+ .clustering("c3").add("val", ByteBuffer.allocate(100)).build();
+
+ CommitLog.instance.add(mutation1);
+ CommitLog.instance.add(mutation2);
+ CommitLog.instance.add(mutation3);
+ CommitLog.instance.sync(true);
+
+ Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted = new HashMap<TableId, IntervalSet<CommitLogPosition>>()
+ {{
+ put(ks1tb1.metadata().id, IntervalSet.empty());
+ put(ks1tb2.metadata().id, IntervalSet.empty());
+ put(ks2tb2.metadata().id, IntervalSet.empty());
+ }};
+
+ List<String> activeSegments = CommitLog.instance.getActiveSegmentNames();
+ File[] files = new File(CommitLog.instance.segmentManager.storageDirectory).tryList((file, name) -> activeSegments.contains(name));
+ ReplayListPropertyReplayer replayer = new ReplayListPropertyReplayer(CommitLog.instance, CommitLogPosition.NONE, cfPersisted, CommitLogReplayer.ReplayFilter.create());
+ replayer.replayFiles(files);
+
+ assertEquals(expectedReplayedMutations, replayer.count);
+ }
+ }
+
+ @Test
public void replayWithBadSyncMarkerCRC() throws IOException
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
@@ -863,7 +1004,7 @@
{
int cellCount = 0;
int max = 1024;
- int discardPosition = (int)(max * .8); // an arbitrary number of entries that we'll skip on the replay
+ int discardPosition = (int) (max * .8); // an arbitrary number of entries that we'll skip on the replay
CommitLogPosition commitLogPosition = null;
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
@@ -948,12 +1089,12 @@
{
DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
- for (int i = 0 ; i < 5 ; i++)
+ for (int i = 0; i < 5; i++)
{
new RowUpdateBuilder(cfs.metadata(), 0, "k")
- .clustering("c" + i).add("val", ByteBuffer.allocate(100))
- .build()
- .apply();
+ .clustering("c" + i).add("val", ByteBuffer.allocate(100))
+ .build()
+ .apply();
if (i == 2)
{
@@ -978,26 +1119,27 @@
}
CommitLog.instance.sync(true);
- System.setProperty("cassandra.replayList", KEYSPACE1 + "." + STANDARD1);
- // Currently we don't attempt to re-flush a memtable that failed, thus make sure data is replayed by commitlog.
- // If retries work subsequent flushes should clear up error and this should change to expect 0.
- assertEquals(1, CommitLog.instance.resetUnsafe(false));
- System.clearProperty("cassandra.replayList");
+ try (WithProperties properties = new WithProperties().set(COMMIT_LOG_REPLAY_LIST, KEYSPACE1 + '.' + STANDARD1))
+ {
+ // Currently we don't attempt to re-flush a memtable that failed, thus make sure data is replayed by commitlog.
+ // If retries work subsequent flushes should clear up error and this should change to expect 0.
+ assertEquals(1, CommitLog.instance.resetUnsafe(false));
+ }
}
public void testOutOfOrderFlushRecovery(BiConsumer<ColumnFamilyStore, Memtable> flushAction, boolean performCompaction)
- throws ExecutionException, InterruptedException, IOException
+ throws ExecutionException, InterruptedException, IOException
{
CommitLog.instance.resetUnsafe(true);
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
- for (int i = 0 ; i < 5 ; i++)
+ for (int i = 0; i < 5; i++)
{
new RowUpdateBuilder(cfs.metadata(), 0, "k")
- .clustering("c" + i).add("val", ByteBuffer.allocate(100))
- .build()
- .apply();
+ .clustering("c" + i).add("val", ByteBuffer.allocate(100))
+ .build()
+ .apply();
Memtable current = cfs.getTracker().getView().getCurrentMemtable();
if (i == 2)
@@ -1012,7 +1154,7 @@
reader.reloadSSTableMetadata();
CommitLog.instance.sync(true);
- System.setProperty("cassandra.replayList", KEYSPACE1 + "." + STANDARD1);
+ COMMIT_LOG_REPLAY_LIST.setString(KEYSPACE1 + '.' + STANDARD1);
// In the absence of error, this should be 0 because forceBlockingFlush/forceRecycleAllSegments would have
// persisted all data in the commit log. Because we know there was an error, there must be something left to
// replay.
@@ -1089,16 +1231,11 @@
int replayed = 0;
- try
+ try (WithProperties properties = new WithProperties().set(COMMITLOG_IGNORE_REPLAY_ERRORS, true))
{
- System.setProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY, "true");
replayed = CommitLog.instance.resetUnsafe(false);
}
- finally
- {
- System.clearProperty(CommitLogReplayer.IGNORE_REPLAY_ERRORS_PROPERTY);
- }
-
+
assertEquals(replayed, 1);
}
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
index bd4b28f..1fc43bd 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
@@ -19,8 +19,6 @@
package org.apache.cassandra.db.compaction;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
import org.junit.After;
import org.junit.BeforeClass;
@@ -44,15 +42,11 @@
private static final String KEYSPACE1 = "Keyspace1";
private static final String LCS_TABLE = "LCS_TABLE";
private static final String STCS_TABLE = "STCS_TABLE";
- private static final String DTCS_TABLE = "DTCS_TABLE";
private static final String TWCS_TABLE = "TWCS_TABLE";
@BeforeClass
public static void loadData() throws ConfigurationException
{
- Map<String, String> stcsOptions = new HashMap<>();
- stcsOptions.put("tombstone_compaction_interval", "1");
-
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
@@ -60,13 +54,10 @@
.compaction(CompactionParams.lcs(Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, STCS_TABLE)
.compaction(CompactionParams.stcs(Collections.emptyMap())),
- SchemaLoader.standardCFMD(KEYSPACE1, DTCS_TABLE)
- .compaction(CompactionParams.create(DateTieredCompactionStrategy.class, Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, TWCS_TABLE)
.compaction(CompactionParams.create(TimeWindowCompactionStrategy.class, Collections.emptyMap())));
Keyspace.open(KEYSPACE1).getColumnFamilyStore(LCS_TABLE).disableAutoCompaction();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(STCS_TABLE).disableAutoCompaction();
- Keyspace.open(KEYSPACE1).getColumnFamilyStore(DTCS_TABLE).disableAutoCompaction();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(TWCS_TABLE).disableAutoCompaction();
}
@@ -76,7 +67,6 @@
Keyspace.open(KEYSPACE1).getColumnFamilyStore(LCS_TABLE).truncateBlocking();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(STCS_TABLE).truncateBlocking();
- Keyspace.open(KEYSPACE1).getColumnFamilyStore(DTCS_TABLE).truncateBlocking();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(TWCS_TABLE).truncateBlocking();
}
@@ -93,12 +83,6 @@
}
@Test(timeout=30000)
- public void testGetNextBackgroundTaskDoesNotBlockDTCS()
- {
- testGetNextBackgroundTaskDoesNotBlock(DTCS_TABLE);
- }
-
- @Test(timeout=30000)
public void testGetNextBackgroundTaskDoesNotBlockTWCS()
{
testGetNextBackgroundTaskDoesNotBlock(TWCS_TABLE);
diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
index 56d6c40..7849796 100644
--- a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
@@ -46,8 +46,10 @@
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexBuilder;
-import org.apache.cassandra.io.sstable.IndexSummaryRedistribution;
+import org.apache.cassandra.io.sstable.IScrubber;
+import org.apache.cassandra.io.sstable.IVerifier;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.FBUtilities;
@@ -143,7 +145,15 @@
Map<TableId, LifecycleTransaction> transactions = ImmutableMap.<TableId, LifecycleTransaction>builder().put(getCurrentColumnFamilyStore().metadata().id, txn).build();
IndexSummaryRedistribution isr = new IndexSummaryRedistribution(transactions, 0, 1000);
MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
- CompactionManager.instance.runIndexSummaryRedistribution(isr, mockActiveCompactions);
+ mockActiveCompactions.beginCompaction(isr);
+ try
+ {
+ isr.redistributeSummaries();
+ }
+ finally
+ {
+ mockActiveCompactions.finishCompaction(isr);
+ }
assertTrue(mockActiveCompactions.finished);
assertNotNull(mockActiveCompactions.holder);
// index redistribution operates over all keyspaces/tables, we always cancel them
@@ -191,7 +201,7 @@
try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstable, OperationType.SCRUB))
{
MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
- CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, true, false, false, mockActiveCompactions);
+ CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, IScrubber.options().skipCorrupted().build(), mockActiveCompactions);
assertTrue(mockActiveCompactions.finished);
assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable));
@@ -214,7 +224,7 @@
SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null);
MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
- CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, new Verifier.Options.Builder().build(), mockActiveCompactions);
+ CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, IVerifier.options().build(), mockActiveCompactions);
assertTrue(mockActiveCompactions.finished);
assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable));
assertFalse(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> false));
@@ -245,4 +255,4 @@
finished = true;
}
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
index 67421ba..51da0c4 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
@@ -86,14 +86,16 @@
assertEquals(1, activeCompactions.size());
assertEquals(activeCompactions.get(0).getCompactionInfo().getSSTables(), toMarkCompacting);
// predicate requires the non-compacting sstables, should not cancel the one currently compacting:
- cfs.runWithCompactionsDisabled(() -> null, (sstable) -> !toMarkCompacting.contains(sstable), false, false, true);
+ cfs.runWithCompactionsDisabled(() -> null, (sstable) -> !toMarkCompacting.contains(sstable),
+ OperationType.P0, false, false, true);
assertEquals(1, activeCompactions.size());
assertFalse(activeCompactions.get(0).isStopRequested());
// predicate requires the compacting ones - make sure stop is requested and that when we abort that
// compaction we actually run the callable (countdown the latch)
CountDownLatch cdl = new CountDownLatch(1);
- Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; }, toMarkCompacting::contains, false, false, true));
+ Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; }, toMarkCompacting::contains,
+ OperationType.P0, false, false, true));
t.start();
while (!activeCompactions.get(0).isStopRequested())
Thread.sleep(100);
@@ -139,13 +141,16 @@
expectedSSTables.add(new HashSet<>(sstables.subList(6, 9)));
assertEquals(compactingSSTables, expectedSSTables);
- cfs.runWithCompactionsDisabled(() -> null, (sstable) -> false, false, false, true);
+ cfs.runWithCompactionsDisabled(() -> null, (sstable) -> false,
+ OperationType.P0, false, false, true);
assertEquals(2, activeCompactions.size());
assertTrue(activeCompactions.stream().noneMatch(CompactionInfo.Holder::isStopRequested));
CountDownLatch cdl = new CountDownLatch(1);
// start a compaction which only needs the sstables where first token is > 50 - these are the sstables compacted by tcts.get(1)
- Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; }, (sstable) -> first(sstable) > 50, false, false, true));
+ Thread t = new Thread(() -> cfs.runWithCompactionsDisabled(() -> { cdl.countDown(); return null; },
+ (sstable) -> first(sstable) > 50,
+ OperationType.P0, false, false, true));
t.start();
activeCompactions = getActiveCompactionsForTable(cfs);
assertEquals(2, activeCompactions.size());
@@ -333,7 +338,8 @@
}
}
assertTrue(foundCompaction);
- cfs.runWithCompactionsDisabled(() -> {compactionsStopped.countDown(); return null;}, (sstable) -> true, false, false, true);
+ cfs.runWithCompactionsDisabled(() -> { compactionsStopped.countDown(); return null; },
+ (sstable) -> true, OperationType.P0, false, false, true);
// wait for the runWithCompactionsDisabled callable
compactionsStopped.await();
assertEquals(1, getActiveCompactionsForTable(cfs).size());
@@ -430,7 +436,8 @@
Set<SSTableReader> sstables = new HashSet<>();
try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
{
- getCurrentColumnFamilyStore().runWithCompactionsDisabled(() -> true, (sstable) -> { sstables.add(sstable); return true;}, false, false, false);
+ getCurrentColumnFamilyStore().runWithCompactionsDisabled(() -> true, (sstable) -> { sstables.add(sstable); return true;},
+ OperationType.P0, false, false, false);
}
// the predicate only gets compacting sstables, and we are only compacting the 2i sstables - with interruptIndexes = false we should see no sstables here
assertTrue(sstables.isEmpty());
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
index 9d81b61..ce9b28a 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java
@@ -204,4 +204,37 @@
assertFalse(evaluator.test(boundary));
assertTrue(evaluator.test(boundary - 1));
}
+
+ @Test
+ public void testDisableNeverPurgeTombstones()
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);
+ cfs.truncateBlocking();
+
+ DecoratedKey key = Util.dk("k1");
+ long timestamp = System.currentTimeMillis();
+ applyMutation(cfs.metadata(), key, timestamp);
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+ Set<SSTableReader> toCompact = Sets.newHashSet(cfs.getLiveSSTables());
+ cfs.setNeverPurgeTombstones(true);
+ applyMutation(cfs.metadata(), key, timestamp + 1);
+
+ try (CompactionController cc = new CompactionController(cfs, toCompact, (int)(System.currentTimeMillis()/1000)))
+ {
+ assertFalse(cc.getPurgeEvaluator(key).test(timestamp));
+ assertFalse(cc.getPurgeEvaluator(key).test(timestamp + 1));
+ assertTrue(cc.getFullyExpiredSSTables().isEmpty());
+
+ cfs.setNeverPurgeTombstones(false);
+ assertFalse(cc.getPurgeEvaluator(key).test(timestamp));
+ assertFalse(cc.getPurgeEvaluator(key).test(timestamp + 1));
+ assertTrue(cc.getFullyExpiredSSTables().isEmpty());
+
+ cc.maybeRefreshOverlaps();
+ assertTrue(cc.getPurgeEvaluator(key).test(timestamp));
+ assertFalse(cc.getPurgeEvaluator(key).test(timestamp + 1));
+ assertTrue(cc.getFullyExpiredSSTables().isEmpty());
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
index ff3f210..330e1c5 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
@@ -17,9 +17,10 @@
*/
package org.apache.cassandra.db.compaction;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS;
import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.assertCommandIssued;
import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.makeRow;
-import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.rows;
+import static org.apache.cassandra.db.transform.DuplicateRowCheckerTest.partition;
import static org.junit.Assert.*;
import java.util.*;
@@ -460,7 +461,7 @@
@Test
public void duplicateRowsTest() throws Throwable
{
- System.setProperty("cassandra.diagnostic_snapshot_interval_nanos", "0");
+ DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS.setLong(0);
// Create a table and insert some data. The actual rows read in the test will be synthetic
// but this creates an sstable on disk to be snapshotted.
createTable("CREATE TABLE %s (pk text, ck1 int, ck2 int, v int, PRIMARY KEY (pk, ck1, ck2))");
@@ -494,7 +495,7 @@
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
DecoratedKey key = cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes("key"));
try (CompactionController controller = new CompactionController(cfs, Integer.MAX_VALUE);
- UnfilteredRowIterator rows = rows(cfs.metadata(), key, false, unfiltereds);
+ UnfilteredRowIterator rows = partition(cfs.metadata(), key, false, unfiltereds);
ISSTableScanner scanner = new Scanner(Collections.singletonList(rows));
CompactionIterator iter = new CompactionIterator(OperationType.COMPACTION,
Collections.singletonList(scanner),
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerBoundaryReloadTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerBoundaryReloadTest.java
new file mode 100644
index 0000000..0d3b0d0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerBoundaryReloadTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.net.UnknownHostException;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.service.StorageService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+public class CompactionStrategyManagerBoundaryReloadTest extends CQLTester
+{
+ @Test
+ public void testNoReload()
+ {
+ createTable("create table %s (id int primary key)");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ List<List<AbstractCompactionStrategy>> strategies = cfs.getCompactionStrategyManager().getStrategies();
+ DiskBoundaries db = cfs.getDiskBoundaries();
+ StorageService.instance.getTokenMetadata().invalidateCachedRings();
+ // make sure the strategy instances are the same (no reload)
+ assertTrue(isSame(strategies, cfs.getCompactionStrategyManager().getStrategies()));
+ // but disk boundaries are not .equal (ring version changed)
+ assertNotEquals(db, cfs.getDiskBoundaries());
+ assertTrue(db.isEquivalentTo(cfs.getDiskBoundaries()));
+
+ db = cfs.getDiskBoundaries();
+ alterTable("alter table %s with comment = 'abcd'");
+ assertTrue(isSame(strategies, cfs.getCompactionStrategyManager().getStrategies()));
+ // disk boundaries don't change because of alter
+ assertEquals(db, cfs.getDiskBoundaries());
+ }
+
+ @Test
+ public void testReload() throws UnknownHostException
+ {
+ createTable("create table %s (id int primary key)");
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ List<List<AbstractCompactionStrategy>> strategies = cfs.getCompactionStrategyManager().getStrategies();
+ DiskBoundaries db = cfs.getDiskBoundaries();
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ tmd.updateNormalToken(tmd.partitioner.getMinimumToken(), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(tmd.partitioner.getMaximumToken(), InetAddressAndPort.getByName("127.0.0.2"));
+ // make sure the strategy instances have been reloaded
+ assertFalse(isSame(strategies,
+ cfs.getCompactionStrategyManager().getStrategies()));
+ assertNotEquals(db, cfs.getDiskBoundaries());
+ db = cfs.getDiskBoundaries();
+
+ strategies = cfs.getCompactionStrategyManager().getStrategies();
+ alterTable("alter table %s with compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': false}");
+ assertFalse(isSame(strategies,
+ cfs.getCompactionStrategyManager().getStrategies()));
+ assertEquals(db, cfs.getDiskBoundaries());
+
+ }
+
+ private boolean isSame(List<List<AbstractCompactionStrategy>> a, List<List<AbstractCompactionStrategy>> b)
+ {
+ if (a.size() != b.size())
+ return false;
+ for (int i = 0; i < a.size(); i++)
+ {
+ if (a.get(i).size() != b.get(i).size())
+ return false;
+ for (int j = 0; j < a.get(i).size(); j++)
+ if (a.get(i).get(j) != b.get(i).get(j))
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
index 9960e8e..8a9f2fb 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
@@ -28,11 +28,14 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -395,6 +398,41 @@
}
}
+ @Test
+ public void testCountsByBuckets()
+ {
+ Assert.assertArrayEquals(
+ new int[] {2, 2, 4},
+ CompactionStrategyManager.sumCountsByBucket(ImmutableList.of(
+ ImmutableMap.of(60000L, 1, 0L, 2, 180000L, 1),
+ ImmutableMap.of(60000L, 1, 0L, 2, 180000L, 1)), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX));
+ Assert.assertArrayEquals(
+ new int[] {1, 1, 3},
+ CompactionStrategyManager.sumCountsByBucket(ImmutableList.of(
+ ImmutableMap.of(60000L, 1, 0L, 1),
+ ImmutableMap.of(0L, 2, 180000L, 1)), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX));
+ Assert.assertArrayEquals(
+ new int[] {1, 1},
+ CompactionStrategyManager.sumCountsByBucket(ImmutableList.of(
+ ImmutableMap.of(60000L, 1, 0L, 1),
+ ImmutableMap.of()), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX));
+ Assert.assertArrayEquals(
+ new int[] {8, 4},
+ CompactionStrategyManager.sumCountsByBucket(ImmutableList.of(
+ ImmutableMap.of(60000L, 2, 0L, 1, 180000L, 4),
+ ImmutableMap.of(60000L, 2, 0L, 1, 180000L, 4)), 2));
+ Assert.assertArrayEquals(
+ new int[] {1, 1, 2},
+ CompactionStrategyManager.sumCountsByBucket(ImmutableList.of(
+ Collections.emptyMap(),
+ ImmutableMap.of(60000L, 1, 0L, 2, 180000L, 1)), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX));
+ Assert.assertArrayEquals(
+ new int[] {},
+ CompactionStrategyManager.sumCountsByBucket(ImmutableList.of(
+ Collections.emptyMap(),
+ Collections.emptyMap()), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX));
+ }
+
private MockCFS createJBODMockCFS(int disks)
{
// Create #disks data directories to simulate JBOD
@@ -464,8 +502,6 @@
return index;
}
-
-
class MockBoundaryManager
{
private final ColumnFamilyStore cfs;
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
index 24b0c3d..ca75453 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
@@ -191,4 +191,12 @@
cfs.getTracker().removeUnsafe(sstables);
}
}
+
+ @Test
+ public void testMajorCompactTask()
+ {
+ //major compact without range/pk specified
+ CompactionTasks compactionTasks = cfs.getCompactionStrategyManager().getMaximalTasks(Integer.MAX_VALUE, false, OperationType.MAJOR_COMPACTION);
+ Assert.assertTrue(compactionTasks.stream().allMatch(task -> task.compactionType.equals(OperationType.MAJOR_COMPACTION)));
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
index 462d406..987ce18 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
@@ -55,7 +55,7 @@
@Test
@BMRules(rules = { @BMRule(name = "One SSTable too big for remaining disk space test",
targetClass = "Directories",
- targetMethod = "hasAvailableDiskSpace",
+ targetMethod = "hasDiskSpaceForCompactionsAndStreams",
condition = "not flagged(\"done\")",
action = "flag(\"done\"); return false;") } )
public void testSSTableNotEnoughDiskSpaceForCompactionGetsDropped() throws Throwable
@@ -80,7 +80,7 @@
@Test
@BMRules(rules = { @BMRule(name = "No disk space with expired SSTables test",
targetClass = "Directories",
- targetMethod = "hasAvailableDiskSpace",
+ targetMethod = "hasDiskSpaceForCompactionsAndStreams",
action = "return false;") } )
public void testExpiredSSTablesStillGetDroppedWithNoDiskSpace() throws Throwable
{
@@ -103,7 +103,7 @@
@Test(expected = RuntimeException.class)
@BMRules(rules = { @BMRule(name = "No disk space with expired SSTables test",
targetClass = "Directories",
- targetMethod = "hasAvailableDiskSpace",
+ targetMethod = "hasDiskSpaceForCompactionsAndStreams",
action = "return false;") } )
public void testRuntimeExceptionWhenNoDiskSpaceForCompaction() throws Throwable
{
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
index 65eea6a..5aef144 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
@@ -19,6 +19,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.FileStore;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -55,9 +56,12 @@
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.schema.CompactionParams;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -108,19 +112,6 @@
waitForMinor(KEYSPACE, currentTable(), SLEEP_TIME, true);
}
-
- @Test
- public void testTriggerMinorCompactionDTCS() throws Throwable
- {
- createTable("CREATE TABLE %s (id text PRIMARY KEY) WITH compaction = {'class':'DateTieredCompactionStrategy', 'min_threshold':2};");
- assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
- execute("insert into %s (id) values ('1') using timestamp 1000"); // same timestamp = same window = minor compaction triggered
- flush();
- execute("insert into %s (id) values ('1') using timestamp 1000");
- flush();
- waitForMinor(KEYSPACE, currentTable(), SLEEP_TIME, true);
- }
-
@Test
public void testTriggerMinorCompactionTWCS() throws Throwable
{
@@ -213,21 +204,21 @@
{
createTable("CREATE TABLE %s (id text PRIMARY KEY)");
Map<String, String> localOptions = new HashMap<>();
- localOptions.put("class", "DateTieredCompactionStrategy");
+ localOptions.put("class", "SizeTieredCompactionStrategy");
getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
- assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), DateTieredCompactionStrategy.class));
+ assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
// Invalidate disk boundaries to ensure that boundary invalidation will not cause the old strategy to be reloaded
getCurrentColumnFamilyStore().invalidateLocalRanges();
// altering something non-compaction related
execute("ALTER TABLE %s WITH gc_grace_seconds = 1000");
// should keep the local compaction strat
- assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), DateTieredCompactionStrategy.class));
+ assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
// Alter keyspace replication settings to force compaction strategy reload
execute("alter keyspace "+keyspace()+" with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }");
// should keep the local compaction strat
- assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), DateTieredCompactionStrategy.class));
+ assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
// altering a compaction option
- execute("ALTER TABLE %s WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold':3}");
+ execute("ALTER TABLE %s WITH compaction = {'class':'SizeTieredCompactionStrategy', 'min_threshold': 3}");
// will use the new option
assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), SizeTieredCompactionStrategy.class));
}
@@ -237,24 +228,23 @@
{
createTable("CREATE TABLE %s (id text PRIMARY KEY)");
Map<String, String> localOptions = new HashMap<>();
- localOptions.put("class", "DateTieredCompactionStrategy");
+ localOptions.put("class", "SizeTieredCompactionStrategy");
localOptions.put("enabled", "false");
getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
localOptions.clear();
- localOptions.put("class", "DateTieredCompactionStrategy");
+ localOptions.put("class", "SizeTieredCompactionStrategy");
// localOptions.put("enabled", "true"); - this is default!
getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
}
-
@Test
public void testSetLocalCompactionStrategyEnable() throws Throwable
{
createTable("CREATE TABLE %s (id text PRIMARY KEY)");
Map<String, String> localOptions = new HashMap<>();
- localOptions.put("class", "DateTieredCompactionStrategy");
+ localOptions.put("class", "LeveledCompactionStrategy");
getCurrentColumnFamilyStore().disableAutoCompaction();
assertFalse(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
@@ -263,8 +253,6 @@
assertTrue(getCurrentColumnFamilyStore().getCompactionStrategyManager().isEnabled());
}
-
-
@Test(expected = IllegalArgumentException.class)
public void testBadLocalCompactionStrategyOptions()
{
@@ -358,13 +346,13 @@
// write enough data to make sure we use an IndexedReader when doing a read, and make sure it fails when reading a corrupt row deletion
DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
int maxSizePre = DatabaseDescriptor.getColumnIndexSizeInKiB();
- DatabaseDescriptor.setColumnIndexSize(1024);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(1024);
prepareWide();
RowUpdateBuilder.deleteRowAt(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, -1, 22, 33).apply();
flush();
readAndValidate(true);
readAndValidate(false);
- DatabaseDescriptor.setColumnIndexSize(maxSizePre);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(maxSizePre);
}
@Test
@@ -373,14 +361,14 @@
// write enough data to make sure we use an IndexedReader when doing a read, and make sure it fails when reading a corrupt standard tombstone
DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
int maxSizePre = DatabaseDescriptor.getColumnIndexSizeInKiB();
- DatabaseDescriptor.setColumnIndexSize(1024);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(1024);
prepareWide();
RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), -1, System.currentTimeMillis() * 1000, 22).clustering(33).delete("b");
rub.build().apply();
flush();
readAndValidate(true);
readAndValidate(false);
- DatabaseDescriptor.setColumnIndexSize(maxSizePre);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(maxSizePre);
}
@Test
@@ -389,7 +377,7 @@
// write enough data to make sure we use an IndexedReader when doing a read, and make sure it fails when reading a corrupt range tombstone
DatabaseDescriptor.setCorruptedTombstoneStrategy(Config.CorruptedTombstoneStrategy.exception);
final int maxSizePreKiB = DatabaseDescriptor.getColumnIndexSizeInKiB();
- DatabaseDescriptor.setColumnIndexSize(1024);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(1024);
prepareWide();
RangeTombstone rt = new RangeTombstone(Slice.ALL, new DeletionTime(System.currentTimeMillis(), -1));
RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, 22).clustering(33).addRangeTombstone(rt);
@@ -397,7 +385,7 @@
flush();
readAndValidate(true);
readAndValidate(false);
- DatabaseDescriptor.setColumnIndexSize(maxSizePreKiB);
+ DatabaseDescriptor.setColumnIndexSizeInKiB(maxSizePreKiB);
}
@@ -770,4 +758,68 @@
if (shouldFind)
fail("No minor compaction triggered in "+maxWaitTime+"ms");
}
+
+ @Test
+ public void testNoDiskspace() throws Throwable
+ {
+ createTable("create table %s (id int primary key, i int) with compaction={'class':'SizeTieredCompactionStrategy'}");
+ getCurrentColumnFamilyStore().disableAutoCompaction();
+ for (int i = 0; i < 10; i++)
+ {
+ execute("insert into %s (id, i) values (?,?)", i, i);
+ getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+ }
+ CompactionInfo.Holder holder = holder(OperationType.COMPACTION);
+ CompactionManager.instance.active.beginCompaction(holder);
+ try
+ {
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ fail("Exception expected");
+ }
+ catch (Exception ignored)
+ {
+ // expected
+ }
+ finally
+ {
+ CompactionManager.instance.active.finishCompaction(holder);
+ }
+ // don't block compactions if there is a huge validation
+ holder = holder(OperationType.VALIDATION);
+ CompactionManager.instance.active.beginCompaction(holder);
+ try
+ {
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ }
+ finally
+ {
+ CompactionManager.instance.active.finishCompaction(holder);
+ }
+ }
+
+ private CompactionInfo.Holder holder(OperationType opType)
+ {
+ CompactionInfo.Holder holder = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ long availableSpace = 0;
+ for (File f : getCurrentColumnFamilyStore().getDirectories().getCFDirectories())
+ availableSpace += PathUtils.tryGetSpace(f.toPath(), FileStore::getUsableSpace);
+
+ return new CompactionInfo(getCurrentColumnFamilyStore().metadata(),
+ opType,
+ +0,
+ +availableSpace * 2,
+ nextTimeUUID(),
+ getCurrentColumnFamilyStore().getLiveSSTables());
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+ return holder;
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index fcf5c51..3c02154 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -31,6 +31,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -61,13 +62,12 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.SchemaTestUtil;
@@ -95,7 +95,7 @@
compactionOptions.put("tombstone_compaction_interval", "1");
// Disable tombstone histogram rounding for tests
- System.setProperty("cassandra.streaminghistogram.roundseconds", "1");
+ CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.setInt(1);
SchemaLoader.prepareServer();
@@ -110,11 +110,16 @@
public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
{
+ return populate(ks, cf, startRowKey, endRowKey, "", ttl);
+ }
+
+ public static long populate(String ks, String cf, int startRowKey, int endRowKey, String suffix, int ttl)
+ {
long timestamp = System.currentTimeMillis();
TableMetadata cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata();
for (int i = startRowKey; i <= endRowKey; i++)
{
- DecoratedKey key = Util.dk(Integer.toString(i));
+ DecoratedKey key = Util.dk(Integer.toString(i) + suffix);
for (int j = 0; j < 10; j++)
{
new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey())
@@ -181,12 +186,12 @@
// disable compaction while flushing
store.disableAutoCompaction();
- //Populate sstable1 with with keys [0..9]
- populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); //ttl=3s
+ //Populate sstable1 with with keys [0a..29a]
+ populate(KEYSPACE1, CF_STANDARD1, 0, 29, "a",3); //ttl=3s
Util.flush(store);
- //Populate sstable2 with with keys [10..19] (keys do not overlap with SSTable1)
- long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 10, 19, 3); //ttl=3s
+ //Populate sstable2 with with keys [0b..29b] (keys do not overlap with SSTable1, but the range is almost fully covered)
+ long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 0, 29, "b", 3); //ttl=3s
Util.flush(store);
assertEquals(2, store.getLiveSSTables().size());
@@ -274,7 +279,7 @@
SSTableReader sstable = sstables.iterator().next();
SSTableId prevGeneration = sstable.descriptor.id;
- String file = new File(sstable.descriptor.filenameFor(Component.DATA)).absolutePath();
+ String file = sstable.descriptor.fileFor(Components.DATA).absolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes
@@ -363,8 +368,8 @@
for (SSTableReader sstable : cfs.getLiveSSTables())
{
StatsMetadata stats = sstable.getSSTableMetadata();
- assertEquals(ByteBufferUtil.bytes("0"), stats.minClusteringValues.get(0));
- assertEquals(ByteBufferUtil.bytes("b"), stats.maxClusteringValues.get(0));
+ assertEquals(ByteBufferUtil.bytes("0"), stats.coveredClustering.start().bufferAt(0));
+ assertEquals(ByteBufferUtil.bytes("b"), stats.coveredClustering.end().bufferAt(0));
}
assertEquals(keys, k);
@@ -565,4 +570,4 @@
CompactionManager.instance.setConcurrentCompactors(1);
assertEquals(1, CompactionManager.instance.getCoreCompactorThreads());
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
deleted file mode 100644
index 4df210a..0000000
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.compaction;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.Pair;
-
-import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.getBuckets;
-import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.newestBucket;
-import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.filterOldSSTables;
-import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.validateOptions;
-
-import static org.junit.Assert.*;
-
-public class DateTieredCompactionStrategyTest extends SchemaLoader
-{
- public static final String KEYSPACE1 = "DateTieredCompactionStrategyTest";
- private static final String CF_STANDARD1 = "Standard1";
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- // Disable tombstone histogram rounding for tests
- System.setProperty("cassandra.streaminghistogram.roundseconds", "1");
-
- SchemaLoader.prepareServer();
-
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1));
- }
-
- @Test
- public void testOptionsValidation() throws ConfigurationException
- {
- Map<String, String> options = new HashMap<>();
- options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, "1825");
- Map<String, String> unvalidated = validateOptions(options);
- assertTrue(unvalidated.isEmpty());
-
- try
- {
- options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "0");
- validateOptions(options);
- fail(String.format("%s == 0 should be rejected", DateTieredCompactionStrategyOptions.BASE_TIME_KEY));
- }
- catch (ConfigurationException e) {}
-
- try
- {
- options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "-1337");
- validateOptions(options);
- fail(String.format("Negative %s should be rejected", DateTieredCompactionStrategyOptions.BASE_TIME_KEY));
- }
- catch (ConfigurationException e)
- {
- options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "1");
- }
-
- try
- {
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, "-1337");
- validateOptions(options);
- fail(String.format("Negative %s should be rejected", DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY));
- }
- catch (ConfigurationException e)
- {
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, "0");
- }
-
- try
- {
- options.put(DateTieredCompactionStrategyOptions.MAX_WINDOW_SIZE_KEY, "-1");
- validateOptions(options);
- fail(String.format("Negative %s should be rejected", DateTieredCompactionStrategyOptions.MAX_WINDOW_SIZE_KEY));
- }
- catch (ConfigurationException e)
- {
- options.put(DateTieredCompactionStrategyOptions.MAX_WINDOW_SIZE_KEY, "0");
- }
-
- options.put("bad_option", "1.0");
- unvalidated = validateOptions(options);
- assertTrue(unvalidated.containsKey("bad_option"));
- }
-
- @Test
- public void testTimeConversions()
- {
- Map<String, String> options = new HashMap<>();
- options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
- options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "SECONDS");
-
- DateTieredCompactionStrategyOptions opts = new DateTieredCompactionStrategyOptions(options);
- assertEquals(opts.maxSSTableAge, TimeUnit.SECONDS.convert(365*1000, TimeUnit.DAYS));
-
- options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
- opts = new DateTieredCompactionStrategyOptions(options);
- assertEquals(opts.maxSSTableAge, TimeUnit.MILLISECONDS.convert(365*1000, TimeUnit.DAYS));
-
- options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MICROSECONDS");
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, "10");
- opts = new DateTieredCompactionStrategyOptions(options);
- assertEquals(opts.maxSSTableAge, TimeUnit.MICROSECONDS.convert(10, TimeUnit.DAYS));
-
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, "0.5");
- opts = new DateTieredCompactionStrategyOptions(options);
- assertEquals(opts.maxSSTableAge, TimeUnit.MICROSECONDS.convert(1, TimeUnit.DAYS) / 2);
-
- options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "HOURS");
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, "0.5");
- opts = new DateTieredCompactionStrategyOptions(options);
- assertEquals(opts.maxSSTableAge, 12);
-
- }
-
- @Test
- public void testGetBuckets()
- {
- List<Pair<String, Long>> pairs = Lists.newArrayList(
- Pair.create("a", 199L),
- Pair.create("b", 299L),
- Pair.create("a", 1L),
- Pair.create("b", 201L)
- );
- List<List<String>> buckets = getBuckets(pairs, 100L, 2, 200L, Long.MAX_VALUE);
- assertEquals(2, buckets.size());
-
- for (List<String> bucket : buckets)
- {
- assertEquals(2, bucket.size());
- assertEquals(bucket.get(0), bucket.get(1));
- }
-
-
- pairs = Lists.newArrayList(
- Pair.create("a", 2000L),
- Pair.create("b", 3600L),
- Pair.create("a", 200L),
- Pair.create("c", 3950L),
- Pair.create("too new", 4125L),
- Pair.create("b", 3899L),
- Pair.create("c", 3900L)
- );
- buckets = getBuckets(pairs, 100L, 3, 4050L, Long.MAX_VALUE);
- // targets (divPosition, size): (40, 100), (39, 100), (12, 300), (3, 900), (0, 2700)
- // in other words: 0 - 2699, 2700 - 3599, 3600 - 3899, 3900 - 3999, 4000 - 4099
- assertEquals(3, buckets.size());
-
- for (List<String> bucket : buckets)
- {
- assertEquals(2, bucket.size());
- assertEquals(bucket.get(0), bucket.get(1));
- }
-
-
- // Test base 1.
- pairs = Lists.newArrayList(
- Pair.create("a", 200L),
- Pair.create("a", 299L),
- Pair.create("b", 2000L),
- Pair.create("b", 2014L),
- Pair.create("c", 3610L),
- Pair.create("c", 3690L),
- Pair.create("d", 3898L),
- Pair.create("d", 3899L),
- Pair.create("e", 3900L),
- Pair.create("e", 3950L),
- Pair.create("too new", 4125L)
- );
- buckets = getBuckets(pairs, 100L, 1, 4050L, Long.MAX_VALUE);
-
- assertEquals(5, buckets.size());
-
- for (List<String> bucket : buckets)
- {
- assertEquals(2, bucket.size());
- assertEquals(bucket.get(0), bucket.get(1));
- }
- }
-
- @Test
- public void testPrepBucket()
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
- cfs.disableAutoCompaction();
-
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
-
- // create 3 sstables
- int numSSTables = 3;
- for (int r = 0; r < numSSTables; r++)
- {
- DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata(), r, key.getKey())
- .clustering("column")
- .add("val", value).build().applyUnsafe();
-
- Util.flush(cfs);
- }
- Util.flush(cfs);
-
- List<SSTableReader> sstrs = new ArrayList<>(cfs.getLiveSSTables());
-
- List<SSTableReader> newBucket = newestBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32, 9, 10, Long.MAX_VALUE, new SizeTieredCompactionStrategyOptions());
- assertTrue("incoming bucket should not be accepted when it has below the min threshold SSTables", newBucket.isEmpty());
-
- newBucket = newestBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32, 10, 10, Long.MAX_VALUE, new SizeTieredCompactionStrategyOptions());
- assertFalse("non-incoming bucket should be accepted when it has at least 2 SSTables", newBucket.isEmpty());
-
- assertEquals("an sstable with a single value should have equal min/max timestamps", sstrs.get(0).getMinTimestamp(), sstrs.get(0).getMaxTimestamp());
- assertEquals("an sstable with a single value should have equal min/max timestamps", sstrs.get(1).getMinTimestamp(), sstrs.get(1).getMaxTimestamp());
- assertEquals("an sstable with a single value should have equal min/max timestamps", sstrs.get(2).getMinTimestamp(), sstrs.get(2).getMaxTimestamp());
- cfs.truncateBlocking();
- }
-
- @Test
- public void testFilterOldSSTables()
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
- cfs.disableAutoCompaction();
-
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
-
- // create 3 sstables
- int numSSTables = 3;
- for (int r = 0; r < numSSTables; r++)
- {
- DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata(), r, key.getKey())
- .clustering("column")
- .add("val", value).build().applyUnsafe();
-
- Util.flush(cfs);
- }
- Util.flush(cfs);
-
- Iterable<SSTableReader> filtered;
- List<SSTableReader> sstrs = new ArrayList<>(cfs.getLiveSSTables());
-
- filtered = filterOldSSTables(sstrs, 0, 2);
- assertEquals("when maxSSTableAge is zero, no sstables should be filtered", sstrs.size(), Iterables.size(filtered));
-
- filtered = filterOldSSTables(sstrs, 1, 2);
- assertEquals("only the newest 2 sstables should remain", 2, Iterables.size(filtered));
-
- filtered = filterOldSSTables(sstrs, 1, 3);
- assertEquals("only the newest sstable should remain", 1, Iterables.size(filtered));
-
- filtered = filterOldSSTables(sstrs, 1, 4);
- assertEquals("no sstables should remain when all are too old", 0, Iterables.size(filtered));
- cfs.truncateBlocking();
- }
-
-
- @Test
- public void testDropExpiredSSTables() throws InterruptedException
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
- cfs.disableAutoCompaction();
-
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
-
- // create 2 sstables
- DecoratedKey key = Util.dk(String.valueOf("expired"));
- new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), 1, key.getKey())
- .clustering("column")
- .add("val", value).build().applyUnsafe();
-
- Util.flush(cfs);
- SSTableReader expiredSSTable = cfs.getLiveSSTables().iterator().next();
- Thread.sleep(10);
-
- key = Util.dk(String.valueOf("nonexpired"));
- new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), key.getKey())
- .clustering("column")
- .add("val", value).build().applyUnsafe();
-
- Util.flush(cfs);
- assertEquals(cfs.getLiveSSTables().size(), 2);
-
- Map<String, String> options = new HashMap<>();
-
- options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
- options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
- options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
- options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
- DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
- for (SSTableReader sstable : cfs.getLiveSSTables())
- dtcs.addSSTable(sstable);
- dtcs.startup();
- assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
- Thread.sleep(2000);
- AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
- assertNotNull(t);
- assertEquals(1, Iterables.size(t.transaction.originals()));
- SSTableReader sstable = t.transaction.originals().iterator().next();
- assertEquals(sstable, expiredSSTable);
- t.transaction.abort();
- cfs.truncateBlocking();
- }
-
- @Test
- public void testSTCSBigWindow()
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
- cfs.disableAutoCompaction();
- ByteBuffer bigValue = ByteBuffer.wrap(new byte[10000]);
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
- int numSSTables = 40;
- // create big sstabels out of half:
- long timestamp = System.currentTimeMillis();
- for (int r = 0; r < numSSTables / 2; r++)
- {
- for (int i = 0; i < 10; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey())
- .clustering("column")
- .add("val", bigValue).build().applyUnsafe();
- }
- Util.flush(cfs);
- }
- // and small ones:
- for (int r = 0; r < numSSTables / 2; r++)
- {
- DecoratedKey key = Util.dk(String.valueOf(r));
- new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey())
- .clustering("column")
- .add("val", value).build().applyUnsafe();
- Util.flush(cfs);
- }
- Map<String, String> options = new HashMap<>();
- options.put(SizeTieredCompactionStrategyOptions.MIN_SSTABLE_SIZE_KEY, "1");
- DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
- for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
- dtcs.addSSTable(sstable);
- AbstractCompactionTask task = dtcs.getNextBackgroundTask(0);
- assertEquals(20, task.transaction.originals().size());
- task.transaction.abort();
- cfs.truncateBlocking();
- }
-}
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index 07a45e7..49f257b 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -45,6 +45,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
@@ -56,23 +57,23 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.SSTableAddedNotification;
import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
-import org.apache.cassandra.repair.ValidationManager;
-import org.apache.cassandra.repair.state.ValidationState;
-import org.apache.cassandra.schema.MockSchema;
-import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.repair.ValidationManager;
import org.apache.cassandra.repair.Validator;
+import org.apache.cassandra.repair.state.ValidationState;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.awaitility.Awaitility;
import static java.util.Collections.singleton;
-import static org.assertj.core.api.Assertions.assertThat;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -92,7 +93,7 @@
public static void defineSchema() throws ConfigurationException
{
// Disable tombstone histogram rounding for tests
- System.setProperty("cassandra.streaminghistogram.roundseconds", "1");
+ CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.setInt(1);
SchemaLoader.prepareServer();
diff --git a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
index 4253731..3f5e4b6 100644
--- a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
@@ -34,6 +34,7 @@
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NEVER_PURGE_TOMBSTONES;
import static org.junit.Assert.assertEquals;
public class NeverPurgeTest extends CQLTester
@@ -41,7 +42,7 @@
@BeforeClass
public static void setUpClass() // method name must match the @BeforeClass annotated method in CQLTester
{
- System.setProperty("cassandra.never_purge_tombstones", "true");
+ NEVER_PURGE_TOMBSTONES.setBoolean(true);
CQLTester.setUpClass();
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java
index b922ca80..5e0ed66 100644
--- a/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.compaction;
import java.util.Iterator;
+import java.util.Map;
import org.junit.After;
import org.junit.Before;
@@ -26,12 +27,15 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.DirectoriesTest;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.CloseableIterator;
@@ -201,7 +205,29 @@
{
wrapped[i] = new LimitableDataDirectory(original[i]);
}
- return new Directories(cfs.metadata(), wrapped);
+ return new Directories(cfs.metadata(), wrapped)
+ {
+ @Override
+ public boolean hasDiskSpaceForCompactionsAndStreams(Map<File, Long> expectedNewWriteSizes, Map<File, Long> totalCompactionWriteRemaining)
+ {
+ return hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSizes, totalCompactionWriteRemaining, file -> {
+ for (DataDirectory location : getWriteableLocations())
+ {
+ if (file.toPath().startsWith(location.location.toPath())) {
+ LimitableDataDirectory directory = (LimitableDataDirectory) location;
+ if (directory.availableSpace != null)
+ {
+ DirectoriesTest.FakeFileStore store = new DirectoriesTest.FakeFileStore();
+ // reverse the computation in Directories.getAvailableSpaceForCompactions
+ store.usableSpace = Math.round(directory.availableSpace / DatabaseDescriptor.getMaxSpaceForCompactionsPerDrive()) + DatabaseDescriptor.getMinFreeSpacePerDriveInBytes();
+ return store;
+ }
+ }
+ }
+ return Directories.getFileStore(file);
+ });
+ }
+ };
}
}
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
index bb1d1f0..9e03ec3 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java
@@ -29,9 +29,8 @@
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
@@ -129,8 +128,7 @@
Util.flush(cfs);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS);
- try(FileChannel fc = new File(filenameToCorrupt).newReadWriteChannel())
+ try(FileChannel fc = sstable.descriptor.fileFor(Components.STATS).newReadWriteChannel())
{
fc.position(0);
fc.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
index fc98a9c..dac331e 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
@@ -29,6 +29,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
@@ -55,7 +56,7 @@
public static void defineSchema() throws ConfigurationException
{
// Disable tombstone histogram rounding for tests
- System.setProperty("cassandra.streaminghistogram.roundseconds", "1");
+ CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.setInt(1);
SchemaLoader.prepareServer();
@@ -93,11 +94,11 @@
@Test
public void testGetBuckets()
{
- List<Pair<String, Long>> pairs = new ArrayList<Pair<String, Long>>();
+ List<Pair<String, Long>> pairs = new ArrayList<>();
String[] strings = { "a", "bbbb", "cccccccc", "cccccccc", "bbbb", "a" };
for (String st : strings)
{
- Pair<String, Long> pair = Pair.create(st, new Long(st.length()));
+ Pair<String, Long> pair = Pair.create(st, (long) st.length());
pairs.add(pair);
}
@@ -117,7 +118,7 @@
String[] strings2 = { "aaa", "bbbbbbbb", "aaa", "bbbbbbbb", "bbbbbbbb", "aaa" };
for (String st : strings2)
{
- Pair<String, Long> pair = Pair.create(st, new Long(st.length()));
+ Pair<String, Long> pair = Pair.create(st, (long) st.length());
pairs.add(pair);
}
@@ -138,7 +139,7 @@
String[] strings3 = { "aaa", "bbbbbbbb", "aaa", "bbbbbbbb", "bbbbbbbb", "aaa" };
for (String st : strings3)
{
- Pair<String, Long> pair = Pair.create(st, new Long(st.length()));
+ Pair<String, Long> pair = Pair.create(st, (long) st.length());
pairs.add(pair);
}
@@ -146,8 +147,9 @@
assertEquals(1, buckets.size());
}
+ @SuppressWarnings("UnnecessaryLocalVariable")
@Test
- public void testPrepBucket() throws Exception
+ public void testPrepBucket()
{
String ksname = KEYSPACE1;
String cfname = "Standard1";
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 0e3fb76..7c7cdb6 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -29,18 +29,23 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.schema.SchemaTestUtil;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.tools.SSTableExpiredBlockers;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -56,7 +61,7 @@
public static void defineSchema() throws ConfigurationException
{
// Disable tombstone histogram rounding for tests
- System.setProperty("cassandra.streaminghistogram.roundseconds", "1");
+ CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.setInt(1);
SchemaLoader.prepareServer();
diff --git a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
index de60286..41a4c47 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java
@@ -28,18 +28,12 @@
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
-
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
@@ -50,10 +44,17 @@
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.utils.Pair;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION;
import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.getWindowBoundsInMillis;
import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.newestBucket;
import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.validateOptions;
import static org.apache.cassandra.utils.FBUtilities.nowInSeconds;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TimeWindowCompactionStrategyTest extends SchemaLoader
{
@@ -65,8 +66,9 @@
public static void defineSchema() throws ConfigurationException
{
// Disable tombstone histogram rounding for tests
- System.setProperty("cassandra.streaminghistogram.roundseconds", "1");
- System.setProperty(TimeWindowCompactionStrategyOptions.UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY, "true");
+ CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.setInt(1);
+ ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.setBoolean(true);
+
SchemaLoader.prepareServer();
diff --git a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java
index a641be1..c4f72de 100644
--- a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java
@@ -18,15 +18,26 @@
package org.apache.cassandra.db.compaction.writers;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
import com.google.common.primitives.Longs;
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
import org.apache.cassandra.db.compaction.CompactionIterator;
@@ -67,6 +78,13 @@
return Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
}
+ @After
+ public void afterTest() {
+ Keyspace ks = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(TABLE);
+ cfs.truncateBlocking();
+ }
+
@Test
public void testDefaultCompactionWriter() throws Throwable
{
@@ -260,4 +278,4 @@
assertRows(execute(String.format("SELECT k, t FROM %s.%s WHERE k = :i", KEYSPACE, TABLE), i), expected);
}
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/filter/SliceTest.java b/test/unit/org/apache/cassandra/db/filter/SliceTest.java
index c83880c..18233f0 100644
--- a/test/unit/org/apache/cassandra/db/filter/SliceTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/SliceTest.java
@@ -18,19 +18,30 @@
* */
package org.apache.cassandra.db.filter;
-
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
-import org.apache.cassandra.db.*;
import org.junit.Test;
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.utils.ByteBufferUtil;
-import static org.apache.cassandra.db.ClusteringPrefix.Kind.*;
-import static org.junit.Assert.*;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_END_BOUND;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_START_BOUND;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_END_BOUND;
+import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_START_BOUND;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
public class SliceTest
{
@@ -48,221 +59,221 @@
// filter falls entirely before sstable
Slice slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// same case, but with empty start
slice = Slice.make(makeBound(sk), makeBound(ek, 1, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// same case, but with missing components for start
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// same case, but with missing components for start and end
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// end of slice matches start of sstable for the first component, but not the second component
slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0)));
// same case, but with missing components for start
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0)));
// same case, but with missing components for start and end
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0));
- assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0)));
// first two components match, but not the last
slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 0));
- assertFalse(slice.intersects(cc, columnNames(1, 1, 1), columnNames(3, 1, 1)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3, 1, 1)));
// all three components in slice end match the start of the sstable
slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 1, 1), columnNames(3, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3, 1, 1)));
// filter falls entirely after sstable
slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek, 4, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// same case, but with empty end
slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// same case, but with missing components for end
slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek, 1));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// same case, but with missing components for start and end
slice = Slice.make(makeBound(sk, 4, 0), makeBound(ek, 1));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
// start of slice matches end of sstable for the first component, but not the second component
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0)));
// start of slice matches end of sstable for the first two components, but not the last component
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 1, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 0)));
// all three components in the slice start match the end of the sstable
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1)));
// slice covers entire sstable (with no matching edges)
slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 2, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// same case, but with empty ends
slice = Slice.make(makeBound(sk), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// same case, but with missing components
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 2, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// slice covers entire sstable (with matching start)
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// slice covers entire sstable (with matching end)
slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// slice covers entire sstable (with matching start and end)
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// slice falls entirely within sstable (with matching start)
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// same case, but with a missing end component
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// slice falls entirely within sstable (with matching end)
slice = Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// same case, but with a missing start component
slice = Slice.make(makeBound(sk, 1, 1), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)));
// slice falls entirely within sstable
slice = Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2)));
// same case, but with a missing start component
slice = Slice.make(makeBound(sk, 1, 1), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2)));
// same case, but with a missing start and end components
slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2)));
// same case, but with an equal first component and missing start and end components
slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2)));
// slice falls entirely within sstable (slice start and end are the same)
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2)));
// slice starts within sstable, empty end
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// same case, but with missing end components
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// slice starts within sstable (matching sstable start), empty end
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// same case, but with missing end components
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 3));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// slice starts within sstable (matching sstable end), empty end
slice = Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// same case, but with missing end components
slice = Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// slice ends within sstable, empty end
slice = Slice.make(makeBound(sk), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// same case, but with missing start components
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 1, 1));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// slice ends within sstable (matching sstable start), empty start
slice = Slice.make(makeBound(sk), makeBound(ek, 1, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// same case, but with missing start components
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// slice ends within sstable (matching sstable end), empty start
slice = Slice.make(makeBound(sk), makeBound(ek, 2, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// same case, but with missing start components
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 2, 0, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)));
// empty min/max column names
slice = Slice.make(makeBound(sk), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(), columnNames()));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek)));
slice = Slice.make(makeBound(sk, 1), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(), columnNames()));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek)));
slice = Slice.make(makeBound(sk), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(), columnNames()));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek)));
slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(), columnNames()));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek)));
slice = Slice.make(makeBound(sk), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 2)));
slice = Slice.make(makeBound(sk), makeBound(ek, 2));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk, 2), makeBound(ek, 3));
- assertFalse(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
// basic check on reversed slices
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 0, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0)));
slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 0, 0, 0));
- assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0)));
slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 1, 1, 0));
- assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2)));
}
@Test
@@ -279,32 +290,32 @@
// slice does intersect
Slice slice = Slice.make(makeBound(sk), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(1), columnNames(1, 2)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2)));
slice = Slice.make(makeBound(sk), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk, 1), makeBound(ek));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 2, 3));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk, 1, 2, 3), makeBound(ek, 2));
- assertTrue(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
// slice does not intersect
slice = Slice.make(makeBound(sk, 2), makeBound(ek, 3, 4, 5));
- assertFalse(slice.intersects(cc, columnNames(), columnNames(1)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1)));
slice = Slice.make(makeBound(sk, 0), makeBound(ek, 0, 1, 2));
- assertFalse(slice.intersects(cc, columnNames(1), columnNames(1, 2)));
+ assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2)));
}
@Test
@@ -353,14 +364,6 @@
return BufferClusteringBound.create(kind, values);
}
- private static List<ByteBuffer> columnNames(Integer ... components)
- {
- List<ByteBuffer> names = new ArrayList<>(components.length);
- for (int component : components)
- names.add(ByteBufferUtil.bytes(component));
- return names;
- }
-
private static Slice s(int start, int finish)
{
return Slice.make(makeBound(INCL_START_BOUND, start),
@@ -382,4 +385,25 @@
for (int i = 0; i < expected.length; i++)
assertEquals(expected[i], slices.get(i));
}
-}
+
+ private void assertSlicesIntersect(ClusteringComparator cc, Slice s1, Slice s2)
+ {
+ assertSlicesIntersectInternal(cc, s1, s2);
+ assertSlicesIntersectInternal(cc, Slice.ALL, s1);
+ assertSlicesIntersectInternal(cc, Slice.ALL, s2);
+ assertSlicesDoNotIntersect(cc, Slice.make(ClusteringBound.exclusiveStartOf(Clustering.EMPTY), ClusteringBound.exclusiveEndOf(Clustering.EMPTY)), s1);
+ assertSlicesDoNotIntersect(cc, Slice.make(ClusteringBound.exclusiveStartOf(Clustering.EMPTY), ClusteringBound.exclusiveEndOf(Clustering.EMPTY)), s2);
+ }
+
+ private void assertSlicesIntersectInternal(ClusteringComparator cc, Slice s1, Slice s2)
+ {
+ assertTrue(String.format("Slice %s should intersect with slice %s", s1.toString(cc), s2.toString(cc)), s1.intersects(cc, s2));
+ assertTrue(String.format("Slice %s should intersect with slice %s", s2.toString(cc), s1.toString(cc)), s2.intersects(cc, s1));
+ }
+
+ private void assertSlicesDoNotIntersect(ClusteringComparator cc, Slice s1, Slice s2)
+ {
+ assertFalse(String.format("Slice %s should not intersect with slice %s", s1.toString(cc), s2.toString(cc)), s1.intersects(cc, s2));
+ assertFalse(String.format("Slice %s should not intersect with slice %s", s2.toString(cc), s1.toString(cc)), s2.intersects(cc, s1));
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailAllowFilteringTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailAllowFilteringTest.java
index c46498c..5eca839 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailAllowFilteringTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailAllowFilteringTest.java
@@ -22,8 +22,11 @@
import org.junit.Before;
import org.junit.Test;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspaceTables;
+import org.apache.cassandra.service.ClientState;
public class GuardrailAllowFilteringTest extends GuardrailTester
{
@@ -33,7 +36,7 @@
public void setupTest()
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, a int, b int)");
- enableState = getGuardrial();
+ enableState = getGuardrail();
}
@After
@@ -47,7 +50,7 @@
guardrails().setAllowFilteringEnabled(allowFilteringEnabled);
}
- private boolean getGuardrial()
+ private boolean getGuardrail()
{
return guardrails().getAllowFilteringEnabled();
}
@@ -91,4 +94,24 @@
SchemaKeyspaceTables.TABLES,
currentTable()));
}
+
+ @Test
+ public void testRequiredAllowFiltering()
+ {
+ setGuardrail(true);
+ assertRequiredAllowFilteringThrows(systemClientState, StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+ assertRequiredAllowFilteringThrows(superClientState, StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+ assertRequiredAllowFilteringThrows(userClientState, StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+
+ setGuardrail(false);
+ assertRequiredAllowFilteringThrows(systemClientState, StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+ assertRequiredAllowFilteringThrows(superClientState, StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+ assertRequiredAllowFilteringThrows(userClientState, StatementRestrictions.CANNOT_USE_ALLOW_FILTERING_MESSAGE);
+ }
+
+ private void assertRequiredAllowFilteringThrows(ClientState state, String message)
+ {
+ String query = "SELECT * FROM %s WHERE a = 5";
+ assertThrows(() -> execute(state, query), InvalidRequestException.class, message);
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailAlterTableTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailAlterTableTest.java
new file mode 100644
index 0000000..1aa7095
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailAlterTableTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Tests the guardrail for disabling user access to the ALTER TABLE statement, {@link Guardrails#alterTableEnabled}.
+ *
+ * NOTE: This test class depends on {@link #currentTable()} method for setup, cleanup, and execution of tests. You'll
+ * need to refactor this if you add tests that make changes to the current table as the test teardown will no longer match
+ * setup.
+ */
+public class GuardrailAlterTableTest extends GuardrailTester
+{
+ public GuardrailAlterTableTest()
+ {
+ super(Guardrails.alterTableEnabled);
+ }
+
+ @Before
+ public void setupTest() throws Throwable
+ {
+ createTable("CREATE TABLE IF NOT EXISTS %s (k INT, c INT, v TEXT, PRIMARY KEY(k, c))");
+ }
+
+ @After
+ public void afterTest() throws Throwable
+ {
+ dropTable("DROP TABLE %s");
+ setGuardrail(true);
+ }
+
+ private void setGuardrail(boolean alterTableEnabled)
+ {
+ guardrails().setAlterTableEnabled(alterTableEnabled);
+ }
+
+ /**
+ * Confirm that ALTER TABLE queries either work (guardrail enabled) or fail (guardrail disabled) appropriately
+ * @throws Throwable
+ */
+ @Test
+ public void testGuardrailEnabledAndDisabled() throws Throwable
+ {
+ setGuardrail(false);
+ assertFails("ALTER TABLE %s ADD test_one text;", "changing columns");
+
+ setGuardrail(true);
+ assertValid("ALTER TABLE %s ADD test_two text;");
+
+ setGuardrail(false);
+ assertFails("ALTER TABLE %s ADD test_three text;", "changing columns");
+ }
+
+ /**
+ * Confirm the guardrail appropriately catches the ALTER DROP case on a column
+ * @throws Throwable
+ */
+ @Test
+ public void testAppliesToAlterDropColumn() throws Throwable
+ {
+ setGuardrail(true);
+ assertValid("ALTER TABLE %s ADD test_one text;");
+
+ setGuardrail(false);
+ assertFails("ALTER TABLE %s DROP test_one", "changing columns");
+
+ setGuardrail(true);
+ assertValid("ALTER TABLE %s DROP test_one");
+ }
+
+ /**
+ * Confirm the guardrail appropriately catches the ALTER RENAME case on a column
+ * @throws Throwable
+ */
+ @Test
+ public void testAppliesToAlterRenameColumn() throws Throwable
+ {
+ setGuardrail(false);
+ assertFails("ALTER TABLE %s RENAME c TO renamed_c", "changing columns");
+
+ setGuardrail(true);
+ assertValid("ALTER TABLE %s RENAME c TO renamed_c");
+ }
+
+ /**
+ * Confirm we can always alter properties via the options map regardless of guardrail state
+ * @throws Throwable
+ */
+ @Test
+ public void testAlterViaMapAlwaysWorks() throws Throwable
+ {
+ setGuardrail(false);
+ assertValid("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
+
+ setGuardrail(true);
+ assertValid("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };");
+ }
+
+ /**
+ * Confirm the other form of ALTER TABLE property map changing always works regardless of guardrail state
+ * @throws Throwable
+ */
+ @Test
+ public void testAlterOptionsAlwaysWorks() throws Throwable
+ {
+ setGuardrail(true);
+ assertValid("ALTER TABLE %s WITH GC_GRACE_SECONDS = 456; ");
+
+ setGuardrail(false);
+ assertValid("ALTER TABLE %s WITH GC_GRACE_SECONDS = 123; ");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java
index 1483e81..b15f85a 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailCollectionSizeTest.java
@@ -28,12 +28,14 @@
import org.junit.After;
import org.junit.Test;
+import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.db.marshal.MapType;
import org.apache.cassandra.db.marshal.SetType;
import static java.nio.ByteBuffer.allocate;
+import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.BYTES;
/**
* Tests the guardrail for the size of collections, {@link Guardrails#collectionSize}.
@@ -53,7 +55,9 @@
Guardrails.collectionSize,
Guardrails::setCollectionSizeThreshold,
Guardrails::getCollectionSizeWarnThreshold,
- Guardrails::getCollectionSizeFailThreshold);
+ Guardrails::getCollectionSizeFailThreshold,
+ bytes -> new DataStorageSpec.LongBytesBound(bytes, BYTES).toString(),
+ size -> new DataStorageSpec.LongBytesBound(size).toBytes());
}
@After
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailColumnValueSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailColumnValueSizeTest.java
new file mode 100644
index 0000000..f0513da
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailColumnValueSizeTest.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+
+import static java.lang.String.format;
+import static java.nio.ByteBuffer.allocate;
+import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.BYTES;
+
+/**
+ * Tests the guardrail for the size of column values, {@link Guardrails#columnValueSize}.
+ */
+public class GuardrailColumnValueSizeTest extends ThresholdTester
+{
+ private static final int WARN_THRESHOLD = 1024; // bytes
+ private static final int FAIL_THRESHOLD = WARN_THRESHOLD * 4; // bytes
+
+ public GuardrailColumnValueSizeTest()
+ {
+ super(WARN_THRESHOLD + "B",
+ FAIL_THRESHOLD + "B",
+ Guardrails.columnValueSize,
+ Guardrails::setColumnValueSizeThreshold,
+ Guardrails::getColumnValueSizeWarnThreshold,
+ Guardrails::getColumnValueSizeFailThreshold,
+ bytes -> new DataStorageSpec.LongBytesBound(bytes, BYTES).toString(),
+ size -> new DataStorageSpec.LongBytesBound(size).toBytes());
+ }
+
+ @Test
+ public void testSimplePartitionKey() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text PRIMARY KEY, v int)");
+
+ // the size of primary key columns is not guarded because they already have a fixed limit of 65535B
+
+ testNoThreshold("INSERT INTO %s (k, v) VALUES (?, 0)");
+ testNoThreshold("UPDATE %s SET v = 1 WHERE k = ?");
+ testNoThreshold("DELETE v FROM %s WHERE k = ?");
+ testNoThreshold("DELETE FROM %s WHERE k = ?");
+ }
+
+ @Test
+ public void testCompositePartitionKey() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k1 text, k2 text, v int, PRIMARY KEY((k1, k2)))");
+
+ // the size of primary key columns is not guarded because they already have a fixed limit of 65535B
+
+ testNoThreshold2("INSERT INTO %s (k1, k2, v) VALUES (?, ?, 0)");
+ testNoThreshold2("UPDATE %s SET v = 1 WHERE k1 = ? AND k2 = ?");
+ testNoThreshold2("DELETE v FROM %s WHERE k1 = ? AND k2 = ?");
+ testNoThreshold2("DELETE FROM %s WHERE k1 = ? AND k2 = ?");
+ }
+
+ @Test
+ public void testSimpleClustering() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c text, v int, PRIMARY KEY(k, c))");
+
+ // the size of primary key columns is not guarded because they already have a fixed limit of 65535B
+
+ testNoThreshold("INSERT INTO %s (k, c, v) VALUES (0, ?, 0)");
+ testNoThreshold("UPDATE %s SET v = 1 WHERE k = 0 AND c = ?");
+ testNoThreshold("DELETE v FROM %s WHERE k = 0 AND c = ?");
+ testNoThreshold("DELETE FROM %s WHERE k = 0 AND c = ?");
+ }
+
+ @Test
+ public void testCompositeClustering() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c1 text, c2 text, v int, PRIMARY KEY(k, c1, c2))");
+
+ // the size of primary key columns is not guarded because they already have a fixed limit of 65535B
+
+ testNoThreshold("DELETE FROM %s WHERE k = 0 AND c1 = ?");
+ testNoThreshold("DELETE FROM %s WHERE k = 0 AND c1 > ?");
+ testNoThreshold("DELETE FROM %s WHERE k = 0 AND c1 < ?");
+ testNoThreshold("DELETE FROM %s WHERE k = 0 AND c1 >= ?");
+ testNoThreshold("DELETE FROM %s WHERE k = 0 AND c1 <= ?");
+
+ testNoThreshold2("INSERT INTO %s (k, c1, c2, v) VALUES (0, ?, ?, 0)");
+ testNoThreshold2("UPDATE %s SET v = 1 WHERE k = 0 AND c1 = ? AND c2 = ?");
+ testNoThreshold2("DELETE v FROM %s WHERE k = 0 AND c1 = ? AND c2 = ?");
+ testNoThreshold2("DELETE FROM %s WHERE k = 0 AND c1 = ? AND c2 = ?");
+ testNoThreshold2("DELETE FROM %s WHERE k = 0 AND c1 = ? AND c2 > ?");
+ testNoThreshold2("DELETE FROM %s WHERE k = 0 AND c1 = ? AND c2 < ?");
+ testNoThreshold2("DELETE FROM %s WHERE k = 0 AND c1 = ? AND c2 >= ?");
+ testNoThreshold2("DELETE FROM %s WHERE k = 0 AND c1 = ? AND c2 <= ?");
+ }
+
+ @Test
+ public void testRegularColumn() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+
+ testThreshold("v", "INSERT INTO %s (k, v) VALUES (0, ?)");
+ testThreshold("v", "UPDATE %s SET v = ? WHERE k = 0");
+ }
+
+ @Test
+ public void testStaticColumn() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c int, s text STATIC, r int, PRIMARY KEY(k, c))");
+
+ testThreshold("s", "INSERT INTO %s (k, s) VALUES (0, ?)");
+ testThreshold("s", "INSERT INTO %s (k, c, s, r) VALUES (0, 0, ?, 0)");
+ testThreshold("s", "UPDATE %s SET s = ? WHERE k = 0");
+ testThreshold("s", "UPDATE %s SET s = ?, r = 0 WHERE k = 0 AND c = 0");
+ }
+
+ @Test
+ public void testTuple() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, t tuple<text, text>)");
+
+ testThreshold2("t", "INSERT INTO %s (k, t) VALUES (0, (?, ?))", 8);
+ testThreshold2("t", "UPDATE %s SET t = (?, ?) WHERE k = 0", 8);
+ }
+
+ @Test
+ public void testUDT() throws Throwable
+ {
+ String udt = createType("CREATE TYPE %s (a text, b text)");
+ createTable(format("CREATE TABLE %%s (k int PRIMARY KEY, u %s)", udt));
+
+ testThreshold("u", "INSERT INTO %s (k, u) VALUES (0, {a: ?})");
+ testThreshold("u", "INSERT INTO %s (k, u) VALUES (0, {b: ?})");
+ testThreshold("u", "UPDATE %s SET u = {a: ?} WHERE k = 0");
+ testThreshold("u", "UPDATE %s SET u = {b: ?} WHERE k = 0");
+ testThreshold("u", "UPDATE %s SET u.a = ? WHERE k = 0");
+ testThreshold("u", "UPDATE %s SET u.b = ? WHERE k = 0");
+ testThreshold2("u", "INSERT INTO %s (k, u) VALUES (0, {a: ?, b: ?})");
+ testThreshold2("u", "UPDATE %s SET u.a = ?, u.b = ? WHERE k = 0");
+ testThreshold2("u", "UPDATE %s SET u = {a: ?, b: ?} WHERE k = 0");
+ }
+
+ @Test
+ public void testFrozenUDT() throws Throwable
+ {
+ String udt = createType("CREATE TYPE %s (a text, b text)");
+ createTable(format("CREATE TABLE %%s (k int PRIMARY KEY, v frozen<%s>)", udt));
+
+ testThreshold("v", "INSERT INTO %s (k, v) VALUES (0, {a: ?})", 8);
+ testThreshold("v", "INSERT INTO %s (k, v) VALUES (0, {b: ?})", 8);
+ testThreshold("v", "UPDATE %s SET v = {a: ?} WHERE k = 0", 8);
+ testThreshold("v", "UPDATE %s SET v = {b: ?} WHERE k = 0", 8);
+ testThreshold2("v", "INSERT INTO %s (k, v) VALUES (0, {a: ?, b: ?})", 8);
+ testThreshold2("v", "UPDATE %s SET v = {a: ?, b: ?} WHERE k = 0", 8);
+ }
+
+ @Test
+ public void testNestedUDT() throws Throwable
+ {
+ String inner = createType("CREATE TYPE %s (c text, d text)");
+ String outer = createType(format("CREATE TYPE %%s (a text, b frozen<%s>)", inner));
+ createTable(format("CREATE TABLE %%s (k int PRIMARY KEY, v %s)", outer));
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, v) VALUES (0, {a: ?, b: {c: ?, d: ?}})",
+ "UPDATE %s SET v = {a: ?, b: {c: ?, d: ?}} WHERE k = 0"))
+ {
+ assertValid(query, allocate(0), allocate(0), allocate(0));
+ assertValid(query, allocate(WARN_THRESHOLD - 8), allocate(0), allocate(0));
+ assertValid(query, allocate(0), allocate(WARN_THRESHOLD - 8), allocate(0));
+ assertValid(query, allocate(0), allocate(0), allocate(WARN_THRESHOLD - 8));
+
+ assertWarns("v", query, allocate(WARN_THRESHOLD + 1), allocate(0), allocate(0));
+ assertWarns("v", query, allocate(0), allocate(WARN_THRESHOLD - 7), allocate(0));
+ assertWarns("v", query, allocate(0), allocate(0), allocate(WARN_THRESHOLD - 7));
+
+ assertFails("v", query, allocate(FAIL_THRESHOLD + 1), allocate(0), allocate(0));
+ assertFails("v", query, allocate(0), allocate(FAIL_THRESHOLD - 7), allocate(0));
+ assertFails("v", query, allocate(0), allocate(0), allocate(FAIL_THRESHOLD - 7));
+ }
+ }
+
+ @Test
+ public void testList() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<text>)");
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, l) VALUES (0, ?)",
+ "UPDATE %s SET l = ? WHERE k = 0",
+ "UPDATE %s SET l = l + ? WHERE k = 0"))
+ {
+ testCollection("l", query, this::list);
+ }
+
+ testThreshold("l", "UPDATE %s SET l[0] = ? WHERE k = 0");
+
+ String query = "UPDATE %s SET l = l - ? WHERE k = 0";
+ assertValid(query, this::list, allocate(1));
+ assertValid(query, this::list, allocate(FAIL_THRESHOLD));
+ assertValid(query, this::list, allocate(FAIL_THRESHOLD + 1)); // Doesn't write anything because we couldn't write
+ }
+
+ @Test
+ public void testFrozenList() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, fl frozen<list<text>>)");
+
+ // the serialized size of a frozen list is the size of its serialized elements, plus a 32-bit integer prefix for
+ // the number of elements, and another 32-bit integer for the size of each element
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, fl) VALUES (0, ?)",
+ "UPDATE %s SET fl = ? WHERE k = 0"))
+ {
+ testFrozenCollection("fl", query, this::list);
+ }
+ }
+
+ @Test
+ public void testSet() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<text>)");
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, s) VALUES (0, ?)",
+ "UPDATE %s SET s = ? WHERE k = 0",
+ "UPDATE %s SET s = s + ? WHERE k = 0",
+ "UPDATE %s SET s = s - ? WHERE k = 0"))
+ {
+ testCollection("s", query, this::set);
+ }
+ }
+
+ @Test
+ public void testSetWithClustering() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c1 int, c2 int, s set<text>, PRIMARY KEY(k, c1, c2))");
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, c1, c2, s) VALUES (0, 0, 0, ?)",
+ "UPDATE %s SET s = ? WHERE k = 0 AND c1 = 0 AND c2 = 0",
+ "UPDATE %s SET s = s + ? WHERE k = 0 AND c1 = 0 AND c2 = 0",
+ "UPDATE %s SET s = s - ? WHERE k = 0 AND c1 = 0 AND c2 = 0"))
+ {
+ testCollection("s", query, this::set);
+ }
+ }
+
+ @Test
+ public void testFrozenSet() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, fs frozen<set<text>>)");
+
+ // the serialized size of a frozen set is the size of its serialized elements, plus a 32-bit integer prefix for
+ // the number of elements, and another 32-bit integer for the size of each element
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, fs) VALUES (0, ?)",
+ "UPDATE %s SET fs = ? WHERE k = 0"))
+ {
+ testFrozenCollection("fs", query, this::set);
+ }
+ }
+
+ @Test
+ public void testMap() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<text, text>)");
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, m) VALUES (0, ?)",
+ "UPDATE %s SET m = ? WHERE k = 0",
+ "UPDATE %s SET m = m + ? WHERE k = 0"))
+ {
+ testMap("m", query);
+ }
+
+ testThreshold2("m", "UPDATE %s SET m[?] = ? WHERE k = 0");
+ testCollection("m", "UPDATE %s SET m = m - ? WHERE k = 0", this::set);
+ }
+
+ @Test
+ public void testMapWithClustering() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int, c1 int, c2 int, mc map<text, text>, PRIMARY KEY(k, c1, c2))");
+ testMap("mc", "INSERT INTO %s (k, c1, c2, mc) VALUES (0, 0, 0, ?)");
+ }
+
+ @Test
+ public void testFrozenMap() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY,fm frozen<map<text, text>>)");
+
+ // the serialized size of a frozen map is the size of the serialized values plus a 32-bit integer prefix for the
+ // number of key-value pairs, and another 32-bit integer for the size of each value
+
+ for (String query : Arrays.asList("INSERT INTO %s (k, fm) VALUES (0, ?)",
+ "UPDATE %s SET fm = ? WHERE k = 0"))
+ {
+ assertValid(query, this::map, allocate(1), allocate(1));
+ assertValid(query, this::map, allocate(WARN_THRESHOLD - 13), allocate(1));
+ assertValid(query, this::map, allocate(1), allocate(WARN_THRESHOLD - 13));
+
+ assertWarns("fm", query, this::map, allocate(WARN_THRESHOLD - 12), allocate(1));
+ assertWarns("fm", query, this::map, allocate(1), allocate(WARN_THRESHOLD - 12));
+
+ assertFails("fm", query, this::map, allocate(FAIL_THRESHOLD - 12), allocate(1));
+ assertFails("fm", query, this::map, allocate(1), allocate(FAIL_THRESHOLD - 12));
+ }
+ }
+
+ @Test
+ public void testBatch() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text, c text, r text, s text STATIC, PRIMARY KEY(k, c))");
+
+ // partition key
+ testNoThreshold("BEGIN BATCH INSERT INTO %s (k, c, r) VALUES (?, '0', '0'); APPLY BATCH;");
+ testNoThreshold("BEGIN BATCH UPDATE %s SET r = '0' WHERE k = ? AND c = '0'; APPLY BATCH;");
+ testNoThreshold("BEGIN BATCH DELETE r FROM %s WHERE k = ? AND c = '0'; APPLY BATCH;");
+ testNoThreshold("BEGIN BATCH DELETE FROM %s WHERE k = ?; APPLY BATCH;");
+
+ // static column
+ testThreshold("s", "BEGIN BATCH INSERT INTO %s (k, s) VALUES ('0', ?); APPLY BATCH;");
+ testThreshold("s", "BEGIN BATCH INSERT INTO %s (k, s, c, r) VALUES ('0', ?, '0', '0'); APPLY BATCH;");
+ testThreshold("s", "BEGIN BATCH UPDATE %s SET s = ? WHERE k = '0'; APPLY BATCH;");
+ testThreshold("s", "BEGIN BATCH UPDATE %s SET s = ?, r = '0' WHERE k = '0' AND c = '0'; APPLY BATCH;");
+
+ // clustering key
+ testNoThreshold("BEGIN BATCH INSERT INTO %s (k, c, r) VALUES ('0', ?, '0'); APPLY BATCH;");
+ testNoThreshold("BEGIN BATCH UPDATE %s SET r = '0' WHERE k = '0' AND c = ?; APPLY BATCH;");
+ testNoThreshold("BEGIN BATCH DELETE r FROM %s WHERE k = '0' AND c = ?; APPLY BATCH;");
+ testNoThreshold("BEGIN BATCH DELETE FROM %s WHERE k = '0' AND c = ?; APPLY BATCH;");
+
+ // regular column
+ testThreshold("r", "BEGIN BATCH INSERT INTO %s (k, c, r) VALUES ('0', '0', ?); APPLY BATCH;");
+ testThreshold("r", "BEGIN BATCH UPDATE %s SET r = ? WHERE k = '0' AND c = '0'; APPLY BATCH;");
+ }
+
+ @Test
+ public void testCASWithIfNotExistsCondition() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text, c text, v text, s text STATIC, PRIMARY KEY(k, c))");
+
+ // partition key
+ testNoThreshold("INSERT INTO %s (k, c, v) VALUES (?, '0', '0') IF NOT EXISTS");
+
+ // clustering key
+ testNoThreshold("INSERT INTO %s (k, c, v) VALUES ('0', ?, '0') IF NOT EXISTS");
+
+ // static column
+ assertValid("INSERT INTO %s (k, s) VALUES ('1', ?) IF NOT EXISTS", allocate(1));
+ assertValid("INSERT INTO %s (k, s) VALUES ('2', ?) IF NOT EXISTS", allocate(WARN_THRESHOLD));
+ assertValid("INSERT INTO %s (k, s) VALUES ('2', ?) IF NOT EXISTS", allocate(WARN_THRESHOLD + 1)); // not applied
+ assertWarns("s", "INSERT INTO %s (k, s) VALUES ('3', ?) IF NOT EXISTS", allocate(WARN_THRESHOLD + 1));
+
+ // regular column
+ assertValid("INSERT INTO %s (k, c, v) VALUES ('4', '0', ?) IF NOT EXISTS", allocate(1));
+ assertValid("INSERT INTO %s (k, c, v) VALUES ('5', '0', ?) IF NOT EXISTS", allocate(WARN_THRESHOLD));
+ assertValid("INSERT INTO %s (k, c, v) VALUES ('5', '0', ?) IF NOT EXISTS", allocate(WARN_THRESHOLD + 1)); // not applied
+ assertWarns("v", "INSERT INTO %s (k, c, v) VALUES ('6', '0', ?) IF NOT EXISTS", allocate(WARN_THRESHOLD + 1));
+ }
+
+ @Test
+ public void testCASWithIfExistsCondition() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text, c text, v text, s text STATIC, PRIMARY KEY(k, c))");
+
+ // partition key, the CAS updates with values beyond the threshold are not applied so they don't come to fail
+ testNoThreshold("UPDATE %s SET v = '0' WHERE k = ? AND c = '0' IF EXISTS");
+
+ // clustering key, the CAS updates with values beyond the threshold are not applied so they don't come to fail
+ testNoThreshold("UPDATE %s SET v = '0' WHERE k = '0' AND c = ? IF EXISTS");
+
+ // static column, only the applied CAS updates can fire the guardrail
+ assertValid("INSERT INTO %s (k, s) VALUES ('0', '0')");
+ testThreshold("s", "UPDATE %s SET s = ? WHERE k = '0' IF EXISTS");
+ assertValid("DELETE FROM %s WHERE k = '0'");
+ testNoThreshold("UPDATE %s SET s = ? WHERE k = '0' IF EXISTS");
+
+ // regular column, only the applied CAS updates can fire the guardrail
+ assertValid("INSERT INTO %s (k, c) VALUES ('0', '0')");
+ testThreshold("v", "UPDATE %s SET v = ? WHERE k = '0' AND c = '0' IF EXISTS");
+ assertValid("DELETE FROM %s WHERE k = '0' AND c = '0'");
+ testNoThreshold("UPDATE %s SET v = ? WHERE k = '0' AND c = '0' IF EXISTS");
+ }
+
+ @Test
+ public void testCASWithColumnsCondition() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+
+ // updates are always accepted for values lesser than the threshold, independently of whether they are applied
+ assertValid("DELETE FROM %s WHERE k = 0");
+ assertValid("UPDATE %s SET v = ? WHERE k = 0 IF v = '0'", allocate(1));
+ assertValid("UPDATE %s SET v = '0' WHERE k = 0");
+ assertValid("UPDATE %s SET v = ? WHERE k = 0 IF v = '0'", allocate(1));
+
+ // updates are always accepted for values equals to the threshold, independently of whether they are applied
+ assertValid("DELETE FROM %s WHERE k = 0");
+ assertValid("UPDATE %s SET v = ? WHERE k = 0 IF v = '0'", allocate(WARN_THRESHOLD));
+ assertValid("UPDATE %s SET v = '0' WHERE k = 0");
+ assertValid("UPDATE %s SET v = ? WHERE k = 0 IF v = '0'", allocate(WARN_THRESHOLD));
+
+ // updates beyond the threshold fail only if the update is applied
+ assertValid("DELETE FROM %s WHERE k = 0");
+ assertValid("UPDATE %s SET v = ? WHERE k = 0 IF v = '0'", allocate(WARN_THRESHOLD + 1));
+ assertValid("UPDATE %s SET v = '0' WHERE k = 0");
+ assertWarns("v", "UPDATE %s SET v = ? WHERE k = 0 IF v = '0'", allocate(WARN_THRESHOLD + 1));
+ }
+
+ @Test
+ public void testSelect() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k text, c text, r text, s text STATIC, PRIMARY KEY(k, c))");
+
+ // the guardail is only checked for writes; reads are excluded
+
+ testNoThreshold("SELECT * FROM %s WHERE k = ?");
+ testNoThreshold("SELECT * FROM %s WHERE k = '0' AND c = ?");
+ testNoThreshold("SELECT * FROM %s WHERE c = ? ALLOW FILTERING");
+ testNoThreshold("SELECT * FROM %s WHERE s = ? ALLOW FILTERING");
+ testNoThreshold("SELECT * FROM %s WHERE r = ? ALLOW FILTERING");
+ }
+
+ /**
+ * Tests that the max column size guardrail threshold is not applied for the specified 1-placeholder CQL query.
+ *
+ * @param query a CQL modification statement with exactly one placeholder
+ */
+ private void testNoThreshold(String query) throws Throwable
+ {
+ assertValid(query, allocate(1));
+
+ assertValid(query, allocate(WARN_THRESHOLD));
+ assertValid(query, allocate(WARN_THRESHOLD + 1));
+
+ assertValid(query, allocate(FAIL_THRESHOLD));
+ assertValid(query, allocate(FAIL_THRESHOLD + 1));
+ }
+
+ /**
+ * Tests that the max column size guardrail threshold is not applied for the specified 2-placeholder CQL query.
+ *
+ * @param query a CQL modification statement with exactly two placeholders
+ */
+ private void testNoThreshold2(String query) throws Throwable
+ {
+ assertValid(query, allocate(1), allocate(1));
+
+ assertValid(query, allocate(WARN_THRESHOLD), allocate(1));
+ assertValid(query, allocate(1), allocate(WARN_THRESHOLD));
+ assertValid(query, allocate((WARN_THRESHOLD)), allocate((WARN_THRESHOLD)));
+ assertValid(query, allocate(WARN_THRESHOLD + 1), allocate(1));
+ assertValid(query, allocate(1), allocate(WARN_THRESHOLD + 1));
+
+ assertValid(query, allocate(FAIL_THRESHOLD), allocate(1));
+ assertValid(query, allocate(1), allocate(FAIL_THRESHOLD));
+ assertValid(query, allocate((FAIL_THRESHOLD)), allocate((FAIL_THRESHOLD)));
+ assertValid(query, allocate(FAIL_THRESHOLD + 1), allocate(1));
+ assertValid(query, allocate(1), allocate(FAIL_THRESHOLD + 1));
+ }
+
+ /**
+ * Tests that the max column size guardrail threshold is applied for the specified 1-placeholder CQL query.
+ *
+ * @param column the name of the column referenced by the query placeholder
+ * @param query a CQL query with exactly one placeholder
+ */
+ private void testThreshold(String column, String query) throws Throwable
+ {
+ testThreshold(column, query, 0);
+ }
+
+ /**
+ * Tests that the max column size guardrail threshold is applied for the specified 1-placeholder CQL query.
+ *
+ * @param column the name of the column referenced by the query placeholder
+ * @param query a CQL query with exactly one placeholder
+ * @param serializationBytes the extra bytes added to the placeholder value by its wrapping column type serializer
+ */
+ private void testThreshold(String column, String query, int serializationBytes) throws Throwable
+ {
+ int warn = WARN_THRESHOLD - serializationBytes;
+ int fail = FAIL_THRESHOLD - serializationBytes;
+
+ assertValid(query, allocate(0));
+ assertValid(query, allocate(warn));
+ assertWarns(column, query, allocate(warn + 1));
+ assertFails(column, query, allocate(fail + 1));
+ }
+
+ /**
+ * Tests that the max column size guardrail threshold is applied for the specified 2-placeholder CQL query.
+ *
+ * @param column the name of the column referenced by the placeholders
+ * @param query a CQL query with exactly two placeholders
+ */
+ private void testThreshold2(String column, String query) throws Throwable
+ {
+ testThreshold2(column, query, 0);
+ }
+
+ /**
+ * Tests that the max column size guardrail threshold is applied for the specified 2-placeholder query.
+ *
+ * @param column the name of the column referenced by the placeholders
+ * @param query a CQL query with exactly two placeholders
+ * @param serializationBytes the extra bytes added to the size of the placeholder value by their wrapping serializer
+ */
+ private void testThreshold2(String column, String query, int serializationBytes) throws Throwable
+ {
+ int warn = WARN_THRESHOLD - serializationBytes;
+ int fail = FAIL_THRESHOLD - serializationBytes;
+
+ assertValid(query, allocate(0), allocate(0));
+ assertValid(query, allocate(warn), allocate(0));
+ assertValid(query, allocate(0), allocate(warn));
+ assertValid(query, allocate(warn / 2), allocate(warn / 2));
+
+ assertWarns(column, query, allocate(warn + 1), allocate(0));
+ assertWarns(column, query, allocate(0), allocate(warn + 1));
+
+ assertFails(column, query, allocate(fail + 1), allocate(0));
+ assertFails(column, query, allocate(0), allocate(fail + 1));
+ }
+
+ private void testCollection(String column, String query, Function<ByteBuffer[], ByteBuffer> collectionBuilder) throws Throwable
+ {
+ assertValid(query, collectionBuilder, allocate(1));
+ assertValid(query, collectionBuilder, allocate(1), allocate(1));
+ assertValid(query, collectionBuilder, allocate(WARN_THRESHOLD));
+ assertValid(query, collectionBuilder, allocate(WARN_THRESHOLD), allocate(1));
+ assertValid(query, collectionBuilder, allocate(1), allocate(WARN_THRESHOLD));
+ assertValid(query, collectionBuilder, allocate(WARN_THRESHOLD), allocate(WARN_THRESHOLD));
+
+ assertWarns(column, query, collectionBuilder, allocate(WARN_THRESHOLD + 1));
+ assertWarns(column, query, collectionBuilder, allocate(WARN_THRESHOLD + 1), allocate(1));
+ assertWarns(column, query, collectionBuilder, allocate(1), allocate(WARN_THRESHOLD + 1));
+
+ assertFails(column, query, collectionBuilder, allocate(FAIL_THRESHOLD + 1));
+ assertFails(column, query, collectionBuilder, allocate(FAIL_THRESHOLD + 1), allocate(1));
+ assertFails(column, query, collectionBuilder, allocate(1), allocate(FAIL_THRESHOLD + 1));
+ }
+
+ private void testFrozenCollection(String column, String query, Function<ByteBuffer[], ByteBuffer> collectionBuilder) throws Throwable
+ {
+ assertValid(query, collectionBuilder, allocate(1));
+ assertValid(query, collectionBuilder, allocate(WARN_THRESHOLD - 8));
+ assertValid(query, collectionBuilder, allocate((WARN_THRESHOLD - 12) / 2), allocate((WARN_THRESHOLD - 12) / 2));
+
+ assertWarns(column, query, collectionBuilder, allocate(WARN_THRESHOLD - 7));
+ assertWarns(column, query, collectionBuilder, allocate(WARN_THRESHOLD - 12), allocate(1));
+
+ assertFails(column, query, collectionBuilder, allocate(FAIL_THRESHOLD - 7));
+ assertFails(column, query, collectionBuilder, allocate(FAIL_THRESHOLD - 12), allocate(1));
+ }
+
+ private void testMap(String column, String query) throws Throwable
+ {
+ assertValid(query, this::map, allocate(1), allocate(1));
+ assertValid(query, this::map, allocate(WARN_THRESHOLD), allocate(1));
+ assertValid(query, this::map, allocate(1), allocate(WARN_THRESHOLD));
+ assertValid(query, this::map, allocate(WARN_THRESHOLD), allocate(WARN_THRESHOLD));
+
+ assertWarns(column, query, this::map, allocate(1), allocate(WARN_THRESHOLD + 1));
+ assertWarns(column, query, this::map, allocate(WARN_THRESHOLD + 1), allocate(1));
+
+ assertFails(column, query, this::map, allocate(FAIL_THRESHOLD + 1), allocate(1));
+ assertFails(column, query, this::map, allocate(1), allocate(FAIL_THRESHOLD + 1));
+ assertFails(column, query, this::map, allocate(FAIL_THRESHOLD + 1), allocate(FAIL_THRESHOLD + 1));
+ }
+
+ private void assertValid(String query, ByteBuffer... values) throws Throwable
+ {
+ assertValid(() -> execute(query, values));
+ }
+
+ private void assertValid(String query, Function<ByteBuffer[], ByteBuffer> collectionBuilder, ByteBuffer... values) throws Throwable
+ {
+ assertValid(() -> execute(query, collectionBuilder.apply(values)));
+ }
+
+ private void assertWarns(String column, String query, Function<ByteBuffer[], ByteBuffer> collectionBuilder, ByteBuffer... values) throws Throwable
+ {
+ assertWarns(column, query, collectionBuilder.apply(values));
+ }
+
+ private void assertWarns(String column, String query, ByteBuffer... values) throws Throwable
+ {
+ String errorMessage = format("Value of column %s has size %s, this exceeds the warning threshold of %s.",
+ column, WARN_THRESHOLD + 1, WARN_THRESHOLD);
+ assertWarns(() -> execute(query, values), errorMessage);
+ }
+
+ private void assertFails(String column, String query, Function<ByteBuffer[], ByteBuffer> collectionBuilder, ByteBuffer... values) throws Throwable
+ {
+ assertFails(column, query, collectionBuilder.apply(values));
+ }
+
+ private void assertFails(String column, String query, ByteBuffer... values) throws Throwable
+ {
+ String errorMessage = format("Value of column %s has size %s, this exceeds the failure threshold of %s.",
+ column, FAIL_THRESHOLD + 1, FAIL_THRESHOLD);
+ assertFails(() -> execute(query, values), errorMessage);
+ }
+
+ private void execute(String query, ByteBuffer... values)
+ {
+ execute(userClientState, query, Arrays.asList(values));
+ }
+
+ private ByteBuffer set(ByteBuffer... values)
+ {
+ return SetType.getInstance(BytesType.instance, true).decompose(ImmutableSet.copyOf(values));
+ }
+
+ private ByteBuffer list(ByteBuffer... values)
+ {
+ return ListType.getInstance(BytesType.instance, true).decompose(ImmutableList.copyOf(values));
+ }
+
+ private ByteBuffer map(ByteBuffer... values)
+ {
+ assert values.length % 2 == 0;
+
+ int size = values.length / 2;
+ Map<ByteBuffer, ByteBuffer> m = new LinkedHashMap<>(size);
+ for (int i = 0; i < size; i++)
+ m.put(values[2 * i], values[(2 * i) + 1]);
+
+ return MapType.getInstance(BytesType.instance, BytesType.instance, true).decompose(m);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailDropKeyspaceTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailDropKeyspaceTest.java
new file mode 100644
index 0000000..de44725
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailDropKeyspaceTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.After;
+import org.junit.Test;
+
+public class GuardrailDropKeyspaceTest extends GuardrailTester
+{
+ private String keyspaceQuery = "CREATE KEYSPACE dkdt WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}";
+
+ private void setGuardrail(boolean enabled)
+ {
+ Guardrails.instance.setDropKeyspaceEnabled(enabled);
+ }
+
+ public GuardrailDropKeyspaceTest()
+ {
+ super(Guardrails.dropKeyspaceEnabled);
+ }
+
+ @After
+ public void afterTest() throws Throwable
+ {
+ setGuardrail(true);
+ execute("DROP KEYSPACE IF EXISTS dkdt");
+ }
+
+ @Test
+ public void testCanDropWhileFeatureEnabled() throws Throwable
+ {
+ setGuardrail(true);
+ createKeyspace(keyspaceQuery);
+ execute("DROP KEYSPACE dkdt");
+ }
+
+ @Test
+ public void testCannotDropWhileFeatureDisabled() throws Throwable
+ {
+ setGuardrail(false);
+ createKeyspace(keyspaceQuery);
+ assertFails("DROP KEYSPACE dkdt", "DROP KEYSPACE functionality is not allowed");
+ }
+
+ @Test
+ public void testIfExistsDoesNotBypassCheck() throws Throwable
+ {
+ setGuardrail(false);
+ createKeyspace(keyspaceQuery);
+ assertFails("DROP KEYSPACE IF EXISTS dkdt", "DROP KEYSPACE functionality is not allowed");
+ }
+
+ @Test
+ public void testToggle() throws Throwable
+ {
+ setGuardrail(false);
+ createKeyspace(keyspaceQuery);
+ assertFails("DROP KEYSPACE IF EXISTS dkdt", "DROP KEYSPACE functionality is not allowed");
+
+ setGuardrail(true);
+ execute("DROP KEYSPACE dkdt");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java
new file mode 100644
index 0000000..865ac23
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.StorageService;
+import org.assertj.core.api.Assertions;
+
+import static java.lang.String.format;
+
+public class GuardrailMaximumReplicationFactorTest extends ThresholdTester
+{
+ private static final int MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD = 2;
+ private static final int MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD = 4;
+ private static final int DISABLED_GUARDRAIL = -1;
+
+ public GuardrailMaximumReplicationFactorTest()
+ {
+ super(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD,
+ MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD,
+ Guardrails.maximumReplicationFactor,
+ Guardrails::setMaximumReplicationFactorThreshold,
+ Guardrails::getMaximumReplicationFactorWarnThreshold,
+ Guardrails::getMaximumReplicationFactorFailThreshold);
+ }
+
+ @After
+ public void cleanupTest() throws Throwable
+ {
+ execute("DROP KEYSPACE IF EXISTS ks");
+ DatabaseDescriptor.setDefaultKeyspaceRF(1);
+ }
+
+ @Override
+ protected long currentValue()
+ {
+ return Long.parseLong((Keyspace.open("ks").getReplicationStrategy()).configOptions.get("datacenter1"));
+ }
+
+ @Override
+ protected List<String> getWarnings()
+ {
+ List<String> warnings = ClientWarn.instance.getWarnings();
+
+ // filtering out non-guardrails produced warnings
+ return warnings == null
+ ? Collections.emptyList()
+ : warnings.stream()
+ .filter(w -> !w.contains("keyspace ks is higher than the number of nodes 1 for datacenter1") &&
+ !w.contains("When increasing replication factor you need to run a full (-full) repair to distribute the data") &&
+ !w.contains("keyspace ks is higher than the number of nodes") &&
+ !w.contains("Your replication factor 3 for keyspace ks is higher than the number of nodes 2 for datacenter datacenter2"))
+ .collect(Collectors.toList());
+ }
+
+ @Test
+ public void testMaxKeyspaceRFDisabled() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(DISABLED_GUARDRAIL, DISABLED_GUARDRAIL);
+ assertMaxThresholdValid("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 6}");
+ assertMaxThresholdValid("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 10}");
+ }
+
+ @Test
+ public void testSimpleStrategyCreate() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 3}", 3);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 5}", 5);
+ }
+
+ @Test
+ public void testSimpleStrategyAlter() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ execute("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 2}");
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 3}", 3);
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 5}", 5);
+ }
+
+ @Test
+ public void testMultipleDatacenter() throws Throwable
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ public static final String RACK1 = ServerTestUtils.RACK1;
+
+ @Override
+ public String getRack(InetAddressAndPort endpoint) { return RACK1; }
+
+ @Override
+ public String getDatacenter(InetAddressAndPort endpoint) { return "datacenter2"; }
+
+ @Override
+ public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; }
+ });
+
+ List<String> twoWarnings = Arrays.asList(format("The keyspace ks has a replication factor of 3, above the warning threshold of %s.", MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD),
+ format("The keyspace ks has a replication factor of 3, above the warning threshold of %s.", MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
+
+ StorageService.instance.getTokenMetadata().updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.0.0.255"));
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertValid("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 2}");
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 3}", 3);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 3, 'datacenter2' : 3}", twoWarnings);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 5}", 5);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 5, 'datacenter2' : 5}", 5);
+ execute("DROP KEYSPACE IF EXISTS ks");
+
+ execute("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 1, 'datacenter2' : 1}");
+ assertValid("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 2}");
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 3}", 3);
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 3, 'datacenter2' : 3}", twoWarnings);
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 5}", 5);
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 5, 'datacenter2' : 5}", 5);
+
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+ }
+
+ @Test
+ public void testMaxKeyspaceRFOnlyWarnBelow() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, DISABLED_GUARDRAIL);
+ assertMaxThresholdValid("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}");
+ assertMaxThresholdValid("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}");
+ }
+
+ @Test
+ public void testMaxKeyspaceRFOnlyWarnAbove() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, DISABLED_GUARDRAIL);
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}", 3);
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 4}", 4);
+ }
+
+ @Test
+ public void testMaxKeyspaceRFOnlyFailBelow() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(DISABLED_GUARDRAIL, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertMaxThresholdValid("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}");
+ assertMaxThresholdValid("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}");
+ }
+
+ @Test
+ public void testMaxKeyspaceRFOnlyFailAbove() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(DISABLED_GUARDRAIL, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 5}", 5);
+ }
+
+ @Test
+ public void testMaxKeyspaceRFOnlyFailAboveAlter() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(DISABLED_GUARDRAIL, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ execute("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}");
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 6}", 6);
+ }
+
+ @Test
+ public void testMaxKeyspaceRFWarnBelow() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertMaxThresholdValid("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}");
+ assertMaxThresholdValid("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}");
+ }
+
+ @Test
+ public void testMaxKeyspaceRFWarnFailBetween() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}", 3);
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 4}", 4);
+ }
+
+ @Test
+ public void testMaxKeyspaceRFFailAbove() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 5}", 5);
+ }
+
+ @Test
+ public void testMaxKeyspaceRFFailAboveAlter() throws Throwable
+ {
+ guardrails().setMaximumReplicationFactorThreshold(MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ execute("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 4}");
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 5}", 5);
+ }
+
+ @Test
+ public void testMaxRFLesserThanDefaultRF()
+ {
+ DatabaseDescriptor.setDefaultKeyspaceRF(3);
+ Assertions.assertThatThrownBy(() -> guardrails().setMaximumReplicationFactorThreshold(1, 2))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("maximum_replication_factor_fail_threshold to be set (2) cannot be lesser than default_keyspace_rf (3)");
+
+ DatabaseDescriptor.setDefaultKeyspaceRF(1);
+ guardrails().setMaximumReplicationFactorThreshold(1, 2);
+ Assertions.assertThatThrownBy(() -> DatabaseDescriptor.setDefaultKeyspaceRF(3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("default_keyspace_rf to be set (3) cannot be greater than maximum_replication_factor_fail_threshold (2)");
+ }
+
+ private void assertWarns(String query, int rf) throws Throwable
+ {
+ assertWarns(query, format("The keyspace ks has a replication factor of %d, above the warning threshold of %s.",
+ rf, MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
+ }
+
+ private void assertFails(String query, int rf) throws Throwable
+ {
+ assertFails(query, format("The keyspace ks has a replication factor of %d, above the failure threshold of %s.",
+ rf, MAXIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumTimestampTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumTimestampTest.java
new file mode 100644
index 0000000..a5c31ad
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumTimestampTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.service.ClientState;
+
+public class GuardrailMaximumTimestampTest extends ThresholdTester
+{
+ public GuardrailMaximumTimestampTest()
+ {
+ super(TimeUnit.DAYS.toSeconds(1) + "s",
+ TimeUnit.DAYS.toSeconds(2) + "s",
+ Guardrails.maximumAllowableTimestamp,
+ Guardrails::setMaximumTimestampThreshold,
+ Guardrails::getMaximumTimestampWarnThreshold,
+ Guardrails::getMaximumTimestampFailThreshold,
+ micros -> new DurationSpec.LongMicrosecondsBound(micros, TimeUnit.MICROSECONDS).toString(),
+ micros -> new DurationSpec.LongMicrosecondsBound(micros).toMicroseconds());
+ }
+
+ @Before
+ public void setupTest()
+ {
+ createTable("CREATE TABLE IF NOT EXISTS %s (k int primary key, v int)");
+ }
+
+ @Test
+ public void testDisabledAllowsAnyTimestamp() throws Throwable
+ {
+ guardrails().setMaximumTimestampThreshold(null, null);
+ assertValid("INSERT INTO %s (k, v) VALUES (2, 2) USING TIMESTAMP " + (Long.MAX_VALUE - 1));
+ }
+
+ @Test
+ public void testEnabledFail() throws Throwable
+ {
+ assertFails("INSERT INTO %s (k, v) VALUES (2, 2) USING TIMESTAMP " + (Long.MAX_VALUE - 1), "maximum_timestamp violated");
+ }
+
+ @Test
+ public void testEnabledInRange() throws Throwable
+ {
+ assertValid("INSERT INTO %s (k, v) VALUES (1, 1) USING TIMESTAMP " + ClientState.getTimestamp());
+ }
+
+ @Test
+ public void testEnabledWarn() throws Throwable
+ {
+ assertWarns("INSERT INTO %s (k, v) VALUES (1, 1) USING TIMESTAMP " + (ClientState.getTimestamp() + (TimeUnit.DAYS.toMicros(1) + 40000)),
+ "maximum_timestamp violated");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java
index d3df983..8817f9a 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java
@@ -89,6 +89,7 @@
{
List<String> warnings = ClientWarn.instance.getWarnings();
+ // filtering out non-guardrails produced warnings
return warnings == null
? Collections.emptyList()
: warnings.stream()
@@ -132,13 +133,21 @@
}
@Test
- public void testSimpleStrategy() throws Throwable
+ public void testSimpleStrategyCreate() throws Throwable
{
guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
- assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 3}",
- format("The keyspace %s has a replication factor of 3, below the warning threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
- assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 1}",
- format("The keyspace %s has a replication factor of 1, below the failure threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 3}", 3);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 1}", 1);
+ }
+
+ @Test
+ public void testSimpleStrategyAlter() throws Throwable
+ {
+ guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
+ execute("CREATE KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 4}");
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 3}", 3);
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': 1}", 1);
}
@Test
@@ -161,17 +170,26 @@
List<String> twoWarnings = Arrays.asList(format("The keyspace %s has a replication factor of 2, below the warning threshold of %d.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD),
format("The keyspace %s has a replication factor of 2, below the warning threshold of %d.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
-
+
StorageService.instance.getTokenMetadata().updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.0.0.255"));
guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
- assertValid("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 4 };");
- assertWarns("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 2 };",
- format("The keyspace %s has a replication factor of 2, below the warning threshold of %d.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
- assertWarns("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 2 };", twoWarnings);
- assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 1 };",
- format("The keyspace %s has a replication factor of 1, below the failure threshold of %d.", KS, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
- assertFails("CREATE KEYSPACE ks1 WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 1, 'datacenter2' : 1 };",
- format("The keyspace ks1 has a replication factor of 1, below the failure threshold of %d.", MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ assertValid("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 4 }");
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 2 }", 2);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 2 }", twoWarnings);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 1 }", 1);
+ execute("DROP KEYSPACE IF EXISTS ks");
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 1, 'datacenter2' : 1 }", 1);
+ execute("DROP KEYSPACE IF EXISTS ks");
+
+ execute("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 5, 'datacenter2' : 5}");
+ assertValid("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 4 }");
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 2 }", 2);
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 2 }", twoWarnings);
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 1 }", 1);
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 1, 'datacenter2' : 1 }", 1);
DatabaseDescriptor.setEndpointSnitch(snitch);
execute("DROP KEYSPACE IF EXISTS ks1");
@@ -189,10 +207,8 @@
public void testMinKeyspaceRFOnlyWarnBelow() throws Throwable
{
guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, DISABLED_GUARDRAIL);
- assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}",
- format("The keyspace %s has a replication factor of 3, below the warning threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
- assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}",
- format("The keyspace %s has a replication factor of 2, below the warning threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}", 3);
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}", 2);
}
@Test
@@ -207,8 +223,7 @@
public void testMinKeyspaceRFOnlyFailBelow() throws Throwable
{
guardrails().setMinimumReplicationFactorThreshold(DISABLED_GUARDRAIL, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
- assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}",
- format("The keyspace %s has a replication factor of 1, below the failure threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}", 1);
}
@Test
@@ -216,8 +231,7 @@
{
guardrails().setMinimumReplicationFactorThreshold(DISABLED_GUARDRAIL, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
execute("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}");
- assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}",
- format("The keyspace %s has a replication factor of 1, below the failure threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}", 1);
}
@Test
@@ -232,18 +246,15 @@
public void testMinKeyspaceRFWarnFailBetween() throws Throwable
{
guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
- assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}",
- format("The keyspace %s has a replication factor of 3, below the warning threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
- assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}",
- format("The keyspace %s has a replication factor of 2, below the warning threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
+ assertWarns("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 3}", 3);
+ assertWarns("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 2}", 2);
}
@Test
public void testMinKeyspaceRFFailBelow() throws Throwable
{
guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
- assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}",
- format("The keyspace %s has a replication factor of 1, below the failure threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ assertFails("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}", 1);
}
@Test
@@ -251,26 +262,33 @@
{
guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
execute("CREATE KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 4}");
- assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}",
- format("The keyspace %s has a replication factor of 1, below the failure threshold of %s.", KS, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
+ assertFails("ALTER KEYSPACE ks WITH replication = { 'class': 'NetworkTopologyStrategy', 'datacenter1': 1}", 1);
}
@Test
public void testMinRFGreaterThanDefaultRF()
{
- try
- {
- DatabaseDescriptor.setDefaultKeyspaceRF(1);
- guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD);
- }
- catch (IllegalArgumentException e)
- {
- String expectedMessage = "";
+ DatabaseDescriptor.setDefaultKeyspaceRF(3);
+ Assertions.assertThatThrownBy(() -> guardrails().setMinimumReplicationFactorThreshold(5, 4))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("minimum_replication_factor_fail_threshold to be set (4) cannot be greater than default_keyspace_rf (3)");
- if(guardrails().getMinimumReplicationFactorFailThreshold() > DatabaseDescriptor.getDefaultKeyspaceRF())
- expectedMessage = format("%s_fail_threshold to be set (%d) cannot be greater than default_keyspace_rf (%d)",
- WHAT, guardrails().getMinimumReplicationFactorFailThreshold(), DatabaseDescriptor.getDefaultKeyspaceRF());
- Assertions.assertThat(e.getMessage()).contains(expectedMessage);
- }
+ DatabaseDescriptor.setDefaultKeyspaceRF(6);
+ guardrails().setMinimumReplicationFactorThreshold(5, 4);
+ Assertions.assertThatThrownBy(() -> DatabaseDescriptor.setDefaultKeyspaceRF(3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("default_keyspace_rf to be set (3) cannot be less than minimum_replication_factor_fail_threshold (4)");
+ }
+
+ private void assertWarns(String query, int rf) throws Throwable
+ {
+ assertWarns(query, format("The keyspace ks has a replication factor of %d, below the warning threshold of %s.",
+ rf, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD));
+ }
+
+ private void assertFails(String query, int rf) throws Throwable
+ {
+ assertFails(query, format("The keyspace ks has a replication factor of %d, below the failure threshold of %s.",
+ rf, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD));
}
}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumTimestampTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumTimestampTest.java
new file mode 100644
index 0000000..87169fc
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumTimestampTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.service.ClientState;
+
+public class GuardrailMinimumTimestampTest extends ThresholdTester
+{
+ public GuardrailMinimumTimestampTest()
+ {
+ super(TimeUnit.DAYS.toSeconds(1) + "s",
+ TimeUnit.DAYS.toSeconds(2) + "s",
+ Guardrails.minimumAllowableTimestamp,
+ Guardrails::setMinimumTimestampThreshold,
+ Guardrails::getMinimumTimestampWarnThreshold,
+ Guardrails::getMinimumTimestampFailThreshold,
+ micros -> new DurationSpec.LongMicrosecondsBound(micros, TimeUnit.MICROSECONDS).toString(),
+ micros -> new DurationSpec.LongMicrosecondsBound(micros).toMicroseconds());
+ }
+
+ @Before
+ public void setupTest()
+ {
+ createTable("CREATE TABLE IF NOT EXISTS %s (k int primary key, v int)");
+ }
+
+ @Test
+ public void testDisabled() throws Throwable
+ {
+ guardrails().setMinimumTimestampThreshold(null, null);
+ assertValid("INSERT INTO %s (k, v) VALUES (1, 1) USING TIMESTAMP 12345");
+ }
+
+ @Test
+ public void testEnabledFailure() throws Throwable
+ {
+ assertFails("INSERT INTO %s (k, v) VALUES (1, 1) USING TIMESTAMP 12345", "minimum_timestamp violated");
+ }
+
+ @Test
+ public void testEnabledInRange() throws Throwable
+ {
+ assertValid("INSERT INTO %s (k, v) VALUES (1, 1) USING TIMESTAMP " + ClientState.getTimestamp());
+ }
+
+ @Test
+ public void testEnabledWarn() throws Throwable
+ {
+ assertWarns("INSERT INTO %s (k, v) VALUES (1, 1) USING TIMESTAMP " + (ClientState.getTimestamp() - (TimeUnit.DAYS.toMicros(1) + 40000)),
+ "minimum_timestamp violated");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailPartitionSizeTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPartitionSizeTest.java
new file mode 100644
index 0000000..cf48390
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailPartitionSizeTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.db.marshal.Int32Type;
+
+import static java.nio.ByteBuffer.allocate;
+import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.BYTES;
+
+/**
+ * Tests the guardrail for the size of partitions, {@link Guardrails#partitionSize}.
+ * <p>
+ * The emission on unredacted log messages is tested in {@link org.apache.cassandra.distributed.test.guardrails.GuardrailPartitionSizeTest}.
+ */
+public class GuardrailPartitionSizeTest extends ThresholdTester
+{
+ private static final int WARN_THRESHOLD = 1024 * 1024; // bytes (1 MiB)
+ private static final int FAIL_THRESHOLD = WARN_THRESHOLD * 2; // bytes (2 MiB)
+ private static final int NUM_CLUSTERINGS = 10;
+ private static final String REDACTED_MESSAGE = "Guardrail partition_size violated: Partition <redacted> has size";
+
+ public GuardrailPartitionSizeTest()
+ {
+ super(WARN_THRESHOLD + "B",
+ FAIL_THRESHOLD + "B",
+ Guardrails.partitionSize,
+ Guardrails::setPartitionSizeThreshold,
+ Guardrails::getPartitionSizeWarnThreshold,
+ Guardrails::getPartitionSizeFailThreshold,
+ bytes -> new DataStorageSpec.LongBytesBound(bytes, BYTES).toString(),
+ size -> new DataStorageSpec.LongBytesBound(size).toBytes());
+ }
+
+ @Test
+ public void testPartitionSizeEnabled() throws Throwable
+ {
+ // Insert enough data to trigger the warning guardrail, but not the failure one.
+ populateTable(WARN_THRESHOLD);
+
+ flush();
+ listener.assertWarned(REDACTED_MESSAGE);
+ listener.clear();
+
+ compact();
+ listener.assertWarned(REDACTED_MESSAGE);
+ listener.clear();
+
+ // Insert enough data to trigger the failure guardrail.
+ populateTable(FAIL_THRESHOLD);
+
+ flush();
+ listener.assertFailed(REDACTED_MESSAGE);
+ listener.clear();
+
+ compact();
+ listener.assertFailed(REDACTED_MESSAGE);
+ listener.clear();
+
+ // remove most of the data to be under the threshold again
+ assertValid("DELETE FROM %s WHERE k = 1 AND c > 0");
+
+ flush();
+ listener.assertNotWarned();
+ listener.assertNotFailed();
+
+ compact();
+ listener.assertNotWarned();
+ listener.assertNotFailed();
+ listener.clear();
+ }
+
+ @Test
+ public void testPartitionSizeDisabled() throws Throwable
+ {
+ guardrails().setPartitionSizeThreshold(null, null);
+
+ populateTable(FAIL_THRESHOLD);
+
+ flush();
+ listener.assertNotWarned();
+ listener.assertNotFailed();
+
+ compact();
+ listener.assertNotWarned();
+ listener.assertNotFailed();
+ }
+
+ private void populateTable(int threshold) throws Throwable
+ {
+ createTable("CREATE TABLE IF NOT EXISTS %s (k int, c int, v blob, PRIMARY KEY(k, c))");
+ disableCompaction();
+
+ for (int i = 0; i < NUM_CLUSTERINGS; i++)
+ {
+ final int c = i;
+ assertValid(() -> execute(userClientState,
+ "INSERT INTO %s (k, c, v) VALUES (1, ?, ?)",
+ Arrays.asList(Int32Type.instance.decompose(c),
+ allocate(threshold / NUM_CLUSTERINGS))));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailSimpleStrategyTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSimpleStrategyTest.java
new file mode 100644
index 0000000..3cc6bc7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailSimpleStrategyTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.After;
+import org.junit.Test;
+
+public class GuardrailSimpleStrategyTest extends GuardrailTester
+{
+ public static String ERROR_MSG = "SimpleStrategy is not allowed";
+
+ public GuardrailSimpleStrategyTest()
+ {
+ super(Guardrails.simpleStrategyEnabled);
+ }
+
+ private void setGuardrail(boolean enabled)
+ {
+ guardrails().setSimpleStrategyEnabled(enabled);
+ }
+
+ @After
+ public void afterTest() throws Throwable
+ {
+ setGuardrail(true);
+ execute("DROP KEYSPACE IF EXISTS test_ss;");
+ }
+
+ @Test
+ public void testCanCreateWithGuardrailEnabled() throws Throwable
+ {
+ assertValid("CREATE KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};");
+ }
+
+ @Test
+ public void testCanAlterWithGuardrailEnabled() throws Throwable
+ {
+ execute("CREATE KEYSPACE test_ss WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1':2, 'datacenter2':0};");
+ assertValid("ALTER KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};");
+ }
+
+ @Test
+ public void testGuardrailBlocksCreate() throws Throwable
+ {
+ setGuardrail(false);
+ assertFails("CREATE KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};", ERROR_MSG);
+ }
+
+ @Test
+ public void testGuardrailBlocksAlter() throws Throwable
+ {
+ setGuardrail(false);
+ execute("CREATE KEYSPACE test_ss WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1':2, 'datacenter2':0};");
+ assertFails("ALTER KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};", ERROR_MSG);
+ }
+
+ @Test
+ public void testToggle() throws Throwable
+ {
+ setGuardrail(false);
+ assertFails("CREATE KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};", ERROR_MSG);
+
+ setGuardrail(true);
+ assertValid("CREATE KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};");
+ execute("ALTER KEYSPACE test_ss WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1':2, 'datacenter2':0};");
+
+ setGuardrail(false);
+ assertFails("ALTER KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};", ERROR_MSG);
+
+ setGuardrail(true);
+ assertValid("ALTER KEYSPACE test_ss WITH replication = {'class': 'SimpleStrategy'};");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java
index 7c94702..f2744ab 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTester.java
@@ -67,6 +67,18 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+/**
+ * This class provides specific utility methods for testing Guardrails that should be used instead of the provided
+ * {@link CQLTester} methods. Many of the methods in CQLTester don't respect the {@link ClientState} provided for a query
+ * and instead use {@link ClientState#forInternalCalls()} which flags as an internal query and thus bypasses auth and
+ * guardrail checks.
+ *
+ * Some GuardrailTester methods and their usage is as follows:
+ * {@link GuardrailTester#assertValid(String)} to confirm the query as structured is valid given the state of the db
+ * {@link GuardrailTester#assertWarns(String, String)} to confirm a query succeeds but warns the text provided
+ * {@link GuardrailTester#assertFails(String, String)} to confirm a query fails with the message provided
+ * {@link GuardrailTester#testExcludedUsers} to confirm superusers are excluded from application of the guardrail
+ */
public abstract class GuardrailTester extends CQLTester
{
// Name used when testing CREATE TABLE that should be aborted (we need to provide it as assertFails, which
@@ -106,7 +118,15 @@
DatabaseDescriptor.setDiagnosticEventsEnabled(true);
systemClientState = ClientState.forInternalCalls();
+
userClientState = ClientState.forExternalCalls(InetSocketAddress.createUnresolved("127.0.0.1", 123));
+ AuthenticatedUser user = new AuthenticatedUser(USERNAME)
+ {
+ @Override
+ public boolean canLogin() { return true; }
+ };
+ userClientState.login(user);
+
superClientState = ClientState.forExternalCalls(InetSocketAddress.createUnresolved("127.0.0.1", 321));
superClientState.login(new AuthenticatedUser(CassandraRoleManager.DEFAULT_SUPERUSER_NAME));
}
@@ -318,6 +338,10 @@
assertFails(function, true, messages, redactedMessages);
}
+ /**
+ * Unlike {@link CQLTester#assertInvalidThrowMessage}, the chain of methods ending here in {@link GuardrailTester}
+ * respect the input ClientState so guardrails permissions will be correctly checked.
+ */
protected void assertFails(CheckedFunction function, boolean thrown, List<String> messages, List<String> redactedMessages) throws Throwable
{
ClientWarn.instance.captureWarnings();
@@ -337,9 +361,17 @@
if (guardrail != null)
{
- String prefix = guardrail.decorateMessage("");
- assertTrue(format("Full error message '%s' doesn't start with the prefix '%s'", e.getMessage(), prefix),
- e.getMessage().startsWith(prefix));
+ String message = e.getMessage();
+ String prefix = guardrail.decorateMessage("").replace(". " + guardrail.reason, "");
+ assertTrue(format("Full error message '%s' doesn't start with the prefix '%s'", message, prefix),
+ message.startsWith(prefix));
+
+ String reason = guardrail.reason;
+ if (reason != null)
+ {
+ assertTrue(format("Full error message '%s' doesn't end with the reason '%s'", message, reason),
+ message.endsWith(reason));
+ }
}
assertTrue(format("Full error message '%s' does not contain expected message '%s'", e.getMessage(), failMessage),
@@ -396,9 +428,16 @@
String warning = warnings.get(i);
if (guardrail != null)
{
- String prefix = guardrail.decorateMessage("");
+ String prefix = guardrail.decorateMessage("").replace(". " + guardrail.reason, "");
assertTrue(format("Warning log message '%s' doesn't start with the prefix '%s'", warning, prefix),
warning.startsWith(prefix));
+
+ String reason = guardrail.reason;
+ if (reason != null)
+ {
+ assertTrue(format("Warning log message '%s' doesn't end with the reason '%s'", warning, reason),
+ warning.endsWith(reason));
+ }
}
assertTrue(format("Warning log message '%s' does not contain expected message '%s'", warning, message),
@@ -478,6 +517,10 @@
return execute(state, query, options);
}
+ /**
+ * Performs execution of query using the input {@link ClientState} (i.e. unlike {@link ClientState#forInternalCalls()}
+ * which may not) to ensure guardrails are approprieately applied to the query provided.
+ */
protected ResultMessage execute(ClientState state, String query, QueryOptions options)
{
QueryState queryState = new QueryState(state);
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailZeroDefaultTTLOnTWCSTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailZeroDefaultTTLOnTWCSTest.java
new file mode 100644
index 0000000..afd00e6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailZeroDefaultTTLOnTWCSTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
+
+public class GuardrailZeroDefaultTTLOnTWCSTest extends GuardrailTester
+{
+ private static final String QUERY = "CREATE TABLE IF NOT EXISTS tb1 (k int PRIMARY KEY, a int, b int) " +
+ "WITH default_time_to_live = 0 " +
+ "AND compaction = {'class': 'TimeWindowCompactionStrategy', 'enabled': true }";
+
+ private static final String VALID_QUERY_1 = "CREATE TABLE IF NOT EXISTS tb2 (k int PRIMARY KEY, a int, b int) " +
+ "WITH default_time_to_live = 1 " +
+ "AND compaction = {'class': 'TimeWindowCompactionStrategy', 'enabled': true }";
+
+ private static final String VALID_QUERY_2 = "CREATE TABLE IF NOT EXISTS tb3 (k int PRIMARY KEY, a int, b int) " +
+ "WITH default_time_to_live = 0";
+
+ public GuardrailZeroDefaultTTLOnTWCSTest()
+ {
+ super(Guardrails.zeroTTLOnTWCSEnabled);
+ }
+
+ @Test
+ public void testGuardrailDisabled() throws Throwable
+ {
+ prepareTest(false, true);
+ assertFails(QUERY, "0 default_time_to_live on a table with " +
+ TimeWindowCompactionStrategy.class.getSimpleName() +
+ " compaction strategy is not allowed");
+ }
+
+ @Test
+ public void testGuardrailEnabledWarnEnabled() throws Throwable
+ {
+ prepareTest(true, true);
+ assertWarns(QUERY, "0 default_time_to_live on a table with " +
+ TimeWindowCompactionStrategy.class.getSimpleName() +
+ " compaction strategy is not recommended");
+ }
+
+ @Test
+ public void testGuardrailEnabledWarnDisabled() throws Throwable
+ {
+ prepareTest(true, false);
+ assertValid(QUERY);
+ }
+
+ @Test
+ public void testGuardrailNotTriggered() throws Throwable
+ {
+ prepareTest(true, true);
+ assertValid(VALID_QUERY_1);
+ assertValid(VALID_QUERY_2);
+
+ prepareTest(false, true);
+ assertValid(VALID_QUERY_1);
+ assertValid(VALID_QUERY_2);
+ }
+
+ @Test
+ public void testExcludedUsers() throws Throwable
+ {
+ for (boolean enabled : new boolean[] { false, true })
+ {
+ for (boolean warned : new boolean[]{ false, true })
+ {
+ prepareTest(enabled, warned);
+ testExcludedUsers(() -> QUERY,
+ () -> VALID_QUERY_1,
+ () -> VALID_QUERY_2);
+ }
+ }
+ }
+
+ private void prepareTest(boolean enabled, boolean warned)
+ {
+ guardrails().setZeroTTLOnTWCSEnabled(enabled);
+ guardrails().setZeroTTLOnTWCSWarned(warned);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailsConfigProviderTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailsConfigProviderTest.java
index e99b736..36fbc7c 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailsConfigProviderTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailsConfigProviderTest.java
@@ -37,6 +37,7 @@
String name = getClass().getCanonicalName() + '$' + CustomProvider.class.getSimpleName();
GuardrailsConfigProvider provider = GuardrailsConfigProvider.build(name);
MaxThreshold guard = new MaxThreshold("test_guardrail",
+ "Some reason",
state -> provider.getOrCreate(state).getTablesWarnThreshold(),
state -> provider.getOrCreate(state).getTablesFailThreshold(),
(isWarn, what, v, t) -> format("%s: for %s, %s > %s",
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailsTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailsTest.java
index a0a5823..f143120 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailsTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailsTest.java
@@ -38,7 +38,8 @@
public class GuardrailsTest extends GuardrailTester
{
public static final int DISABLED = -1;
-
+ public static final String REASON = "Testing";
+ public static final String FEATURE = "Feature name";
private void testDisabledThreshold(Threshold guard) throws Throwable
{
@@ -58,17 +59,18 @@
public void testDisabledMaxThreshold() throws Throwable
{
Threshold.ErrorMessageProvider errorMessageProvider = (isWarn, what, v, t) -> "Should never trigger";
- testDisabledThreshold(new MaxThreshold("x", state -> DISABLED, state -> DISABLED, errorMessageProvider));
+ testDisabledThreshold(new MaxThreshold("x", REASON, state -> DISABLED, state -> DISABLED, errorMessageProvider));
}
@Test
public void testMaxThreshold() throws Throwable
{
MaxThreshold guard = new MaxThreshold("x",
- state -> 10,
- state -> 100,
- (isWarn, what, v, t) -> format("%s: for %s, %s > %s",
- isWarn ? "Warning" : "Aborting", what, v, t));
+ REASON,
+ state -> 10,
+ state -> 100,
+ (isWarn, featureName, v, t) -> format("%s: for %s, %s > %s",
+ isWarn ? "Warning" : "Aborting", featureName, v, t));
assertTrue(guard.enabled(userClientState));
@@ -91,10 +93,11 @@
public void testWarnOnlyMaxThreshold() throws Throwable
{
MaxThreshold guard = new MaxThreshold("x",
- state -> 10,
- state -> DISABLED,
- (isWarn, what, v, t) -> format("%s: for %s, %s > %s",
- isWarn ? "Warning" : "Aborting", what, v, t));
+ REASON,
+ state -> 10,
+ state -> DISABLED,
+ (isWarn, featureName, v, t) -> format("%s: for %s, %s > %s",
+ isWarn ? "Warning" : "Aborting", featureName, v, t));
assertTrue(guard.enabled(userClientState));
@@ -109,10 +112,11 @@
public void testFailOnlyMaxThreshold() throws Throwable
{
MaxThreshold guard = new MaxThreshold("x",
- state -> DISABLED,
- state -> 10,
- (isWarn, what, v, t) -> format("%s: for %s, %s > %s",
- isWarn ? "Warning" : "Aborting", what, v, t));
+ REASON,
+ state -> DISABLED,
+ state -> 10,
+ (isWarn, featureName, v, t) -> format("%s: for %s, %s > %s",
+ isWarn ? "Warning" : "Aborting", featureName, v, t));
assertTrue(guard.enabled(userClientState));
@@ -127,10 +131,11 @@
public void testMaxThresholdUsers() throws Throwable
{
MaxThreshold guard = new MaxThreshold("x",
- state -> 10,
- state -> 100,
- (isWarn, what, v, t) -> format("%s: for %s, %s > %s",
- isWarn ? "Warning" : "Failure", what, v, t));
+ REASON,
+ state -> 10,
+ state -> 100,
+ (isWarn, featureName, v, t) -> format("%s: for %s, %s > %s",
+ isWarn ? "Warning" : "Failure", featureName, v, t));
// value under both thresholds
assertValid(() -> guard.guard(5, "x", false, null));
@@ -156,13 +161,14 @@
public void testDisabledMinThreshold() throws Throwable
{
Threshold.ErrorMessageProvider errorMessageProvider = (isWarn, what, v, t) -> "Should never trigger";
- testDisabledThreshold(new MinThreshold("x", state -> DISABLED, state -> DISABLED, errorMessageProvider));
+ testDisabledThreshold(new MinThreshold("x", REASON, state -> DISABLED, state -> DISABLED, errorMessageProvider));
}
@Test
public void testMinThreshold() throws Throwable
{
MinThreshold guard = new MinThreshold("x",
+ REASON,
state -> 100,
state -> 10,
(isWarn, what, v, t) -> format("%s: for %s, %s < %s",
@@ -189,6 +195,7 @@
public void testWarnOnlyMinThreshold() throws Throwable
{
MinThreshold guard = new MinThreshold("x",
+ REASON,
state -> 10,
state -> DISABLED,
(isWarn, what, v, t) -> format("%s: for %s, %s < %s",
@@ -207,6 +214,7 @@
public void testFailOnlyMinThreshold() throws Throwable
{
MinThreshold guard = new MinThreshold("x",
+ REASON,
state -> DISABLED,
state -> 10,
(isWarn, what, v, t) -> format("%s: for %s, %s < %s",
@@ -225,6 +233,7 @@
public void testMinThresholdUsers() throws Throwable
{
MinThreshold guard = new MinThreshold("x",
+ REASON,
state -> 100,
state -> 10,
(isWarn, what, v, t) -> format("%s: for %s, %s < %s",
@@ -251,35 +260,53 @@
}
@Test
- public void testDisableFlag() throws Throwable
+ public void testEnableFlag() throws Throwable
{
- assertFails(() -> new DisableFlag("x", state -> true, "X").ensureEnabled(userClientState), "X is not allowed");
- assertValid(() -> new DisableFlag("x", state -> false, "X").ensureEnabled(userClientState));
+ assertFails(() -> new EnableFlag("x", REASON, state -> false, "X").ensureEnabled(userClientState), "X is not allowed");
+ assertValid(() -> new EnableFlag("x", REASON, state -> true, "X").ensureEnabled(userClientState));
- assertFails(() -> new DisableFlag("x", state -> true, "X").ensureEnabled("Y", userClientState), "Y is not allowed");
- assertValid(() -> new DisableFlag("x", state -> false, "X").ensureEnabled("Y", userClientState));
+ assertFails(() -> new EnableFlag("x", REASON, state -> false, "X").ensureEnabled("Y", userClientState), "Y is not allowed");
+ assertValid(() -> new EnableFlag("x", REASON, state -> true, "X").ensureEnabled("Y", userClientState));
}
@Test
- public void testDisableFlagUsers() throws Throwable
+ public void testEnableFlagUsers() throws Throwable
{
- DisableFlag enabled = new DisableFlag("x", state -> false, "X");
+ EnableFlag enabled = new EnableFlag("x", REASON, state -> true, "X");
assertValid(() -> enabled.ensureEnabled(null));
assertValid(() -> enabled.ensureEnabled(userClientState));
assertValid(() -> enabled.ensureEnabled(systemClientState));
assertValid(() -> enabled.ensureEnabled(superClientState));
- DisableFlag disabled = new DisableFlag("x", state -> true, "X");
+ EnableFlag disabled = new EnableFlag("x", REASON, state -> false, "X");
assertFails(() -> disabled.ensureEnabled(userClientState), "X is not allowed");
assertValid(() -> disabled.ensureEnabled(systemClientState));
assertValid(() -> disabled.ensureEnabled(superClientState));
}
@Test
+ public void testEnableFlagWarn() throws Throwable
+ {
+ EnableFlag disabledGuard = new EnableFlag("x", null, state -> true, state -> false, FEATURE);
+
+ assertFails(() -> disabledGuard.ensureEnabled(null), false, FEATURE + " is not allowed");
+ assertFails(() -> disabledGuard.ensureEnabled(userClientState), FEATURE + " is not allowed");
+ assertValid(() -> disabledGuard.ensureEnabled(systemClientState));
+ assertValid(() -> disabledGuard.ensureEnabled(superClientState));
+
+ EnableFlag enabledGuard = new EnableFlag("x", null, state -> true, state -> true, FEATURE);
+ assertWarns(() -> enabledGuard.ensureEnabled(null), FEATURE + " is not recommended");
+ assertWarns(() -> enabledGuard.ensureEnabled(userClientState), FEATURE + " is not recommended");
+ assertValid(() -> enabledGuard.ensureEnabled(systemClientState));
+ assertValid(() -> enabledGuard.ensureEnabled(superClientState));
+ }
+
+ @Test
public void testValuesWarned() throws Throwable
{
// Using a sorted set below to ensure the order in the warning message checked below is not random
Values<Integer> warned = new Values<>("x",
+ REASON,
state -> insertionOrderedSet(4, 6, 20),
state -> Collections.emptySet(),
state -> Collections.emptySet(),
@@ -300,6 +327,7 @@
{
// Using a sorted set below to ensure the order in the error message checked below are not random
Values<Integer> ignored = new Values<>("x",
+ REASON,
state -> Collections.emptySet(),
state -> insertionOrderedSet(4, 6, 20),
state -> Collections.emptySet(),
@@ -334,6 +362,7 @@
{
// Using a sorted set below to ensure the order in the error message checked below are not random
Values<Integer> disallowed = new Values<>("x",
+ REASON,
state -> Collections.emptySet(),
state -> Collections.emptySet(),
state -> insertionOrderedSet(4, 6, 20),
@@ -359,6 +388,7 @@
public void testValuesUsers() throws Throwable
{
Values<Integer> disallowed = new Values<>("x",
+ REASON,
state -> Collections.singleton(2),
state -> Collections.singleton(3),
state -> Collections.singleton(4),
@@ -396,6 +426,7 @@
public void testPredicates() throws Throwable
{
Predicates<Integer> guard = new Predicates<>("x",
+ REASON,
state -> x -> x > 10,
state -> x -> x > 100,
(isWarn, value) -> format("%s: %s", isWarn ? "Warning" : "Aborting", value));
@@ -412,6 +443,7 @@
public void testPredicatesUsers() throws Throwable
{
Predicates<Integer> guard = new Predicates<>("x",
+ REASON,
state -> x -> x > 10,
state -> x -> x > 100,
(isWarn, value) -> format("%s: %s", isWarn ? "Warning" : "Aborting", value));
diff --git a/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java b/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java
index fae305a..7f79078 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/ThresholdTester.java
@@ -28,11 +28,9 @@
import org.junit.Before;
import org.junit.Test;
-import org.apache.cassandra.config.DataStorageSpec;
import org.assertj.core.api.Assertions;
import static java.lang.String.format;
-import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.BYTES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
@@ -89,14 +87,18 @@
Threshold threshold,
TriConsumer<Guardrails, String, String> setter,
Function<Guardrails, String> warnGetter,
- Function<Guardrails, String> failGetter)
+ Function<Guardrails, String> failGetter,
+ Function<Long, String> stringFormatter,
+ ToLongFunction<String> stringParser)
{
super(threshold);
- this.warnThreshold = new DataStorageSpec.LongBytesBound(warnThreshold).toBytes();
- this.failThreshold = new DataStorageSpec.LongBytesBound(failThreshold).toBytes();
- this.setter = (g, w, a) -> setter.accept(g, w == null ? null : new DataStorageSpec.LongBytesBound(w, BYTES).toString(), a == null ? null : new DataStorageSpec.LongBytesBound(a, BYTES).toString());
- this.warnGetter = g -> new DataStorageSpec.LongBytesBound(warnGetter.apply(g)).toBytes();
- this.failGetter = g -> new DataStorageSpec.LongBytesBound(failGetter.apply(g)).toBytes();
+ this.warnThreshold = stringParser.applyAsLong(warnThreshold);
+ this.failThreshold = stringParser.applyAsLong(failThreshold);
+ this.setter = (g, w, f) -> setter.accept(g,
+ w == null ? null : stringFormatter.apply(w),
+ f == null ? null : stringFormatter.apply(f));
+ this.warnGetter = g -> stringParser.applyAsLong(warnGetter.apply(g));
+ this.failGetter = g -> stringParser.applyAsLong(failGetter.apply(g));
maxValue = Long.MAX_VALUE - 1;
disabledValue = null;
}
@@ -247,10 +249,9 @@
value, name, disabledValue);
if (expectedMessage == null && value < 0)
- expectedMessage = format("Invalid data storage: value must be non-negative");
+ expectedMessage = "value must be non-negative";
- assertEquals(format("Exception message '%s' does not contain '%s'", e.getMessage(), expectedMessage),
- expectedMessage, e.getMessage());
+ Assertions.assertThat(e).hasMessageContaining(expectedMessage);
}
}
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
index 1b121c1..20af2a0 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
@@ -85,9 +85,9 @@
@Test
public void testIdentityMap()
{
- Integer one = new Integer(1);
- Integer two = new Integer(2);
- Integer three = new Integer(3);
+ Integer one = Integer.valueOf(1);
+ Integer two = Integer.valueOf(2);
+ Integer three = Integer.valueOf(3);
Map<Integer, Integer> identity = Helpers.identityMap(set(one, two, three));
Assert.assertEquals(3, identity.size());
Assert.assertSame(one, identity.get(1));
@@ -124,7 +124,7 @@
failure = false;
try
{
- Map<Integer, Integer> notIdentity = ImmutableMap.of(1, new Integer(1), 2, 2, 3, 3);
+ Map<Integer, Integer> notIdentity = ImmutableMap.of(Integer.MIN_VALUE, Integer.valueOf(Integer.MIN_VALUE), 2, 2, 3, 3);
Helpers.replace(notIdentity, a, b);
}
catch (AssertionError e)
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
index 2812353..8afe77e 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
@@ -41,15 +41,22 @@
import org.junit.Test;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiTableReader;
+import org.apache.cassandra.io.sstable.format.bti.PartitionIndex;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
@@ -57,7 +64,7 @@
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.MockSchema;
-import org.apache.cassandra.utils.AlwaysPresentFilter;
+import org.apache.cassandra.utils.FilterFactory;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -839,22 +846,17 @@
assertNotNull(tmpFiles);
assertEquals(numNewFiles - 1, tmpFiles.size());
- File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
- File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));
+ List<File> sstableFiles = sstable2.descriptor.getFormat().primaryComponents().stream().map(sstable2.descriptor::fileFor).collect(Collectors.toList());
- assertTrue(tmpFiles.contains(ssTable2DataFile));
- assertTrue(tmpFiles.contains(ssTable2IndexFile));
+ for (File f : tmpFiles) assertTrue(tmpFiles.contains(f));
List<File> files = directories.sstableLister(Directories.OnTxnErr.THROW).listFiles();
List<File> filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
assertNotNull(files);
assertNotNull(filesNoTmp);
- assertTrue(files.contains(ssTable2DataFile));
- assertTrue(files.contains(ssTable2IndexFile));
-
- assertFalse(filesNoTmp.contains(ssTable2DataFile));
- assertFalse(filesNoTmp.contains(ssTable2IndexFile));
+ for (File f : tmpFiles) assertTrue(files.contains(f));
+ for (File f : tmpFiles) assertFalse(filesNoTmp.contains(f));
log.finish();
@@ -865,8 +867,8 @@
filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
assertNotNull(filesNoTmp);
- assertTrue(filesNoTmp.contains(ssTable2DataFile));
- assertTrue(filesNoTmp.contains(ssTable2IndexFile));
+
+ for (File f : tmpFiles) assertTrue(filesNoTmp.contains(f));
sstable1.selfRef().release();
sstable2.selfRef().release();
@@ -1261,37 +1263,82 @@
private static SSTableReader sstable(File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException
{
- Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), SSTableFormat.Type.BIG);
- Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
- for (Component component : components)
+ Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), DatabaseDescriptor.getSelectedSSTableFormat());
+ if (BigFormat.isSelected())
{
- File file = new File(descriptor.filenameFor(component));
- if (!file.exists())
- assertTrue(file.createFileIfNotExists());
+ Set<Component> components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC);
+ for (Component component : components)
+ {
+ File file = descriptor.fileFor(component);
+ if (!file.exists())
+ assertTrue(file.createFileIfNotExists());
- Util.setFileLength(file, size);
+ Util.setFileLength(file, size);
+ }
+
+ FileHandle dFile = new FileHandle.Builder(descriptor.fileFor(Components.DATA)).complete();
+ FileHandle iFile = new FileHandle.Builder(descriptor.fileFor(Components.PRIMARY_INDEX)).complete();
+
+ DecoratedKey key = MockSchema.readerBounds(generation);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
+ StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice())
+ .get(MetadataType.STATS);
+ SSTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components)
+ .setTableMetadataRef(cfs.metadata)
+ .setDataFile(dFile)
+ .setIndexFile(iFile)
+ .setIndexSummary(MockSchema.indexSummary.sharedCopy())
+ .setFilter(FilterFactory.AlwaysPresent)
+ .setMaxDataAge(1L)
+ .setStatsMetadata(metadata)
+ .setOpenReason(SSTableReader.OpenReason.NORMAL)
+ .setSerializationHeader(header)
+ .setFirst(key)
+ .setLast(key)
+ .build(cfs, false, false);
+ return reader;
}
+ else if (BtiFormat.isSelected())
+ {
+ Set<Component> components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC);
+ for (Component component : components)
+ {
+ File file = descriptor.fileFor(component);
+ if (!file.exists())
+ assertTrue(file.createFileIfNotExists());
- FileHandle dFile = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).complete();
- FileHandle iFile = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).complete();
+ Util.setFileLength(file, size);
+ }
- SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
- StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
- .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header)
- .get(MetadataType.STATS);
- SSTableReader reader = SSTableReader.internalOpen(descriptor,
- components,
- cfs.metadata,
- iFile,
- dFile,
- MockSchema.indexSummary.sharedCopy(),
- new AlwaysPresentFilter(),
- 1L,
- metadata,
- SSTableReader.OpenReason.NORMAL,
- header);
- reader.first = reader.last = MockSchema.readerBounds(generation);
- return reader;
+ FileHandle dFile = new FileHandle.Builder(descriptor.fileFor(Components.DATA)).complete();
+ FileHandle iFile = new FileHandle.Builder(descriptor.fileFor(BtiFormat.Components.PARTITION_INDEX)).complete();
+ FileHandle rFile = new FileHandle.Builder(descriptor.fileFor(BtiFormat.Components.ROW_INDEX)).complete();
+
+ DecoratedKey key = MockSchema.readerBounds(generation);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
+ StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice())
+ .get(MetadataType.STATS);
+ SSTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
+ .setTableMetadataRef(cfs.metadata)
+ .setDataFile(dFile)
+ .setPartitionIndex(new PartitionIndex(iFile, 0, 0, MockSchema.readerBounds(generation), MockSchema.readerBounds(generation)))
+ .setRowIndexFile(rFile)
+ .setFilter(FilterFactory.AlwaysPresent)
+ .setMaxDataAge(1L)
+ .setStatsMetadata(metadata)
+ .setOpenReason(SSTableReader.OpenReason.NORMAL)
+ .setSerializationHeader(header)
+ .setFirst(key)
+ .setLast(key)
+ .build(cfs, false, false);
+ return reader;
+ }
+ else
+ {
+ throw Util.testMustBeImplementedForSSTableFormat();
+ }
}
private static void assertFiles(String dirPath, Set<String> expectedFiles) throws IOException
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
index 0420957..85c8ae0 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
@@ -24,14 +24,11 @@
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import org.apache.cassandra.io.util.File;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.Assert;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SerializationHeader;
@@ -43,8 +40,11 @@
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableRewriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@@ -160,18 +160,16 @@
File directory = txn.originals().iterator().next().descriptor.directory;
Descriptor desc = cfs.newSSTableDescriptor(directory);
TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
- rewriter.switchWriter(SSTableWriter.create(metadata,
- desc,
- 0,
- 0,
- null,
- false,
- 0,
- SerializationHeader.make(cfs.metadata(), txn.originals()),
- cfs.indexManager.listIndexes(),
- txn));
+ rewriter.switchWriter(desc.getFormat().getWriterFactory().builder(desc)
+ .setTableMetadataRef(metadata)
+ .setSerializationHeader(SerializationHeader.make(cfs.metadata(), txn.originals()))
+ .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+ .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator))
+ .addDefaultComponents()
+ .build(txn, cfs));
while (ci.hasNext())
{
+ ci.setTargetDirectory(rewriter.currentWriter().getFilename());
rewriter.append(ci.next());
if (nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index e39f71f..d6e3742 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -41,7 +41,18 @@
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.InitialSSTableAddedNotification;
+import org.apache.cassandra.notifications.MemtableDiscardedNotification;
+import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.notifications.MemtableSwitchedNotification;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableDeletingNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
+import org.apache.cassandra.notifications.SSTableMetadataChanged;
+import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -162,7 +173,8 @@
Assert.assertTrue(listener.received.get(0) instanceof InitialSSTableAddedNotification);
for (SSTableReader reader : readers)
- Assert.assertTrue(reader.isKeyCacheEnabled());
+ if (reader instanceof KeyCacheSupport<?>)
+ Assert.assertTrue(((KeyCacheSupport<?>)reader).getKeyCache().isEnabled());
Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount());
}
@@ -184,7 +196,10 @@
Assert.assertEquals(3, tracker.view.get().sstables.size());
for (SSTableReader reader : readers)
- Assert.assertTrue(reader.isKeyCacheEnabled());
+ {
+ if (reader instanceof KeyCacheSupport<?>)
+ Assert.assertTrue(((KeyCacheSupport<?>)reader).getKeyCache().isEnabled());
+ }
Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount());
Assert.assertEquals(1, listener.senders.size());
@@ -316,7 +331,8 @@
Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
Assert.assertEquals(Optional.of(prev2), ((SSTableAddedNotification) listener.received.get(1)).memtable());
listener.received.clear();
- Assert.assertTrue(reader.isKeyCacheEnabled());
+ if (reader instanceof KeyCacheSupport<?>)
+ Assert.assertTrue(((KeyCacheSupport<?>) reader).getKeyCache().isEnabled());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
// test invalidated CFS
diff --git a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
index d22a8ac..69c1eb5 100644
--- a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java
@@ -47,7 +47,7 @@
{
private static final String KEYSPACE1 = "DynamicCompositeType";
private static final String CF_STANDARDDYNCOMPOSITE = "StandardDynamicComposite";
- private static Map<Byte, AbstractType<?>> aliases = new HashMap<>();
+ public static Map<Byte, AbstractType<?>> aliases = new HashMap<>();
private static final DynamicCompositeType comparator;
static
@@ -60,7 +60,7 @@
}
private static final int UUID_COUNT = 3;
- private static final UUID[] uuids = new UUID[UUID_COUNT];
+ public static final UUID[] uuids = new UUID[UUID_COUNT];
static
{
for (int i = 0; i < UUID_COUNT; ++i)
@@ -320,13 +320,12 @@
assert !TypeParser.parse("DynamicCompositeType(a => BytesType)").isCompatibleWith(TypeParser.parse("DynamicCompositeType(a => BytesType, b => AsciiType)"));
}
- private ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, boolean lastIsOne)
+ private static ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, boolean lastIsOne)
{
return createDynamicCompositeKey(s, uuid, i, lastIsOne, false);
}
- private ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, boolean lastIsOne,
- final boolean reversed)
+ public static ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, boolean lastIsOne, boolean reversed)
{
String intType = (reversed ? "ReversedType(IntegerType)" : "IntegerType");
ByteBuffer bytes = ByteBufferUtil.bytes(s);
diff --git a/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java b/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java
index bab0611..e89a722 100644
--- a/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/JsonConversionTest.java
@@ -23,16 +23,14 @@
import java.nio.ByteBuffer;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.JsonUtils;
import org.junit.Test;
public class JsonConversionTest
{
- private static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper();
-
@Test
public void testMap() throws Exception
{
@@ -311,7 +309,7 @@
private static void assertBytebufferPositionAndOutput(String json, String typeString) throws Exception
{
AbstractType<?> type = TypeParser.parse(typeString);
- Object jsonObject = JSON_OBJECT_MAPPER.readValue(json, Object.class);
+ Object jsonObject = JsonUtils.JSON_OBJECT_MAPPER.readValue(json, Object.class);
ByteBuffer bb = type.fromJSONObject(jsonObject).bindAndGet(QueryOptions.DEFAULT);
int position = bb.position();
diff --git a/test/unit/org/apache/cassandra/db/marshal/TimeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/TimeTypeTest.java
index d61d2c6..26a2b15 100644
--- a/test/unit/org/apache/cassandra/db/marshal/TimeTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/TimeTypeTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -29,11 +29,12 @@
public class TimeTypeTest extends CQLTester
{
+ @SuppressWarnings({ "PointlessArithmeticExpression", "WrapperTypeMayBePrimitive" })
@Test
public void testComparison()
{
Long t1 = TimeSerializer.timeStringToLong("01:00:00.123456789");
- Long t2 = new Long((1L * 60L * 60L * 1000L * 1000L * 1000L) + 123456789);
+ Long t2 = 1L * 60L * 60L * 1000L * 1000L * 1000L + 123456789L;
ByteBuffer b1 = TimeSerializer.instance.serialize(t1);
ByteBuffer b2 = TimeSerializer.instance.serialize(t2);
assert TimeType.instance.compare(b1, b2) == 0 : "Failed == comparison";
@@ -41,7 +42,7 @@
b2 = TimeSerializer.instance.serialize(123456789L);
assert TimeType.instance.compare(b1, b2) > 0 : "Failed > comparison";
- t2 = new Long(2L * 60L * 60L * 1000L * 1000L * 1000L + 123456789);
+ t2 = 2L * 60L * 60L * 1000L * 1000L * 1000L + 123456789L;
b2 = TimeSerializer.instance.serialize(t2);
assert TimeType.instance.compare(b1, b2) < 0 : "Failed < comparison";
diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java
index 4d25a1f..474b867 100644
--- a/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java
@@ -204,7 +204,7 @@
qt().forAll(tupleWithValueGen(baseGen)).checkAssert(pair -> {
TupleType tuple = pair.left;
ByteBuffer value = pair.right;
- Assertions.assertThat(TupleType.buildValue(tuple.split(value)))
+ Assertions.assertThat(TupleType.buildValue(tuple.split(ByteBufferAccessor.instance, value)))
.as("TupleType.buildValue(split(value)) == value")
.isEqualTo(value);
});
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java
new file mode 100644
index 0000000..b2cfa3e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableQuickTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+
+@RunWith(Parameterized.class)
+public class MemtableQuickTest extends CQLTester
+{
+ static final Logger logger = LoggerFactory.getLogger(MemtableQuickTest.class);
+
+ static final int partitions = 50_000;
+ static final int rowsPerPartition = 4;
+
+ static final int deletedPartitionsStart = 20_000;
+ static final int deletedPartitionsEnd = deletedPartitionsStart + 10_000;
+
+ static final int deletedRowsStart = 40_000;
+ static final int deletedRowsEnd = deletedRowsStart + 5_000;
+
+ @Parameterized.Parameter()
+ public String memtableClass;
+
+ @Parameterized.Parameters(name = "{0}")
+ public static List<Object> parameters()
+ {
+ return ImmutableList.of("skiplist",
+ "skiplist_sharded",
+ "skiplist_sharded_locking",
+ "trie");
+ }
+
+ @BeforeClass
+ public static void setUp()
+ {
+ CQLTester.setUpClass();
+ CQLTester.prepareServer();
+ CQLTester.disablePreparedReuseForTest();
+ logger.info("setupClass done.");
+ }
+
+ @Test
+ public void testMemtable() throws Throwable
+ {
+
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" +
+ " with compression = {'enabled': false}" +
+ " and memtable = '" + memtableClass + "'");
+ execute("use " + keyspace + ';');
+
+ String writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)";
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ Util.flush(cfs);
+
+ long i;
+ long limit = partitions;
+ logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition);
+ for (i = 0; i < limit; ++i)
+ {
+ for (long j = 0; j < rowsPerPartition; ++j)
+ execute(writeStatement, i, j, i + j);
+ }
+
+ logger.info("Deleting partitions between {} and {}", deletedPartitionsStart, deletedPartitionsEnd);
+ for (i = deletedPartitionsStart; i < deletedPartitionsEnd; ++i)
+ {
+ // no partition exists, but we will create a tombstone
+ execute("DELETE FROM " + table + " WHERE userid = ?", i);
+ }
+
+ logger.info("Deleting rows between {} and {}", deletedRowsStart, deletedRowsEnd);
+ for (i = deletedRowsStart; i < deletedRowsEnd; ++i)
+ {
+ // no row exists, but we will create a tombstone (and partition)
+ execute("DELETE FROM " + table + " WHERE userid = ? AND picid = ?", i, 0L);
+ }
+
+ logger.info("Reading {} partitions", partitions);
+ for (i = 0; i < limit; ++i)
+ {
+ UntypedResultSet result = execute("SELECT * FROM " + table + " WHERE userid = ?", i);
+ if (i >= deletedPartitionsStart && i < deletedPartitionsEnd)
+ assertEmpty(result);
+ else
+ {
+ int start = 0;
+ if (i >= deletedRowsStart && i < deletedRowsEnd)
+ start = 1;
+ Object[][] rows = new Object[rowsPerPartition - start][];
+ for (long j = start; j < rowsPerPartition; ++j)
+ rows[(int) (j - start)] = row(i, j, i + j);
+ assertRows(result, rows);
+ }
+ }
+
+ int deletedPartitions = deletedPartitionsEnd - deletedPartitionsStart;
+ int deletedRows = deletedRowsEnd - deletedRowsStart;
+ logger.info("Selecting *");
+ UntypedResultSet result = execute("SELECT * FROM " + table);
+ assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows);
+
+ Util.flush(cfs);
+
+ logger.info("Selecting *");
+ result = execute("SELECT * FROM " + table);
+ assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeHeapBuffersTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeHeapBuffersTest.java
new file mode 100644
index 0000000..4497e8c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeHeapBuffersTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.utils.memory.MemtablePool;
+import org.apache.cassandra.utils.memory.SlabPool;
+
+public class MemtableSizeHeapBuffersTest extends MemtableSizeTestBase
+{
+ // Overrides CQLTester.setUpClass to run before it
+ @BeforeClass
+ public static void setUpClass()
+ {
+ setup(Config.MemtableAllocationType.heap_buffers);
+ }
+
+ @Override
+ void checkMemtablePool()
+ {
+ MemtablePool memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
+ logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
+ Assert.assertTrue(memoryPool instanceof SlabPool);
+ Assert.assertEquals(0, memoryPool.offHeap.limit);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeOffheapBuffersTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeOffheapBuffersTest.java
new file mode 100644
index 0000000..022f4f1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeOffheapBuffersTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.utils.memory.MemtablePool;
+import org.apache.cassandra.utils.memory.SlabPool;
+
+public class MemtableSizeOffheapBuffersTest extends MemtableSizeTestBase
+{
+ // Overrides CQLTester.setUpClass to run before it
+ @BeforeClass
+ public static void setUpClass()
+ {
+ setup(Config.MemtableAllocationType.offheap_buffers);
+ }
+
+
+ @Override
+ void checkMemtablePool()
+ {
+ MemtablePool memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
+ logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
+ Assert.assertTrue(memoryPool instanceof SlabPool);
+ Assert.assertTrue(memoryPool.offHeap.limit > 0);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeOffheapObjectsTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeOffheapObjectsTest.java
new file mode 100644
index 0000000..559f456
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeOffheapObjectsTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.utils.memory.MemtablePool;
+import org.apache.cassandra.utils.memory.NativePool;
+
+public class MemtableSizeOffheapObjectsTest extends MemtableSizeTestBase
+{
+ // Overrides CQLTester.setUpClass to run before it
+ @BeforeClass
+ public static void setUpClass()
+ {
+ setup(Config.MemtableAllocationType.offheap_objects);
+ }
+
+ @Override
+ void checkMemtablePool()
+ {
+ MemtablePool memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
+ logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
+ Assert.assertTrue(memoryPool instanceof NativePool);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java
new file mode 100644
index 0000000..7760527
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeTestBase.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.lang.reflect.Field;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.utils.FBUtilities;
+import org.github.jamm.MemoryMeter;
+
+// Note: This test can be run in idea with the allocation type configured in the test yaml and memtable using the
+// value memtableClass is initialized with.
+@RunWith(Parameterized.class)
+public abstract class MemtableSizeTestBase extends CQLTester
+{
+ // Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of
+ // partitions).
+ static MemoryMeter meter = new MemoryMeter().ignoreKnownSingletons()
+ .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE);
+
+ static final Logger logger = LoggerFactory.getLogger(MemtableSizeTestBase.class);
+
+ static final int partitions = 50_000;
+ static final int rowsPerPartition = 4;
+
+ static final int deletedPartitions = 10_000;
+ static final int deletedRows = 5_000;
+
+ @Parameterized.Parameter(0)
+ public String memtableClass = "skiplist";
+
+ @Parameterized.Parameters(name = "{0}")
+ public static List<Object> parameters()
+ {
+ return ImmutableList.of("skiplist",
+ "skiplist_sharded",
+ "trie");
+ }
+
+ // Must be within 3% of the real usage. We are actually more precise than this, but the threshold is set higher to
+ // avoid flakes. For on-heap allocators we allow for extra overheads below.
+ final int MAX_DIFFERENCE_PERCENT = 3;
+ // Slab overhead, added when the memtable uses heap_buffers.
+ final int SLAB_OVERHEAD = 1024 * 1024;
+
+ public static void setup(Config.MemtableAllocationType allocationType)
+ {
+ ServerTestUtils.daemonInitialization();
+ try
+ {
+ Field confField = DatabaseDescriptor.class.getDeclaredField("conf");
+ confField.setAccessible(true);
+ Config conf = (Config) confField.get(null);
+ conf.memtable_allocation_type = allocationType;
+ conf.memtable_cleanup_threshold = 0.8f; // give us more space to fit test data without flushing
+ }
+ catch (NoSuchFieldException | IllegalAccessException e)
+ {
+ throw new RuntimeException(e);
+ }
+
+ CQLTester.setUpClass();
+ CQLTester.prepareServer();
+ logger.info("setupClass done, allocation type {}", allocationType);
+ }
+
+ void checkMemtablePool()
+ {
+ // overridden by instances
+ }
+
+ @Test
+ public void testSize() throws Throwable
+ {
+ // Make sure memtables use the correct allocation type, i.e. that setup has worked.
+ // If this fails, make sure the test is not reusing an already-initialized JVM.
+ checkMemtablePool();
+
+ CQLTester.disablePreparedReuseForTest();
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ try
+ {
+ String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" +
+ " with compression = {'enabled': false}" +
+ " and memtable = '" + memtableClass + "'");
+ execute("use " + keyspace + ';');
+
+ String writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)";
+ forcePreparedValues();
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ Util.flush(cfs);
+
+ Memtable memtable = cfs.getTracker().getView().getCurrentMemtable();
+ long deepSizeBefore = meter.measureDeep(memtable);
+ logger.info("Memtable deep size before {}", FBUtilities.prettyPrintMemory(deepSizeBefore));
+ long i;
+ long limit = partitions;
+ logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition);
+ for (i = 0; i < limit; ++i)
+ {
+ for (long j = 0; j < rowsPerPartition; ++j)
+ execute(writeStatement, i, j, i + j);
+ }
+
+ logger.info("Deleting {} partitions", deletedPartitions);
+ limit += deletedPartitions;
+ for (; i < limit; ++i)
+ {
+ // no partition exists, but we will create a tombstone
+ execute("DELETE FROM " + table + " WHERE userid = ?", i);
+ }
+
+ logger.info("Deleting {} rows", deletedRows);
+ limit += deletedRows;
+ for (; i < limit; ++i)
+ {
+ // no row exists, but we will create a tombstone (and partition)
+ execute("DELETE FROM " + table + " WHERE userid = ? AND picid = ?", i, 0L);
+ }
+
+ Assert.assertSame("Memtable flushed during test. Test was not carried out correctly.",
+ memtable,
+ cfs.getTracker().getView().getCurrentMemtable());
+
+ Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable);
+ long actualHeap = usage.ownsOnHeap;
+ logger.info(String.format("Memtable in %s mode: %d ops, %s serialized bytes, %s",
+ DatabaseDescriptor.getMemtableAllocationType(),
+ memtable.operationCount(),
+ FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()),
+ usage));
+
+ long deepSizeAfter = meter.measureDeep(memtable);
+ logger.info("Memtable deep size {}", FBUtilities.prettyPrintMemory(deepSizeAfter));
+
+ long expectedHeap = deepSizeAfter - deepSizeBefore;
+ long max_difference = MAX_DIFFERENCE_PERCENT * expectedHeap / 100;
+ long trie_overhead = memtable instanceof TrieMemtable ? ((TrieMemtable) memtable).unusedReservedMemory() : 0;
+ switch (DatabaseDescriptor.getMemtableAllocationType())
+ {
+ case heap_buffers:
+ max_difference += SLAB_OVERHEAD;
+ actualHeap += trie_overhead; // adjust trie memory with unused buffer space if on-heap
+ break;
+ case unslabbed_heap_buffers:
+ actualHeap += trie_overhead; // adjust trie memory with unused buffer space if on-heap
+ break;
+ }
+ String message = String.format("Expected heap usage close to %s, got %s, %s difference.\n",
+ FBUtilities.prettyPrintMemory(expectedHeap),
+ FBUtilities.prettyPrintMemory(actualHeap),
+ FBUtilities.prettyPrintMemory(expectedHeap - actualHeap));
+ logger.info(message);
+ if (Math.abs(actualHeap - expectedHeap) > max_difference)
+ {
+ // Under Java 11, it seems the meter can reach into phantom reference queues and count more space than
+ // is actually reachable. Unfortunately ignoreNonStrongReferences() does not help (worse, it throws
+ // exceptions trying to get a phantom referrent). Retrying the measurement appears to clear these up.
+ Thread.sleep(50);
+ long secondPass = meter.measureDeep(memtable);
+ logger.error("Deep size first pass {} second pass {}",
+ FBUtilities.prettyPrintMemory(deepSizeAfter),
+ FBUtilities.prettyPrintMemory(secondPass));
+ expectedHeap = secondPass - deepSizeBefore;
+ }
+
+ Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= max_difference);
+ }
+ finally
+ {
+ execute(String.format("DROP KEYSPACE IF EXISTS %s", keyspace));
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/MemtableSizeUnslabbedTest.java b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeUnslabbedTest.java
new file mode 100644
index 0000000..b59a474
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/MemtableSizeUnslabbedTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.utils.memory.HeapPool;
+import org.apache.cassandra.utils.memory.MemtablePool;
+
+public class MemtableSizeUnslabbedTest extends MemtableSizeTestBase
+{
+ // Overrides CQLTester.setUpClass to run before it
+ @BeforeClass
+ public static void setUpClass()
+ {
+ setup(Config.MemtableAllocationType.unslabbed_heap_buffers);
+ }
+
+ @Override
+ void checkMemtablePool()
+ {
+ MemtablePool memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
+ logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
+ Assert.assertTrue(memoryPool instanceof HeapPool);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java b/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java
new file mode 100644
index 0000000..ef5079b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/memtable/ShardedMemtableConfigTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.io.IOException;
+import javax.management.Attribute;
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.InvalidAttributeValueException;
+import javax.management.MBeanException;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.db.memtable.AbstractShardedMemtable.SHARDED_MEMTABLE_CONFIG_OBJECT_NAME;
+import static org.junit.Assert.assertEquals;
+
+public class ShardedMemtableConfigTest extends CQLTester
+{
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ startJMXServer();
+ createMBeanServerConnection();
+ }
+
+ @Test
+ public void testDefaultShardCountSetByJMX() throws MalformedObjectNameException, ReflectionException, AttributeNotFoundException, InstanceNotFoundException, MBeanException, IOException, InvalidAttributeValueException, InterruptedException
+ {
+ // check the default, but also make sure the class is initialized if the default memtable is not sharded
+ assertEquals(FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
+ jmxConnection.setAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), new Attribute("DefaultShardCount", "7"));
+ assertEquals(7, AbstractShardedMemtable.getDefaultShardCount());
+ assertEquals("7", jmxConnection.getAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), "DefaultShardCount"));
+ }
+
+ @Test
+ public void testAutoShardCount() throws MalformedObjectNameException, ReflectionException, AttributeNotFoundException, InstanceNotFoundException, MBeanException, IOException, InvalidAttributeValueException
+ {
+ AbstractShardedMemtable.getDefaultShardCount(); // initialize class
+ jmxConnection.setAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), new Attribute("DefaultShardCount", "auto"));
+ assertEquals(FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
+ assertEquals(Integer.toString(FBUtilities.getAvailableProcessors()),
+ jmxConnection.getAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), "DefaultShardCount"));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java b/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java
index 04196f6..8b755af 100644
--- a/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java
+++ b/test/unit/org/apache/cassandra/db/partitions/AtomicBTreePartitionMemtableAccountingTest.java
@@ -322,7 +322,7 @@
OpOrder.Group writeOp = opOrder.getCurrent();
Cloner cloner = allocator.cloner(writeOp);
- partition.addAllWithSizeDelta(update, cloner, writeOp, indexer);
+ partition.addAll(update, cloner, writeOp, indexer);
opOrder.newBarrier().issue();
assertThat(allocator.onHeap().owns()).isGreaterThanOrEqualTo(0L);
@@ -338,7 +338,7 @@
opOrder.newBarrier().issue();
OpOrder.Group writeOp = opOrder.getCurrent();
Cloner cloner = recreatedAllocator.cloner(writeOp);
- recreated.addAllWithSizeDelta(update, cloner, writeOp, indexer);
+ recreated.addAll(update, cloner, writeOp, indexer);
}
// offheap allocators don't release on heap memory, so expect the same
diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
index a559478..c95b2db 100644
--- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
@@ -41,17 +41,14 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.concurrent.FutureTask;
-import org.apache.cassandra.concurrent.ImmediateExecutor;
-import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.concurrent.Future;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.FutureTask;
+import org.apache.cassandra.concurrent.ImmediateExecutor;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -78,7 +75,10 @@
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.Util;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -665,14 +665,23 @@
@Override
public boolean apply(SSTableReader sstable)
{
- cdl.countDown();
- if (cdl.getCount() > 0)
- throw new PendingAntiCompaction.SSTableAcquisitionException("blah");
return true;
}
};
+
CompactionManager.instance.active.beginCompaction(holder);
- PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, nextTimeUUID(), 10, 1, acp);
+ PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, nextTimeUUID(), 10, 1, acp)
+ {
+ protected PendingAntiCompaction.AcquireResult acquireSSTables()
+ {
+ cdl.countDown();
+ if (cdl.getCount() > 0)
+ throw new PendingAntiCompaction.SSTableAcquisitionException("blah");
+ else
+ CompactionManager.instance.active.finishCompaction(holder);
+ return super.acquireSSTables();
+ }
+ };
Future f = es.submit(acquisitionCallable);
cdl.await();
assertNotNull(f.get());
diff --git a/test/unit/org/apache/cassandra/db/rows/RowsMergingTest.java b/test/unit/org/apache/cassandra/db/rows/RowsMergingTest.java
index 8049af3..01e335d 100644
--- a/test/unit/org/apache/cassandra/db/rows/RowsMergingTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/RowsMergingTest.java
@@ -26,12 +26,14 @@
import com.google.common.base.Joiner;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BTREE_BRANCH_SHIFT;
+
public class RowsMergingTest extends CQLTester
{
@BeforeClass
public static void setUpClass()
{
- System.setProperty("cassandra.btree.branchshift", "2");
+ BTREE_BRANCH_SHIFT.setInt(2);
CQLTester.setUpClass();
}
diff --git a/test/unit/org/apache/cassandra/db/rows/RowsTest.java b/test/unit/org/apache/cassandra/db/rows/RowsTest.java
index cfeebfd..8742fd8 100644
--- a/test/unit/org/apache/cassandra/db/rows/RowsTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/RowsTest.java
@@ -200,6 +200,12 @@
{
this.hasLegacyCounterShards |= hasLegacyCounterShards;
}
+
+ @Override
+ public void updatePartitionDeletion(DeletionTime dt)
+ {
+ update(dt);
+ }
}
private static long secondToTs(int now)
@@ -648,4 +654,4 @@
liveCell(b, 1),
liveCell(a));
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
index 0c4a79d..e3c42cd 100644
--- a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java
@@ -159,7 +159,7 @@
UnfilteredRowIterator iterator = throttled.next();
assertFalse(throttled.hasNext());
assertFalse(iterator.hasNext());
- assertEquals(Int32Type.instance.getSerializer().deserialize(iterator.staticRow().cells().iterator().next().buffer()), new Integer(160));
+ assertEquals(Int32Type.instance.getSerializer().deserialize(iterator.staticRow().cells().iterator().next().buffer()), Integer.valueOf(160));
}
// test opt out
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java
new file mode 100644
index 0000000..86df1cb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import java.math.BigInteger;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.SSTableReadsListener;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class UnfilteredRowIteratorWithLowerBoundTest
+{
+ private static final String KEYSPACE = "ks";
+ private static final String TABLE = "tbl";
+ private static final String SLICES_TABLE = "tbl_slices";
+ private static TableMetadata tableMetadata;
+ private static TableMetadata slicesTableMetadata;
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ DatabaseDescriptor.daemonInitialization();
+
+ tableMetadata =
+ TableMetadata.builder(KEYSPACE, TABLE)
+ .addPartitionKeyColumn("k", UTF8Type.instance)
+ .addStaticColumn("s", UTF8Type.instance)
+ .addClusteringColumn("i", IntegerType.instance)
+ .addRegularColumn("v", UTF8Type.instance)
+ .build();
+
+ slicesTableMetadata = TableMetadata.builder(KEYSPACE, SLICES_TABLE)
+ .addPartitionKeyColumn("k", UTF8Type.instance)
+ .addClusteringColumn("c1", Int32Type.instance)
+ .addClusteringColumn("c2", Int32Type.instance)
+ .addRegularColumn("v", IntegerType.instance)
+ .build();
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), tableMetadata, slicesTableMetadata);
+ }
+
+ @Before
+ public void truncate()
+ {
+ Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).truncateBlocking();
+ Keyspace.open(KEYSPACE).getColumnFamilyStore(SLICES_TABLE).truncateBlocking();
+ }
+
+ @Test
+ public void testLowerBoundApplicableSingleColumnAsc()
+ {
+ String query = "INSERT INTO %s.%s (k, i) VALUES ('k1', %s)";
+ SSTableReader sstable = createSSTable(tableMetadata, KEYSPACE, TABLE, query);
+ assertEquals(Slice.make(Util.clustering(tableMetadata.comparator, BigInteger.valueOf(0)),
+ Util.clustering(tableMetadata.comparator, BigInteger.valueOf(9))),
+ sstable.getSSTableMetadata().coveredClustering);
+ DecoratedKey key = tableMetadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+ Slice slice1 = Slice.make(Util.clustering(tableMetadata.comparator, BigInteger.valueOf(3)).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(tableMetadata, key, slice1, sstable, false));
+ assertTrue(lowerBoundApplicable(tableMetadata, key, slice1, sstable, true));
+
+ Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(tableMetadata.comparator, BigInteger.valueOf(3)).asEndBound());
+ assertTrue(lowerBoundApplicable(tableMetadata, key, slice2, sstable, false));
+ assertFalse(lowerBoundApplicable(tableMetadata, key, slice2, sstable, true));
+
+ // corner cases
+ Slice slice3 = Slice.make(Util.clustering(tableMetadata.comparator, BigInteger.valueOf(0)).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(tableMetadata, key, slice3, sstable, false));
+ assertTrue(lowerBoundApplicable(tableMetadata, key, slice3, sstable, true));
+
+ Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(tableMetadata.comparator, BigInteger.valueOf(9)).asEndBound());
+ assertTrue(lowerBoundApplicable(tableMetadata, key, slice4, sstable, false));
+ assertFalse(lowerBoundApplicable(tableMetadata, key, slice4, sstable, true));
+ }
+
+ @Test
+ public void testLowerBoundApplicableSingleColumnDesc()
+ {
+ String TABLE_REVERSED = "tbl_reversed";
+ String createTable = String.format(
+ "CREATE TABLE %s.%s (k text, i varint, v int, primary key (k, i)) WITH CLUSTERING ORDER BY (i DESC)",
+ KEYSPACE, TABLE_REVERSED);
+ QueryProcessor.executeOnceInternal(createTable);
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_REVERSED);
+ TableMetadata metadata = cfs.metadata();
+ String query = "INSERT INTO %s.%s (k, i) VALUES ('k1', %s)";
+ SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE_REVERSED, query);
+ assertEquals(Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(9)),
+ Util.clustering(metadata.comparator, BigInteger.valueOf(0))),
+ sstable.getSSTableMetadata().coveredClustering);
+ DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+ Slice slice1 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(8)).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false));
+ assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true));
+
+ Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(8)).asEndBound());
+ assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false));
+ assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true));
+
+ // corner cases
+ Slice slice3 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(9)).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false));
+ assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true));
+
+ Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(0)).asEndBound());
+ assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false));
+ assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true));
+ }
+
+ @Test
+ public void testLowerBoundApplicableMultipleColumnsAsc()
+ {
+ String query = "INSERT INTO %s.%s (k, c1, c2) VALUES ('k1', 0, %s)";
+ SSTableReader sstable = createSSTable(slicesTableMetadata, KEYSPACE, SLICES_TABLE, query);
+ assertEquals(Slice.make(Util.clustering(slicesTableMetadata.comparator, 0, 0),
+ Util.clustering(slicesTableMetadata.comparator, 0, 9)),
+ sstable.getSSTableMetadata().coveredClustering);
+ DecoratedKey key = slicesTableMetadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+ Slice slice1 = Slice.make(Util.clustering(slicesTableMetadata.comparator, 0, 3).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice1, sstable, false));
+ assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice1, sstable, true));
+
+ Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(slicesTableMetadata.comparator, 0, 3).asEndBound());
+ assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice2, sstable, false));
+ assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice2, sstable, true));
+
+ // corner cases
+ Slice slice3 = Slice.make(Util.clustering(slicesTableMetadata.comparator, 0, 0).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice3, sstable, false));
+ assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice3, sstable, true));
+
+ Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(slicesTableMetadata.comparator, 0, 9).asEndBound());
+ assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice4, sstable, false));
+ assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice4, sstable, true));
+ }
+
+ @Test
+ public void testLowerBoundApplicableMultipleColumnsDesc()
+ {
+ String TABLE_REVERSED = "tbl_slices_reversed";
+ String createTable = String.format(
+ "CREATE TABLE %s.%s (k text, c1 int, c2 int, v int, primary key (k, c1, c2)) WITH CLUSTERING ORDER BY (c1 ASC, c2 DESC)",
+ KEYSPACE, TABLE_REVERSED);
+ QueryProcessor.executeOnceInternal(createTable);
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_REVERSED);
+ TableMetadata metadata = cfs.metadata();
+
+ String query = "INSERT INTO %s.%s (k, c1, c2) VALUES ('k1', 0, %s)";
+ SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE_REVERSED, query);
+ assertEquals(Slice.make(Util.clustering(metadata.comparator, 0, 9),
+ Util.clustering(metadata.comparator, 0, 0)),
+ sstable.getSSTableMetadata().coveredClustering);
+ DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+
+ Slice slice1 = Slice.make(Util.clustering(metadata.comparator, 0, 8).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false));
+ assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true));
+
+ Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, 0, 8).asEndBound());
+ assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false));
+ assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true));
+
+ // corner cases
+ Slice slice3 = Slice.make(Util.clustering(metadata.comparator, 0, 9).asStartBound(), ClusteringBound.TOP);
+ assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false));
+ assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true));
+
+ Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, 0, 0).asEndBound());
+ assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false));
+ assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true));
+ }
+
+ private SSTableReader createSSTable(TableMetadata metadata, String keyspace, String table, String query)
+ {
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ for (int i = 0; i < 10; i++)
+ QueryProcessor.executeInternal(String.format(query, keyspace, table, i));
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+ DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1"));
+ ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, key));
+ assertEquals(1, view.sstables.size());
+ return view.sstables.get(0);
+ }
+
+ private boolean lowerBoundApplicable(TableMetadata metadata, DecoratedKey key, Slice slice, SSTableReader sstable, boolean isReversed)
+ {
+ Slices.Builder slicesBuilder = new Slices.Builder(metadata.comparator);
+ slicesBuilder.add(slice);
+ Slices slices = slicesBuilder.build();
+ ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, isReversed);
+
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata,
+ FBUtilities.nowInSeconds(),
+ ColumnFilter.all(metadata),
+ RowFilter.NONE,
+ DataLimits.NONE,
+ key,
+ filter);
+
+ try (UnfilteredRowIteratorWithLowerBound iter = new UnfilteredRowIteratorWithLowerBound(key,
+ sstable,
+ slices,
+ isReversed,
+ ColumnFilter.all(metadata),
+ Mockito.mock(SSTableReadsListener.class)))
+ {
+ return iter.lowerBound() != null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java
index a65b2ea..898da7c 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java
@@ -24,8 +24,6 @@
import java.util.Collections;
import java.util.Queue;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,19 +35,20 @@
import io.netty.channel.DefaultFileRegion;
import io.netty.channel.embedded.EmbeddedChannel;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.SharedDefaultFileRegion;
import org.apache.cassandra.net.AsyncStreamingOutputPlus;
+import org.apache.cassandra.net.SharedDefaultFileRegion;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionInfo;
import org.apache.cassandra.streaming.StreamCoordinator;
@@ -58,6 +57,7 @@
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.streaming.StreamSummary;
+import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.messages.StreamMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -149,7 +149,6 @@
CassandraStreamHeader header =
CassandraStreamHeader.builder()
- .withSSTableFormat(sstable.descriptor.formatType)
.withSSTableVersion(sstable.descriptor.version)
.withSSTableLevel(0)
.withEstimatedKeys(sstable.estimatedKeys())
@@ -209,7 +208,7 @@
StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
- streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED));
+ streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, null));
StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer);
session.init(future);
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
index 93b6e71..6adfded 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.streaming;
+import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@@ -132,7 +133,7 @@
int count = 0;
DecoratedKey key;
- try (KeyIterator iter = new KeyIterator(sstable.descriptor, sstable.metadata()))
+ try (KeyIterator iter = sstable.keyIterator())
{
do
{
@@ -140,6 +141,10 @@
count++;
} while (iter.hasNext() && count < i);
}
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
return key;
}
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java
index 0e5187c..a0e4e51 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java
@@ -27,6 +27,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
@@ -39,9 +40,8 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.schema.KeyspaceParams;
@@ -144,9 +144,9 @@
SerializationHeader.Component serializationHeader = SerializationHeader.makeWithoutStats(metadata).toComponent();
ComponentManifest componentManifest = entireSSTable ? ComponentManifest.create(sstable.descriptor) : null;
DecoratedKey firstKey = entireSSTable ? sstable.first : null;
+
return CassandraStreamHeader.builder()
- .withSSTableFormat(SSTableFormat.Type.BIG)
- .withSSTableVersion(BigFormat.latestVersion)
+ .withSSTableVersion(sstable.descriptor.version)
.withSSTableLevel(0)
.withEstimatedKeys(10)
.withCompressionInfo(compressionInfo)
@@ -166,8 +166,7 @@
TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build();
CassandraStreamHeader header =
CassandraStreamHeader.builder()
- .withSSTableFormat(SSTableFormat.Type.BIG)
- .withSSTableVersion(BigFormat.latestVersion)
+ .withSSTableVersion(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion())
.withSSTableLevel(0)
.withEstimatedKeys(0)
.withSections(Collections.emptyList())
@@ -184,12 +183,11 @@
String ddl = "CREATE TABLE tbl (k INT PRIMARY KEY, v INT)";
TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build();
- ComponentManifest manifest = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Component.DATA, 100L); }});
+ ComponentManifest manifest = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Components.DATA, 100L); }});
CassandraStreamHeader header =
CassandraStreamHeader.builder()
- .withSSTableFormat(SSTableFormat.Type.BIG)
- .withSSTableVersion(BigFormat.latestVersion)
+ .withSSTableVersion(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion())
.withSSTableLevel(0)
.withEstimatedKeys(0)
.withSections(Collections.emptyList())
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
index e7795c8..fac5706 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
@@ -53,10 +54,12 @@
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.StreamSummary;
import org.apache.cassandra.streaming.StreamingChannel;
import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.OutgoingStream;
@@ -69,6 +72,8 @@
import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
public class CassandraStreamManagerTest
{
@@ -90,7 +95,9 @@
public void createKeyspace() throws Exception
{
keyspace = String.format("ks_%s", System.currentTimeMillis());
- tbm = CreateTableStatement.parse(String.format("CREATE TABLE %s (k INT PRIMARY KEY, v INT)", table), keyspace).build();
+ tbm = CreateTableStatement.parse(String.format("CREATE TABLE %s (k INT PRIMARY KEY, v INT)", table), keyspace)
+ .compaction(CompactionParams.stcs(Collections.emptyMap()))
+ .build();
SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), tbm);
cfs = Schema.instance.getColumnFamilyStoreInstance(tbm.id);
}
@@ -240,8 +247,34 @@
done.set(true);
t.join(20);
}
- Assert.assertFalse(failed.get());
- Assert.assertTrue(checkCount.get() >= 2);
+ assertFalse(failed.get());
+ assertTrue(checkCount.get() >= 2);
cfs.truncateBlocking();
}
+
+ @Test
+ public void checkAvailableDiskSpaceAndCompactions()
+ {
+ assertTrue(StreamSession.checkAvailableDiskSpaceAndCompactions(createSummaries(), nextTimeUUID(), null, false));
+ }
+
+ @Test
+ public void checkAvailableDiskSpaceAndCompactionsFailing()
+ {
+ int threshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
+ ActiveRepairService.instance.setRepairPendingCompactionRejectThreshold(1);
+ assertFalse(StreamSession.checkAvailableDiskSpaceAndCompactions(createSummaries(), nextTimeUUID(), null, false));
+ ActiveRepairService.instance.setRepairPendingCompactionRejectThreshold(threshold);
+ }
+
+ private Collection<StreamSummary> createSummaries()
+ {
+ Collection<StreamSummary> summaries = new ArrayList<>();
+ for (int i = 0; i < 10; i++)
+ {
+ StreamSummary summary = new StreamSummary(tbm.id, i, (i + 1) * 10);
+ summaries.add(summary);
+ }
+ return summaries;
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java
index 4909263..f5bfe4f 100644
--- a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java
@@ -18,44 +18,49 @@
package org.apache.cassandra.db.streaming;
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.LinkedHashMap;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBufferFixed;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.serializers.SerializationUtils;
-import static org.junit.Assert.assertNotEquals;
-
public class ComponentManifestTest
{
+ @BeforeClass
+ public static void beforeClass()
+ {
+ DatabaseDescriptor.clientInitialization();
+ }
+
@Test
public void testSerialization()
{
- ComponentManifest expected = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Component.DATA, 100L); }});
- SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializer);
+ ComponentManifest expected = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Components.DATA, 100L); }});
+ SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializers.get(BigFormat.getInstance().name()));
}
- @Test(expected = EOFException.class)
- public void testSerialization_FailsOnBadBytes() throws IOException
- {
- ByteBuffer buf = ByteBuffer.allocate(512);
- ComponentManifest expected = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Component.DATA, 100L); }});
-
- DataOutputBufferFixed out = new DataOutputBufferFixed(buf);
-
- ComponentManifest.serializer.serialize(expected, out, MessagingService.VERSION_40);
-
- buf.putInt(0, -100);
-
- DataInputBuffer in = new DataInputBuffer(out.buffer(), false);
- ComponentManifest actual = ComponentManifest.serializer.deserialize(in, MessagingService.VERSION_40);
- assertNotEquals(expected, actual);
- }
+ // Propose removing this test which now fails on VIntOutOfRange
+ // We don't safely check if the bytes are bad so I don't understand what is being tested
+ // There is no checksum
+// @Test(expected = EOFException.class)
+// public void testSerialization_FailsOnBadBytes() throws IOException
+// {
+// ByteBuffer buf = ByteBuffer.allocate(512);
+// ComponentManifest expected = new ComponentManifest(new LinkedHashMap<Component, Long>() {{ put(Components.DATA, 100L); }});
+//
+// DataOutputBufferFixed out = new DataOutputBufferFixed(buf);
+//
+// ComponentManifest.serializer.serialize(expected, out, MessagingService.VERSION_40);
+//
+// buf.putInt(0, -100);
+//
+// DataInputBuffer in = new DataInputBuffer(out.buffer(), false);
+// ComponentManifest actual = ComponentManifest.serializer.deserialize(in, MessagingService.VERSION_40);
+// assertNotEquals(expected, actual);
+// }
}
diff --git a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java
index 8b63ba5..8107fd9 100644
--- a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java
@@ -56,10 +56,10 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.IndexSummaryManager;
-import org.apache.cassandra.io.sstable.IndexSummaryRedistribution;
import org.apache.cassandra.io.sstable.SSTableUtils;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
@@ -71,7 +71,6 @@
import org.apache.cassandra.schema.SchemaTestUtil;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.OutgoingStream;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionInfo;
@@ -80,6 +79,7 @@
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.streaming.StreamSummary;
+import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
import org.apache.cassandra.streaming.messages.StreamMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -324,7 +324,7 @@
StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, Collections.emptyList(), streamCoordinator);
InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
- streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED));
+ streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, null));
StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer);
session.init(future);
diff --git a/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java b/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java
index e44cbcd..e7f04b1 100644
--- a/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java
+++ b/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java
@@ -38,6 +38,7 @@
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -59,7 +60,7 @@
public void setup() throws Throwable
{
DatabaseDescriptor.setSnapshotOnDuplicateRowDetection(true);
- System.setProperty("cassandra.diagnostic_snapshot_interval_nanos", "0");
+ DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS.setLong(0);
// Create a table and insert some data. The actual rows read in the test will be synthetic
// but this creates an sstable on disk to be snapshotted.
createTable("CREATE TABLE %s (pk text, ck1 int, ck2 int, v int, PRIMARY KEY (pk, ck1, ck2))");
@@ -196,10 +197,10 @@
return BTreeRow.noCellLiveRow(Clustering.make(clusteringByteBuffers), LivenessInfo.create(0, 0));
}
- public static UnfilteredRowIterator rows(TableMetadata metadata,
- DecoratedKey key,
- boolean isReversedOrder,
- Unfiltered... unfiltereds)
+ public static UnfilteredRowIterator partition(TableMetadata metadata,
+ DecoratedKey key,
+ boolean isReversedOrder,
+ Unfiltered... unfiltereds)
{
Iterator<Unfiltered> iterator = Iterators.forArray(unfiltereds);
return new AbstractUnfilteredRowIterator(metadata,
@@ -227,7 +228,7 @@
public static UnfilteredPartitionIterator iter(TableMetadata metadata, boolean isReversedOrder, Unfiltered... unfiltereds)
{
DecoratedKey key = metadata.partitioner.decorateKey(bytes("key"));
- UnfilteredRowIterator rowIter = rows(metadata, key, isReversedOrder, unfiltereds);
+ UnfilteredRowIterator rowIter = partition(metadata, key, isReversedOrder, unfiltereds);
return new SingletonUnfilteredPartitionIterator(rowIter);
}
}
diff --git a/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java b/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java
new file mode 100644
index 0000000..94903a7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/CollectionMergeTrieTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.*;
+import static org.apache.cassandra.db.tries.MergeTrieTest.removeDuplicates;
+
+public class CollectionMergeTrieTest
+{
+ private static final int COUNT = 15000;
+ private static final Random rand = new Random();
+
+ @Test
+ public void testDirect()
+ {
+ ByteComparable[] src1 = generateKeys(rand, COUNT);
+ ByteComparable[] src2 = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+ InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+
+ content1.putAll(content2);
+ // construct directly, trie.merge() will defer to mergeWith on two sources
+ Trie<ByteBuffer> union = new CollectionMergeTrie<>(ImmutableList.of(trie1, trie2), x -> x.iterator().next());
+
+ assertSameContent(union, content1);
+ }
+
+ @Test
+ public void testWithDuplicates()
+ {
+ ByteComparable[] src1 = generateKeys(rand, COUNT);
+ ByteComparable[] src2 = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+ InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+
+ addToInMemoryTrie(generateKeys(new Random(5), COUNT), content1, trie1, true);
+ addToInMemoryTrie(generateKeys(new Random(5), COUNT), content2, trie2, true);
+
+ content1.putAll(content2);
+ Trie<ByteBuffer> union = new CollectionMergeTrie<>(ImmutableList.of(trie1, trie2), x -> x.iterator().next());
+
+ assertSameContent(union, content1);
+ }
+
+ @Test
+ public void testDistinct()
+ {
+ ByteComparable[] src1 = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+
+ ByteComparable[] src2 = generateKeys(rand, COUNT);
+ src2 = removeDuplicates(src2, content1);
+ SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+
+ content1.putAll(content2);
+ Trie<ByteBuffer> union = new CollectionMergeTrie.Distinct<>(ImmutableList.of(trie1, trie2));
+
+ assertSameContent(union, content1);
+ }
+
+ @Test
+ public void testMultiple()
+ {
+ for (int i = 0; i < 10; ++i)
+ {
+ testMultiple(rand.nextInt(10) + 5, COUNT / 10);
+ }
+ }
+
+ @Test
+ public void testMerge1()
+ {
+ testMultiple(1, COUNT / 10);
+ }
+
+ @Test
+ public void testMerge2()
+ {
+ testMultiple(2, COUNT / 10);
+ }
+
+ @Test
+ public void testMerge3()
+ {
+ testMultiple(3, COUNT / 10);
+ }
+
+ @Test
+ public void testMerge5()
+ {
+ testMultiple(5, COUNT / 10);
+ }
+
+ @Test
+ public void testMerge0()
+ {
+ testMultiple(0, COUNT / 10);
+ }
+
+ public void testMultiple(int mergeCount, int count)
+ {
+ testMultipleDistinct(mergeCount, count);
+ testMultipleWithDuplicates(mergeCount, count);
+ }
+
+ public void testMultipleDistinct(int mergeCount, int count)
+ {
+ List<Trie<ByteBuffer>> tries = new ArrayList<>(mergeCount);
+ SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+
+ for (int i = 0; i < mergeCount; ++i)
+ {
+ ByteComparable[] src = removeDuplicates(generateKeys(rand, count), content);
+ Trie<ByteBuffer> trie = makeInMemoryTrie(src, content, true);
+ tries.add(trie);
+ }
+
+ Trie<ByteBuffer> union = Trie.mergeDistinct(tries);
+ assertSameContent(union, content);
+ }
+
+ public void testMultipleWithDuplicates(int mergeCount, int count)
+ {
+ List<Trie<ByteBuffer>> tries = new ArrayList<>(mergeCount);
+ SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ ByteComparable[][] keys = new ByteComparable[count][];
+ for (int i = 0; i < mergeCount; ++i)
+ keys[i] = generateKeys(rand, count);
+
+ for (int i = 0; i < mergeCount; ++i)
+ {
+ ByteComparable[] src = Arrays.copyOf(keys[i], count + count / 10);
+ // add duplicates from other tries
+ if (mergeCount > 1)
+ {
+ for (int j = count; j < src.length; ++j)
+ src[j] = keys[randomButNot(rand, mergeCount, i)][rand.nextInt(count)];
+ }
+
+ Trie<ByteBuffer> trie = makeInMemoryTrie(keys[i], content, true);
+ tries.add(trie);
+ }
+
+ Trie<ByteBuffer> union = Trie.merge(tries, x -> x.iterator().next());
+ assertSameContent(union, content);
+
+ try
+ {
+ union = Trie.mergeDistinct(tries);
+ assertSameContent(union, content);
+ Assert.fail("Expected assertion error for duplicate keys.");
+ }
+ catch (AssertionError e)
+ {
+ // correct path
+ }
+ }
+
+ private int randomButNot(Random rand, int bound, int avoid)
+ {
+ int r;
+ do
+ {
+ r = rand.nextInt(bound);
+ }
+ while (r == avoid);
+ return r;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieApplyTest.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieApplyTest.java
new file mode 100644
index 0000000..4b0b7c4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieApplyTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+public class InMemoryTrieApplyTest extends InMemoryTrieTestBase
+{
+ @Override
+ boolean usePut()
+ {
+ return false;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java
new file mode 100644
index 0000000..51b23d8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTriePutTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.junit.Assert.fail;
+
+public class InMemoryTriePutTest extends InMemoryTrieTestBase
+{
+ @Override
+ boolean usePut()
+ {
+ return true;
+ }
+
+ @Test
+ public void testLongKey_StackOverflow() throws InMemoryTrie.SpaceExhaustedException
+ {
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.ON_HEAP);
+ Random rand = new Random(1);
+ byte[] key = new byte[40960];
+ rand.nextBytes(key);
+ ByteBuffer buf = ByteBuffer.wrap(key);
+
+ try
+ {
+ trie.putRecursive(ByteComparable.fixedLength(buf), "value", (x, y) -> y);
+ Assert.fail("StackOverflowError expected with a recursive put for very long keys!");
+ }
+ catch (StackOverflowError soe)
+ {
+ // Expected.
+ }
+ // Using non-recursive put should work.
+ putSimpleResolve(trie, ByteComparable.fixedLength(buf), "value", (x, y) -> y, false);
+ }
+
+ // This tests that trie space allocation works correctly close to the 2G limit. It is normally disabled because
+ // the test machines don't provide enough heap memory (test requires ~8G heap to finish). Run it manually when
+ // InMemoryTrie.allocateBlock is modified.
+ @Ignore
+ @Test
+ public void testOver1GSize() throws InMemoryTrie.SpaceExhaustedException
+ {
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.ON_HEAP);
+ trie.advanceAllocatedPos(0x20000000);
+ String t1 = "test1";
+ String t2 = "testing2";
+ String t3 = "onemoretest3";
+ trie.putRecursive(ByteComparable.of(t1), t1, (x, y) -> y);
+ Assert.assertEquals(t1, trie.get(ByteComparable.of(t1)));
+ Assert.assertNull(trie.get(ByteComparable.of(t2)));
+ Assert.assertFalse(trie.reachedAllocatedSizeThreshold());
+
+ trie.advanceAllocatedPos(InMemoryTrie.ALLOCATED_SIZE_THRESHOLD + 0x1000);
+ trie.putRecursive(ByteComparable.of(t2), t2, (x, y) -> y);
+ Assert.assertEquals(t1, trie.get(ByteComparable.of(t1)));
+ Assert.assertEquals(t2, trie.get(ByteComparable.of(t2)));
+ Assert.assertNull(trie.get(ByteComparable.of(t3)));
+ Assert.assertTrue(trie.reachedAllocatedSizeThreshold());
+
+ trie.advanceAllocatedPos(0x7FFFFEE0); // close to 2G
+ Assert.assertEquals(t1, trie.get(ByteComparable.of(t1)));
+ Assert.assertEquals(t2, trie.get(ByteComparable.of(t2)));
+ Assert.assertNull(trie.get(ByteComparable.of(t3)));
+ Assert.assertTrue(trie.reachedAllocatedSizeThreshold());
+
+ try
+ {
+ trie.putRecursive(ByteComparable.of(t3), t3, (x, y) -> y); // should put it over the edge
+ fail("InMemoryTrie.SpaceExhaustedError was expected");
+ }
+ catch (InMemoryTrie.SpaceExhaustedException e)
+ {
+ // expected
+ }
+
+ Assert.assertEquals(t1, trie.get(ByteComparable.of(t1)));
+ Assert.assertEquals(t2, trie.get(ByteComparable.of(t2)));
+ Assert.assertNull(trie.get(ByteComparable.of(t3)));
+ Assert.assertTrue(trie.reachedAllocatedSizeThreshold());
+
+ try
+ {
+ trie.advanceAllocatedPos(Integer.MAX_VALUE);
+ fail("InMemoryTrie.SpaceExhaustedError was expected");
+ }
+ catch (InMemoryTrie.SpaceExhaustedException e)
+ {
+ // expected
+ }
+
+ Assert.assertEquals(t1, trie.get(ByteComparable.of(t1)));
+ Assert.assertEquals(t2, trie.get(ByteComparable.of(t2)));
+ Assert.assertNull(trie.get(ByteComparable.of(t3)));
+ Assert.assertTrue(trie.reachedAllocatedSizeThreshold());
+
+ trie.discardBuffers();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java
new file mode 100644
index 0000000..d1c1711
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java
@@ -0,0 +1,631 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multiset;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.ObjectSizes;
+
+import static org.junit.Assert.assertEquals;
+
+public abstract class InMemoryTrieTestBase
+{
+ // Set this to true (in combination with smaller count) to dump the tries while debugging a problem.
+ // Do not commit the code with VERBOSE = true.
+ private static final boolean VERBOSE = false;
+
+ private static final int COUNT = 100000;
+ private static final int KEY_CHOICE = 25;
+ private static final int MIN_LENGTH = 10;
+ private static final int MAX_LENGTH = 50;
+
+ Random rand = new Random();
+
+ static final ByteComparable.Version VERSION = InMemoryTrie.BYTE_COMPARABLE_VERSION;
+
+ abstract boolean usePut();
+
+ @Test
+ public void testSingle()
+ {
+ ByteComparable e = ByteComparable.of("test");
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ putSimpleResolve(trie, e, "test", (x, y) -> y);
+ System.out.println("Trie " + trie.dump());
+ assertEquals("test", trie.get(e));
+ assertEquals(null, trie.get(ByteComparable.of("teste")));
+ }
+
+ @Test
+ public void testSplitMulti()
+ {
+ testEntries(new String[] { "testing", "tests", "trials", "trial", "aaaa", "aaaab", "abdddd", "abeeee" });
+ }
+
+ @Test
+ public void testSplitMultiBug()
+ {
+ testEntriesHex(new String[] { "0c4143aeff", "0c4143ae69ff" });
+ }
+
+
+ @Test
+ public void testSparse00bug()
+ {
+ String[] tests = new String[] {
+ "40bd256e6fd2adafc44033303000",
+ "40bdd47ec043641f2b403131323400",
+ "40bd00bf5ae8cf9d1d403133323800",
+ };
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ for (String test : tests)
+ {
+ ByteComparable e = ByteComparable.fixedLength(ByteBufferUtil.hexToBytes(test));
+ System.out.println("Adding " + asString(e) + ": " + test);
+ putSimpleResolve(trie, e, test, (x, y) -> y);
+ }
+
+ System.out.println(trie.dump());
+
+ for (String test : tests)
+ assertEquals(test, trie.get(ByteComparable.fixedLength(ByteBufferUtil.hexToBytes(test))));
+
+ Arrays.sort(tests);
+
+ int idx = 0;
+ for (String s : trie.values())
+ {
+ if (s != tests[idx])
+ throw new AssertionError("" + s + "!=" + tests[idx]);
+ ++idx;
+ }
+ assertEquals(tests.length, idx);
+ }
+
+ @Test
+ public void testUpdateContent()
+ {
+ String[] tests = new String[] {"testing", "tests", "trials", "trial", "testing", "trial", "trial"};
+ String[] values = new String[] {"testing", "tests", "trials", "trial", "t2", "x2", "y2"};
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ for (int i = 0; i < tests.length; ++i)
+ {
+ String test = tests[i];
+ String v = values[i];
+ ByteComparable e = ByteComparable.of(test);
+ System.out.println("Adding " + asString(e) + ": " + v);
+ putSimpleResolve(trie, e, v, (x, y) -> "" + x + y);
+ System.out.println("Trie " + trie.dump());
+ }
+
+ for (int i = 0; i < tests.length; ++i)
+ {
+ String test = tests[i];
+ assertEquals(Stream.iterate(0, x -> x + 1)
+ .limit(tests.length)
+ .filter(x -> tests[x] == test)
+ .map(x -> values[x])
+ .reduce("", (x, y) -> "" + x + y),
+ trie.get(ByteComparable.of(test)));
+ }
+ }
+
+ static class SpecStackEntry
+ {
+ Object[] children;
+ int curChild;
+ Object content;
+ SpecStackEntry parent;
+
+ public SpecStackEntry(Object[] spec, Object content, SpecStackEntry parent)
+ {
+ this.children = spec;
+ this.content = content;
+ this.parent = parent;
+ this.curChild = -1;
+ }
+ }
+
+ public static class CursorFromSpec implements Trie.Cursor<ByteBuffer>
+ {
+ SpecStackEntry stack;
+ int depth;
+
+ CursorFromSpec(Object[] spec)
+ {
+ stack = new SpecStackEntry(spec, null, null);
+ depth = 0;
+ }
+
+ public int advance()
+ {
+ SpecStackEntry current = stack;
+ while (current != null && ++current.curChild >= current.children.length)
+ {
+ current = current.parent;
+ --depth;
+ }
+ if (current == null)
+ {
+ assert depth == -1;
+ return depth;
+ }
+
+ Object child = current.children[current.curChild];
+ if (child instanceof Object[])
+ stack = new SpecStackEntry((Object[]) child, null, current);
+ else
+ stack = new SpecStackEntry(new Object[0], child, current);
+
+ return ++depth;
+ }
+
+ public int advanceMultiple()
+ {
+ if (++stack.curChild >= stack.children.length)
+ return skipChildren();
+
+ Object child = stack.children[stack.curChild];
+ while (child instanceof Object[])
+ {
+ stack = new SpecStackEntry((Object[]) child, null, stack);
+ ++depth;
+ if (stack.children.length == 0)
+ return depth;
+ child = stack.children[0];
+ }
+ stack = new SpecStackEntry(new Object[0], child, stack);
+
+
+ return ++depth;
+ }
+
+ public int skipChildren()
+ {
+ --depth;
+ stack = stack.parent;
+ return advance();
+ }
+
+ public int depth()
+ {
+ return depth;
+ }
+
+ public ByteBuffer content()
+ {
+ return (ByteBuffer) stack.content;
+ }
+
+ public int incomingTransition()
+ {
+ SpecStackEntry parent = stack.parent;
+ return parent != null ? parent.curChild + 0x30 : -1;
+ }
+ }
+
+ static Trie<ByteBuffer> specifiedTrie(Object[] nodeDef)
+ {
+ return new Trie<ByteBuffer>()
+ {
+ @Override
+ protected Cursor<ByteBuffer> cursor()
+ {
+ return new CursorFromSpec(nodeDef);
+ }
+ };
+ }
+
+ @Test
+ public void testEntriesNullChildBug()
+ {
+ Object[] trieDef = new Object[]
+ {
+ new Object[] { // 0
+ ByteBufferUtil.bytes(1), // 01
+ ByteBufferUtil.bytes(2) // 02
+ },
+ // If requestChild returns null, bad things can happen (DB-2982)
+ null, // 1
+ ByteBufferUtil.bytes(3), // 2
+ new Object[] { // 3
+ ByteBufferUtil.bytes(4), // 30
+ // Also try null on the Remaining.ONE path
+ null // 31
+ },
+ ByteBufferUtil.bytes(5), // 4
+ // Also test requestUniqueDescendant returning null
+ new Object[] { // 5
+ new Object[] { // 50
+ new Object[] { // 500
+ null // 5000
+ }
+ }
+ },
+ ByteBufferUtil.bytes(6) // 6
+ };
+
+ SortedMap<ByteComparable, ByteBuffer> expected = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ expected.put(comparable("00"), ByteBufferUtil.bytes(1));
+ expected.put(comparable("01"), ByteBufferUtil.bytes(2));
+ expected.put(comparable("2"), ByteBufferUtil.bytes(3));
+ expected.put(comparable("30"), ByteBufferUtil.bytes(4));
+ expected.put(comparable("4"), ByteBufferUtil.bytes(5));
+ expected.put(comparable("6"), ByteBufferUtil.bytes(6));
+
+ Trie<ByteBuffer> trie = specifiedTrie(trieDef);
+ System.out.println(trie.dump());
+ assertSameContent(trie, expected);
+ }
+
+ static ByteComparable comparable(String s)
+ {
+ ByteBuffer b = ByteBufferUtil.bytes(s);
+ return ByteComparable.fixedLength(b);
+ }
+
+ @Test
+ public void testDirect()
+ {
+ ByteComparable[] src = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ InMemoryTrie<ByteBuffer> trie = makeInMemoryTrie(src, content, usePut());
+ int keysize = Arrays.stream(src)
+ .mapToInt(src1 -> ByteComparable.length(src1, VERSION))
+ .sum();
+ long ts = ObjectSizes.measureDeep(content);
+ long onh = ObjectSizes.measureDeep(trie.contentArrays);
+ System.out.format("Trie size on heap %,d off heap %,d measured %,d keys %,d treemap %,d\n",
+ trie.sizeOnHeap(), trie.sizeOffHeap(), onh, keysize, ts);
+ System.out.format("per entry on heap %.2f off heap %.2f measured %.2f keys %.2f treemap %.2f\n",
+ trie.sizeOnHeap() * 1.0 / COUNT, trie.sizeOffHeap() * 1.0 / COUNT, onh * 1.0 / COUNT, keysize * 1.0 / COUNT, ts * 1.0 / COUNT);
+ if (VERBOSE)
+ System.out.println("Trie " + trie.dump(ByteBufferUtil::bytesToHex));
+
+ assertSameContent(trie, content);
+ checkGet(trie, content);
+
+ trie.discardBuffers();
+ }
+
+ @Test
+ public void testPrefixEvolution()
+ {
+ testEntries(new String[] { "testing",
+ "test",
+ "tests",
+ "tester",
+ "testers",
+ // test changing type with prefix
+ "types",
+ "types1",
+ "types",
+ "types2",
+ "types3",
+ "types4",
+ "types",
+ "types5",
+ "types6",
+ "types7",
+ "types8",
+ "types",
+ // test adding prefix to chain
+ "chain123",
+ "chain",
+ // test adding prefix to sparse
+ "sparse1",
+ "sparse2",
+ "sparse3",
+ "sparse",
+ // test adding prefix to split
+ "split1",
+ "split2",
+ "split3",
+ "split4",
+ "split5",
+ "split6",
+ "split7",
+ "split8",
+ "split"
+ });
+ }
+
+ @Test
+ public void testPrefixUnsafeMulti()
+ {
+ // Make sure prefixes on inside a multi aren't overwritten by embedded metadata node.
+
+ testEntries(new String[] { "test89012345678901234567890",
+ "test8",
+ "test89",
+ "test890",
+ "test8901",
+ "test89012",
+ "test890123",
+ "test8901234",
+ });
+ }
+
+ private void testEntries(String[] tests)
+ {
+ for (Function<String, ByteComparable> mapping :
+ ImmutableList.<Function<String, ByteComparable>>of(ByteComparable::of,
+ s -> ByteComparable.fixedLength(s.getBytes())))
+ {
+ testEntries(tests, mapping);
+ }
+ }
+
+ private void testEntriesHex(String[] tests)
+ {
+ testEntries(tests, s -> ByteComparable.fixedLength(ByteBufferUtil.hexToBytes(s)));
+ // Run the other translations just in case.
+ testEntries(tests);
+ }
+
+ private void testEntries(String[] tests, Function<String, ByteComparable> mapping)
+
+ {
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ for (String test : tests)
+ {
+ ByteComparable e = mapping.apply(test);
+ System.out.println("Adding " + asString(e) + ": " + test);
+ putSimpleResolve(trie, e, test, (x, y) -> y);
+ System.out.println("Trie\n" + trie.dump());
+ }
+
+ for (String test : tests)
+ assertEquals(test, trie.get(mapping.apply(test)));
+ }
+
+ static InMemoryTrie<ByteBuffer> makeInMemoryTrie(ByteComparable[] src,
+ Map<ByteComparable, ByteBuffer> content,
+ boolean usePut)
+
+ {
+ InMemoryTrie<ByteBuffer> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ addToInMemoryTrie(src, content, trie, usePut);
+ return trie;
+ }
+
+ static void addToInMemoryTrie(ByteComparable[] src,
+ Map<ByteComparable, ByteBuffer> content,
+ InMemoryTrie<ByteBuffer> trie,
+ boolean usePut)
+
+ {
+ for (ByteComparable b : src)
+ {
+ // Note: Because we don't ensure order when calling resolve, just use a hash of the key as payload
+ // (so that all sources have the same value).
+ int payload = asString(b).hashCode();
+ ByteBuffer v = ByteBufferUtil.bytes(payload);
+ content.put(b, v);
+ if (VERBOSE)
+ System.out.println("Adding " + asString(b) + ": " + ByteBufferUtil.bytesToHex(v));
+ putSimpleResolve(trie, b, v, (x, y) -> y, usePut);
+ if (VERBOSE)
+ System.out.println(trie.dump(ByteBufferUtil::bytesToHex));
+ }
+ }
+
+ static void checkGet(InMemoryTrie<ByteBuffer> trie, Map<ByteComparable, ByteBuffer> items)
+ {
+ for (Map.Entry<ByteComparable, ByteBuffer> en : items.entrySet())
+ {
+ assertEquals(en.getValue(), trie.get(en.getKey()));
+ }
+ }
+
+ static void assertSameContent(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+ {
+ assertMapEquals(trie, map);
+ assertForEachEntryEquals(trie, map);
+ assertValuesEqual(trie, map);
+ assertForEachValueEquals(trie, map);
+ assertUnorderedValuesEqual(trie, map);
+ }
+
+ private static void assertValuesEqual(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+ {
+ assertIterablesEqual(trie.values(), map.values());
+ }
+
+ private static void assertUnorderedValuesEqual(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+ {
+ Multiset<ByteBuffer> unordered = HashMultiset.create();
+ StringBuilder errors = new StringBuilder();
+ for (ByteBuffer b : trie.valuesUnordered())
+ unordered.add(b);
+
+ for (ByteBuffer b : map.values())
+ if (!unordered.remove(b))
+ errors.append("\nMissing value in valuesUnordered: " + ByteBufferUtil.bytesToHex(b));
+
+ for (ByteBuffer b : unordered)
+ errors.append("\nExtra value in valuesUnordered: " + ByteBufferUtil.bytesToHex(b));
+
+ assertEquals("", errors.toString());
+ }
+
+ private static void assertForEachEntryEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+ {
+ Iterator<Map.Entry<ByteComparable, ByteBuffer>> it = map.entrySet().iterator();
+ trie.forEachEntry((key, value) -> {
+ Assert.assertTrue("Map exhausted first, key " + asString(key), it.hasNext());
+ Map.Entry<ByteComparable, ByteBuffer> entry = it.next();
+ assertEquals(0, ByteComparable.compare(entry.getKey(), key, Trie.BYTE_COMPARABLE_VERSION));
+ assertEquals(entry.getValue(), value);
+ });
+ Assert.assertFalse("Trie exhausted first", it.hasNext());
+ }
+
+ private static void assertForEachValueEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+ {
+ Iterator<ByteBuffer> it = map.values().iterator();
+ trie.forEachValue(value -> {
+ Assert.assertTrue("Map exhausted first, value " + ByteBufferUtil.bytesToHex(value), it.hasNext());
+ ByteBuffer entry = it.next();
+ assertEquals(entry, value);
+ });
+ Assert.assertFalse("Trie exhausted first", it.hasNext());
+ }
+
+ static void assertMapEquals(Trie<ByteBuffer> trie, SortedMap<ByteComparable, ByteBuffer> map)
+ {
+ assertMapEquals(trie.entrySet(), map.entrySet());
+ }
+
+ static void assertMapEquals(Iterable<Map.Entry<ByteComparable, ByteBuffer>> container1,
+ Iterable<Map.Entry<ByteComparable, ByteBuffer>> container2)
+ {
+ Iterator<Map.Entry<ByteComparable, ByteBuffer>> it1 = container1.iterator();
+ Iterator<Map.Entry<ByteComparable, ByteBuffer>> it2 = container2.iterator();
+ List<ByteComparable> failedAt = new ArrayList<>();
+ StringBuilder b = new StringBuilder();
+ while (it1.hasNext() && it2.hasNext())
+ {
+ Map.Entry<ByteComparable, ByteBuffer> en1 = it1.next();
+ Map.Entry<ByteComparable, ByteBuffer> en2 = it2.next();
+ b.append(String.format("TreeSet %s:%s\n", asString(en2.getKey()), ByteBufferUtil.bytesToHex(en2.getValue())));
+ b.append(String.format("Trie %s:%s\n", asString(en1.getKey()), ByteBufferUtil.bytesToHex(en1.getValue())));
+ if (ByteComparable.compare(en1.getKey(), en2.getKey(), VERSION) != 0 || ByteBufferUtil.compareUnsigned(en1.getValue(), en2.getValue()) != 0)
+ failedAt.add(en1.getKey());
+ }
+ while (it1.hasNext())
+ {
+ Map.Entry<ByteComparable, ByteBuffer> en1 = it1.next();
+ b.append(String.format("Trie %s:%s\n", asString(en1.getKey()), ByteBufferUtil.bytesToHex(en1.getValue())));
+ failedAt.add(en1.getKey());
+ }
+ while (it2.hasNext())
+ {
+ Map.Entry<ByteComparable, ByteBuffer> en2 = it2.next();
+ b.append(String.format("TreeSet %s:%s\n", asString(en2.getKey()), ByteBufferUtil.bytesToHex(en2.getValue())));
+ failedAt.add(en2.getKey());
+ }
+ if (!failedAt.isEmpty())
+ {
+ String message = "Failed at " + Lists.transform(failedAt, InMemoryTrieTestBase::asString);
+ System.err.println(message);
+ System.err.println(b);
+ Assert.fail(message);
+ }
+ }
+
+ static <E extends Comparable<E>> void assertIterablesEqual(Iterable<E> expectedIterable, Iterable<E> actualIterable)
+ {
+ Iterator<E> expected = expectedIterable.iterator();
+ Iterator<E> actual = actualIterable.iterator();
+ while (actual.hasNext() && expected.hasNext())
+ {
+ Assert.assertEquals(actual.next(), expected.next());
+ }
+ if (expected.hasNext())
+ Assert.fail("Remaing values in expected, starting with " + expected.next());
+ else if (actual.hasNext())
+ Assert.fail("Remaing values in actual, starting with " + actual.next());
+ }
+
+ static ByteComparable[] generateKeys(Random rand, int count)
+ {
+ ByteComparable[] sources = new ByteComparable[count];
+ TreeSet<ByteComparable> added = new TreeSet<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ for (int i = 0; i < count; ++i)
+ {
+ sources[i] = generateKey(rand);
+ if (!added.add(sources[i]))
+ --i;
+ }
+
+ // note: not sorted!
+ return sources;
+ }
+
+ static ByteComparable generateKey(Random rand)
+ {
+ return generateKey(rand, MIN_LENGTH, MAX_LENGTH);
+ }
+
+ static ByteComparable generateKey(Random rand, int minLength, int maxLength)
+ {
+ int len = rand.nextInt(maxLength - minLength + 1) + minLength;
+ byte[] bytes = new byte[len];
+ int p = 0;
+ int length = bytes.length;
+ while (p < length)
+ {
+ int seed = rand.nextInt(KEY_CHOICE);
+ Random r2 = new Random(seed);
+ int m = r2.nextInt(5) + 2 + p;
+ if (m > length)
+ m = length;
+ while (p < m)
+ bytes[p++] = (byte) r2.nextInt(256);
+ }
+ return ByteComparable.fixedLength(bytes);
+ }
+
+ static String asString(ByteComparable bc)
+ {
+ return bc != null ? bc.byteComparableAsString(VERSION) : "null";
+ }
+
+ <T, M> void putSimpleResolve(InMemoryTrie<T> trie,
+ ByteComparable key,
+ T value,
+ Trie.MergeResolver<T> resolver)
+ {
+ putSimpleResolve(trie, key, value, resolver, usePut());
+ }
+
+ static <T, M> void putSimpleResolve(InMemoryTrie<T> trie,
+ ByteComparable key,
+ T value,
+ Trie.MergeResolver<T> resolver,
+ boolean usePut)
+ {
+ try
+ {
+ trie.putSingleton(key,
+ value,
+ (existing, update) -> existing != null ? resolver.resolve(existing, update) : update,
+ usePut);
+ }
+ catch (InMemoryTrie.SpaceExhaustedException e)
+ {
+ // Should not happen, test stays well below size limit.
+ throw new AssertionError(e);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java
new file mode 100644
index 0000000..c8b6ed5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.VERSION;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKeys;
+
+public class InMemoryTrieThreadedTest
+{
+ private static final int COUNT = 300000;
+ private static final int OTHERS = COUNT / 10;
+ private static final int PROGRESS_UPDATE = COUNT / 15;
+ private static final int READERS = 8;
+ private static final int WALKERS = 2;
+ private static final Random rand = new Random();
+
+ static String value(ByteComparable b)
+ {
+ return b.byteComparableAsString(VERSION);
+ }
+
+ @Test
+ public void testThreaded() throws InterruptedException
+ {
+ ByteComparable[] src = generateKeys(rand, COUNT + OTHERS);
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.ON_HEAP);
+ ConcurrentLinkedQueue<Throwable> errors = new ConcurrentLinkedQueue<>();
+ List<Thread> threads = new ArrayList<>();
+ AtomicBoolean writeCompleted = new AtomicBoolean(false);
+ AtomicInteger writeProgress = new AtomicInteger(0);
+
+ for (int i = 0; i < WALKERS; ++i)
+ threads.add(new Thread(() -> {
+ try
+ {
+ while (!writeCompleted.get())
+ {
+ int min = writeProgress.get();
+ int count = 0;
+ for (Map.Entry<ByteComparable, String> en : trie.entrySet())
+ {
+ String v = value(en.getKey());
+ Assert.assertEquals(en.getKey().byteComparableAsString(VERSION), v, en.getValue());
+ ++count;
+ }
+ Assert.assertTrue("Got only " + count + " while progress is at " + min, count >= min);
+ }
+ }
+ catch (Throwable t)
+ {
+ t.printStackTrace();
+ errors.add(t);
+ }
+ }));
+
+ for (int i = 0; i < READERS; ++i)
+ {
+ threads.add(new Thread(() -> {
+ try
+ {
+ Random r = ThreadLocalRandom.current();
+ while (!writeCompleted.get())
+ {
+ int min = writeProgress.get();
+
+ for (int i1 = 0; i1 < PROGRESS_UPDATE; ++i1)
+ {
+ int index = r.nextInt(COUNT + OTHERS);
+ ByteComparable b = src[index];
+ String v = value(b);
+ String result = trie.get(b);
+ if (result != null)
+ {
+ Assert.assertTrue("Got not added " + index + " when COUNT is " + COUNT,
+ index < COUNT);
+ Assert.assertEquals("Failed " + index, v, result);
+ }
+ else if (index < min)
+ Assert.fail("Failed index " + index + " while progress is at " + min);
+ }
+ }
+ }
+ catch (Throwable t)
+ {
+ t.printStackTrace();
+ errors.add(t);
+ }
+ }));
+ }
+
+ threads.add(new Thread(() -> {
+ try
+ {
+ for (int i = 0; i < COUNT; i++)
+ {
+ ByteComparable b = src[i];
+
+ // Note: Because we don't ensure order when calling resolve, just use a hash of the key as payload
+ // (so that all sources have the same value).
+ String v = value(b);
+ if (i % 2 == 0)
+ trie.apply(Trie.singleton(b, v), (x, y) -> y);
+ else
+ trie.putRecursive(b, v, (x, y) -> y);
+
+ if (i % PROGRESS_UPDATE == 0)
+ writeProgress.set(i);
+ }
+ }
+ catch (Throwable t)
+ {
+ t.printStackTrace();
+ errors.add(t);
+ }
+ finally
+ {
+ writeCompleted.set(true);
+ }
+ }));
+
+ for (Thread t : threads)
+ t.start();
+
+ for (Thread t : threads)
+ t.join();
+
+ if (!errors.isEmpty())
+ Assert.fail("Got errors:\n" + errors);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java b/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java
new file mode 100644
index 0000000..cc08401
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/MergeTrieTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.*;
+
+public class MergeTrieTest
+{
+ private static final int COUNT = 15000;
+ Random rand = new Random();
+
+ @Test
+ public void testDirect()
+ {
+ ByteComparable[] src1 = generateKeys(rand, COUNT);
+ ByteComparable[] src2 = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+ InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+
+ content1.putAll(content2);
+ Trie<ByteBuffer> union = trie1.mergeWith(trie2, (x, y) -> x);
+
+ assertSameContent(union, content1);
+ }
+
+ @Test
+ public void testWithDuplicates()
+ {
+ ByteComparable[] src1 = generateKeys(rand, COUNT);
+ ByteComparable[] src2 = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+ InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+
+ addToInMemoryTrie(generateKeys(new Random(5), COUNT), content1, trie1, true);
+ addToInMemoryTrie(generateKeys(new Random(5), COUNT), content2, trie2, true);
+
+ content1.putAll(content2);
+ Trie<ByteBuffer> union = trie1.mergeWith(trie2, (x, y) -> y);
+
+ assertSameContent(union, content1);
+ }
+
+ @Test
+ public void testDistinct()
+ {
+ ByteComparable[] src1 = generateKeys(rand, COUNT);
+ SortedMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+
+ ByteComparable[] src2 = generateKeys(rand, COUNT);
+ src2 = removeDuplicates(src2, content1);
+ SortedMap<ByteComparable, ByteBuffer> content2 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, VERSION));
+ InMemoryTrie<ByteBuffer> trie2 = makeInMemoryTrie(src2, content2, true);
+
+ content1.putAll(content2);
+ Trie<ByteBuffer> union = new MergeTrie.Distinct<>(trie1, trie2);
+
+ assertSameContent(union, content1);
+ }
+
+ static ByteComparable[] removeDuplicates(ByteComparable[] keys, SortedMap<ByteComparable, ByteBuffer> content1)
+ {
+ return Arrays.stream(keys)
+ .filter(key -> !content1.containsKey(key))
+ .toArray(ByteComparable[]::new);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java b/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java
new file mode 100644
index 0000000..07bae0e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/SlicedTrieTest.java
@@ -0,0 +1,527 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.googlecode.concurrenttrees.common.Iterables;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.asString;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.assertSameContent;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.generateKeys;
+import static org.apache.cassandra.db.tries.InMemoryTrieTestBase.makeInMemoryTrie;
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+
+public class SlicedTrieTest
+{
+ public static final ByteComparable[] BOUNDARIES = toByteComparable(new String[]{
+ "test1",
+ "test11",
+ "test12",
+ "test13",
+ "test2",
+ "test21",
+ "te",
+ "s",
+ "q",
+ "\000",
+ "\777",
+ "\777\000",
+ "\000\777",
+ "\000\000",
+ "\000\000\000",
+ "\000\000\777",
+ "\777\777"
+ });
+ public static final ByteComparable[] KEYS = toByteComparable(new String[]{
+ "test1",
+ "test2",
+ "test55",
+ "test123",
+ "test124",
+ "test12",
+ "test21",
+ "tease",
+ "sort",
+ "sorting",
+ "square",
+ "\777\000",
+ "\000\777",
+ "\000\000",
+ "\000\000\000",
+ "\000\000\777",
+ "\777\777"
+ });
+ public static final Comparator<ByteComparable> BYTE_COMPARABLE_COMPARATOR = (bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, Trie.BYTE_COMPARABLE_VERSION);
+ private static final int COUNT = 15000;
+ Random rand = new Random();
+
+ @Test
+ public void testIntersectRangeDirect()
+ {
+ testIntersectRange(COUNT);
+ }
+
+ public void testIntersectRange(int count)
+ {
+ ByteComparable[] src1 = generateKeys(rand, count);
+ NavigableMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>((bytes1, bytes2) -> ByteComparable.compare(bytes1, bytes2, Trie.BYTE_COMPARABLE_VERSION));
+
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(src1, content1, true);
+
+ checkEqualRange(content1, trie1, null, true, null, true);
+ checkEqualRange(content1, trie1, InMemoryTrieTestBase.generateKey(rand), true, null, true);
+ checkEqualRange(content1, trie1, null, true, InMemoryTrieTestBase.generateKey(rand), true);
+ for (int i = 0; i < 4; ++i)
+ {
+ ByteComparable l = rand.nextBoolean() ? InMemoryTrieTestBase.generateKey(rand) : src1[rand.nextInt(src1.length)];
+ ByteComparable r = rand.nextBoolean() ? InMemoryTrieTestBase.generateKey(rand) : src1[rand.nextInt(src1.length)];
+ int cmp = ByteComparable.compare(l, r, Trie.BYTE_COMPARABLE_VERSION);
+ if (cmp > 0)
+ {
+ ByteComparable t = l;
+ l = r;
+ r = t; // swap
+ }
+
+ boolean includeLeft = (i & 1) != 0 || cmp == 0;
+ boolean includeRight = (i & 2) != 0 || cmp == 0;
+ checkEqualRange(content1, trie1, l, includeLeft, r, includeRight);
+ checkEqualRange(content1, trie1, null, includeLeft, r, includeRight);
+ checkEqualRange(content1, trie1, l, includeLeft, null, includeRight);
+ }
+ }
+
+ private static ByteComparable[] toByteComparable(String[] keys)
+ {
+ return Arrays.stream(keys)
+ .map(x -> ByteComparable.fixedLength(x.getBytes(StandardCharsets.UTF_8)))
+ .toArray(ByteComparable[]::new);
+ }
+
+ @Test
+ public void testSingletonSubtrie()
+ {
+ Arrays.sort(BOUNDARIES, (a, b) -> ByteComparable.compare(a, b, ByteComparable.Version.OSS50));
+ for (int li = -1; li < BOUNDARIES.length; ++li)
+ {
+ ByteComparable l = li < 0 ? null : BOUNDARIES[li];
+ for (int ri = Math.max(0, li); ri <= BOUNDARIES.length; ++ri)
+ {
+ ByteComparable r = ri == BOUNDARIES.length ? null : BOUNDARIES[ri];
+
+ for (int i = li == ri ? 3 : 0; i < 4; ++i)
+ {
+ boolean includeLeft = (i & 1) != 0;
+ boolean includeRight = (i & 2) != 0;
+
+ for (ByteComparable key : KEYS)
+ {
+ int cmp1 = l != null ? ByteComparable.compare(key, l, ByteComparable.Version.OSS50) : 1;
+ int cmp2 = r != null ? ByteComparable.compare(r, key, ByteComparable.Version.OSS50) : 1;
+ Trie<Boolean> ix = new SlicedTrie<>(Trie.singleton(key, true), l, includeLeft, r, includeRight);
+ boolean expected = true;
+ if (cmp1 < 0 || cmp1 == 0 && !includeLeft)
+ expected = false;
+ if (cmp2 < 0 || cmp2 == 0 && !includeRight)
+ expected = false;
+ boolean actual = com.google.common.collect.Iterables.getFirst(ix.values(), false);
+ if (expected != actual)
+ {
+ System.err.println("Intersection");
+ System.err.println(ix.dump());
+ Assert.fail(String.format("Failed on range %s%s,%s%s key %s expected %s got %s\n",
+ includeLeft ? "[" : "(",
+ l != null ? l.byteComparableAsString(ByteComparable.Version.OSS50) : null,
+ r != null ? r.byteComparableAsString(ByteComparable.Version.OSS50) : null,
+ includeRight ? "]" : ")",
+ key.byteComparableAsString(ByteComparable.Version.OSS50),
+ expected,
+ actual));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMemtableSubtrie()
+ {
+ Arrays.sort(BOUNDARIES, BYTE_COMPARABLE_COMPARATOR);
+ NavigableMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+ InMemoryTrie<ByteBuffer> trie1 = makeInMemoryTrie(KEYS, content1, true);
+
+ for (int li = -1; li < BOUNDARIES.length; ++li)
+ {
+ ByteComparable l = li < 0 ? null : BOUNDARIES[li];
+ for (int ri = Math.max(0, li); ri <= BOUNDARIES.length; ++ri)
+ {
+ ByteComparable r = ri == BOUNDARIES.length ? null : BOUNDARIES[ri];
+ for (int i = 0; i < 4; ++i)
+ {
+ boolean includeLeft = (i & 1) != 0;
+ boolean includeRight = (i & 2) != 0;
+ if ((!includeLeft || !includeRight) && li == ri)
+ continue;
+ checkEqualRange(content1, trie1, l, includeLeft, r, includeRight);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMergeSubtrie()
+ {
+ testMergeSubtrie(2);
+ }
+
+ @Test
+ public void testCollectionMergeSubtrie3()
+ {
+ testMergeSubtrie(3);
+ }
+
+ @Test
+ public void testCollectionMergeSubtrie5()
+ {
+ testMergeSubtrie(5);
+ }
+
+ public void testMergeSubtrie(int mergeCount)
+ {
+ Arrays.sort(BOUNDARIES, BYTE_COMPARABLE_COMPARATOR);
+ NavigableMap<ByteComparable, ByteBuffer> content1 = new TreeMap<>(BYTE_COMPARABLE_COMPARATOR);
+ List<Trie<ByteBuffer>> tries = new ArrayList<>();
+ for (int i = 0; i < mergeCount; ++i)
+ {
+ tries.add(makeInMemoryTrie(Arrays.copyOfRange(KEYS,
+ KEYS.length * i / mergeCount,
+ KEYS.length * (i + 1) / mergeCount),
+ content1,
+ true));
+ }
+ Trie<ByteBuffer> trie1 = Trie.mergeDistinct(tries);
+
+ for (int li = -1; li < BOUNDARIES.length; ++li)
+ {
+ ByteComparable l = li < 0 ? null : BOUNDARIES[li];
+ for (int ri = Math.max(0, li); ri <= BOUNDARIES.length; ++ri)
+ {
+ ByteComparable r = ri == BOUNDARIES.length ? null : BOUNDARIES[ri];
+ for (int i = 0; i < 4; ++i)
+ {
+ boolean includeLeft = (i & 1) != 0;
+ boolean includeRight = (i & 2) != 0;
+ if ((!includeLeft || !includeRight) && li == ri)
+ continue;
+ checkEqualRange(content1, trie1, l, includeLeft, r, includeRight);
+ }
+ }
+ }
+ }
+
+ public void checkEqualRange(NavigableMap<ByteComparable, ByteBuffer> content1,
+ Trie<ByteBuffer> t1,
+ ByteComparable l,
+ boolean includeLeft,
+ ByteComparable r,
+ boolean includeRight)
+ {
+ System.out.println(String.format("Intersection with %s%s:%s%s", includeLeft ? "[" : "(", asString(l), asString(r), includeRight ? "]" : ")"));
+ SortedMap<ByteComparable, ByteBuffer> imap = l == null
+ ? r == null
+ ? content1
+ : content1.headMap(r, includeRight)
+ : r == null
+ ? content1.tailMap(l, includeLeft)
+ : content1.subMap(l, includeLeft, r, includeRight);
+
+ Trie<ByteBuffer> intersection = t1.subtrie(l, includeLeft, r, includeRight);
+ assertSameContent(intersection, imap);
+
+ if (l == null || r == null)
+ return;
+
+ // Test intersecting intersection.
+ intersection = t1.subtrie(l, includeLeft, null, false).subtrie(null, false, r, includeRight);
+ assertSameContent(intersection, imap);
+
+ intersection = t1.subtrie(null, false, r, includeRight).subtrie(l, includeLeft, null, false);
+ assertSameContent(intersection, imap);
+ }
+
+ /**
+ * Extract the values of the provide trie into a list.
+ */
+ private static <T> List<T> toList(Trie<T> trie)
+ {
+ return Iterables.toList(trie.values());
+ }
+
+ /**
+ * Creates a simple trie with a root having the provided number of childs, where each child is a leaf whose content
+ * is simply the value of the transition leading to it.
+ *
+ * In other words, {@code singleLevelIntTrie(4)} creates the following trie:
+ * Root
+ * t= 0 1 2 3
+ * | | | |
+ * 0 1 2 3
+ */
+ private static Trie<Integer> singleLevelIntTrie(int childs)
+ {
+ return new Trie<Integer>()
+ {
+ @Override
+ protected Cursor<Integer> cursor()
+ {
+ return new singleLevelCursor();
+ }
+
+ class singleLevelCursor implements Cursor<Integer>
+ {
+ int current = -1;
+
+ @Override
+ public int advance()
+ {
+ ++current;
+ return depth();
+ }
+
+ @Override
+ public int skipChildren()
+ {
+ return advance();
+ }
+
+ @Override
+ public int depth()
+ {
+ if (current == -1)
+ return 0;
+ if (current < childs)
+ return 1;
+ return -1;
+ }
+
+ @Override
+ public int incomingTransition()
+ {
+ return current;
+ }
+
+ @Override
+ public Integer content()
+ {
+ return current;
+ }
+ }
+ };
+ }
+
+ /** Creates a single byte {@link ByteComparable} with the provide value */
+ private static ByteComparable of(int value)
+ {
+ assert value >= 0 && value <= Byte.MAX_VALUE;
+ return ByteComparable.fixedLength(new byte[]{ (byte)value });
+ }
+
+ @Test
+ public void testSimpleIntersectionII()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(of(3), true, of(7), true);
+ assertEquals(asList(3, 4, 5, 6, 7), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleIntersectionEI()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(of(3), false, of(7), true);
+ assertEquals(asList(4, 5, 6, 7), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleIntersectionIE()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(of(3), true, of(7), false);
+ assertEquals(asList(3, 4, 5, 6), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleIntersectionEE()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(of(3), false, of(7), false);
+ assertEquals(asList(4, 5, 6), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleLeftIntersectionE()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(of(3), false, null, true);
+ assertEquals(asList(4, 5, 6, 7, 8, 9), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleLeftIntersectionI()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(of(3), true, null, true);
+ assertEquals(asList(3, 4, 5, 6, 7, 8, 9), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleRightIntersectionE()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(null, true, of(7), false);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleRightIntersectionI()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(null, true, of(7), true);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleNoIntersection()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(null, true, null, true);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(intersection));
+
+ // The two boolean flags don't have a meaning when the bound does not exist. For completeness, also test
+ // with them set to false.
+ intersection = trie.subtrie(null, false, null, false);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(intersection));
+ }
+
+ @Test
+ public void testSimpleEmptyIntersectionLeft()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(ByteComparable.EMPTY, true, null, true);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(intersection));
+
+ intersection = trie.subtrie(ByteComparable.EMPTY, false, null, true);
+ assertEquals(asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(intersection));
+
+ intersection = trie.subtrie(ByteComparable.EMPTY, true, of(5), true);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5), toList(intersection));
+
+ intersection = trie.subtrie(ByteComparable.EMPTY, false, of(5), true);
+ assertEquals(asList(0, 1, 2, 3, 4, 5), toList(intersection));
+
+ }
+
+ @Test
+ public void testSimpleEmptyIntersectionRight()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(10);
+ assertEquals(asList(-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9), toList(trie));
+
+ Trie<Integer> intersection = trie.subtrie(null, true, ByteComparable.EMPTY, true);
+ assertEquals(asList(-1), toList(intersection));
+
+ intersection = trie.subtrie(null, true, ByteComparable.EMPTY, false);
+ assertEquals(asList(), toList(intersection));
+
+ intersection = trie.subtrie(ByteComparable.EMPTY, true, ByteComparable.EMPTY, true);
+ assertEquals(asList(-1), toList(intersection));
+
+ intersection = trie.subtrie(ByteComparable.EMPTY, false, ByteComparable.EMPTY, true);
+ assertEquals(asList(), toList(intersection));
+
+ intersection = trie.subtrie(ByteComparable.EMPTY, true, ByteComparable.EMPTY, false);
+ assertEquals(asList(), toList(intersection));
+
+ // (empty, empty) is an invalid call as the "(empty" is greater than "empty)"
+ }
+
+ @Test
+ public void testSubtrieOnSubtrie()
+ {
+ Trie<Integer> trie = singleLevelIntTrie(15);
+
+ // non-overlapping
+ Trie<Integer> intersection = trie.subtrie(of(0), of(4)).subtrie(of(4), of(8));
+ assertEquals(asList(), toList(intersection));
+ // touching
+ intersection = trie.subtrie(of(0), true, of(3), true).subtrie(of(3), of(8));
+ assertEquals(asList(3), toList(intersection));
+ // overlapping 1
+ intersection = trie.subtrie(of(0), of(4)).subtrie(of(2), of(8));
+ assertEquals(asList(2, 3), toList(intersection));
+ // overlapping 2
+ intersection = trie.subtrie(of(0), of(4)).subtrie(of(1), of(8));
+ assertEquals(asList(1, 2, 3), toList(intersection));
+ // covered
+ intersection = trie.subtrie(of(0), of(4)).subtrie(of(0), of(8));
+ assertEquals(asList(0, 1, 2, 3), toList(intersection));
+ // covered 2
+ intersection = trie.subtrie(of(4), true, of(8), true).subtrie(of(0), of(8));
+ assertEquals(asList(4, 5, 6, 7), toList(intersection));
+ // covered 3
+ intersection = trie.subtrie(of(1), false, of(4), true).subtrie(of(0), of(8));
+ assertEquals(asList(2, 3, 4), toList(intersection));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/TrieToDot.java b/test/unit/org/apache/cassandra/db/tries/TrieToDot.java
new file mode 100644
index 0000000..fd47c0b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/TrieToDot.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.function.Function;
+
+import org.agrona.DirectBuffer;
+
+/**
+ * A class for dumping the structure of a trie to a graphviz/dot representation for making trie graphs.
+ */
+class TrieToDot<T> extends TriePathReconstructor implements Trie.Walker<T, String>
+{
+ private final StringBuilder b;
+ private final Function<T, String> contentToString;
+ private final Function<Integer, String> transitionToString;
+ private final boolean useMultiByte;
+ private int prevPos;
+ private int currNodeTextPos;
+
+ public TrieToDot(Function<T, String> contentToString,
+ Function<Integer, String> transitionToString,
+ boolean useMultiByte)
+ {
+ this.contentToString = contentToString;
+ this.transitionToString = transitionToString;
+ this.useMultiByte = useMultiByte;
+ this.b = new StringBuilder();
+ b.append("digraph G {\n" +
+ " splines=curved");
+ addNodeDefinition(nodeString(0));
+ }
+
+ @Override
+ public void resetPathLength(int newLength)
+ {
+ super.resetPathLength(newLength);
+ prevPos = newLength;
+ }
+
+ private void newLineAndIndent()
+ {
+ b.append('\n');
+ for (int i = 0; i < prevPos + 1; ++i)
+ b.append(" ");
+ }
+
+ @Override
+ public void addPathByte(int nextByte)
+ {
+ newLineAndIndent();
+ super.addPathByte(nextByte);
+ b.append(nodeString(prevPos));
+ b.append(" -> ");
+ String newNode = nodeString(keyPos);
+ b.append(newNode);
+ b.append(" [label=\"");
+ for (int i = prevPos; i < keyPos - 1; ++i)
+ b.append(transitionToString.apply(keyBytes[i] & 0xFF));
+ b.append(transitionToString.apply(nextByte));
+ b.append("\"]");
+ addNodeDefinition(newNode);
+ }
+
+ private void addNodeDefinition(String newNode)
+ {
+ prevPos = keyPos;
+ newLineAndIndent();
+ currNodeTextPos = b.length();
+ b.append(String.format("%s [shape=circle label=\"\"]", newNode));
+ }
+
+ private String nodeString(int keyPos)
+ {
+ StringBuilder b = new StringBuilder();
+ b.append("Node_");
+ for (int i = 0; i < keyPos; ++i)
+ b.append(transitionToString.apply(keyBytes[i] & 0xFF));
+ return b.toString();
+ }
+
+ @Override
+ public void addPathBytes(DirectBuffer buffer, int pos, int count)
+ {
+ if (useMultiByte)
+ {
+ super.addPathBytes(buffer, pos, count);
+ }
+ else
+ {
+ for (int i = 0; i < count; ++i)
+ addPathByte(buffer.getByte(pos + i) & 0xFF);
+ }
+ }
+
+ @Override
+ public void content(T content)
+ {
+ b.replace(currNodeTextPos, b.length(), String.format("%s [shape=doublecircle label=\"%s\"]", nodeString(keyPos), contentToString.apply(content)));
+ }
+
+ @Override
+ public String complete()
+ {
+ b.append("\n}\n");
+ return b.toString();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java b/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java
new file mode 100644
index 0000000..41c66aa
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/TrieToDotTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.compress.BufferType;
+
+public class TrieToDotTest
+{
+ @Test
+ public void testToDotContent() throws Exception
+ {
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ String s = "Trie node types and manipulation mechanisms. The main purpose of this is to allow for handling tries directly as" +
+ " they are on disk without any serialization, and to enable the creation of such files.";
+ s = s.toLowerCase();
+ for (String word : s.split("[^a-z]+"))
+ trie.putRecursive(InMemoryTrieTestBase.comparable(word), word, (x, y) -> y);
+
+ System.out.println(trie.process(new TrieToDot(Object::toString,
+ x -> Character.toString((char) ((int) x)),
+ true)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/TrieToMermaid.java b/test/unit/org/apache/cassandra/db/tries/TrieToMermaid.java
new file mode 100644
index 0000000..a75d301
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/TrieToMermaid.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.tries;
+
+import java.util.function.Function;
+
+import org.agrona.DirectBuffer;
+
+/**
+ * A class for dumping the structure of a trie to a graphviz/dot representation for making trie graphs.
+ */
+class TrieToMermaid<T> extends TriePathReconstructor implements Trie.Walker<T, String>
+{
+ private final StringBuilder b;
+ private final Function<T, String> contentToString;
+ private final Function<Integer, String> transitionToString;
+ private final boolean useMultiByte;
+ private int prevPos;
+ private int currNodeTextPos;
+
+ public TrieToMermaid(Function<T, String> contentToString,
+ Function<Integer, String> transitionToString,
+ boolean useMultiByte)
+ {
+ this.contentToString = contentToString;
+ this.transitionToString = transitionToString;
+ this.useMultiByte = useMultiByte;
+ this.b = new StringBuilder();
+ b.append("graph TD");
+ newLineAndIndent();
+ addNodeDefinition(nodeString(0));
+ newLineAndIndent();
+ b.append("style " + nodeString(0) + " fill:darkgrey");
+ }
+
+ @Override
+ public void resetPathLength(int newLength)
+ {
+ super.resetPathLength(newLength);
+ prevPos = newLength;
+ }
+
+ private void newLineAndIndent()
+ {
+ b.append('\n');
+ for (int i = 0; i < prevPos + 1; ++i)
+ b.append(" ");
+ }
+
+ @Override
+ public void addPathByte(int nextByte)
+ {
+ newLineAndIndent();
+ super.addPathByte(nextByte);
+ b.append(nodeString(prevPos));
+ String newNode = nodeString(keyPos);
+ b.append(" --\"");
+ for (int i = prevPos; i < keyPos - 1; ++i)
+ b.append(transitionToString.apply(keyBytes[i] & 0xFF));
+ b.append(transitionToString.apply(nextByte));
+ b.append("\"--> ");
+ addNodeDefinition(newNode);
+ }
+
+ private void addNodeDefinition(String newNode)
+ {
+ prevPos = keyPos;
+ currNodeTextPos = b.length();
+ b.append(String.format("%s(( ))", newNode));
+ }
+
+ private String nodeString(int keyPos)
+ {
+ StringBuilder b = new StringBuilder();
+ b.append("Node_");
+ for (int i = 0; i < keyPos; ++i)
+ b.append(transitionToString.apply(keyBytes[i] & 0xFF));
+ return b.toString();
+ }
+
+ @Override
+ public void addPathBytes(DirectBuffer buffer, int pos, int count)
+ {
+ if (useMultiByte)
+ {
+ super.addPathBytes(buffer, pos, count);
+ }
+ else
+ {
+ for (int i = 0; i < count; ++i)
+ addPathByte(buffer.getByte(pos + i) & 0xFF);
+ }
+ }
+
+ @Override
+ public void content(T content)
+ {
+ b.replace(currNodeTextPos, b.length(), String.format("%s(((%s)))", nodeString(keyPos), contentToString.apply(content)));
+ }
+
+ @Override
+ public String complete()
+ {
+ b.append("\n");
+ return b.toString();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java b/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java
new file mode 100644
index 0000000..bd691bb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/tries/TrieToMermaidTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.tries;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.compress.BufferType;
+
+public class TrieToMermaidTest
+{
+ @Test
+ public void testToMermaidContent() throws Exception
+ {
+ InMemoryTrie<String> trie = new InMemoryTrie<>(BufferType.OFF_HEAP);
+ // This was used as a basis the graphs in BTIFormat.md
+ String s = "a allow an and any are as node of on the this to trie types with without";
+ s = s.toLowerCase();
+ for (String word : s.split("[^a-z]+"))
+ trie.putRecursive(InMemoryTrieTestBase.comparable(word), word, (x, y) -> y);
+
+ System.out.println(trie.process(new TrieToMermaid(Object::toString,
+ x -> Character.toString((char) ((int) x)),
+ false)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/virtual/ClientsTableKeyspaceColTest.java b/test/unit/org/apache/cassandra/db/virtual/ClientsTableKeyspaceColTest.java
new file mode 100644
index 0000000..c282cac
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/virtual/ClientsTableKeyspaceColTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class ClientsTableKeyspaceColTest
+{
+
+ private static Cluster cluster;
+ private static EmbeddedCassandraService cassandra;
+ private static final String KS_NAME = "vts";
+
+ @BeforeClass
+ public static void setupClass() throws IOException
+ {
+ cassandra = ServerTestUtils.startEmbeddedCassandraService();
+
+ cluster = Cluster.builder().addContactPoint("127.0.0.1")
+ .withPort(DatabaseDescriptor.getNativeTransportPort())
+ .build();
+
+ ClientsTable table = new ClientsTable(KS_NAME);
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(table)));
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ try
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+ finally
+ {
+ if (cassandra != null)
+ cassandra.stop();
+ }
+ }
+
+ @Test
+ public void testWithoutConnectingToKeyspace()
+ {
+ try (Session session = cluster.connect())
+ {
+ List<Row> rows = session.execute("SELECT * from " + KS_NAME + ".clients").all();
+ assertTrue("At least one client should be returned.", rows.size() > 0);
+ for (Row r : rows)
+ assertNull(r.getString("keyspace_name")); // No keyspace is specifed while connecting. It should be null.
+ }
+ }
+
+ @Test
+ public void testChangingKeyspace()
+ {
+ String keyspace1 = "system_distributed";
+ String keyspace2 = "system_auth";
+ try (Session session = cluster.connect(keyspace1))
+ {
+
+ InetAddress poolConnection = null;
+ int port = -1;
+ for (Row r : session.execute("SELECT * from " + KS_NAME + ".clients").all())
+ {
+ // Keyspace is used for pool connection only (control connection is not using keyspace).
+ // Using r["keyspace_name"] == keyspace1 as a hint to identify a pool connection as we can't identify
+ // control connection based on information in this table.
+ String keyspace = r.getString("keyspace_name");
+ if (keyspace1.equals(keyspace))
+ {
+ poolConnection = r.getInet("address");
+ port = r.getInt("port");
+ break;
+ }
+ }
+
+ assertNotEquals(-1, port);
+ assertNotNull(poolConnection);
+
+ session.execute("USE " + keyspace2);
+
+ String usedKeyspace = null;
+ for (Row r : session.execute("SELECT * from " + KS_NAME + ".clients").all())
+ {
+ if (poolConnection.equals(r.getInet("address")) && port == r.getInt("port"))
+ {
+ usedKeyspace = r.getString("keyspace_name");
+ break;
+ }
+ }
+
+ assertEquals(keyspace2, usedKeyspace);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/virtual/CredentialsCacheKeysTableTest.java b/test/unit/org/apache/cassandra/db/virtual/CredentialsCacheKeysTableTest.java
index 45132a9..7aa1ec4 100644
--- a/test/unit/org/apache/cassandra/db/virtual/CredentialsCacheKeysTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/CredentialsCacheKeysTableTest.java
@@ -26,6 +26,7 @@
import com.datastax.driver.core.EndPoint;
import com.datastax.driver.core.PlainTextAuthProvider;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.auth.AuthTestUtils;
import org.apache.cassandra.auth.AuthenticatedUser;
import org.apache.cassandra.auth.IAuthenticator;
@@ -48,6 +49,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
// high value is used for convenient debugging
DatabaseDescriptor.setCredentialsValidity(20_000);
diff --git a/test/unit/org/apache/cassandra/db/virtual/LogMessagesTableTest.java b/test/unit/org/apache/cassandra/db/virtual/LogMessagesTableTest.java
new file mode 100644
index 0000000..f3655a4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/virtual/LogMessagesTableTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.time.Instant;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.LoggingEvent;
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable.DataSet;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable.Partition;
+import org.apache.cassandra.dht.LocalPartitioner;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.LOGS_VIRTUAL_TABLE_MAX_ROWS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class LogMessagesTableTest extends CQLTester
+{
+ private String keyspace = createKeyspaceName();
+ private LogMessagesTable table;
+
+ @BeforeClass
+ public static void setup()
+ {
+ CQLTester.setUpClass();
+ }
+
+ @Test
+ public void testTruncate() throws Throwable
+ {
+ registerVirtualTable();
+
+ int numberOfRows = 100;
+ List<LoggingEvent> loggingEvents = getLoggingEvents(numberOfRows);
+ loggingEvents.forEach(table::add);
+
+ execute(query("truncate %s"));
+
+ assertTrue(executeNet(query("select timestamp from %s")).all().isEmpty());
+ }
+
+ @Test
+ public void empty() throws Throwable
+ {
+ registerVirtualTable();
+ assertEmpty(execute(query("select * from %s")));
+ }
+
+ @Test
+ public void testInsert()
+ {
+ registerVirtualTable();
+
+ int numberOfRows = 1000;
+ List<LoggingEvent> loggingEvents = getLoggingEvents(numberOfRows);
+ loggingEvents.forEach(table::add);
+
+ assertEquals(numberOfRows, numberOfPartitions());
+ }
+
+ @Test
+ public void testLimitedCapacity() throws Throwable
+ {
+ registerVirtualTable(100);
+
+ int numberOfRows = 1000;
+ List<LoggingEvent> loggingEvents = getLoggingEvents(numberOfRows);
+ loggingEvents.forEach(table::add);
+
+ // even we inserted 1000 rows, only 100 are present as its capacity is bounded
+ assertEquals(100, numberOfPartitions());
+
+ // the first record in the table will be the last one which we inserted
+ LoggingEvent firstEvent = loggingEvents.get(999);
+ assertRowsNet(executeNet(query("select timestamp from %s limit 1")),
+ new Object[] { new Date(firstEvent.getTimeStamp()) });
+
+ // the last record in the table will be 900th we inserted
+ List<Row> all = executeNet(query("select timestamp from %s")).all();
+ assertEquals(100, all.size());
+ Row row = all.get(all.size() - 1);
+ Date timestamp = row.getTimestamp(0);
+ assertEquals(loggingEvents.get(900).getTimeStamp(), timestamp.getTime());
+ }
+
+ @Test
+ public void testMultipleLogsInSameMillisecond()
+ {
+ registerVirtualTable(10);
+ List<LoggingEvent> loggingEvents = getLoggingEvents(10, Instant.now(), 5);
+ loggingEvents.forEach(table::add);
+
+ // 2 partitions, 5 rows in each
+ assertEquals(2, numberOfPartitions());
+ }
+
+ @Test
+ public void testResolvingBufferSize()
+ {
+ LOGS_VIRTUAL_TABLE_MAX_ROWS.setInt(-1);
+ assertEquals(LogMessagesTable.LOGS_VIRTUAL_TABLE_DEFAULT_ROWS, LogMessagesTable.resolveBufferSize());
+
+ LOGS_VIRTUAL_TABLE_MAX_ROWS.setInt(0);
+ assertEquals(LogMessagesTable.LOGS_VIRTUAL_TABLE_DEFAULT_ROWS, LogMessagesTable.resolveBufferSize());
+
+ LOGS_VIRTUAL_TABLE_MAX_ROWS.setInt(1000001);
+ assertEquals(LogMessagesTable.LOGS_VIRTUAL_TABLE_DEFAULT_ROWS, LogMessagesTable.resolveBufferSize());
+
+ LOGS_VIRTUAL_TABLE_MAX_ROWS.setInt(999);
+ assertEquals(LogMessagesTable.LOGS_VIRTUAL_TABLE_DEFAULT_ROWS, LogMessagesTable.resolveBufferSize());
+
+ LOGS_VIRTUAL_TABLE_MAX_ROWS.setInt(50001);
+ assertEquals(50001, LogMessagesTable.resolveBufferSize());
+ }
+
+ private void registerVirtualTable()
+ {
+ registerVirtualTable(LogMessagesTable.LOGS_VIRTUAL_TABLE_MIN_ROWS);
+ }
+
+ private void registerVirtualTable(int size)
+ {
+ table = new LogMessagesTable(keyspace, size);
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(keyspace, ImmutableList.of(table)));
+ }
+
+ private int numberOfPartitions()
+ {
+ DataSet data = table.data();
+
+ Iterator<Partition> partitions = data.getPartitions(DataRange.allData(new LocalPartitioner(TimestampType.instance)));
+
+ int numberOfPartitions = 0;
+
+ while (partitions.hasNext())
+ {
+ partitions.next();
+ numberOfPartitions += 1;
+ }
+
+ return numberOfPartitions;
+ }
+
+ private String query(String query)
+ {
+ return String.format(query, table.toString());
+ }
+
+ private List<LoggingEvent> getLoggingEvents(int size)
+ {
+ return getLoggingEvents(size, Instant.now(), 1);
+ }
+
+ private List<LoggingEvent> getLoggingEvents(int size, Instant firstTimestamp, int logsInMillisecond)
+ {
+ List<LoggingEvent> logs = new LinkedList<>();
+ int partitions = size / logsInMillisecond;
+
+ for (int i = 0; i < partitions; i++)
+ {
+ long timestamp = firstTimestamp.toEpochMilli();
+ firstTimestamp = firstTimestamp.plusSeconds(1);
+
+ for (int j = 0; j < logsInMillisecond; j++)
+ logs.add(getLoggingEvent(timestamp));
+ }
+
+ return logs;
+ }
+
+ private LoggingEvent getLoggingEvent(long timestamp)
+ {
+ LoggingEvent event = new LoggingEvent();
+ event.setLevel(Level.INFO);
+ event.setMessage("message " + timestamp);
+ event.setLoggerName("logger " + timestamp);
+ event.setThreadName(Thread.currentThread().getName());
+ event.setTimeStamp(timestamp);
+
+ return event;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/virtual/NetworkPermissionsCacheKeysTableTest.java b/test/unit/org/apache/cassandra/db/virtual/NetworkPermissionsCacheKeysTableTest.java
index f944884..e935bbb 100644
--- a/test/unit/org/apache/cassandra/db/virtual/NetworkPermissionsCacheKeysTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/NetworkPermissionsCacheKeysTableTest.java
@@ -24,6 +24,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.auth.AuthTestUtils;
import org.apache.cassandra.auth.AuthenticatedUser;
import org.apache.cassandra.auth.DCPermissions;
@@ -46,6 +47,8 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
+
// high value is used for convenient debugging
DatabaseDescriptor.setPermissionsValidity(20_000);
diff --git a/test/unit/org/apache/cassandra/db/virtual/PermissionsCacheKeysTableTest.java b/test/unit/org/apache/cassandra/db/virtual/PermissionsCacheKeysTableTest.java
index 2171b2d..4295175 100644
--- a/test/unit/org/apache/cassandra/db/virtual/PermissionsCacheKeysTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/PermissionsCacheKeysTableTest.java
@@ -28,6 +28,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.auth.AuthTestUtils;
import org.apache.cassandra.auth.AuthenticatedUser;
import org.apache.cassandra.auth.DataResource;
@@ -52,6 +53,7 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
// high value is used for convenient debugging
DatabaseDescriptor.setPermissionsValidity(20_000);
diff --git a/test/unit/org/apache/cassandra/db/virtual/RolesCacheKeysTableTest.java b/test/unit/org/apache/cassandra/db/virtual/RolesCacheKeysTableTest.java
index 40c3037..de14aef 100644
--- a/test/unit/org/apache/cassandra/db/virtual/RolesCacheKeysTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/RolesCacheKeysTableTest.java
@@ -24,6 +24,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.auth.AuthTestUtils;
import org.apache.cassandra.auth.AuthenticatedUser;
import org.apache.cassandra.auth.IRoleManager;
@@ -46,6 +47,7 @@
@BeforeClass
public static void setUpClass()
{
+ ServerTestUtils.daemonInitialization();
// high value is used for convenient debugging
DatabaseDescriptor.setRolesValidity(20_000);
diff --git a/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
index f642a2f..6e8a136 100644
--- a/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
@@ -73,11 +73,14 @@
List<SSTableReader> sstables = IntStream.range(0, 10)
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
.collect(Collectors.toList());
+
+ String directory = String.format("/some/datadir/%s/%s-%s", cfs.metadata.keyspace, cfs.metadata.name, cfs.metadata.id.asUUID());
+
CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
{
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
+ return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, directory);
}
public boolean isGlobal()
@@ -90,7 +93,7 @@
UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks");
assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal,
OperationType.COMPACTION.toString().toLowerCase(), bytesCompacted, sstables.size(),
- bytesTotal, CompactionInfo.Unit.BYTES.toString()));
+ directory, bytesTotal, CompactionInfo.Unit.BYTES.toString()));
CompactionManager.instance.active.finishCompaction(compactionHolder);
result = execute("SELECT * FROM vts.sstable_tasks");
diff --git a/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java
new file mode 100644
index 0000000..0983230
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Collections;
+import java.util.Date;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.Clock;
+
+public class SnapshotsTableTest extends CQLTester
+{
+ private static final String KS_NAME = "vts";
+ private static final String SNAPSHOT_TTL = "snapshotTtl";
+ private static final String SNAPSHOT_NO_TTL = "snapshotNoTtl";
+ private static final DurationSpec.IntSecondsBound ttl = new DurationSpec.IntSecondsBound("4h");
+
+ @Before
+ public void before() throws Throwable
+ {
+ SnapshotsTable vtable = new SnapshotsTable(KS_NAME);
+ VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(vtable)));
+
+ createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))");
+
+ for (int i = 0; i != 10; ++i)
+ execute("INSERT INTO %s (pk, ck) VALUES (?, ?)", i, i);
+
+ flush();
+ }
+
+ @After
+ public void after()
+ {
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_NO_TTL, KEYSPACE);
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_TTL, KEYSPACE);
+
+ schemaChange(String.format("DROP TABLE %s", KEYSPACE + "." + currentTable()));
+ }
+
+ @Test
+ public void testSnapshots() throws Throwable
+ {
+ Instant now = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()).truncatedTo(ChronoUnit.MILLIS);
+ Date createdAt = new Date(now.toEpochMilli());
+ Date expiresAt = new Date(now.plusSeconds(ttl.toSeconds()).toEpochMilli());
+
+ getCurrentColumnFamilyStore(KEYSPACE).snapshot(SNAPSHOT_NO_TTL, null, false, false, null, null, now);
+ getCurrentColumnFamilyStore(KEYSPACE).snapshot(SNAPSHOT_TTL, null, false, false, ttl, null, now);
+
+ // query all from snapshots virtual table
+ UntypedResultSet result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots");
+ assertRowsIgnoringOrder(result,
+ row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false),
+ row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
+
+ // query with conditions
+ result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where ephemeral = false");
+ assertRows(result,
+ row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false),
+ row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
+
+ result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where size_on_disk > 1000");
+ assertRows(result,
+ row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false),
+ row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
+
+ result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where name = ?", SNAPSHOT_TTL);
+ assertRows(result,
+ row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
+
+ // clear some snapshots
+ StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_NO_TTL, KEYSPACE);
+
+ result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots");
+ assertRowsIgnoringOrder(result,
+ row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java b/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java
index c8e3d89..56d0193 100644
--- a/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java
@@ -96,9 +96,9 @@
assertRows(execute(t("select id, follower, operation, peers, status, progress_percentage, last_updated_at, failure_cause, success_message from %s")),
new Object[] { state.id(), true, "Repair", Collections.emptyList(), "start", 0F, new Date(state.lastUpdatedAtMillis()), null, null });
- state.handleStreamEvent(new StreamEvent.SessionPreparedEvent(state.id(), new SessionInfo(PEER2, 1, PEER1, Collections.emptyList(), Collections.emptyList(), StreamSession.State.PREPARING), StreamSession.PrepareDirection.ACK));
+ state.handleStreamEvent(new StreamEvent.SessionPreparedEvent(state.id(), new SessionInfo(PEER2, 1, PEER1, Collections.emptyList(), Collections.emptyList(), StreamSession.State.PREPARING, null), StreamSession.PrepareDirection.ACK));
- state.onSuccess(new StreamState(state.id(), StreamOperation.REPAIR, ImmutableSet.of(new SessionInfo(PEER2, 1, PEER1, Collections.emptyList(), Collections.emptyList(), StreamSession.State.COMPLETE))));
+ state.onSuccess(new StreamState(state.id(), StreamOperation.REPAIR, ImmutableSet.of(new SessionInfo(PEER2, 1, PEER1, Collections.emptyList(), Collections.emptyList(), StreamSession.State.COMPLETE, null))));
assertRows(execute(t("select id, follower, operation, peers, status, progress_percentage, last_updated_at, failure_cause, success_message from %s")),
new Object[] { state.id(), true, "Repair", Arrays.asList(address(127, 0, 0, 2).toString()), "success", 100F, new Date(state.lastUpdatedAtMillis()), null, null });
}
@@ -121,8 +121,8 @@
StreamResultFuture future = state.future();
state.phase.start();
- SessionInfo s1 = new SessionInfo(PEER2, 0, FBUtilities.getBroadcastAddressAndPort(), Arrays.asList(streamSummary()), Arrays.asList(streamSummary(), streamSummary()), StreamSession.State.PREPARING);
- SessionInfo s2 = new SessionInfo(PEER3, 0, FBUtilities.getBroadcastAddressAndPort(), Arrays.asList(streamSummary()), Arrays.asList(streamSummary(), streamSummary()), StreamSession.State.PREPARING);
+ SessionInfo s1 = new SessionInfo(PEER2, 0, FBUtilities.getBroadcastAddressAndPort(), Arrays.asList(streamSummary()), Arrays.asList(streamSummary(), streamSummary()), StreamSession.State.PREPARING, null);
+ SessionInfo s2 = new SessionInfo(PEER3, 0, FBUtilities.getBroadcastAddressAndPort(), Arrays.asList(streamSummary()), Arrays.asList(streamSummary(), streamSummary()), StreamSession.State.PREPARING, null);
// we only update stats on ACK
state.handleStreamEvent(new StreamEvent.SessionPreparedEvent(state.id(), s1, StreamSession.PrepareDirection.ACK));
diff --git a/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java
index 5242d55..c8ca9ba 100644
--- a/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/SystemPropertiesTableTest.java
@@ -37,11 +37,13 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+// checkstyle: suppress below 'blockSystemPropertyUsage'
+
public class SystemPropertiesTableTest extends CQLTester
{
private static final String KS_NAME = "vts";
private static final Map<String, String> ORIGINAL_ENV_MAP = System.getenv();
- private static final String TEST_PROP = "org.apache.cassandra.db.virtual.SystemPropertiesTableTest";
+ private static final String TEST_PROP = "cassandra.SystemPropertiesTableTest";
private SystemPropertiesTable table;
diff --git a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
index 5b5365d..c24690b 100644
--- a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
+++ b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
@@ -27,6 +27,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@@ -36,6 +37,8 @@
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.FBUtilities;
/**
@@ -124,5 +127,11 @@
{
return 0;
}
+
+ @Override
+ public ByteSource asComparableBytes(ByteComparable.Version version)
+ {
+ return IntegerType.instance.asComparableBytes(IntegerType.instance.decompose(token), version);
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
index c4e5db8..9859487 100644
--- a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
+++ b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java
@@ -34,6 +34,8 @@
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
public class LengthPartitioner implements IPartitioner
{
@@ -95,6 +97,11 @@
return new BigIntegerToken(new BigInteger(ByteBufferUtil.getArray(bytes)));
}
+ public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
+ {
+ return fromByteArray(IntegerType.instance.fromComparableBytes(comparableBytes, version));
+ }
+
public String toString(Token token)
{
BigIntegerToken bigIntegerToken = (BigIntegerToken) token;
@@ -129,15 +136,15 @@
public Map<Token, Float> describeOwnership(List<Token> sortedTokens)
{
// allTokens will contain the count and be returned, sorted_ranges is shorthand for token<->token math.
- Map<Token, Float> allTokens = new HashMap<Token, Float>();
- List<Range<Token>> sortedRanges = new ArrayList<Range<Token>>();
+ Map<Token, Float> allTokens = new HashMap<>();
+ List<Range<Token>> sortedRanges = new ArrayList<>();
// this initializes the counts to 0 and calcs the ranges in order.
Token lastToken = sortedTokens.get(sortedTokens.size() - 1);
for (Token node : sortedTokens)
{
- allTokens.put(node, new Float(0.0));
- sortedRanges.add(new Range<Token>(lastToken, node));
+ allTokens.put(node, 0.0F);
+ sortedRanges.add(new Range<>(lastToken, node));
lastToken = node;
}
@@ -154,7 +161,7 @@
}
// Sum every count up and divide count/total for the fractional ownership.
- Float total = new Float(0.0);
+ Float total = 0.0F;
for (Float f : allTokens.values())
total += f;
for (Map.Entry<Token, Float> row : allTokens.entrySet())
diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java b/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java
index e580461..c139760 100644
--- a/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java
+++ b/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java
@@ -91,13 +91,13 @@
}
@Override
- public void warn(String msg, Throwable th)
+ public void warn(Throwable th, String msg)
{
// We can only guarantee that ownership stdev won't increase above the warn threshold for racks==1 or racks==rf
if (racks == 1 || racks == rf)
fail(msg);
else
- super.warn(msg, th);
+ super.warn(th, msg);
}
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java b/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java
index 5e897b6..9647519 100644
--- a/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java
+++ b/test/unit/org/apache/cassandra/diag/store/DiagnosticEventMemoryStoreTest.java
@@ -66,15 +66,15 @@
assertEquals(3, res.size());
Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
- assertEquals(new Long(1), entry.getKey());
+ assertEquals(Long.valueOf(1), entry.getKey());
assertSame(e1, entry.getValue());
entry = res.pollFirstEntry();
- assertEquals(new Long(2), entry.getKey());
+ assertEquals(Long.valueOf(2), entry.getKey());
assertSame(e2, entry.getValue());
entry = res.pollFirstEntry();
- assertEquals(new Long(3), entry.getKey());
+ assertEquals(Long.valueOf(3), entry.getKey());
assertSame(e3, entry.getValue());
}
@@ -95,11 +95,11 @@
assertEquals(2, res.size());
Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
- assertEquals(new Long(1), entry.getKey());
+ assertEquals(Long.valueOf(1), entry.getKey());
assertSame(e1, entry.getValue());
entry = res.pollLastEntry();
- assertEquals(new Long(2), entry.getKey());
+ assertEquals(Long.valueOf(2), entry.getKey());
assertSame(e2, entry.getValue());
}
@@ -122,11 +122,11 @@
assertEquals(2, res.size());
Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
- assertEquals(new Long(2), entry.getKey());
+ assertEquals(Long.valueOf(2), entry.getKey());
assertSame(e2, entry.getValue());
entry = res.pollLastEntry();
- assertEquals(new Long(3), entry.getKey());
+ assertEquals(Long.valueOf(3), entry.getKey());
assertSame(e3, entry.getValue());
}
@@ -151,11 +151,11 @@
assertEquals(2, res.size());
Map.Entry<Long, DiagnosticEvent> entry = res.pollFirstEntry();
- assertEquals(new Long(4), entry.getKey());
+ assertEquals(Long.valueOf(4), entry.getKey());
assertSame(e2, entry.getValue());
entry = res.pollFirstEntry();
- assertEquals(new Long(5), entry.getKey());
+ assertEquals(Long.valueOf(5), entry.getKey());
assertSame(e3, entry.getValue());
store.store(new TestEvent1()); // 7
@@ -164,7 +164,7 @@
res = store.scan(4L, 2);
assertEquals(2, res.size());
- assertEquals(new Long(7), res.pollFirstEntry().getKey());
- assertEquals(new Long(8), res.pollLastEntry().getKey());
+ assertEquals(Long.valueOf(7), res.pollFirstEntry().getKey());
+ assertEquals(Long.valueOf(8), res.pollLastEntry().getKey());
}
}
diff --git a/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java b/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
index 77fabef..8ec3dae 100644
--- a/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
+++ b/test/unit/org/apache/cassandra/gms/FailureDetectorTest.java
@@ -37,6 +37,7 @@
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_LOCAL_PAUSE_IN_MS;
import static org.junit.Assert.assertFalse;
public class FailureDetectorTest
@@ -45,7 +46,7 @@
public static void setup()
{
// slow unit tests can cause problems with FailureDetector's GC pause handling
- System.setProperty("cassandra.max_local_pause_in_ms", "20000");
+ MAX_LOCAL_PAUSE_IN_MS.setLong(20000);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
diff --git a/test/unit/org/apache/cassandra/gms/GossiperTest.java b/test/unit/org/apache/cassandra/gms/GossiperTest.java
index 68841ea..4614836 100644
--- a/test/unit/org/apache/cassandra/gms/GossiperTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java
@@ -22,12 +22,15 @@
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
import com.google.common.net.InetAddresses;
import org.junit.After;
import org.junit.AfterClass;
@@ -46,20 +49,26 @@
import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.CassandraGenerators;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
+import org.assertj.core.api.Assertions;
+import org.quicktheories.core.Gen;
+import org.quicktheories.impl.Constraint;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.quicktheories.QuickTheory.qt;
public class GossiperTest
{
static
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
}
@@ -419,6 +428,89 @@
}
}
+ @Test
+ public void orderingComparator()
+ {
+ qt().forAll(epStateMapGen()).checkAssert(map -> {
+ Comparator<Map.Entry<InetAddressAndPort, EndpointState>> comp = Gossiper.stateOrderMap();
+ List<Map.Entry<InetAddressAndPort, EndpointState>> elements = new ArrayList<>(map.entrySet());
+ for (int i = 0; i < elements.size(); i++)
+ {
+ for (int j = 0; j < elements.size(); j++)
+ {
+ Map.Entry<InetAddressAndPort, EndpointState> e1 = elements.get(i);
+ boolean e1Bootstrapping = VersionedValue.BOOTSTRAPPING_STATUS.contains(Gossiper.getGossipStatus(e1.getValue()));
+ Map.Entry<InetAddressAndPort, EndpointState> e2 = elements.get(j);
+ boolean e2Bootstrapping = VersionedValue.BOOTSTRAPPING_STATUS.contains(Gossiper.getGossipStatus(e2.getValue()));
+ Ordering ordering = Ordering.compare(comp, e1, e2);
+
+ if (e1Bootstrapping == e2Bootstrapping)
+ {
+ // check generation
+ Ordering sub = Ordering.compare(e1.getValue().getHeartBeatState().getGeneration(), e2.getValue().getHeartBeatState().getGeneration());
+ if (sub == Ordering.EQ)
+ {
+ // check addressWPort
+ sub = Ordering.compare(e1.getKey(), e2.getKey());
+ }
+ Assertions.assertThat(ordering)
+ .describedAs("Both elements bootstrap check were equal: %s == %s", e1Bootstrapping, e2Bootstrapping)
+ .isEqualTo(sub);
+ }
+ else if (e1Bootstrapping)
+ {
+ Assertions.assertThat(ordering).isEqualTo(Ordering.GT);
+ }
+ else
+ {
+ Assertions.assertThat(ordering).isEqualTo(Ordering.LT);
+ }
+ }
+ }
+ });
+ }
+
+ enum Ordering
+ {
+ LT, EQ, GT;
+
+ static <T> Ordering compare(Comparator<T> comparator, T a, T b)
+ {
+ int rc = comparator.compare(a, b);
+ if (rc < 0) return LT;
+ if (rc == 0) return EQ;
+ return GT;
+ }
+
+ static <T extends Comparable<T>> Ordering compare(T a, T b)
+ {
+ return compare(Comparator.naturalOrder(), a, b);
+ }
+ }
+
+ private static Gen<Map<InetAddressAndPort, EndpointState>> epStateMapGen()
+ {
+ Gen<InetAddressAndPort> addressAndPorts = CassandraGenerators.INET_ADDRESS_AND_PORT_GEN;
+ Gen<EndpointState> states = CassandraGenerators.endpointStates();
+ Constraint sizeGen = Constraint.between(2, 10);
+ Gen<Map<InetAddressAndPort, EndpointState>> mapGen = rs -> {
+ int size = Math.toIntExact(rs.next(sizeGen));
+ Map<InetAddressAndPort, EndpointState> map = Maps.newHashMapWithExpectedSize(size);
+ for (int i = 0; i < size; i++)
+ {
+ while (true)
+ {
+ InetAddressAndPort address = addressAndPorts.generate(rs);
+ if (map.containsKey(address)) continue;
+ map.put(address, states.generate(rs));
+ break;
+ }
+ }
+ return map;
+ };
+ return mapGen;
+ }
+
static class SimpleStateChangeListener implements IEndpointStateChangeSubscriber
{
static class OnChangeParams
diff --git a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
index 0d6d199..1f75876 100644
--- a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
+++ b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java
@@ -39,6 +39,7 @@
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -54,7 +55,7 @@
@BeforeClass
public static void setUp() throws ConfigurationException
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
SchemaLoader.prepareServer();
StorageService.instance.initServer();
}
diff --git a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
index 381edb8..33d9a12 100644
--- a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
+++ b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
@@ -33,11 +33,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -49,6 +49,8 @@
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.AUTO_BOOTSTRAP;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
import static org.apache.cassandra.net.MockMessagingService.verb;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
@@ -61,7 +63,7 @@
@BeforeClass
public static void setUp() throws ConfigurationException
{
- System.setProperty("cassandra.config", "cassandra-seeds.yaml");
+ CASSANDRA_CONFIG.setString("cassandra-seeds.yaml");
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
@@ -171,8 +173,7 @@
}, 1);
- System.setProperty(Config.PROPERTY_PREFIX + "auto_bootstrap", "false");
- try
+ try (WithProperties properties = new WithProperties().set(AUTO_BOOTSTRAP, false))
{
StorageService.instance.checkForEndpointCollision(SystemKeyspace.getOrInitializeLocalHostId(), SystemKeyspace.loadHostIds().keySet());
}
@@ -180,7 +181,6 @@
{
assertEquals("Unable to gossip with any peers", e.getMessage());
}
- System.clearProperty(Config.PROPERTY_PREFIX + "auto_bootstrap");
}
@Test
@@ -209,9 +209,10 @@
}, 1);
- System.setProperty(Config.PROPERTY_PREFIX + "auto_bootstrap", "false");
- StorageService.instance.checkForEndpointCollision(SystemKeyspace.getOrInitializeLocalHostId(), SystemKeyspace.loadHostIds().keySet());
- System.clearProperty(Config.PROPERTY_PREFIX + "auto_bootstrap");
+ try (WithProperties properties = new WithProperties().set(AUTO_BOOTSTRAP, false))
+ {
+ StorageService.instance.checkForEndpointCollision(SystemKeyspace.getOrInitializeLocalHostId(), SystemKeyspace.loadHostIds().keySet());
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java
index cc29163..da9f7dd 100644
--- a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java
+++ b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java
@@ -17,27 +17,19 @@
*/
package org.apache.cassandra.hints;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.zip.CRC32;
-import org.apache.cassandra.io.util.File;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class ChecksummedDataInputTest
{
diff --git a/test/unit/org/apache/cassandra/hints/DTestSerializer.java b/test/unit/org/apache/cassandra/hints/DTestSerializer.java
index 61bd77c..0221c52 100644
--- a/test/unit/org/apache/cassandra/hints/DTestSerializer.java
+++ b/test/unit/org/apache/cassandra/hints/DTestSerializer.java
@@ -45,7 +45,7 @@
}
UUIDSerializer.serializer.serialize(message.hostId, out, version);
- out.writeUnsignedVInt(0);
+ out.writeUnsignedVInt32(0);
message.unknownTableID.serialize(out);
}
diff --git a/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java b/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
index e24ff76..f814245 100644
--- a/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintWriteTTLTest.java
@@ -44,6 +44,8 @@
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_MAX_HINT_TTL;
+
public class HintWriteTTLTest
{
private static int TTL = 500;
@@ -84,7 +86,7 @@
@BeforeClass
public static void setupClass() throws Exception
{
- System.setProperty("cassandra.maxHintTTL", Integer.toString(TTL));
+ CASSANDRA_MAX_HINT_TTL.setInt(TTL);
SchemaLoader.prepareServer();
TableMetadata tbm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "ks").gcGraceSeconds(GC_GRACE).build();
SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), tbm);
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 441d7b0..3c33327 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -24,10 +24,21 @@
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadLocalRandom;
@@ -36,35 +47,58 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer;
import org.apache.cassandra.index.sasi.analyzer.NoOpAnalyzer;
@@ -80,11 +114,16 @@
import org.apache.cassandra.index.sasi.utils.RangeIterator;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.IndexSummaryManager;
import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
@@ -94,13 +133,8 @@
import org.apache.cassandra.utils.Pair;
import org.assertj.core.api.Assertions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Uninterruptibles;
-
-import org.junit.*;
-
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
import static org.apache.cassandra.db.ColumnFamilyStoreTest.getSnapshotManifestAndSchemaFileSizes;
public class SASIIndexTest
@@ -108,7 +142,7 @@
private static final IPartitioner PARTITIONER;
static {
- System.setProperty("cassandra.config", "cassandra-murmur.yaml");
+ CASSANDRA_CONFIG.setString("cassandra-murmur.yaml");
PARTITIONER = Murmur3Partitioner.instance;
}
@@ -192,7 +226,7 @@
sstable.getKeyspaceName(),
sstable.getColumnFamilyName(),
sstable.descriptor.id,
- sstable.descriptor.formatType);
+ sstable.descriptor.version.format);
Set<Component> components = snapshotSSTables.get(snapshotSSTable);
@@ -201,8 +235,8 @@
for (Component c : components)
{
- long componentSize = Files.size(Paths.get(snapshotSSTable.filenameFor(c)));
- if (Component.Type.fromRepresentation(c.name) == Component.Type.SECONDARY_INDEX)
+ long componentSize = snapshotSSTable.fileFor(c).length();
+ if (Component.Type.fromRepresentation(c.name, sstable.descriptor.version.format) == Components.Types.SECONDARY_INDEX)
indexSize += componentSize;
else
tableSize += componentSize;
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
index ad0caff..6707aa5 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
@@ -18,25 +18,38 @@
package org.apache.cassandra.index.sasi.disk;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ThreadLocalRandom;
+import com.google.common.util.concurrent.Futures;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.sasi.SASIIndex;
import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.File;
@@ -49,11 +62,7 @@
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.utils.ByteBufferUtil;
-import com.google.common.util.concurrent.Futures;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
public class PerSSTableIndexWriterTest extends SchemaLoader
{
@@ -63,7 +72,7 @@
@BeforeClass
public static void loadSchema() throws ConfigurationException
{
- System.setProperty("cassandra.config", "cassandra-murmur.yaml");
+ CASSANDRA_CONFIG.setString("cassandra-murmur.yaml");
SchemaLoader.loadSchema();
SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(KS_NAME,
KeyspaceParams.simpleTransient(1),
@@ -112,7 +121,9 @@
Map.Entry<DecoratedKey, Row> key = keyIterator.next();
- indexWriter.startPartition(key.getKey(), position++);
+
+ indexWriter.startPartition(key.getKey(), position, position);
+ position++;
indexWriter.nextUnfilteredCluster(key.getValue());
}
@@ -126,7 +137,7 @@
for (String segment : segments)
Assert.assertTrue(new File(segment).exists());
- String indexFile = indexWriter.indexes.get(column).filename(true);
+ File indexFile = indexWriter.indexes.get(column).file(true);
// final flush
indexWriter.complete();
@@ -134,7 +145,7 @@
for (String segment : segments)
Assert.assertFalse(new File(segment).exists());
- OnDiskIndex index = new OnDiskIndex(new File(indexFile), Int32Type.instance, keyPosition -> {
+ OnDiskIndex index = new OnDiskIndex(indexFile, Int32Type.instance, keyPosition -> {
ByteBuffer key = ByteBufferUtil.bytes(String.format(keyFormat, keyPosition));
return cfs.metadata().partitioner.decorateKey(key);
});
@@ -233,7 +244,7 @@
Assert.assertFalse(new File(segment).exists());
// and combined index doesn't exist either
- Assert.assertFalse(new File(index.outputFile).exists());
+ Assert.assertFalse(index.outputFile.exists());
}
private static void populateSegment(TableMetadata metadata, PerSSTableIndexWriter.Index index, Map<Long, Set<Integer>> data)
diff --git a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
index 8620f5c..5ffebcc 100644
--- a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
@@ -43,6 +43,8 @@
import org.junit.*;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
+
public class OperationTest extends SchemaLoader
{
private static final String KS_NAME = "sasi";
@@ -57,7 +59,7 @@
@BeforeClass
public static void loadSchema() throws ConfigurationException
{
- System.setProperty("cassandra.config", "cassandra-murmur.yaml");
+ CASSANDRA_CONFIG.setString("cassandra-murmur.yaml");
SchemaLoader.loadSchema();
SchemaLoader.createKeyspace(KS_NAME,
KeyspaceParams.simpleTransient(1),
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java b/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java
index dcd79b9..332906b 100644
--- a/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java
@@ -17,20 +17,21 @@
*/
package org.apache.cassandra.index.sasi.utils;
-import java.io.*;
+import java.io.IOException;
+import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
+import org.junit.Assert;
+import org.junit.Test;
+
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
public class MappedBufferTest
{
@Test
@@ -461,7 +462,7 @@
file.getFD().sync();
- try (MappedBuffer buffer = new MappedBuffer(new ChannelProxy(tmp.absolutePath(), file.getChannel())))
+ try (MappedBuffer buffer = new MappedBuffer(new ChannelProxy(tmp, file.getChannel())))
{
Assert.assertEquals(numValues * 8, buffer.limit());
Assert.assertEquals(numValues * 8, buffer.capacity());
@@ -530,7 +531,7 @@
try
{
- return new MappedBuffer(new ChannelProxy(testFile.absolutePath(), file.getChannel()), numPageBits);
+ return new MappedBuffer(new ChannelProxy(testFile, file.getChannel()), numPageBits);
}
finally
{
diff --git a/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java b/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java
deleted file mode 100644
index afcf2a5..0000000
--- a/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-package org.apache.cassandra.io;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-
-import org.junit.Test;
-
-import org.apache.cassandra.io.sstable.BloomFilterTracker;
-
-import static org.junit.Assert.assertEquals;
-
-public class BloomFilterTrackerTest
-{
- @Test
- public void testAddingFalsePositives()
- {
- BloomFilterTracker bft = new BloomFilterTracker();
- assertEquals(0L, bft.getFalsePositiveCount());
- assertEquals(0L, bft.getRecentFalsePositiveCount());
- bft.addFalsePositive();
- bft.addFalsePositive();
- assertEquals(2L, bft.getFalsePositiveCount());
- assertEquals(2L, bft.getRecentFalsePositiveCount());
- assertEquals(0L, bft.getRecentFalsePositiveCount());
- assertEquals(2L, bft.getFalsePositiveCount()); // sanity check
- }
-
- @Test
- public void testAddingTruePositives()
- {
- BloomFilterTracker bft = new BloomFilterTracker();
- assertEquals(0L, bft.getTruePositiveCount());
- assertEquals(0L, bft.getRecentTruePositiveCount());
- bft.addTruePositive();
- bft.addTruePositive();
- assertEquals(2L, bft.getTruePositiveCount());
- assertEquals(2L, bft.getRecentTruePositiveCount());
- assertEquals(0L, bft.getRecentTruePositiveCount());
- assertEquals(2L, bft.getTruePositiveCount()); // sanity check
- }
-
- @Test
- public void testAddingToOneLeavesTheOtherAlone()
- {
- BloomFilterTracker bft = new BloomFilterTracker();
- bft.addFalsePositive();
- assertEquals(0L, bft.getTruePositiveCount());
- assertEquals(0L, bft.getRecentTruePositiveCount());
- bft.addTruePositive();
- assertEquals(1L, bft.getFalsePositiveCount());
- assertEquals(1L, bft.getRecentFalsePositiveCount());
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
index be8b162..5ecba47 100644
--- a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
+++ b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java
@@ -21,26 +21,34 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.CompactionInterruptedException;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.io.sstable.IndexSummaryManager;
-import org.apache.cassandra.io.sstable.IndexSummaryRedistribution;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport;
+import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
public class DiskSpaceMetricsTest extends CQLTester
{
/**
@@ -67,6 +75,7 @@
@Test
public void summaryRedistribution() throws Throwable
{
+ Assume.assumeTrue(IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
createTable("CREATE TABLE %s (pk bigint, PRIMARY KEY (pk)) WITH min_index_interval=1");
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
@@ -102,18 +111,36 @@
private void assertDiskSpaceEqual(ColumnFamilyStore cfs)
{
+ Set<SSTableReader> liveSSTables = cfs.getTracker().getView().liveSSTables();
long liveDiskSpaceUsed = cfs.metric.liveDiskSpaceUsed.getCount();
- long actual = 0;
- for (SSTableReader sstable : cfs.getTracker().getView().liveSSTables())
- actual += sstable.bytesOnDisk();
+ long actual = liveSSTables.stream().mapToLong(SSTableReader::bytesOnDisk).sum();
+ long uncompressedLiveDiskSpaceUsed = cfs.metric.uncompressedLiveDiskSpaceUsed.getCount();
+ long actualUncompressed = liveSSTables.stream().mapToLong(SSTableReader::logicalBytesOnDisk).sum();
- Assert.assertEquals("bytes on disk does not match current metric liveDiskSpaceUsed", actual, liveDiskSpaceUsed);
+ assertEquals("bytes on disk does not match current metric LiveDiskSpaceUsed", actual, liveDiskSpaceUsed);
+ assertEquals("bytes on disk does not match current metric UncompressedLiveDiskSpaceUsed", actualUncompressed, uncompressedLiveDiskSpaceUsed);
+
+ // Keyspace-level metrics should be equivalent to table-level metrics, as there is only one table.
+ assertEquals(cfs.keyspace.metric.liveDiskSpaceUsed.getValue().longValue(), liveDiskSpaceUsed);
+ assertEquals(cfs.keyspace.metric.uncompressedLiveDiskSpaceUsed.getValue().longValue(), uncompressedLiveDiskSpaceUsed);
+ assertEquals(cfs.keyspace.metric.unreplicatedLiveDiskSpaceUsed.getValue().longValue(), liveDiskSpaceUsed);
+ assertEquals(cfs.keyspace.metric.unreplicatedUncompressedLiveDiskSpaceUsed.getValue().longValue(), uncompressedLiveDiskSpaceUsed);
+
+ // Global load metrics should be internally consistent, given there is no replication, but slightly greater
+ // than table and keyspace-level metrics, given the global versions account for non-user tables.
+ long globalLoad = StorageMetrics.load.getCount();
+ assertEquals(globalLoad, StorageMetrics.unreplicatedLoad.getValue().longValue());
+ assertThat(globalLoad).isGreaterThan(liveDiskSpaceUsed);
+
+ long globalUncompressedLoad = StorageMetrics.uncompressedLoad.getCount();
+ assertEquals(globalUncompressedLoad, StorageMetrics.unreplicatedUncompressedLoad.getValue().longValue());
+ assertThat(globalUncompressedLoad).isGreaterThan(uncompressedLiveDiskSpaceUsed);
// totalDiskSpaceUsed is based off SStable delete, which is async: LogTransaction's tidy enqueues in ScheduledExecutors.nonPeriodicTasks
// wait for there to be no more pending sstable releases
LifecycleTransaction.waitForDeletions();
long totalDiskSpaceUsed = cfs.metric.totalDiskSpaceUsed.getCount();
- Assert.assertEquals("bytes on disk does not match current metric totalDiskSpaceUsed", actual, totalDiskSpaceUsed);
+ assertEquals("bytes on disk does not match current metric totalDiskSpaceUsed", actual, totalDiskSpaceUsed);
}
private static void indexDownsampleCancelLastSSTable(ColumnFamilyStore cfs)
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index c398ac4..2bf127f 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@ -20,11 +20,10 @@
import java.io.EOFException;
import java.io.IOException;
-import java.io.RandomAccessFile; //checkstyle: permit this import
+import java.io.RandomAccessFile;
import java.util.Arrays;
import java.util.Random;
-import org.assertj.core.api.Assertions;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,9 +33,16 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.utils.SyncUtil;
+import org.assertj.core.api.Assertions;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
@@ -98,7 +104,7 @@
File f = FileUtils.createTempFile("compressed6791_", "3");
String filename = f.absolutePath();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
- try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata",
+ try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, new File(filename + ".metadata"),
null, SequentialWriterOption.DEFAULT,
CompressionParams.snappy(32),
sstableMetadataCollector))
@@ -119,9 +125,8 @@
writer.finish();
}
- try (FileHandle.Builder builder = new FileHandle.Builder(filename)
- .withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
- FileHandle fh = builder.complete();
+ try (CompressionMetadata compressionMetadata = CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true);
+ FileHandle fh = new FileHandle.Builder(f).withCompressionMetadata(compressionMetadata).complete();
RandomAccessReader reader = fh.createReader())
{
String res = reader.readLine();
@@ -151,7 +156,7 @@
try
{
writeSSTable(file, CompressionParams.snappy(chunkLength), 10);
- CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), true);
+ CompressionMetadata metadata = CompressionMetadata.open(new File(filename + ".metadata"), file.length(), true);
long chunks = 2761628520L;
long midPosition = (chunks / 2L) * chunkLength;
@@ -177,9 +182,8 @@
final String filename = f.absolutePath();
writeSSTable(f, compressed ? CompressionParams.snappy() : null, junkSize);
- CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), true) : null;
- try (FileHandle.Builder builder = new FileHandle.Builder(filename).mmapped(usemmap).withCompressionMetadata(compressionMetadata);
- FileHandle fh = builder.complete();
+ try (CompressionMetadata compressionMetadata = compressed ? CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true) : null;
+ FileHandle fh = new FileHandle.Builder(f).mmapped(usemmap).withCompressionMetadata(compressionMetadata).complete();
RandomAccessReader reader = fh.createReader())
{
String expected = "The quick brown fox jumps over the lazy dog";
@@ -203,7 +207,7 @@
final String filename = f.absolutePath();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
try(SequentialWriter writer = params != null
- ? new CompressedSequentialWriter(f, filename + ".metadata",
+ ? new CompressedSequentialWriter(f, new File(filename + ".metadata"),
null, SequentialWriterOption.DEFAULT,
params, sstableMetadataCollector)
: new SequentialWriter(f))
@@ -243,7 +247,7 @@
assertTrue(metadata.createFileIfNotExists());
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
- try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.path(),
+ try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata,
null, SequentialWriterOption.DEFAULT,
CompressionParams.snappy(), sstableMetadataCollector))
{
@@ -252,16 +256,14 @@
}
// open compression metadata and get chunk information
- CompressionMetadata meta = new CompressionMetadata(metadata.path(), file.length(), true);
- CompressionMetadata.Chunk chunk = meta.chunkFor(0);
-
- try (FileHandle.Builder builder = new FileHandle.Builder(file.path()).withCompressionMetadata(meta);
- FileHandle fh = builder.complete();
+ try (CompressionMetadata meta = CompressionMetadata.open(metadata, file.length(), true);
+ FileHandle fh = new FileHandle.Builder(file).withCompressionMetadata(meta).complete();
RandomAccessReader reader = fh.createReader())
{// read and verify compressed data
assertEquals(CONTENT, reader.readLine());
Random random = new Random();
- try(RandomAccessFile checksumModifier = new RandomAccessFile(file.toJavaIOFile(), "rw"))
+ CompressionMetadata.Chunk chunk = meta.chunkFor(0);
+ try (RandomAccessFile checksumModifier = new RandomAccessFile(file.toJavaIOFile(), "rw"))
{
byte[] checksum = new byte[4];
@@ -312,4 +314,4 @@
file.write(checksum);
SyncUtil.sync(file.getFD());
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 2b50633..afa469c 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -21,29 +21,39 @@
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
-
-import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH;
-import static org.apache.commons.io.FileUtils.readFileToByteArray;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
import com.google.common.io.Files;
import org.junit.After;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.Assert;
-
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.io.util.DataPosition;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.io.util.SequentialWriterTest;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH;
+import static org.apache.commons.io.FileUtils.readFileToByteArray;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
public class CompressedSequentialWriterTest extends SequentialWriterTest
{
private CompressionParams compressionParameters;
@@ -117,7 +127,7 @@
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
- try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata",
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, new File(filename + ".metadata"),
null, SequentialWriterOption.DEFAULT,
compressionParameters,
sstableMetadataCollector))
@@ -150,8 +160,8 @@
}
assert f.exists();
- try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
- FileHandle fh = builder.complete();
+ try (CompressionMetadata compressionMetadata = CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true);
+ FileHandle fh = new FileHandle.Builder(f).withCompressionMetadata(compressionMetadata).complete();
RandomAccessReader reader = fh.createReader())
{
assertEquals(dataPre.length + rawPost.length, reader.length());
@@ -217,7 +227,7 @@
compressionParameters = new CompressionParams(MockCompressor.class.getTypeName(),
MockCompressor.paramsFor(ratio, extra),
DEFAULT_CHUNK_LENGTH, ratio);
- try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, f.path() + ".metadata",
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, new File(f.path() + ".metadata"),
null, SequentialWriterOption.DEFAULT,
compressionParameters,
sstableMetadataCollector))
@@ -228,8 +238,8 @@
}
assert f.exists();
- try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
- FileHandle fh = builder.complete();
+ try (CompressionMetadata compressionMetadata = CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true);
+ FileHandle fh = new FileHandle.Builder(f).withCompressionMetadata(compressionMetadata).complete();
RandomAccessReader reader = fh.createReader())
{
assertEquals(size, reader.length());
@@ -276,7 +286,7 @@
final int bufferSize = 48;
final int writeSize = 64;
byte[] toWrite = new byte[writeSize];
- try (SequentialWriter writer = new CompressedSequentialWriter(tempFile, offsetsFile.path(),
+ try (SequentialWriter writer = new CompressedSequentialWriter(tempFile, offsetsFile,
null, SequentialWriterOption.DEFAULT,
CompressionParams.lz4(bufferSize),
new MetadataCollector(new ClusteringComparator(UTF8Type.instance))))
@@ -330,7 +340,7 @@
private TestableCSW(File file, File offsetsFile) throws IOException
{
- this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile.path(),
+ this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile,
null, SequentialWriterOption.DEFAULT,
CompressionParams.lz4(BUFFER_SIZE, MAX_COMPRESSED),
new MetadataCollector(new ClusteringComparator(UTF8Type.instance))));
@@ -385,4 +395,4 @@
}
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java b/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java
new file mode 100644
index 0000000..321fe57
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.compress;
+
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.schema.CompressionParams;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+
+public class CompressionMetadataTest
+{
+ File chunksIndexFile = new File("/path/to/metadata");
+ CompressionParams params = CompressionParams.zstd();
+ long dataLength = 1000;
+ long compressedFileLength = 100;
+
+ private CompressionMetadata newCompressionMetadata(Memory memory)
+ {
+ return new CompressionMetadata(chunksIndexFile,
+ params,
+ memory,
+ memory.size(),
+ dataLength,
+ compressedFileLength);
+ }
+
+ @Test
+ public void testMemoryIsFreed()
+ {
+ Memory memory = Memory.allocate(10);
+ CompressionMetadata cm = newCompressionMetadata(memory);
+
+ cm.close();
+ assertThat(cm.isCleanedUp()).isTrue();
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(memory::size);
+ }
+
+ @Test
+ public void testMemoryIsShared()
+ {
+ Memory memory = Memory.allocate(10);
+ CompressionMetadata cm = newCompressionMetadata(memory);
+
+ CompressionMetadata copy = cm.sharedCopy();
+ assertThat(copy).isNotSameAs(cm);
+
+ cm.close();
+ assertThat(cm.isCleanedUp()).isFalse();
+ assertThat(copy.isCleanedUp()).isFalse();
+ assertThat(memory.size()).isEqualTo(10); // expected that no expection is thrown since memory should not be released yet
+
+ copy.close();
+ assertThat(cm.isCleanedUp()).isTrue();
+ assertThat(copy.isCleanedUp()).isTrue();
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(memory::size);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileChannel.java b/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileChannel.java
new file mode 100644
index 0000000..4fecd24
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileChannel.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.filesystem;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+public class ForwardingFileChannel extends FileChannel
+{
+ protected final FileChannel delegate;
+
+ public ForwardingFileChannel(FileChannel delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ protected FileChannel delegate()
+ {
+ return delegate;
+ }
+
+ @Override
+ public int read(ByteBuffer dst) throws IOException
+ {
+ return delegate().read(dst);
+ }
+
+ @Override
+ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException
+ {
+ return delegate().read(dsts, offset, length);
+ }
+
+ @Override
+ public int write(ByteBuffer src) throws IOException
+ {
+ return delegate().write(src);
+ }
+
+ @Override
+ public long write(ByteBuffer[] srcs, int offset, int length) throws IOException
+ {
+ return delegate().write(srcs, offset, length);
+ }
+
+ @Override
+ public long position() throws IOException
+ {
+ return delegate().position();
+ }
+
+ @Override
+ public FileChannel position(long newPosition) throws IOException
+ {
+ return delegate().position(newPosition);
+ }
+
+ @Override
+ public long size() throws IOException
+ {
+ return delegate().size();
+ }
+
+ @Override
+ public FileChannel truncate(long size) throws IOException
+ {
+ return delegate().truncate(size);
+ }
+
+ @Override
+ public void force(boolean metaData) throws IOException
+ {
+ delegate().force(metaData);
+ }
+
+ @Override
+ public long transferTo(long position, long count, WritableByteChannel target) throws IOException
+ {
+ return delegate().transferTo(position, count, target);
+ }
+
+ @Override
+ public long transferFrom(ReadableByteChannel src, long position, long count) throws IOException
+ {
+ return delegate().transferFrom(src, position, count);
+ }
+
+ @Override
+ public int read(ByteBuffer dst, long position) throws IOException
+ {
+ return delegate().read(dst, position);
+ }
+
+ @Override
+ public int write(ByteBuffer src, long position) throws IOException
+ {
+ return delegate().write(src, position);
+ }
+
+ @Override
+ public MappedByteBuffer map(MapMode mode, long position, long size) throws IOException
+ {
+ return delegate().map(mode, position, size);
+ }
+
+ @Override
+ public FileLock lock(long position, long size, boolean shared) throws IOException
+ {
+ return delegate().lock(position, size, shared);
+ }
+
+ @Override
+ public FileLock tryLock(long position, long size, boolean shared) throws IOException
+ {
+ return delegate().tryLock(position, size, shared);
+ }
+
+ @Override
+ protected void implCloseChannel() throws IOException
+ {
+ // .close(), and .isOpen() are "final", so need to leverage implCloseChannel
+ // to close the underline channel
+ delegate().close();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystem.java b/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystem.java
new file mode 100644
index 0000000..39c682c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystem.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.filesystem;
+
+import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.WatchService;
+import java.nio.file.attribute.UserPrincipalLookupService;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+
+public class ForwardingFileSystem extends FileSystem
+{
+ protected final FileSystem delegate;
+
+ public ForwardingFileSystem(FileSystem delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ protected FileSystem delegate()
+ {
+ return delegate;
+ }
+
+ protected Path wrap(Path p)
+ {
+ return p;
+ }
+
+ protected Path unwrap(Path p)
+ {
+ return p;
+ }
+
+ @Override
+ public FileSystemProvider provider()
+ {
+ return delegate().provider();
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ delegate().close();
+ }
+
+ @Override
+ public boolean isOpen()
+ {
+ return delegate().isOpen();
+ }
+
+ @Override
+ public boolean isReadOnly()
+ {
+ return delegate().isReadOnly();
+ }
+
+ @Override
+ public String getSeparator()
+ {
+ return delegate().getSeparator();
+ }
+
+ @Override
+ public Iterable<Path> getRootDirectories()
+ {
+ return Iterables.transform(delegate().getRootDirectories(), this::wrap);
+ }
+
+ @Override
+ public Iterable<FileStore> getFileStores()
+ {
+ return delegate().getFileStores();
+ }
+
+ @Override
+ public Set<String> supportedFileAttributeViews()
+ {
+ return delegate().supportedFileAttributeViews();
+ }
+
+ @Override
+ public Path getPath(String first, String... more)
+ {
+ return wrap(delegate().getPath(first, more));
+ }
+
+ @Override
+ public PathMatcher getPathMatcher(String syntaxAndPattern)
+ {
+ PathMatcher matcher = delegate().getPathMatcher(syntaxAndPattern);
+ return path -> matcher.matches(unwrap(path));
+ }
+
+ @Override
+ public UserPrincipalLookupService getUserPrincipalLookupService()
+ {
+ return delegate().getUserPrincipalLookupService();
+ }
+
+ @Override
+ public WatchService newWatchService() throws IOException
+ {
+ return delegate().newWatchService();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystemProvider.java b/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystemProvider.java
new file mode 100644
index 0000000..fd1f6d9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystemProvider.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.filesystem;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.FileChannel;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.AccessMode;
+import java.nio.file.CopyOption;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileStore;
+import java.nio.file.FileSystem;
+import java.nio.file.LinkOption;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.FileAttributeView;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import com.google.common.collect.Iterators;
+
+public class ForwardingFileSystemProvider extends FileSystemProvider
+{
+ protected final FileSystemProvider delegate;
+
+ public ForwardingFileSystemProvider(FileSystemProvider delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ protected FileSystemProvider delegate()
+ {
+ return delegate;
+ }
+
+ protected Path wrap(Path p)
+ {
+ return p;
+ }
+
+ protected Path unwrap(Path p)
+ {
+ return p;
+ }
+
+ @Override
+ public String getScheme()
+ {
+ return delegate().getScheme();
+ }
+
+ @Override
+ public FileSystem newFileSystem(URI uri, Map<String, ?> env) throws IOException
+ {
+ return delegate().newFileSystem(uri, env);
+ }
+
+ @Override
+ public FileSystem getFileSystem(URI uri)
+ {
+ return delegate().getFileSystem(uri);
+ }
+
+ @Override
+ public Path getPath(URI uri)
+ {
+ return wrap(delegate().getPath(uri));
+ }
+
+ @Override
+ public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException
+ {
+ return delegate().newByteChannel(unwrap(path), options, attrs);
+ }
+
+ @Override
+ public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException
+ {
+ DirectoryStream<Path> stream = delegate().newDirectoryStream(unwrap(dir), filter);
+ return new DirectoryStream<Path>()
+ {
+ @Override
+ public Iterator<Path> iterator()
+ {
+ return Iterators.transform(stream.iterator(), ForwardingFileSystemProvider.this::wrap);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ stream.close();
+ }
+ };
+ }
+
+ @Override
+ public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException
+ {
+ delegate().createDirectory(unwrap(dir), attrs);
+ }
+
+ @Override
+ public void delete(Path path) throws IOException
+ {
+ delegate().delete(unwrap(path));
+ }
+
+ @Override
+ public void copy(Path source, Path target, CopyOption... options) throws IOException
+ {
+ delegate().copy(unwrap(source), unwrap(target), options);
+ }
+
+ @Override
+ public void move(Path source, Path target, CopyOption... options) throws IOException
+ {
+ delegate().move(unwrap(source), unwrap(target), options);
+ }
+
+ @Override
+ public boolean isSameFile(Path path, Path path2) throws IOException
+ {
+ return delegate().isSameFile(unwrap(path), unwrap(path2));
+ }
+
+ @Override
+ public boolean isHidden(Path path) throws IOException
+ {
+ return delegate().isHidden(unwrap(path));
+ }
+
+ @Override
+ public FileStore getFileStore(Path path) throws IOException
+ {
+ return delegate().getFileStore(unwrap(path));
+ }
+
+ @Override
+ public void checkAccess(Path path, AccessMode... modes) throws IOException
+ {
+ delegate().checkAccess(unwrap(path), modes);
+ }
+
+ @Override
+ public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options)
+ {
+ return delegate().getFileAttributeView(unwrap(path), type, options);
+ }
+
+ @Override
+ public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException
+ {
+ return delegate().readAttributes(unwrap(path), type, options);
+ }
+
+ @Override
+ public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException
+ {
+ return delegate().readAttributes(unwrap(path), attributes, options);
+ }
+
+ @Override
+ public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException
+ {
+ delegate().setAttribute(unwrap(path), attribute, value, options);
+ }
+
+ @Override
+ public FileSystem newFileSystem(Path path, Map<String, ?> env) throws IOException
+ {
+ return delegate().newFileSystem(unwrap(path), env);
+ }
+
+ @Override
+ public InputStream newInputStream(Path path, OpenOption... options) throws IOException
+ {
+ return delegate().newInputStream(unwrap(path), options);
+ }
+
+ @Override
+ public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException
+ {
+ return delegate().newOutputStream(unwrap(path), options);
+ }
+
+ @Override
+ public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException
+ {
+ return delegate().newFileChannel(unwrap(path), options, attrs);
+ }
+
+ @Override
+ public AsynchronousFileChannel newAsynchronousFileChannel(Path path, Set<? extends OpenOption> options, ExecutorService executor, FileAttribute<?>... attrs) throws IOException
+ {
+ return delegate().newAsynchronousFileChannel(unwrap(path), options, executor, attrs);
+ }
+
+ @Override
+ public void createSymbolicLink(Path link, Path target, FileAttribute<?>... attrs) throws IOException
+ {
+ delegate().createSymbolicLink(unwrap(link), target, attrs);
+ }
+
+ @Override
+ public void createLink(Path link, Path existing) throws IOException
+ {
+ delegate().createLink(unwrap(link), unwrap(existing));
+ }
+
+ @Override
+ public boolean deleteIfExists(Path path) throws IOException
+ {
+ return delegate().deleteIfExists(unwrap(path));
+ }
+
+ @Override
+ public Path readSymbolicLink(Path link) throws IOException
+ {
+ return wrap(delegate().readSymbolicLink(unwrap(link)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/filesystem/ForwardingPath.java b/test/unit/org/apache/cassandra/io/filesystem/ForwardingPath.java
new file mode 100644
index 0000000..7dd0037
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/filesystem/ForwardingPath.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.filesystem;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.util.Iterator;
+
+import com.google.common.collect.Iterators;
+
+public class ForwardingPath implements Path
+{
+ protected final Path delegate;
+
+ public ForwardingPath(Path delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ protected Path delegate()
+ {
+ return delegate;
+ }
+
+ protected Path wrap(Path a)
+ {
+ return a;
+ }
+
+ protected Path unwrap(Path p)
+ {
+ return p;
+ }
+
+ @Override
+ public FileSystem getFileSystem()
+ {
+ return delegate().getFileSystem();
+ }
+
+ @Override
+ public boolean isAbsolute()
+ {
+ return delegate().isAbsolute();
+ }
+
+ @Override
+ public Path getRoot()
+ {
+ return wrap(delegate().getRoot());
+ }
+
+ @Override
+ public Path getFileName()
+ {
+ return wrap(delegate().getFileName());
+ }
+
+ @Override
+ public Path getParent()
+ {
+ Path parent = delegate().getParent();
+ if (parent == null)
+ return null;
+ return wrap(parent);
+ }
+
+ @Override
+ public int getNameCount()
+ {
+ return delegate().getNameCount();
+ }
+
+ @Override
+ public Path getName(int index)
+ {
+ return wrap(delegate().getName(index));
+ }
+
+ @Override
+ public Path subpath(int beginIndex, int endIndex)
+ {
+ return wrap(delegate().subpath(beginIndex, endIndex));
+ }
+
+ @Override
+ public boolean startsWith(Path other)
+ {
+ return delegate().startsWith(unwrap(other));
+ }
+
+ @Override
+ public boolean startsWith(String other)
+ {
+ return delegate().startsWith(other);
+ }
+
+ @Override
+ public boolean endsWith(Path other)
+ {
+ return delegate().endsWith(unwrap(other));
+ }
+
+ @Override
+ public boolean endsWith(String other)
+ {
+ return delegate().endsWith(other);
+ }
+
+ @Override
+ public Path normalize()
+ {
+ return wrap(delegate().normalize());
+ }
+
+ @Override
+ public Path resolve(Path other)
+ {
+ return wrap(delegate().resolve(unwrap(other)));
+ }
+
+ @Override
+ public Path resolve(String other)
+ {
+ return wrap(delegate().resolve(other));
+ }
+
+ @Override
+ public Path resolveSibling(Path other)
+ {
+ return wrap(delegate().resolveSibling(unwrap(other)));
+ }
+
+ @Override
+ public Path resolveSibling(String other)
+ {
+ return wrap(delegate().resolveSibling(other));
+ }
+
+ @Override
+ public Path relativize(Path other)
+ {
+ return wrap(delegate().relativize(unwrap(other)));
+ }
+
+ @Override
+ public URI toUri()
+ {
+ return delegate().toUri();
+ }
+
+ @Override
+ public Path toAbsolutePath()
+ {
+ return wrap(delegate().toAbsolutePath());
+ }
+
+ @Override
+ public Path toRealPath(LinkOption... options) throws IOException
+ {
+ return wrap(delegate().toRealPath(options));
+ }
+
+ @Override
+ public File toFile()
+ {
+ return delegate().toFile();
+ }
+
+ @Override
+ public WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException
+ {
+ return delegate().register(watcher, events, modifiers);
+ }
+
+ @Override
+ public WatchKey register(WatchService watcher, WatchEvent.Kind<?>... events) throws IOException
+ {
+ return delegate().register(watcher, events);
+ }
+
+ @Override
+ public Iterator<Path> iterator()
+ {
+ return Iterators.transform(delegate().iterator(), this::wrap);
+ }
+
+ @Override
+ public int compareTo(Path other)
+ {
+ return delegate().compareTo(unwrap(other));
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ return delegate().equals(obj instanceof Path ? unwrap((Path) obj) : obj);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return delegate().hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return delegate().toString();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/filesystem/ListenableFileSystem.java b/test/unit/org/apache/cassandra/io/filesystem/ListenableFileSystem.java
new file mode 100644
index 0000000..659d34d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/filesystem/ListenableFileSystem.java
@@ -0,0 +1,856 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.filesystem;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+public class ListenableFileSystem extends ForwardingFileSystem
+{
+ @FunctionalInterface
+ public interface PathFilter
+ {
+ boolean accept(Path entry) throws IOException;
+ }
+
+ public interface Listener
+ {
+ }
+
+ public interface OnPreOpen extends Listener
+ {
+ void preOpen(Path path, Set<? extends OpenOption> options, FileAttribute<?>[] attrs) throws IOException;
+ }
+
+ public interface OnPostOpen extends Listener
+ {
+ void postOpen(Path path, Set<? extends OpenOption> options, FileAttribute<?>[] attrs, FileChannel channel) throws IOException;
+ }
+
+ public interface OnPreRead extends Listener
+ {
+ void preRead(Path path, FileChannel channel, long position, ByteBuffer dst) throws IOException;
+ }
+
+ public interface OnPostRead extends Listener
+ {
+ void postRead(Path path, FileChannel channel, long position, ByteBuffer dst, int read) throws IOException;
+ }
+
+ public interface OnPreTransferTo extends Listener
+ {
+ void preTransferTo(Path path, FileChannel channel, long position, long count, WritableByteChannel target) throws IOException;
+ }
+
+ public interface OnPostTransferTo extends Listener
+ {
+ void postTransferTo(Path path, FileChannel channel, long position, long count, WritableByteChannel target, long transfered) throws IOException;
+ }
+
+ public interface OnPreTransferFrom extends Listener
+ {
+ void preTransferFrom(Path path, FileChannel channel, ReadableByteChannel src, long position, long count) throws IOException;
+ }
+
+ public interface OnPostTransferFrom extends Listener
+ {
+ void postTransferFrom(Path path, FileChannel channel, ReadableByteChannel src, long position, long count, long transfered) throws IOException;
+ }
+
+ public interface OnPreWrite extends Listener
+ {
+ void preWrite(Path path, FileChannel channel, long position, ByteBuffer src) throws IOException;
+ }
+
+ public interface OnPostWrite extends Listener
+ {
+ void postWrite(Path path, FileChannel channel, long position, ByteBuffer src, int wrote) throws IOException;
+ }
+
+ public interface OnPrePosition extends Listener
+ {
+ void prePosition(Path path, FileChannel channel, long position, long newPosition) throws IOException;
+ }
+
+ public interface OnPostPosition extends Listener
+ {
+ void postPosition(Path path, FileChannel channel, long position, long newPosition) throws IOException;
+ }
+
+ public interface OnPreTruncate extends Listener
+ {
+ void preTruncate(Path path, FileChannel channel, long size, long targetSize) throws IOException;
+ }
+
+ public interface OnPostTruncate extends Listener
+ {
+ void postTruncate(Path path, FileChannel channel, long size, long targetSize, long newSize) throws IOException;
+ }
+
+ public interface OnPreForce extends Listener
+ {
+ void preForce(Path path, FileChannel channel, boolean metaData) throws IOException;
+ }
+
+ public interface OnPostForce extends Listener
+ {
+ void postForce(Path path, FileChannel channel, boolean metaData) throws IOException;
+ }
+
+ public interface Unsubscribable extends AutoCloseable
+ {
+ @Override
+ void close();
+ }
+
+ private final List<OnPreOpen> onPreOpen = new CopyOnWriteArrayList<>();
+ private final List<OnPostOpen> onPostOpen = new CopyOnWriteArrayList<>();
+ private final List<OnPreTransferTo> onPreTransferTo = new CopyOnWriteArrayList<>();
+ private final List<OnPostTransferTo> onPostTransferTo = new CopyOnWriteArrayList<>();
+ private final List<OnPreRead> onPreRead = new CopyOnWriteArrayList<>();
+ private final List<OnPostRead> onPostRead = new CopyOnWriteArrayList<>();
+ private final List<OnPreWrite> onPreWrite = new CopyOnWriteArrayList<>();
+ private final List<OnPostWrite> onPostWrite = new CopyOnWriteArrayList<>();
+ private final List<OnPreTransferFrom> onPreTransferFrom = new CopyOnWriteArrayList<>();
+ private final List<OnPostTransferFrom> onPostTransferFrom = new CopyOnWriteArrayList<>();
+
+ private final List<OnPreForce> onPreForce = new CopyOnWriteArrayList<>();
+ private final List<OnPostForce> onPostForce = new CopyOnWriteArrayList<>();
+ private final List<OnPreTruncate> onPreTruncate = new CopyOnWriteArrayList<>();
+ private final List<OnPostTruncate> onPostTruncate = new CopyOnWriteArrayList<>();
+ private final List<OnPrePosition> onPrePosition = new CopyOnWriteArrayList<>();
+ private final List<OnPostPosition> onPostPosition = new CopyOnWriteArrayList<>();
+ private final List<List<? extends Listener>> lists = Arrays.asList(onPreOpen, onPostOpen,
+ onPreRead, onPostRead,
+ onPreTransferTo, onPostTransferTo,
+ onPreWrite, onPostWrite,
+ onPreTransferFrom, onPostTransferFrom,
+ onPreForce, onPostForce,
+ onPreTruncate, onPostTruncate,
+ onPrePosition, onPostPosition);
+ private final ListenableFileSystemProvider provider;
+
+ public ListenableFileSystem(FileSystem delegate)
+ {
+ super(delegate);
+ this.provider = new ListenableFileSystemProvider(super.provider());
+ }
+
+ public Unsubscribable listen(Listener listener)
+ {
+ List<List<? extends Listener>> matches = new ArrayList<>(1);
+ if (listener instanceof OnPreOpen)
+ {
+ onPreOpen.add((OnPreOpen) listener);
+ matches.add(onPreOpen);
+ }
+ if (listener instanceof OnPostOpen)
+ {
+ onPostOpen.add((OnPostOpen) listener);
+ matches.add(onPostOpen);
+ }
+ if (listener instanceof OnPreRead)
+ {
+ onPreRead.add((OnPreRead) listener);
+ matches.add(onPreRead);
+ }
+ if (listener instanceof OnPostRead)
+ {
+ onPostRead.add((OnPostRead) listener);
+ matches.add(onPostRead);
+ }
+ if (listener instanceof OnPreTransferTo)
+ {
+ onPreTransferTo.add((OnPreTransferTo) listener);
+ matches.add(onPreTransferTo);
+ }
+ if (listener instanceof OnPostTransferTo)
+ {
+ onPostTransferTo.add((OnPostTransferTo) listener);
+ matches.add(onPostTransferTo);
+ }
+ if (listener instanceof OnPreWrite)
+ {
+ onPreWrite.add((OnPreWrite) listener);
+ matches.add(onPreWrite);
+ }
+ if (listener instanceof OnPostWrite)
+ {
+ onPostWrite.add((OnPostWrite) listener);
+ matches.add(onPostWrite);
+ }
+ if (listener instanceof OnPreTransferFrom)
+ {
+ onPreTransferFrom.add((OnPreTransferFrom) listener);
+ matches.add(onPreTransferFrom);
+ }
+ if (listener instanceof OnPostTransferFrom)
+ {
+ onPostTransferFrom.add((OnPostTransferFrom) listener);
+ matches.add(onPostTransferFrom);
+ }
+ if (listener instanceof OnPreForce)
+ {
+ onPreForce.add((OnPreForce) listener);
+ matches.add(onPreForce);
+ }
+ if (listener instanceof OnPostForce)
+ {
+ onPostForce.add((OnPostForce) listener);
+ matches.add(onPostForce);
+ }
+ if (listener instanceof OnPreTruncate)
+ {
+ onPreTruncate.add((OnPreTruncate) listener);
+ matches.add(onPreTruncate);
+ }
+ if (listener instanceof OnPostTruncate)
+ {
+ onPostTruncate.add((OnPostTruncate) listener);
+ matches.add(onPostTruncate);
+ }
+ if (listener instanceof OnPrePosition)
+ {
+ onPrePosition.add((OnPrePosition) listener);
+ matches.add(onPrePosition);
+ }
+ if (listener instanceof OnPostPosition)
+ {
+ onPostPosition.add((OnPostPosition) listener);
+ matches.add(onPostPosition);
+ }
+ if (matches.isEmpty())
+ throw new IllegalArgumentException("Unable to find a listenable type for " + listener.getClass());
+ return () -> remove(matches, listener);
+ }
+
+ public Unsubscribable onPreOpen(OnPreOpen callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPreOpen(PathFilter filter, OnPreOpen callback)
+ {
+ return onPreOpen((path, options, attrs) -> {
+ if (filter.accept(path))
+ callback.preOpen(path, options, attrs);
+ });
+ }
+
+ public Unsubscribable onPostOpen(OnPostOpen callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPostOpen(PathFilter filter, OnPostOpen callback)
+ {
+ return onPostOpen((path, options, attrs, channel) -> {
+ if (filter.accept(path))
+ callback.postOpen(path, options, attrs, channel);
+ });
+ }
+
+ public Unsubscribable onPreRead(OnPreRead callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPreRead(PathFilter filter, OnPreRead callback)
+ {
+ return onPreRead((path, channel, position, dst) -> {
+ if (filter.accept(path))
+ callback.preRead(path, channel, position, dst);
+ });
+ }
+
+ public Unsubscribable onPostRead(OnPostRead callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPostRead(PathFilter filter, OnPostRead callback)
+ {
+ return onPostRead((path, channel, position, dst, read) -> {
+ if (filter.accept(path))
+ callback.postRead(path, channel, position, dst, read);
+ });
+ }
+
+ public Unsubscribable onPreTransferTo(OnPreTransferTo callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPreTransferTo(PathFilter filter, OnPreTransferTo callback)
+ {
+ return onPreTransferTo((path, channel, position, count, target) -> {
+ if (filter.accept(path))
+ callback.preTransferTo(path, channel, position, count, target);
+ });
+ }
+
+ public Unsubscribable onPostTransferTo(OnPostTransferTo callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPostTransferTo(PathFilter filter, OnPostTransferTo callback)
+ {
+ return onPostTransferTo((path, channel, position, count, target, transfered) -> {
+ if (filter.accept(path))
+ callback.postTransferTo(path, channel, position, count, target, transfered);
+ });
+ }
+
+ public Unsubscribable onPreTransferFrom(OnPreTransferFrom callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPreTransferFrom(PathFilter filter, OnPreTransferFrom callback)
+ {
+ return onPreTransferFrom((path, channel, src, position, count) -> {
+ if (filter.accept(path))
+ callback.preTransferFrom(path, channel, src, position, count);
+ });
+ }
+
+ public Unsubscribable onPostTransferFrom(OnPostTransferFrom callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPostTransferFrom(PathFilter filter, OnPostTransferFrom callback)
+ {
+ return onPostTransferFrom((path, channel, src, position, count, transfered) -> {
+ if (filter.accept(path))
+ callback.postTransferFrom(path, channel, src, position, count, transfered);
+ });
+ }
+
+ public Unsubscribable onPreWrite(OnPreWrite callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPreWrite(PathFilter filter, OnPreWrite callback)
+ {
+ return onPreWrite((path, channel, position, src) -> {
+ if (filter.accept(path))
+ callback.preWrite(path, channel, position, src);
+ });
+ }
+
+ public Unsubscribable onPostWrite(OnPostWrite callback)
+ {
+ return listen(callback);
+ }
+
+ public Unsubscribable onPostWrite(PathFilter filter, OnPostWrite callback)
+ {
+ return onPostWrite((path, channel, position, src, wrote) -> {
+ if (filter.accept(path))
+ callback.postWrite(path, channel, position, src, wrote);
+ });
+ }
+
+ public Unsubscribable onPrePosition(OnPrePosition callbackk)
+ {
+ return listen(callbackk);
+ }
+
+ public Unsubscribable onPrePosition(PathFilter filter, OnPrePosition callbackk)
+ {
+ return onPrePosition((path, channel, position, newPosition) -> {
+ if (filter.accept(path))
+ callbackk.prePosition(path, channel, position, newPosition);
+ });
+ }
+
+ public Unsubscribable onPostPosition(OnPostPosition callbackk)
+ {
+ return listen(callbackk);
+ }
+
+ public Unsubscribable onPostPosition(PathFilter filter, OnPostPosition callbackk)
+ {
+ return onPostPosition((path, channel, position, newPosition) -> {
+ if (filter.accept(path))
+ callbackk.postPosition(path, channel, position, newPosition);
+ });
+ }
+
+ public Unsubscribable onPreTruncate(OnPreTruncate callbackk)
+ {
+ return listen(callbackk);
+ }
+
+ public Unsubscribable onPreTruncate(PathFilter filter, OnPreTruncate callbackk)
+ {
+ return onPreTruncate((path, channel, size, targetSize) -> {
+ if (filter.accept(path))
+ callbackk.preTruncate(path, channel, size, targetSize);
+ });
+ }
+
+ public Unsubscribable onPostTruncate(OnPostTruncate callbackk)
+ {
+ return listen(callbackk);
+ }
+
+ public Unsubscribable onPostTruncate(PathFilter filter, OnPostTruncate callbackk)
+ {
+ return onPostTruncate((path, channel, size, targetSize, newSize) -> {
+ if (filter.accept(path))
+ callbackk.postTruncate(path, channel, size, targetSize, newSize);
+ });
+ }
+
+ public Unsubscribable onPreForce(OnPreForce callbackk)
+ {
+ return listen(callbackk);
+ }
+
+ public Unsubscribable onPreForce(PathFilter filter, OnPreForce callback)
+ {
+ return onPreForce((path, channel, metadata) -> {
+ if (filter.accept(path))
+ callback.preForce(path, channel, metadata);
+ });
+ }
+
+ public Unsubscribable onPostForce(OnPostForce callbackk)
+ {
+ return listen(callbackk);
+ }
+
+ public Unsubscribable onPostForce(PathFilter filter, OnPostForce callback)
+ {
+ return onPostForce((path, channel, metadata) -> {
+ if (filter.accept(path))
+ callback.postForce(path, channel, metadata);
+ });
+ }
+
+ public void remove(Listener listener)
+ {
+ remove(lists, listener);
+ }
+
+ private static void remove(List<List<? extends Listener>> lists, Listener listener)
+ {
+ lists.forEach(l -> l.remove(listener));
+ }
+
+ public void clearListeners()
+ {
+ lists.forEach(List::clear);
+ }
+
+ private interface ListenerAction<T>
+ {
+ void accept(T value) throws IOException;
+ }
+
+ private <T> void notifyListeners(List<T> listeners, ListenerAction<T> fn) throws IOException
+ {
+ for (T listener : listeners)
+ fn.accept(listener);
+ }
+
+ @Override
+ protected Path wrap(Path p)
+ {
+ return p instanceof ListenablePath ? p : new ListenablePath(p);
+ }
+
+ @Override
+ protected Path unwrap(Path p)
+ {
+ return p instanceof ListenablePath ? ((ListenablePath) p).delegate : p;
+ }
+
+ @Override
+ public ListenableFileSystemProvider provider()
+ {
+ return provider;
+ }
+
+
+ private class ListenableFileSystemProvider extends ForwardingFileSystemProvider
+ {
+ ListenableFileSystemProvider(FileSystemProvider delegate)
+ {
+ super(delegate);
+ }
+
+ @Override
+ protected Path wrap(Path a)
+ {
+ return ListenableFileSystem.this.wrap(a);
+ }
+
+ @Override
+ protected Path unwrap(Path p)
+ {
+ return ListenableFileSystem.this.unwrap(p);
+ }
+
+ @Override
+ public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException
+ {
+ int len = options.length;
+ Set<OpenOption> opts = new HashSet<>(len + 3);
+ if (len == 0)
+ {
+ opts.add(StandardOpenOption.CREATE);
+ opts.add(StandardOpenOption.TRUNCATE_EXISTING);
+ }
+ else
+ {
+ for (OpenOption opt : options)
+ {
+ if (opt == StandardOpenOption.READ)
+ throw new IllegalArgumentException("READ not allowed");
+ opts.add(opt);
+ }
+ }
+ opts.add(StandardOpenOption.WRITE);
+ return Channels.newOutputStream(newFileChannel(path, opts));
+ }
+
+ @Override
+ public InputStream newInputStream(Path path, OpenOption... options) throws IOException
+ {
+ for (OpenOption opt : options)
+ {
+ // All OpenOption values except for APPEND and WRITE are allowed
+ if (opt == StandardOpenOption.APPEND ||
+ opt == StandardOpenOption.WRITE)
+ throw new UnsupportedOperationException("'" + opt + "' not allowed");
+ }
+ Set<OpenOption> opts = new HashSet<>(Arrays.asList(options));
+ return Channels.newInputStream(newFileChannel(path, opts));
+ }
+
+ @Override
+ public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException
+ {
+ return newFileChannel(path, options, attrs);
+ }
+
+ @Override
+ public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException
+ {
+ notifyListeners(onPreOpen, l -> l.preOpen(path, options, attrs));
+ ListenableFileChannel channel = new ListenableFileChannel(path, delegate().newFileChannel(unwrap(path), options, attrs));
+ notifyListeners(onPostOpen, l -> l.postOpen(path, options, attrs, channel));
+ return channel;
+ }
+
+ @Override
+ public AsynchronousFileChannel newAsynchronousFileChannel(Path path, Set<? extends OpenOption> options, ExecutorService executor, FileAttribute<?>... attrs) throws IOException
+ {
+ throw new UnsupportedOperationException("TODO");
+ }
+
+ // block the APIs that try to switch FileSystem based off schema
+ @Override
+ public FileSystem newFileSystem(URI uri, Map<String, ?> env) throws IOException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileSystem newFileSystem(Path path, Map<String, ?> env) throws IOException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getScheme()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileSystem getFileSystem(URI uri)
+ {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private class ListenablePath extends ForwardingPath
+ {
+ public ListenablePath(Path delegate)
+ {
+ super(delegate);
+ }
+
+ @Override
+ protected Path wrap(Path a)
+ {
+ return ListenableFileSystem.this.wrap(a);
+ }
+
+ @Override
+ protected Path unwrap(Path p)
+ {
+ return ListenableFileSystem.this.unwrap(p);
+ }
+
+ @Override
+ public FileSystem getFileSystem()
+ {
+ return ListenableFileSystem.this;
+ }
+
+ @Override
+ public File toFile()
+ {
+ if (delegate().getFileSystem() == FileSystems.getDefault())
+ return delegate().toFile();
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private class ListenableFileChannel extends ForwardingFileChannel
+ {
+ private final AtomicReference<Mapped> mutable = new AtomicReference<>();
+ private final Path path;
+
+ ListenableFileChannel(Path path, FileChannel delegate)
+ {
+ super(delegate);
+ this.path = path;
+ }
+
+ @Override
+ public int read(ByteBuffer dst) throws IOException
+ {
+ long position = position();
+ notifyListeners(onPreRead, l -> l.preRead(path, this, position, dst));
+ int read = super.read(dst);
+ notifyListeners(onPostRead, l -> l.postRead(path, this, position, dst, read));
+ return read;
+ }
+
+ @Override
+ public int read(ByteBuffer dst, long position) throws IOException
+ {
+ notifyListeners(onPreRead, l -> l.preRead(path, this, position, dst));
+ int read = super.read(dst, position);
+ notifyListeners(onPostRead, l -> l.postRead(path, this, position, dst, read));
+ return read;
+ }
+
+ @Override
+ public int write(ByteBuffer src) throws IOException
+ {
+ long position = position();
+ notifyListeners(onPreWrite, l -> l.preWrite(path, this, position, src));
+ int write = super.write(src);
+ notifyListeners(onPostWrite, l -> l.postWrite(path, this, position, src, write));
+ return write;
+ }
+
+ @Override
+ public int write(ByteBuffer src, long position) throws IOException
+ {
+ notifyListeners(onPreWrite, l -> l.preWrite(path, this, position, src));
+ int write = super.write(src, position);
+ notifyListeners(onPostWrite, l -> l.postWrite(path, this, position, src, write));
+ return write;
+ }
+
+ @Override
+ public FileChannel position(long newPosition) throws IOException
+ {
+ long position = position();
+ notifyListeners(onPrePosition, l -> l.prePosition(path, this, position, newPosition));
+ super.position(newPosition);
+ notifyListeners(onPostPosition, l -> l.postPosition(path, this, position, newPosition));
+ return this;
+ }
+
+ @Override
+ public FileChannel truncate(long size) throws IOException
+ {
+ long currentSize = this.size();
+ notifyListeners(onPreTruncate, l -> l.preTruncate(path, this, currentSize, size));
+ super.truncate(size);
+ long latestSize = this.size();
+ notifyListeners(onPostTruncate, l -> l.postTruncate(path, this, currentSize, size, latestSize));
+ return this;
+ }
+
+ @Override
+ public void force(boolean metaData) throws IOException
+ {
+ notifyListeners(onPreForce, l -> l.preForce(path, this, metaData));
+ super.force(metaData);
+ notifyListeners(onPostForce, l -> l.postForce(path, this, metaData));
+ }
+
+ @Override
+ public long transferTo(long position, long count, WritableByteChannel target) throws IOException
+ {
+ notifyListeners(onPreTransferTo, l -> l.preTransferTo(path, this, position, count, target));
+ long transfered = super.transferTo(position, count, target);
+ notifyListeners(onPostTransferTo, l -> l.postTransferTo(path, this, position, count, target, transfered));
+ return transfered;
+ }
+
+ @Override
+ public long transferFrom(ReadableByteChannel src, long position, long count) throws IOException
+ {
+ notifyListeners(onPreTransferFrom, l -> l.preTransferFrom(path, this, src, position, count));
+ long transfered = super.transferFrom(src, position, count);
+ notifyListeners(onPostTransferFrom, l -> l.postTransferFrom(path, this, src, position, count, transfered));
+ return transfered;
+ }
+
+ @Override
+ public MappedByteBuffer map(MapMode mode, long position, long size) throws IOException
+ {
+ // this behavior isn't 100% correct... if you mix access with FileChanel and ByteBuffer you will get different
+ // results than with a real mmap solution... This limitation is due to ByteBuffer being private, so not able
+ // to create custom BBs to mimc this access...
+ if (mode == MapMode.READ_WRITE && mutable.get() != null)
+ throw new UnsupportedOperationException("map called twice with mode READ_WRITE; first was " + mutable.get() + ", now " + new Mapped(mode, null, position, Math.toIntExact(size)));
+
+ int isize = Math.toIntExact(size);
+ MappedByteBuffer bb = (MappedByteBuffer) ByteBuffer.allocateDirect(isize);
+
+ Mapped mapped = new Mapped(mode, bb, position, isize);
+ if (mode == MapMode.READ_ONLY)
+ {
+ ByteBufferUtil.readFully(this, mapped.bb, position);
+ mapped.bb.flip();
+
+ Runnable forcer = () -> {
+ };
+ MemoryUtil.setAttachment(bb, forcer);
+ }
+ else if (mode == MapMode.READ_WRITE)
+ {
+ if (delegate().size() - position > 0)
+ {
+ ByteBufferUtil.readFully(this, mapped.bb, position);
+ mapped.bb.flip();
+ }
+ // with real files the FD gets copied so the close of the channel does not block the BB mutation
+ // from flushing... it's possible to support this use case, but kept things simplier for now by
+ // failing if the backing channel was closed.
+ Runnable forcer = () -> {
+ ByteBuffer local = bb.duplicate();
+ local.position(0);
+ long pos = position;
+ try
+ {
+ while (local.hasRemaining())
+ {
+ int wrote = write(local, pos);
+ if (wrote == -1)
+ throw new EOFException();
+ pos += wrote;
+ }
+ }
+ catch (IOException e)
+ {
+ throw new UncheckedIOException(e);
+ }
+ };
+ MemoryUtil.setAttachment(bb, forcer);
+ if (!mutable.compareAndSet(null, mapped))
+ throw new UnsupportedOperationException("map called twice");
+ }
+ else
+ {
+ throw new UnsupportedOperationException("Unsupported mode: " + mode);
+ }
+ return mapped.bb;
+ }
+
+ @Override
+ protected void implCloseChannel() throws IOException
+ {
+ super.implCloseChannel();
+ mutable.set(null);
+ }
+ }
+
+ private static class Mapped
+ {
+ final FileChannel.MapMode mode;
+ final MappedByteBuffer bb;
+ final long position;
+ final int size;
+
+ private Mapped(FileChannel.MapMode mode, MappedByteBuffer bb, long position, int size)
+ {
+ this.mode = mode;
+ this.bb = bb;
+ this.position = position;
+ this.size = size;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Mapped{" +
+ "mode=" + mode +
+ ", position=" + position +
+ ", size=" + size +
+ '}';
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java
deleted file mode 100644
index 14d3c5e..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-package org.apache.cassandra.io.sstable;
-
-import org.apache.cassandra.io.util.File;
-import java.io.IOException;
-
-import org.junit.BeforeClass;
-
-import org.junit.Assert;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
-
-public class BigTableWriterTest extends AbstractTransactionalTest
-{
- public static final String KEYSPACE1 = "BigTableWriterTest";
- public static final String CF_STANDARD = "Standard1";
-
- private static ColumnFamilyStore cfs;
-
- @BeforeClass
- public static void defineSchema() throws Exception
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD, 0, Int32Type.instance, AsciiType.instance, Int32Type.instance));
- cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
- }
-
- protected TestableTransaction newTest() throws IOException
- {
- return new TestableBTW();
- }
-
- private static class TestableBTW extends TestableTransaction
- {
- final File file;
- final Descriptor descriptor;
- final SSTableTxnWriter writer;
-
- private TestableBTW()
- {
- this(cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables()));
- }
-
- private TestableBTW(Descriptor desc)
- {
- this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false,
- new SerializationHeader(true, cfs.metadata(),
- cfs.metadata().regularAndStaticColumns(),
- EncodingStats.NO_STATS)));
- }
-
- private TestableBTW(Descriptor desc, SSTableTxnWriter sw)
- {
- super(sw);
- this.file = new File(desc.filenameFor(Component.DATA));
- this.descriptor = desc;
- this.writer = sw;
-
- for (int i = 0; i < 100; i++)
- {
- UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), i);
- for (int j = 0; j < 10; j++)
- update.newRow(j).add("val", SSTableRewriterTest.random(0, 1000));
- writer.append(update.build().unfilteredIterator());
- }
- }
-
- protected void assertInProgress() throws Exception
- {
- assertExists(Component.DATA, Component.PRIMARY_INDEX);
- assertNotExists(Component.FILTER, Component.SUMMARY);
- Assert.assertTrue(file.length() > 0);
- }
-
- protected void assertPrepared() throws Exception
- {
- assertExists(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.SUMMARY);
- }
-
- protected void assertAborted() throws Exception
- {
- assertNotExists(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.SUMMARY);
- Assert.assertFalse(file.exists());
- }
-
- protected void assertCommitted() throws Exception
- {
- assertPrepared();
- }
-
- @Override
- protected boolean commitCanThrow()
- {
- return true;
- }
-
- private void assertExists(Component ... components)
- {
- for (Component component : components)
- Assert.assertTrue(new File(descriptor.filenameFor(component)).exists());
- }
-
- private void assertNotExists(Component ... components)
- {
- for (Component component : components)
- Assert.assertFalse(component.toString(), new File(descriptor.filenameFor(component)).exists());
- }
- }
-
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
index 1851314..5d81d6c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
@@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
@@ -55,6 +56,7 @@
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.*;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -852,7 +854,7 @@
CQLSSTableWriter writer = CQLSSTableWriter.builder()
.inDirectory(dataDir)
.forTable(schema)
- .using("INSERT INTO " + qualifiedTable + " (k, c1, c2, v) VALUES (?, ?, ?, textAsBlob(?))")
+ .using("INSERT INTO " + qualifiedTable + " (k, c1, c2, v) VALUES (?, ?, ?, text_as_blob(?))")
.build();
writer.addRow(1, 2, 3, "abc");
@@ -1116,6 +1118,122 @@
assertEquals(0, filtered.size());
}
+ @Test
+ public void testWriteWithTimestamps() throws Exception
+ {
+ long now = currentTimeMillis();
+ long then = now - 1000;
+ final String schema = "CREATE TABLE " + qualifiedTable + " ("
+ + " k int,"
+ + " v1 int,"
+ + " v2 int,"
+ + " v3 text,"
+ + " PRIMARY KEY (k)"
+ + ")";
+
+ CQLSSTableWriter writer = CQLSSTableWriter.builder()
+ .inDirectory(dataDir)
+ .forTable(schema)
+ .using("INSERT INTO " + qualifiedTable +
+ " (k, v1, v2, v3) VALUES (?,?,?,?) using timestamp ?" )
+ .build();
+
+ // Note that, all other things being equal, Cassandra will sort these rows lexicographically, so we use "higher" values in the
+ // row we expect to "win" so that we're sure that it isn't just accidentally picked due to the row sorting.
+ writer.addRow( 1, 4, 5, "b", now); // This write should be the one found at the end because it has a higher timestamp
+ writer.addRow( 1, 2, 3, "a", then);
+ writer.close();
+ loadSSTables(dataDir, keyspace);
+
+ UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
+ assertEquals(1, resultSet.size());
+
+ Iterator<UntypedResultSet.Row> iter = resultSet.iterator();
+ UntypedResultSet.Row r1 = iter.next();
+ assertEquals(1, r1.getInt("k"));
+ assertEquals(4, r1.getInt("v1"));
+ assertEquals(5, r1.getInt("v2"));
+ assertEquals("b", r1.getString("v3"));
+ assertFalse(iter.hasNext());
+ }
+ @Test
+ public void testWriteWithTtl() throws Exception
+ {
+ final String schema = "CREATE TABLE " + qualifiedTable + " ("
+ + " k int,"
+ + " v1 int,"
+ + " v2 int,"
+ + " v3 text,"
+ + " PRIMARY KEY (k)"
+ + ")";
+
+ CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder()
+ .inDirectory(dataDir)
+ .forTable(schema)
+ .using("INSERT INTO " + qualifiedTable +
+ " (k, v1, v2, v3) VALUES (?,?,?,?) using TTL ?");
+ CQLSSTableWriter writer = builder.build();
+ // add a row that _should_ show up - 1 hour TTL
+ writer.addRow( 1, 2, 3, "a", 3600);
+ // Insert a row with a TTL of 1 second - should not appear in results once we sleep
+ writer.addRow( 2, 4, 5, "b", 1);
+ writer.close();
+ Thread.sleep(1200); // Slightly over 1 second, just to make sure
+ loadSSTables(dataDir, keyspace);
+
+ UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
+ assertEquals(1, resultSet.size());
+
+ Iterator<UntypedResultSet.Row> iter = resultSet.iterator();
+ UntypedResultSet.Row r1 = iter.next();
+ assertEquals(1, r1.getInt("k"));
+ assertEquals(2, r1.getInt("v1"));
+ assertEquals(3, r1.getInt("v2"));
+ assertEquals("a", r1.getString("v3"));
+ assertFalse(iter.hasNext());
+ }
+ @Test
+ public void testWriteWithTimestampsAndTtl() throws Exception
+ {
+ final String schema = "CREATE TABLE " + qualifiedTable + " ("
+ + " k int,"
+ + " v1 int,"
+ + " v2 int,"
+ + " v3 text,"
+ + " PRIMARY KEY (k)"
+ + ")";
+
+ CQLSSTableWriter writer = CQLSSTableWriter.builder()
+ .inDirectory(dataDir)
+ .forTable(schema)
+ .using("INSERT INTO " + qualifiedTable +
+ " (k, v1, v2, v3) VALUES (?,?,?,?) using timestamp ? AND TTL ?" )
+ .build();
+ // NOTE: It would be easier to make this a timestamp in the past, but Cassandra also has a _local_ deletion time
+ // which is based on the server's timestamp, so simply setting the timestamp to some time in the past
+ // doesn't actually do what you'd think it would do.
+ long oneSecondFromNow = TimeUnit.MILLISECONDS.toMicros(currentTimeMillis() + 1000);
+ // Insert some rows with a timestamp of 1 second from now, and different TTLs
+ // add a row that _should_ show up - 1 hour TTL
+ writer.addRow( 1, 2, 3, "a", oneSecondFromNow, 3600);
+ // Insert a row "two seconds ago" with a TTL of 1 second - should not appear in results
+ writer.addRow( 2, 4, 5, "b", oneSecondFromNow, 1);
+ writer.close();
+ loadSSTables(dataDir, keyspace);
+ UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
+ Thread.sleep(1200);
+ resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
+ assertEquals(1, resultSet.size());
+
+ Iterator<UntypedResultSet.Row> iter = resultSet.iterator();
+ UntypedResultSet.Row r1 = iter.next();
+ assertEquals(1, r1.getInt("k"));
+ assertEquals(2, r1.getInt("v1"));
+ assertEquals(3, r1.getInt("v2"));
+ assertEquals("a", r1.getString("v3"));
+ assertFalse(iter.hasNext());
+ }
+
private static void loadSSTables(File dataDir, String ks) throws ExecutionException, InterruptedException
{
SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client()
diff --git a/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java
new file mode 100644
index 0000000..a9b7630
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.util.HashSet;
+import java.util.function.Function;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.sstable.Component.Type;
+import org.apache.cassandra.io.sstable.SSTableFormatTest.Format1;
+import org.apache.cassandra.io.sstable.SSTableFormatTest.Format2;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.mockito.Mockito;
+
+import static org.apache.cassandra.io.sstable.SSTableFormatTest.factory;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+
+public class ComponentTest
+{
+ private static final String SECOND = "second";
+ private static final String FIRST = "first";
+
+ static
+ {
+ DatabaseDescriptor.daemonInitialization(() -> {
+ Config config = DatabaseDescriptor.loadConfig();
+ SSTableFormatTest.configure(new Config.SSTableConfig(), new BigFormat.BigFormatFactory(), factory("first", Format1.class), factory("second", Format2.class));
+ return config;
+ });
+ }
+
+ @Test
+ public void testTypes()
+ {
+ Function<Type, Component> componentFactory = Mockito.mock(Function.class);
+
+ // do not allow to define a type with the same name or repr as the existing type for this or parent format
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> Type.createSingleton(Components.Types.TOC.name, Components.Types.TOC.repr + "x", Format1.class));
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> Type.createSingleton(Components.Types.TOC.name + "x", Components.Types.TOC.repr, Format2.class));
+
+ // allow to define a format with other name and repr
+ Type t1 = Type.createSingleton("ONE", "One.db", Format1.class);
+
+ // allow to define a format with the same name and repr for two different formats
+ Type t2f1 = Type.createSingleton("TWO", "Two.db", Format1.class);
+ Type t2f2 = Type.createSingleton("TWO", "Two.db", Format2.class);
+ assertThat(t2f1).isNotEqualTo(t2f2);
+
+ assertThatExceptionOfType(NullPointerException.class).isThrownBy(() -> Type.createSingleton(null, "-Three.db", Format1.class));
+
+ assertThat(Type.fromRepresentation("should be custom", BigFormat.getInstance())).isSameAs(Components.Types.CUSTOM);
+ assertThat(Type.fromRepresentation(Components.Types.TOC.repr, BigFormat.getInstance())).isSameAs(Components.Types.TOC);
+ assertThat(Type.fromRepresentation(t1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(t1);
+ assertThat(Type.fromRepresentation(t2f1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(t2f1);
+ assertThat(Type.fromRepresentation(t2f2.repr, DatabaseDescriptor.getSSTableFormats().get(SECOND))).isSameAs(t2f2);
+ }
+
+ @Test
+ public void testComponents()
+ {
+ Type t3f1 = Type.createSingleton("THREE", "Three.db", Format1.class);
+ Type t3f2 = Type.createSingleton("THREE", "Three.db", Format2.class);
+ Type t4f1 = Type.create("FOUR", ".*-Four.db", Format1.class);
+ Type t4f2 = Type.create("FOUR", ".*-Four.db", Format2.class);
+
+ Component c1 = t3f1.getSingleton();
+ Component c2 = t3f2.getSingleton();
+
+ assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t3f1.createComponent(t3f1.repr));
+ assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t3f2.createComponent(t3f2.repr));
+ assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t4f1.getSingleton());
+ assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t4f2.getSingleton());
+
+ assertThat(Component.parse(t3f1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(c1);
+ assertThat(Component.parse(t3f2.repr, DatabaseDescriptor.getSSTableFormats().get("second"))).isSameAs(c2);
+ assertThat(c1).isNotEqualTo(c2);
+ assertThat(c1.type).isSameAs(t3f1);
+ assertThat(c2.type).isSameAs(t3f2);
+
+ Component c3 = Component.parse("abc-Four.db", DatabaseDescriptor.getSSTableFormats().get(FIRST));
+ Component c4 = Component.parse("abc-Four.db", DatabaseDescriptor.getSSTableFormats().get("second"));
+ assertThat(c3.type).isSameAs(t4f1);
+ assertThat(c4.type).isSameAs(t4f2);
+ assertThat(c3.name).isEqualTo("abc-Four.db");
+ assertThat(c4.name).isEqualTo("abc-Four.db");
+ assertThat(c3).isNotEqualTo(c4);
+ assertThat(c3).isNotEqualTo(c1);
+ assertThat(c4).isNotEqualTo(c2);
+
+ Component c5 = Component.parse("abc-Five.db", DatabaseDescriptor.getSSTableFormats().get(FIRST));
+ assertThat(c5.type).isSameAs(Components.Types.CUSTOM);
+ assertThat(c5.name).isEqualTo("abc-Five.db");
+
+ Component c6 = Component.parse("Data.db", DatabaseDescriptor.getSSTableFormats().get("second"));
+ assertThat(c6.type).isSameAs(Components.Types.DATA);
+ assertThat(c6).isSameAs(Components.DATA);
+
+ HashSet<Component> s1 = Sets.newHashSet(Component.getSingletonsFor(Format1.class));
+ HashSet<Component> s2 = Sets.newHashSet(Component.getSingletonsFor(Format2.class));
+ assertThat(s1).contains(c1, Components.DATA, Components.STATS, Components.COMPRESSION_INFO);
+ assertThat(s2).contains(c2, Components.DATA, Components.STATS, Components.COMPRESSION_INFO);
+ assertThat(s1).doesNotContain(c2);
+ assertThat(s2).doesNotContain(c1);
+
+ assertThat(Sets.newHashSet(Component.getSingletonsFor(DatabaseDescriptor.getSSTableFormats().get(FIRST)))).isEqualTo(s1);
+ assertThat(Sets.newHashSet(Component.getSingletonsFor(DatabaseDescriptor.getSSTableFormats().get("second")))).isEqualTo(s2);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
index 405f3da..1720ea0 100644
--- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.util.UUID;
-import org.apache.cassandra.io.util.File;
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -29,11 +28,15 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class DescriptorTest
{
@@ -84,19 +87,19 @@
private void testFromFilenameFor(File dir)
{
- checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG));
+ checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), DatabaseDescriptor.getSelectedSSTableFormat()));
// secondary index
String idxName = "myidx";
File idxDir = new File(dir.absolutePath() + File.pathSeparator() + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName);
- checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), SSTableFormat.Type.BIG));
+ checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), DatabaseDescriptor.getSelectedSSTableFormat()));
}
private void checkFromFilename(Descriptor original)
{
- File file = new File(original.filenameFor(Component.DATA));
+ File file = original.fileFor(Components.DATA);
- Pair<Descriptor, Component> pair = Descriptor.fromFilenameWithComponent(file);
+ Pair<Descriptor, Component> pair = Descriptor.fromFileWithComponent(file);
Descriptor desc = pair.left;
assertEquals(original.directory, desc.directory);
@@ -104,7 +107,7 @@
assertEquals(original.cfname, desc.cfname);
assertEquals(original.version, desc.version);
assertEquals(original.id, desc.id);
- assertEquals(Component.DATA, pair.right);
+ assertEquals(Components.DATA, pair.right);
}
@Test
@@ -112,8 +115,8 @@
{
// Descriptor should be equal when parent directory points to the same directory
File dir = new File(".");
- Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG);
- Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG);
+ Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), DatabaseDescriptor.getSelectedSSTableFormat());
+ Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), DatabaseDescriptor.getSelectedSSTableFormat());
assertEquals(desc1, desc2);
assertEquals(desc1.hashCode(), desc2.hashCode());
}
@@ -121,16 +124,16 @@
@Test
public void validateNames()
{
- String[] names = {
- "ma-1-big-Data.db",
- // 2ndary index
- ".idx1" + File.pathSeparator() + "ma-1-big-Data.db",
- };
+ final SSTableFormat<?, ?> ssTableFormat = DatabaseDescriptor.getSelectedSSTableFormat();
+ String name = ssTableFormat.name();
+ final Version version = ssTableFormat.getLatestVersion();
+ String[] fileNames = { version + "-1-" + name + "-Data.db",
+ // 2ndary index
+ ".idx1" + File.pathSeparator() + version + "-1-" + name + "-Data.db",
+ };
- for (String name : names)
- {
- assertNotNull(Descriptor.fromFilename(name));
- }
+ for (String fileName : fileNames)
+ assertNotNull(Descriptor.fromFileWithComponent(new File(fileName), false).left);
}
@Test
@@ -146,7 +149,7 @@
{
try
{
- Descriptor d = Descriptor.fromFilename(name);
+ Descriptor d = Descriptor.fromFile(new File(name));
Assert.fail(name);
} catch (Throwable e) {
//good
@@ -160,12 +163,13 @@
// from Cassandra dirs
String[] filePaths = new String[]{
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/snapshots/snapshot/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/backups/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/snapshots/snapshot/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/nb-1-big-TOC.txt",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/snapshots/snapshot/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/backups/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/snapshots/snapshot/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(filePaths, "ks1", "tab1");
@@ -173,12 +177,12 @@
// indexes
String[] filePathsIndexes = new String[]{
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/snapshots/snapshot/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/backups/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/snapshots/snapshot/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-3424234234324/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/snapshots/snapshot/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/backups/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/snapshots/snapshot/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/ks1/tab1-34234234234234234234234234234234/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(filePathsIndexes, "ks1", "tab1.index");
@@ -186,23 +190,23 @@
// what if even a snapshot of a keyspace and table called snapshots is called snapshots?
String[] filePathsWithSnapshotKeyspaceAndTable = new String[]{
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/snapshots/snapshots/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/backups/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/snapshots/snapshots/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/backups/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(filePathsWithSnapshotKeyspaceAndTable, "snapshots", "snapshots");
String[] filePathsWithSnapshotKeyspaceAndTableWithIndices = new String[]{
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/snapshots/snapshots/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/backups/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/snapshots/snapshots/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-742738427389478/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/snapshots/snapshots/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/backups/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/snapshots/snapshots/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/snapshots/snapshots-74273842738947874273842738947878/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(filePathsWithSnapshotKeyspaceAndTableWithIndices, "snapshots", "snapshots.index");
@@ -210,12 +214,12 @@
// what if keyspace and table is called backups?
String[] filePathsWithBackupsKeyspaceAndTable = new String[]{
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/snapshots/snapshots/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/backups/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/snapshots/snapshots/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/backups/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(filePathsWithBackupsKeyspaceAndTable, "backups", "backups");
@@ -224,11 +228,12 @@
String[] legacyFilePathsWithBackupsKeyspaceAndTable = new String[]{
"/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/nb-1-big-TOC.txt",
//"/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/snapshots/snapshots/na-1-big-Index.db", #not supported (CASSANDRA-14013)
"/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/backups/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/nb-22-big-Index.db",
- //"/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/snapshots/snapshots/nb-22-big-Index.db", #not supported (CASSANDRA-14013)
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/backups/nb-22-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ //"/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db", #not supported (CASSANDRA-14013)
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(legacyFilePathsWithBackupsKeyspaceAndTable, "backups", "backups");
@@ -236,53 +241,53 @@
// what if even a snapshot of a keyspace and table called backups is called snapshots?
String[] filePathsWithBackupsKeyspaceAndTableWithIndices = new String[]{
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/snapshots/snapshots/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/backups/.index/na-1-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/snapshots/snapshots/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-742738427389478/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/snapshots/snapshots/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/backups/.index/na-1-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/snapshots/snapshots/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/path/to/cassandra/data/dir2/dir5/dir6/backups/backups-74273842738947874273842738947878/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
};
testKeyspaceTableParsing(filePathsWithBackupsKeyspaceAndTableWithIndices, "backups", "backups.index");
String[] outsideOfCassandra = new String[]{
- "/tmp/some/path/tests/keyspace/table-3424234234234/na-1-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-3424234234234/snapshots/snapshots/na-1-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-3424234234234/backups/na-1-big-Index.db",
- "/tmp/tests/keyspace/table-3424234234234/na-1-big-Index.db",
- "/keyspace/table-3424234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-3424234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-3424234234234/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-3424234234234/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/tests/keyspace/table-3424234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/keyspace/table-3424234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db"
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/na-1-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/snapshots/snapshots/na-1-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/backups/na-1-big-Index.db",
+ "/tmp/tests/keyspace/table-34234234234234234234234234234234/na-1-big-Index.db",
+ "/keyspace/table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/tests/keyspace/table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/keyspace/table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db"
};
testKeyspaceTableParsing(outsideOfCassandra, "keyspace", "table");
String[] outsideOfCassandraUppercaseKeyspaceAndTable = new String[]{
- "/tmp/some/path/tests/Keyspace/Table-23424324234234/na-1-big-Index.db",
- "/tmp/some/path/tests/Keyspace/Table-23424324234234/snapshots/snapshots/na-1-big-Index.db",
- "/tmp/some/path/tests/Keyspace/Table-23424324234234/backups/na-1-big-Index.db",
- "/tmp/tests/Keyspace/Table-23424324234234/na-1-big-Index.db",
- "/Keyspace/Table-23424324234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/Keyspace/Table-23424324234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/Keyspace/Table-23424324234234/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/Keyspace/Table-23424324234234/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/tests/Keyspace/Table-23424324234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/Keyspace/Table-23424324234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db"
+ "/tmp/some/path/tests/Keyspace/Table-34234234234234234234234234234234/na-1-big-Index.db",
+ "/tmp/some/path/tests/Keyspace/Table-34234234234234234234234234234234/snapshots/snapshots/na-1-big-Index.db",
+ "/tmp/some/path/tests/Keyspace/Table-34234234234234234234234234234234/backups/na-1-big-Index.db",
+ "/tmp/tests/Keyspace/Table-34234234234234234234234234234234/na-1-big-Index.db",
+ "/Keyspace/Table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/Keyspace/Table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/Keyspace/Table-34234234234234234234234234234234/snapshots/snapshots/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/Keyspace/Table-34234234234234234234234234234234/backups/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/tests/Keyspace/Table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/Keyspace/Table-34234234234234234234234234234234/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db"
};
testKeyspaceTableParsing(outsideOfCassandraUppercaseKeyspaceAndTable, "Keyspace", "Table");
String[] outsideOfCassandraIndexes = new String[]{
- "/tmp/some/path/tests/keyspace/table-32423423423423/.index/na-1-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-32423423423423/snapshots/snapshots/.index/na-1-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-32423423423423/backups/.index/na-1-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-32423423423423/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-32423423423423/snapshots/snapshots/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
- "/tmp/some/path/tests/keyspace/table-32423423423423/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db"
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/.index/na-1-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/snapshots/snapshots/.index/na-1-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/backups/.index/na-1-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/snapshots/snapshots/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db",
+ "/tmp/some/path/tests/keyspace/table-34234234234234234234234234234234/backups/.index/nb-3g1m_0nuf_3vj5m2k1125165rxa7-big-Index.db"
};
testKeyspaceTableParsing(outsideOfCassandraIndexes, "keyspace", "table.index");
@@ -299,10 +304,10 @@
{
for (String filePath : filePaths)
{
- Descriptor descriptor = Descriptor.fromFilename(filePath);
+ Descriptor descriptor = Descriptor.fromFile(new File(filePath));
Assert.assertNotNull(descriptor);
- Assert.assertEquals(expectedKeyspace, descriptor.ksname);
- Assert.assertEquals(expectedTable, descriptor.cfname);
+ Assert.assertEquals(String.format("Expected keyspace not found for %s", filePath), expectedKeyspace, descriptor.ksname);
+ Assert.assertEquals(String.format("Expected table not found for %s", filePath), expectedTable, descriptor.cfname);
}
}
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
deleted file mode 100644
index c494e3c..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
+++ /dev/null
@@ -1,788 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Sets;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.compaction.CompactionInfo;
-import org.apache.cassandra.db.compaction.CompactionInterruptedException;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.metrics.RestorableMeter;
-import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.SchemaTestUtil;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-import static com.google.common.collect.ImmutableMap.of;
-import static java.util.Arrays.asList;
-import static org.apache.cassandra.Util.assertOnDiskState;
-import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-import static org.apache.cassandra.io.sstable.IndexSummaryRedistribution.DOWNSAMPLE_THESHOLD;
-import static org.apache.cassandra.io.sstable.IndexSummaryRedistribution.UPSAMPLE_THRESHOLD;
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class IndexSummaryManagerTest
-{
- private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManagerTest.class);
-
- int originalMinIndexInterval;
- int originalMaxIndexInterval;
- long originalCapacity;
-
- private static final String KEYSPACE1 = "IndexSummaryManagerTest";
- // index interval of 8, no key caching
- private static final String CF_STANDARDLOWiINTERVAL = "StandardLowIndexInterval";
- private static final String CF_STANDARDRACE = "StandardRace";
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWiINTERVAL)
- .minIndexInterval(8)
- .maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDRACE)
- .minIndexInterval(8)
- .maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING));
- }
-
- @Before
- public void beforeTest()
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- originalMinIndexInterval = cfs.metadata().params.minIndexInterval;
- originalMaxIndexInterval = cfs.metadata().params.maxIndexInterval;
- originalCapacity = IndexSummaryManager.instance.getMemoryPoolCapacityInMB();
- }
-
- @After
- public void afterTest()
- {
- for (CompactionInfo.Holder holder : CompactionManager.instance.active.getCompactions())
- {
- holder.stop();
- }
-
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
-
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build());
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMaxIndexInterval).build());
-
- IndexSummaryManager.instance.setMemoryPoolCapacityInMB(originalCapacity);
- }
-
- private static long totalOffHeapSize(List<SSTableReader> sstables)
- {
- long total = 0;
- for (SSTableReader sstable : sstables)
- total += sstable.getIndexSummaryOffHeapSize();
- return total;
- }
-
- private static List<SSTableReader> resetSummaries(ColumnFamilyStore cfs, List<SSTableReader> sstables, long originalOffHeapSize) throws IOException
- {
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), originalOffHeapSize * sstables.size());
- }
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
-
- return sstables;
- }
-
- private void validateData(ColumnFamilyStore cfs, int numPartition)
- {
- for (int i = 0; i < numPartition; i++)
- {
- Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, String.format("%3d", i)).build());
- Cell<?> cell = row.getCell(cfs.metadata().getColumn(ByteBufferUtil.bytes("val")));
- assertNotNull(cell);
- assertEquals(100, cell.buffer().array().length);
-
- }
- }
-
- private Comparator<SSTableReader> hotnessComparator = new Comparator<SSTableReader>()
- {
- public int compare(SSTableReader o1, SSTableReader o2)
- {
- return Double.compare(o1.getReadMeter().fifteenMinuteRate(), o2.getReadMeter().fifteenMinuteRate());
- }
- };
-
- private void createSSTables(String ksname, String cfname, int numSSTables, int numPartition)
- {
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- cfs.truncateBlocking();
- cfs.disableAutoCompaction();
-
- ArrayList<Future> futures = new ArrayList<>(numSSTables);
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
- for (int sstable = 0; sstable < numSSTables; sstable++)
- {
- for (int p = 0; p < numPartition; p++)
- {
-
- String key = String.format("%3d", p);
- new RowUpdateBuilder(cfs.metadata(), 0, key)
- .clustering("column")
- .add("val", value)
- .build()
- .applyUnsafe();
- }
- futures.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
- }
- for (Future future : futures)
- {
- try
- {
- future.get();
- }
- catch (InterruptedException | ExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
- assertEquals(numSSTables, cfs.getLiveSSTables().size());
- validateData(cfs, numPartition);
- }
-
- @Test
- public void testChangeMinIndexInterval() throws IOException
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- int numSSTables = 1;
- int numRows = 256;
- createSSTables(ksname, cfname, numSSTables, numRows);
-
- List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- for (SSTableReader sstable : sstables)
- assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
-
- // double the min_index_interval
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build());
- IndexSummaryManager.instance.redistributeSummaries();
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize());
- }
-
- // return min_index_interval to its original value
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build());
- IndexSummaryManager.instance.redistributeSummaries();
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize());
- }
-
- // halve the min_index_interval, but constrain the available space to exactly what we have now; as a result,
- // the summary shouldn't change
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval / 2).build());
- SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- long summarySpace = sstable.getIndexSummaryOffHeapSize();
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), summarySpace);
- }
-
- sstable = cfs.getLiveSSTables().iterator().next();
- assertEquals(originalMinIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / originalMinIndexInterval, sstable.getIndexSummarySize());
-
- // keep the min_index_interval the same, but now give the summary enough space to grow by 50%
- double previousInterval = sstable.getEffectiveIndexInterval();
- int previousSize = sstable.getIndexSummarySize();
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace * 1.5));
- }
- sstable = cfs.getLiveSSTables().iterator().next();
- assertEquals(previousSize * 1.5, (double) sstable.getIndexSummarySize(), 1);
- assertEquals(previousInterval * (1.0 / 1.5), sstable.getEffectiveIndexInterval(), 0.001);
-
- // return min_index_interval to it's original value (double it), but only give the summary enough space
- // to have an effective index interval of twice the new min
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build());
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace / 2.0));
- }
- sstable = cfs.getLiveSSTables().iterator().next();
- assertEquals(originalMinIndexInterval * 2, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / (originalMinIndexInterval * 2), sstable.getIndexSummarySize());
-
- // raise the min_index_interval above our current effective interval, but set the max_index_interval lower
- // than what we actually have space for (meaning the index summary would ideally be smaller, but this would
- // result in an effective interval above the new max)
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 4).build());
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMinIndexInterval * 4).build());
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10);
- }
- sstable = cfs.getLiveSSTables().iterator().next();
- assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- }
-
- @Test
- public void testChangeMaxIndexInterval() throws IOException
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- int numSSTables = 1;
- int numRows = 256;
- createSSTables(ksname, cfname, numSSTables, numRows);
-
- List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10);
- }
- sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
-
- // halve the max_index_interval
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval / 2).build());
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 1);
- }
- sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- {
- assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
- assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummarySize());
- }
-
- // return max_index_interval to its original value
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval * 2).build());
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 1);
- }
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01);
- assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummarySize());
- }
- }
-
- @Test(timeout = 10000)
- public void testRedistributeSummaries() throws IOException
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- int numSSTables = 4;
- int numRows = 256;
- createSSTables(ksname, cfname, numSSTables, numRows);
-
- int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata().params.minIndexInterval) / cfs.metadata().params.maxIndexInterval;
-
- List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize();
-
- // there should be enough space to not downsample anything
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables));
- }
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
- assertEquals(singleSummaryOffHeapSpace * numSSTables, totalOffHeapSize(sstables));
- validateData(cfs, numRows);
-
- // everything should get cut in half
- assert sstables.size() == 4;
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
- }
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // everything should get cut to a quarter
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 4)));
- }
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL / 4, sstable.getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // upsample back up to half
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2) + 4));
- }
- assert sstables.size() == 4;
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // upsample back up to the original index summary
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables));
- }
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // make two of the four sstables cold, only leave enough space for three full index summaries,
- // so the two cold sstables should get downsampled to be half of their original size
- sstables.get(0).overrideReadMeter(new RestorableMeter(50.0, 50.0));
- sstables.get(1).overrideReadMeter(new RestorableMeter(50.0, 50.0));
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3));
- }
- Collections.sort(sstables, hotnessComparator);
- assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // small increases or decreases in the read rate don't result in downsampling or upsampling
- double lowerRate = 50.0 * (DOWNSAMPLE_THESHOLD + (DOWNSAMPLE_THESHOLD * 0.10));
- double higherRate = 50.0 * (UPSAMPLE_THRESHOLD - (UPSAMPLE_THRESHOLD * 0.10));
- sstables.get(0).overrideReadMeter(new RestorableMeter(lowerRate, lowerRate));
- sstables.get(1).overrideReadMeter(new RestorableMeter(higherRate, higherRate));
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3));
- }
- Collections.sort(sstables, hotnessComparator);
- assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // reset, and then this time, leave enough space for one of the cold sstables to not get downsampled
- sstables = resetSummaries(cfs, sstables, singleSummaryOffHeapSpace);
- sstables.get(0).overrideReadMeter(new RestorableMeter(1.0, 1.0));
- sstables.get(1).overrideReadMeter(new RestorableMeter(2.0, 2.0));
- sstables.get(2).overrideReadMeter(new RestorableMeter(1000.0, 1000.0));
- sstables.get(3).overrideReadMeter(new RestorableMeter(1000.0, 1000.0));
-
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3) + 50);
- }
- Collections.sort(sstables, hotnessComparator);
-
- if (sstables.get(0).getIndexSummarySamplingLevel() == minSamplingLevel)
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(1).getIndexSummarySamplingLevel());
- else
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(0).getIndexSummarySamplingLevel());
-
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel());
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
-
- // Cause a mix of upsampling and downsampling. We'll leave enough space for two full index summaries. The two
- // coldest sstables will get downsampled to 4/128 of their size, leaving us with 1 and 92/128th index
- // summaries worth of space. The hottest sstable should get a full index summary, and the one in the middle
- // should get the remainder.
- sstables.get(0).overrideReadMeter(new RestorableMeter(0.0, 0.0));
- sstables.get(1).overrideReadMeter(new RestorableMeter(0.0, 0.0));
- sstables.get(2).overrideReadMeter(new RestorableMeter(92, 92));
- sstables.get(3).overrideReadMeter(new RestorableMeter(128.0, 128.0));
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL))));
- }
- Collections.sort(sstables, hotnessComparator);
- assertEquals(1, sstables.get(0).getIndexSummarySize()); // at the min sampling level
- assertEquals(1, sstables.get(0).getIndexSummarySize()); // at the min sampling level
- assertTrue(sstables.get(2).getIndexSummarySamplingLevel() > minSamplingLevel);
- assertTrue(sstables.get(2).getIndexSummarySamplingLevel() < BASE_SAMPLING_LEVEL);
- assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
-
- // Don't leave enough space for even the minimal index summaries
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10);
- }
- for (SSTableReader sstable : sstables)
- assertEquals(1, sstable.getIndexSummarySize()); // at the min sampling level
- validateData(cfs, numRows);
- }
-
- @Test
- public void testRebuildAtSamplingLevel() throws IOException
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL;
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- cfs.truncateBlocking();
- cfs.disableAutoCompaction();
-
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
-
- int numRows = 256;
- for (int row = 0; row < numRows; row++)
- {
- String key = String.format("%3d", row);
- new RowUpdateBuilder(cfs.metadata(), 0, key)
- .clustering("column")
- .add("val", value)
- .build()
- .applyUnsafe();
- }
-
- Util.flush(cfs);
-
- List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
- assertEquals(1, sstables.size());
- SSTableReader original = sstables.get(0);
-
- SSTableReader sstable = original;
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN))
- {
- for (int samplingLevel = 1; samplingLevel < BASE_SAMPLING_LEVEL; samplingLevel++)
- {
- sstable = sstable.cloneWithNewSummarySamplingLevel(cfs, samplingLevel);
- assertEquals(samplingLevel, sstable.getIndexSummarySamplingLevel());
- int expectedSize = (numRows * samplingLevel) / (cfs.metadata().params.minIndexInterval * BASE_SAMPLING_LEVEL);
- assertEquals(expectedSize, sstable.getIndexSummarySize(), 1);
- txn.update(sstable, true);
- txn.checkpoint();
- }
- txn.finish();
- }
- }
-
- @Test
- public void testJMXFunctions() throws IOException
- {
- IndexSummaryManager manager = IndexSummaryManager.instance;
-
- // resize interval
- manager.setResizeIntervalInMinutes(-1);
- assertEquals(-1, DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
- assertNull(manager.getTimeToNextResize(TimeUnit.MINUTES));
-
- manager.setResizeIntervalInMinutes(10);
- assertEquals(10, manager.getResizeIntervalInMinutes());
- assertEquals(10, DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
- assertEquals(10, manager.getTimeToNextResize(TimeUnit.MINUTES), 1);
- manager.setResizeIntervalInMinutes(15);
- assertEquals(15, manager.getResizeIntervalInMinutes());
- assertEquals(15, DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
- assertEquals(15, manager.getTimeToNextResize(TimeUnit.MINUTES), 2);
-
- // memory pool capacity
- assertTrue(manager.getMemoryPoolCapacityInMB() >= 0);
- manager.setMemoryPoolCapacityInMB(10);
- assertEquals(10, manager.getMemoryPoolCapacityInMB());
-
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- cfs.truncateBlocking();
- cfs.disableAutoCompaction();
-
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
-
- int numSSTables = 2;
- int numRows = 10;
- for (int sstable = 0; sstable < numSSTables; sstable++)
- {
- for (int row = 0; row < numRows; row++)
- {
- String key = String.format("%3d", row);
- new RowUpdateBuilder(cfs.metadata(), 0, key)
- .clustering("column")
- .add("val", value)
- .build()
- .applyUnsafe();
- }
- Util.flush(cfs);
- }
-
- assertTrue(manager.getAverageIndexInterval() >= cfs.metadata().params.minIndexInterval);
- Map<String, Integer> intervals = manager.getIndexIntervals();
- for (Map.Entry<String, Integer> entry : intervals.entrySet())
- if (entry.getKey().contains(CF_STANDARDLOWiINTERVAL))
- assertEquals(cfs.metadata().params.minIndexInterval, entry.getValue(), 0.001);
-
- manager.setMemoryPoolCapacityInMB(0);
- manager.redistributeSummaries();
- assertTrue(manager.getAverageIndexInterval() > cfs.metadata().params.minIndexInterval);
- intervals = manager.getIndexIntervals();
- for (Map.Entry<String, Integer> entry : intervals.entrySet())
- {
- if (entry.getKey().contains(CF_STANDARDLOWiINTERVAL))
- assertTrue(entry.getValue() >= cfs.metadata().params.minIndexInterval);
- }
- }
-
- @Test
- public void testCancelIndex() throws Exception
- {
- testCancelIndexHelper((cfs) -> CompactionManager.instance.stopCompaction("INDEX_SUMMARY"));
- }
-
- @Test
- public void testCancelIndexInterrupt() throws Exception
- {
- testCancelIndexHelper((cfs) -> CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfs.metadata()), (sstable) -> true, false));
- }
-
- public void testCancelIndexHelper(Consumer<ColumnFamilyStore> cancelFunction) throws Exception
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- cfs.disableAutoCompaction();
- final int numSSTables = 8;
- int numRows = 256;
- createSSTables(ksname, cfname, numSSTables, numRows);
-
- List<SSTableReader> allSSTables = new ArrayList<>(cfs.getLiveSSTables());
- List<SSTableReader> sstables = allSSTables.subList(0, 4);
- List<SSTableReader> compacting = allSSTables.subList(4, 8);
-
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- final long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize();
-
- // everything should get cut in half
- final AtomicReference<CompactionInterruptedException> exception = new AtomicReference<>();
- // barrier to control when redistribution runs
- final CountDownLatch barrier = new CountDownLatch(1);
- CompactionInfo.Holder ongoingCompaction = new CompactionInfo.Holder()
- {
- public CompactionInfo getCompactionInfo()
- {
- return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting);
- }
-
- public boolean isGlobal()
- {
- return false;
- }
- };
- try (LifecycleTransaction ignored = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN))
- {
- CompactionManager.instance.active.beginCompaction(ongoingCompaction);
-
- Thread t = NamedThreadFactory.createAnonymousThread(new Runnable()
- {
- public void run()
- {
- try
- {
- // Don't leave enough space for even the minimal index summaries
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- IndexSummaryManager.redistributeSummaries(new ObservableRedistribution(of(cfs.metadata.id, txn),
- 0,
- singleSummaryOffHeapSpace,
- barrier));
- }
- }
- catch (CompactionInterruptedException ex)
- {
- exception.set(ex);
- }
- catch (IOException ignored)
- {
- }
- }
- });
-
- t.start();
- while (CompactionManager.instance.getActiveCompactions() < 2 && t.isAlive())
- Thread.sleep(1);
- // to ensure that the stop condition check in IndexSummaryRedistribution::redistributeSummaries
- // is made *after* the halt request is made to the CompactionManager, don't allow the redistribution
- // to proceed until stopCompaction has been called.
- cancelFunction.accept(cfs);
- // allows the redistribution to proceed
- barrier.countDown();
- t.join();
- }
- finally
- {
- CompactionManager.instance.active.finishCompaction(ongoingCompaction);
- }
-
- assertNotNull("Expected compaction interrupted exception", exception.get());
- assertTrue("Expected no active compactions", CompactionManager.instance.active.getCompactions().isEmpty());
-
- Set<SSTableReader> beforeRedistributionSSTables = new HashSet<>(allSSTables);
- Set<SSTableReader> afterCancelSSTables = new HashSet<>(cfs.getLiveSSTables());
- Set<SSTableReader> disjoint = Sets.symmetricDifference(beforeRedistributionSSTables, afterCancelSSTables);
- assertTrue(String.format("Mismatched files before and after cancelling redistribution: %s",
- Joiner.on(",").join(disjoint)),
- disjoint.isEmpty());
- assertOnDiskState(cfs, 8);
- validateData(cfs, numRows);
- }
-
- @Test
- public void testPauseIndexSummaryManager() throws Exception
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- int numSSTables = 4;
- int numRows = 256;
- createSSTables(ksname, cfname, numSSTables, numRows);
-
- List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize();
-
- // everything should get cut in half
- assert sstables.size() == numSSTables;
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
- {
- try (AutoCloseable toresume = CompactionManager.instance.pauseGlobalCompaction())
- {
- sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata().id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
- fail("The redistribution should fail - we got paused before adding to active compactions, but after marking compacting");
- }
- }
- catch (CompactionInterruptedException e)
- {
- // expected
- }
- for (SSTableReader sstable : sstables)
- assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
- validateData(cfs, numRows);
- assertOnDiskState(cfs, numSSTables);
- }
-
- private static List<SSTableReader> redistributeSummaries(List<SSTableReader> compacting,
- Map<TableId, LifecycleTransaction> transactions,
- long memoryPoolBytes)
- throws IOException
- {
- long nonRedistributingOffHeapSize = compacting.stream().mapToLong(SSTableReader::getIndexSummaryOffHeapSize).sum();
- return IndexSummaryManager.redistributeSummaries(new IndexSummaryRedistribution(transactions,
- nonRedistributingOffHeapSize,
- memoryPoolBytes));
- }
-
- private static class ObservableRedistribution extends IndexSummaryRedistribution
- {
- CountDownLatch barrier;
-
- ObservableRedistribution(Map<TableId, LifecycleTransaction> transactions,
- long nonRedistributingOffHeapSize,
- long memoryPoolBytes,
- CountDownLatch barrier)
- {
- super(transactions, nonRedistributingOffHeapSize, memoryPoolBytes);
- this.barrier = barrier;
- }
-
- public List<SSTableReader> redistributeSummaries() throws IOException
- {
- try
- {
- barrier.await();
- }
- catch (InterruptedException e)
- {
- throw new RuntimeException("Interrupted waiting on test barrier");
- }
- return super.redistributeSummaries();
- }
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
deleted file mode 100644
index 57e4d4e..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.metrics.RestorableMeter;
-import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.SchemaTestUtil;
-
-import static org.junit.Assert.assertEquals;
-
-public class IndexSummaryRedistributionTest
-{
- private static final String KEYSPACE1 = "IndexSummaryRedistributionTest";
- private static final String CF_STANDARD = "Standard";
-
- @BeforeClass
- public static void defineSchema() throws ConfigurationException
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
- .minIndexInterval(8)
- .maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING));
- }
-
- @Test
- public void testMetricsLoadAfterRedistribution() throws IOException
- {
- String ksname = KEYSPACE1;
- String cfname = CF_STANDARD;
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- int numSSTables = 1;
- int numRows = 1024 * 10;
- long load = StorageMetrics.load.getCount();
- StorageMetrics.load.dec(load); // reset the load metric
- createSSTables(ksname, cfname, numSSTables, numRows);
-
- List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
- for (SSTableReader sstable : sstables)
- sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
-
- long oldSize = 0;
- for (SSTableReader sstable : sstables)
- {
- assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- oldSize += sstable.bytesOnDisk();
- }
-
- load = StorageMetrics.load.getCount();
- long others = load - oldSize; // Other SSTables size, e.g. schema and other system SSTables
-
- int originalMinIndexInterval = cfs.metadata().params.minIndexInterval;
- // double the min_index_interval
- SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build());
- IndexSummaryManager.instance.redistributeSummaries();
-
- long newSize = 0;
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
- assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize());
- newSize += sstable.bytesOnDisk();
- }
- newSize += others;
- load = StorageMetrics.load.getCount();
-
- // new size we calculate should be almost the same as the load in metrics
- assertEquals(newSize, load, newSize / 10);
- }
-
- private void createSSTables(String ksname, String cfname, int numSSTables, int numRows)
- {
- Keyspace keyspace = Keyspace.open(ksname);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- cfs.truncateBlocking();
- cfs.disableAutoCompaction();
-
- ArrayList<Future> futures = new ArrayList<>(numSSTables);
- ByteBuffer value = ByteBuffer.wrap(new byte[100]);
- for (int sstable = 0; sstable < numSSTables; sstable++)
- {
- for (int row = 0; row < numRows; row++)
- {
- String key = String.format("%3d", row);
- new RowUpdateBuilder(cfs.metadata(), 0, key)
- .clustering("column")
- .add("val", value)
- .build()
- .applyUnsafe();
- }
- futures.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
- }
- for (Future future : futures)
- {
- try
- {
- future.get();
- }
- catch (InterruptedException | ExecutionException e)
- {
- throw new RuntimeException(e);
- }
- }
- assertEquals(numSSTables, cfs.getLiveSSTables().size());
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
deleted file mode 100644
index d0680f8..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.google.common.collect.Lists;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.Assume;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.RandomPartitioner;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-
-import static org.apache.cassandra.io.sstable.IndexSummaryBuilder.downsample;
-import static org.apache.cassandra.io.sstable.IndexSummaryBuilder.entriesAtSamplingLevel;
-import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.junit.Assert.*;
-
-public class IndexSummaryTest
-{
- private final static Random random = new Random();
-
- @BeforeClass
- public static void initDD()
- {
- DatabaseDescriptor.daemonInitialization();
-
- final long seed = nanoTime();
- System.out.println("Using seed: " + seed);
- random.setSeed(seed);
- }
-
- IPartitioner partitioner = Util.testPartitioner();
-
- @BeforeClass
- public static void setup()
- {
- final long seed = nanoTime();
- System.out.println("Using seed: " + seed);
- random.setSeed(seed);
- }
-
- @Test
- public void testIndexSummaryKeySizes() throws IOException
- {
- // On Circle CI we normally don't have enough off-heap memory for this test so ignore it
- Assume.assumeTrue(System.getenv("CIRCLECI") == null);
-
- testIndexSummaryProperties(32, 100);
- testIndexSummaryProperties(64, 100);
- testIndexSummaryProperties(100, 100);
- testIndexSummaryProperties(1000, 100);
- testIndexSummaryProperties(10000, 100);
- }
-
- private void testIndexSummaryProperties(int keySize, int numKeys) throws IOException
- {
- final int minIndexInterval = 1;
- final List<DecoratedKey> keys = new ArrayList<>(numKeys);
-
- try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
- {
- for (int i = 0; i < numKeys; i++)
- {
- byte[] randomBytes = new byte[keySize];
- random.nextBytes(randomBytes);
- DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
- keys.add(key);
- builder.maybeAddEntry(key, i);
- }
-
- try(IndexSummary indexSummary = builder.build(partitioner))
- {
- assertEquals(numKeys, keys.size());
- assertEquals(minIndexInterval, indexSummary.getMinIndexInterval());
- assertEquals(numKeys, indexSummary.getMaxNumberOfEntries());
- assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
-
- for (int i = 0; i < numKeys; i++)
- assertEquals(keys.get(i).getKey(), ByteBuffer.wrap(indexSummary.getKey(i)));
- }
- }
- }
-
- /**
- * Test an index summary whose total size is bigger than 2GiB,
- * the index summary builder should log an error but it should still
- * create an index summary, albeit one that does not cover the entire sstable.
- */
- @Test
- public void testLargeIndexSummary() throws IOException
- {
- // On Circle CI we normally don't have enough off-heap memory for this test so ignore it
- Assume.assumeTrue(System.getenv("CIRCLECI") == null);
-
- final int numKeys = 1000000;
- final int keySize = 3000;
- final int minIndexInterval = 1;
-
- try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
- {
- for (int i = 0; i < numKeys; i++)
- {
- byte[] randomBytes = new byte[keySize];
- random.nextBytes(randomBytes);
- DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
- builder.maybeAddEntry(key, i);
- }
-
- try (IndexSummary indexSummary = builder.build(partitioner))
- {
- assertNotNull(indexSummary);
- assertEquals(numKeys, indexSummary.getMaxNumberOfEntries());
- assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
- }
- }
- }
-
- /**
- * Test an index summary whose total size is bigger than 2GiB,
- * having updated IndexSummaryBuilder.defaultExpectedKeySize to match the size,
- * the index summary should be downsampled automatically.
- */
- @Test
- public void testLargeIndexSummaryWithExpectedSizeMatching() throws IOException
- {
- // On Circle CI we normally don't have enough off-heap memory for this test so ignore it
- Assume.assumeTrue(System.getenv("CIRCLECI") == null);
-
- final int numKeys = 1000000;
- final int keySize = 3000;
- final int minIndexInterval = 1;
-
- long oldExpectedKeySize = IndexSummaryBuilder.defaultExpectedKeySize;
- IndexSummaryBuilder.defaultExpectedKeySize = 3000;
-
- try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
- {
- for (int i = 0; i < numKeys; i++)
- {
- byte[] randomBytes = new byte[keySize];
- random.nextBytes(randomBytes);
- DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
- builder.maybeAddEntry(key, i);
- }
-
- try (IndexSummary indexSummary = builder.build(partitioner))
- {
- assertNotNull(indexSummary);
- assertEquals(minIndexInterval * 2, indexSummary.getMinIndexInterval());
- assertEquals(numKeys / 2, indexSummary.getMaxNumberOfEntries());
- assertEquals(numKeys + 2, indexSummary.getEstimatedKeyCount());
- }
- }
- finally
- {
- IndexSummaryBuilder.defaultExpectedKeySize = oldExpectedKeySize;
- }
- }
-
- @Test
- public void testGetKey()
- {
- Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
- for (int i = 0; i < 100; i++)
- assertEquals(random.left.get(i).getKey(), ByteBuffer.wrap(random.right.getKey(i)));
- random.right.close();
- }
-
- @Test
- public void testBinarySearch()
- {
- Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
- for (int i = 0; i < 100; i++)
- assertEquals(i, random.right.binarySearch(random.left.get(i)));
- random.right.close();
- }
-
- @Test
- public void testGetPosition()
- {
- Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 2);
- for (int i = 0; i < 50; i++)
- assertEquals(i*2, random.right.getPosition(i));
- random.right.close();
- }
-
- @Test
- public void testSerialization() throws IOException
- {
- Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
- DataOutputBuffer dos = new DataOutputBuffer();
- IndexSummary.serializer.serialize(random.right, dos);
- // write junk
- dos.writeUTF("JUNK");
- dos.writeUTF("JUNK");
- FileUtils.closeQuietly(dos);
- DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray()));
- IndexSummary is = IndexSummary.serializer.deserialize(dis, partitioner, 1, 1);
- for (int i = 0; i < 100; i++)
- assertEquals(i, is.binarySearch(random.left.get(i)));
- // read the junk
- assertEquals(dis.readUTF(), "JUNK");
- assertEquals(dis.readUTF(), "JUNK");
- is.close();
- FileUtils.closeQuietly(dis);
- random.right.close();
- }
-
- @Test
- public void testAddEmptyKey() throws Exception
- {
- IPartitioner p = new RandomPartitioner();
- try (IndexSummaryBuilder builder = new IndexSummaryBuilder(1, 1, BASE_SAMPLING_LEVEL))
- {
- builder.maybeAddEntry(p.decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER), 0);
- IndexSummary summary = builder.build(p);
- assertEquals(1, summary.size());
- assertEquals(0, summary.getPosition(0));
- assertArrayEquals(new byte[0], summary.getKey(0));
-
- DataOutputBuffer dos = new DataOutputBuffer();
- IndexSummary.serializer.serialize(summary, dos);
- DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray()));
- IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, 1, 1);
-
- assertEquals(1, loaded.size());
- assertEquals(summary.getPosition(0), loaded.getPosition(0));
- assertArrayEquals(summary.getKey(0), summary.getKey(0));
- summary.close();
- loaded.close();
- }
- }
-
- private Pair<List<DecoratedKey>, IndexSummary> generateRandomIndex(int size, int interval)
- {
- List<DecoratedKey> list = Lists.newArrayList();
- try (IndexSummaryBuilder builder = new IndexSummaryBuilder(list.size(), interval, BASE_SAMPLING_LEVEL))
- {
- for (int i = 0; i < size; i++)
- {
- UUID uuid = UUID.randomUUID();
- DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.bytes(uuid));
- list.add(key);
- }
- Collections.sort(list);
- for (int i = 0; i < size; i++)
- builder.maybeAddEntry(list.get(i), i);
- IndexSummary summary = builder.build(partitioner);
- return Pair.create(list, summary);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void testDownsamplePatterns()
- {
- assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(0));
- assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(1));
-
- assertEquals(Arrays.asList(1, 0), Downsampling.getSamplingPattern(2));
- assertEquals(Arrays.asList(3, 1, 2, 0), Downsampling.getSamplingPattern(4));
- assertEquals(Arrays.asList(7, 3, 5, 1, 6, 2, 4, 0), Downsampling.getSamplingPattern(8));
- assertEquals(Arrays.asList(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0), Downsampling.getSamplingPattern(16));
- }
-
- private static boolean shouldSkip(int index, List<Integer> startPoints)
- {
- for (int start : startPoints)
- {
- if ((index - start) % BASE_SAMPLING_LEVEL == 0)
- return true;
- }
- return false;
- }
-
- @Test
- public void testDownsample()
- {
- final int NUM_KEYS = 4096;
- final int INDEX_INTERVAL = 128;
- final int ORIGINAL_NUM_ENTRIES = NUM_KEYS / INDEX_INTERVAL;
-
-
- Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(NUM_KEYS, INDEX_INTERVAL);
- List<DecoratedKey> keys = random.left;
- IndexSummary original = random.right;
-
- // sanity check on the original index summary
- for (int i = 0; i < ORIGINAL_NUM_ENTRIES; i++)
- assertEquals(keys.get(i * INDEX_INTERVAL).getKey(), ByteBuffer.wrap(original.getKey(i)));
-
- List<Integer> samplePattern = Downsampling.getSamplingPattern(BASE_SAMPLING_LEVEL);
-
- // downsample by one level, then two levels, then three levels...
- int downsamplingRound = 1;
- for (int samplingLevel = BASE_SAMPLING_LEVEL - 1; samplingLevel >= 1; samplingLevel--)
- {
- try (IndexSummary downsampled = downsample(original, samplingLevel, 128, partitioner);)
- {
- assertEquals(entriesAtSamplingLevel(samplingLevel, original.getMaxNumberOfEntries()), downsampled.size());
-
- int sampledCount = 0;
- List<Integer> skipStartPoints = samplePattern.subList(0, downsamplingRound);
- for (int i = 0; i < ORIGINAL_NUM_ENTRIES; i++)
- {
- if (!shouldSkip(i, skipStartPoints))
- {
- assertEquals(keys.get(i * INDEX_INTERVAL).getKey(), ByteBuffer.wrap(downsampled.getKey(sampledCount)));
- sampledCount++;
- }
- }
-
- testPosition(original, downsampled, keys);
- downsamplingRound++;
- }
- }
-
- // downsample one level each time
- IndexSummary previous = original;
- downsamplingRound = 1;
- for (int downsampleLevel = BASE_SAMPLING_LEVEL - 1; downsampleLevel >= 1; downsampleLevel--)
- {
- IndexSummary downsampled = downsample(previous, downsampleLevel, 128, partitioner);
- if (previous != original)
- previous.close();
- assertEquals(entriesAtSamplingLevel(downsampleLevel, original.getMaxNumberOfEntries()), downsampled.size());
-
- int sampledCount = 0;
- List<Integer> skipStartPoints = samplePattern.subList(0, downsamplingRound);
- for (int i = 0; i < ORIGINAL_NUM_ENTRIES; i++)
- {
- if (!shouldSkip(i, skipStartPoints))
- {
- assertEquals(keys.get(i * INDEX_INTERVAL).getKey(), ByteBuffer.wrap(downsampled.getKey(sampledCount)));
- sampledCount++;
- }
- }
-
- testPosition(original, downsampled, keys);
- previous = downsampled;
- downsamplingRound++;
- }
- previous.close();
- original.close();
- }
-
- private void testPosition(IndexSummary original, IndexSummary downsampled, List<DecoratedKey> keys)
- {
- for (DecoratedKey key : keys)
- {
- long orig = SSTableReader.getIndexScanPositionFromBinarySearchResult(original.binarySearch(key), original);
- int binarySearch = downsampled.binarySearch(key);
- int index = SSTableReader.getIndexSummaryIndexFromBinarySearchResult(binarySearch);
- int scanFrom = (int) SSTableReader.getIndexScanPositionFromBinarySearchResult(index, downsampled);
- assert scanFrom <= orig;
- int effectiveInterval = downsampled.getEffectiveIndexIntervalAfterIndex(index);
- DecoratedKey k = null;
- for (int i = 0 ; k != key && i < effectiveInterval && scanFrom < keys.size() ; i++, scanFrom ++)
- k = keys.get(scanFrom);
- assert k == key;
- }
- }
-
- @Test
- public void testOriginalIndexLookup()
- {
- for (int i = BASE_SAMPLING_LEVEL; i >= 1; i--)
- assertEquals(i, Downsampling.getOriginalIndexes(i).size());
-
- ArrayList<Integer> full = new ArrayList<>();
- for (int i = 0; i < BASE_SAMPLING_LEVEL; i++)
- full.add(i);
-
- assertEquals(full, Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL));
- // the entry at index 127 is the first to go
- assertEquals(full.subList(0, full.size() - 1), Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL - 1));
-
- // spot check a few values (these depend on BASE_SAMPLING_LEVEL being 128)
- assertEquals(128, BASE_SAMPLING_LEVEL);
- assertEquals(Arrays.asList(0, 32, 64, 96), Downsampling.getOriginalIndexes(4));
- assertEquals(Arrays.asList(0, 64), Downsampling.getOriginalIndexes(2));
- assertEquals(Arrays.asList(0), Downsampling.getOriginalIndexes(1));
- }
-
- @Test
- public void testGetNumberOfSkippedEntriesAfterIndex()
- {
- int indexInterval = 128;
- for (int i = 0; i < BASE_SAMPLING_LEVEL; i++)
- assertEquals(indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(i, BASE_SAMPLING_LEVEL, indexInterval));
-
- // with one round of downsampling, only the last summary entry has been removed, so only the last index will have
- // double the gap until the next sample
- for (int i = 0; i < BASE_SAMPLING_LEVEL - 2; i++)
- assertEquals(indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(i, BASE_SAMPLING_LEVEL - 1, indexInterval));
- assertEquals(indexInterval * 2, Downsampling.getEffectiveIndexIntervalAfterIndex(BASE_SAMPLING_LEVEL - 2, BASE_SAMPLING_LEVEL - 1, indexInterval));
-
- // at samplingLevel=2, the retained summary points are [0, 64] (assumes BASE_SAMPLING_LEVEL is 128)
- assertEquals(128, BASE_SAMPLING_LEVEL);
- assertEquals(64 * indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(0, 2, indexInterval));
- assertEquals(64 * indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(1, 2, indexInterval));
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 063afe5..432e4b0 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -19,24 +19,19 @@
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
-import com.google.common.collect.Lists;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import com.google.common.collect.Lists;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,31 +41,36 @@
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
import org.apache.cassandra.db.compaction.AbstractCompactionTask;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.repair.PendingAntiCompaction;
-import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
-import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
-import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.db.rows.RangeTombstoneMarker;
import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.OutgoingStream;
-import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.TimeUUID;
import static java.util.Collections.singleton;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_LEGACY_SSTABLE_ROOT;
import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -85,8 +85,6 @@
{
private static final Logger logger = LoggerFactory.getLogger(LegacySSTableTest.class);
- public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root";
-
public static File LEGACY_SSTABLE_ROOT;
/**
@@ -94,7 +92,7 @@
* See {@link #testGenerateSstables()} to generate sstables.
* Take care on commit as you need to add the sstable files using {@code git add -f}
*/
- public static final String[] legacyVersions = {"nb", "na", "me", "md", "mc", "mb", "ma"};
+ public static final String[] legacyVersions = {"da", "nc", "nb", "na", "me", "md", "mc", "mb", "ma"};
// 1200 chars
static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
@@ -113,8 +111,8 @@
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
- String scp = System.getProperty(LEGACY_SSTABLE_PROP);
- Assert.assertNotNull("System property " + LEGACY_SSTABLE_PROP + " not set", scp);
+ String scp = TEST_LEGACY_SSTABLE_ROOT.getString();
+ Assert.assertNotNull("System property " + TEST_LEGACY_SSTABLE_ROOT.getKey() + " not set", scp);
LEGACY_SSTABLE_ROOT = new File(scp).toAbsolute();
Assert.assertTrue("System property " + LEGACY_SSTABLE_ROOT + " does not specify a directory", LEGACY_SSTABLE_ROOT.isDirectory());
@@ -144,8 +142,9 @@
*/
protected Descriptor getDescriptor(String legacyVersion, String table) throws IOException
{
- Path file = Files.list(getTableDir(legacyVersion, table).toPath()).findFirst().orElseThrow(() -> new RuntimeException(String.format("No files for version=%s and table=%s", legacyVersion, table)));
- return Descriptor.fromFilename(new File(file));
+ File[] files = getTableDir(legacyVersion, table).list();
+ Preconditions.checkArgument(files.length > 0, "No files for version=%s and table=%s", legacyVersion, table);
+ return Descriptor.fromFileWithComponent(files[0]).left;
}
@Test
@@ -332,7 +331,7 @@
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkVersion(true).build()))
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(true).build()))
{
verifier.verify();
if (!sstable.descriptor.version.isLatestVersion())
@@ -344,7 +343,7 @@
// make sure we don't throw any exception if not checking version:
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkVersion(false).build()))
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(false).build()))
{
verifier.verify();
}
@@ -420,7 +419,7 @@
private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception
{
String table = String.format(tablePattern, legacyVersion);
- SSTableReader sstable = SSTableReader.open(getDescriptor(legacyVersion, table));
+ SSTableReader sstable = SSTableReader.open(null, getDescriptor(legacyVersion, table));
IPartitioner p = sstable.getPartitioner();
List<Range<Token>> ranges = new ArrayList<>();
ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100"))));
@@ -462,6 +461,11 @@
private static void verifyCache(String legacyVersion, long startCount) throws InterruptedException, java.util.concurrent.ExecutionException
{
+ // Only perform test if format uses cache.
+ SSTableReader sstable = Iterables.getFirst(Keyspace.open("legacy_tables").getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)).getLiveSSTables(), null);
+ if (!(sstable instanceof KeyCacheSupport))
+ return;
+
//For https://issues.apache.org/jira/browse/CASSANDRA-10778
//Validate whether the key cache successfully saves in the presence of old keys as
//well as loads the correct number of keys
@@ -594,7 +598,7 @@
/**
* Generates sstables for 8 CQL tables (see {@link #createTables(String)}) in <i>current</i>
* sstable format (version) into {@code test/data/legacy-sstables/VERSION}, where
- * {@code VERSION} matches {@link Version#getVersion() BigFormat.latestVersion.getVersion()}.
+ * {@code VERSION} matches {@link Version#version BigFormat.latestVersion.getVersion()}.
* <p>
* Run this test alone (e.g. from your IDE) when a new version is introduced or format changed
* during development. I.e. remove the {@code @Ignore} annotation temporarily.
@@ -604,6 +608,7 @@
@Test
public void testGenerateSstables() throws Throwable
{
+ SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
Random rand = new Random();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 128; i++)
@@ -616,31 +621,31 @@
{
String valPk = Integer.toString(pk);
QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_simple (pk, val) VALUES ('%s', '%s')",
- BigFormat.latestVersion, valPk, "foo bar baz"));
+ format.getLatestVersion(), valPk, "foo bar baz"));
QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_simple_counter SET val = val + 1 WHERE pk = '%s'",
- BigFormat.latestVersion, valPk));
+ format.getLatestVersion(), valPk));
for (int ck = 0; ck < 50; ck++)
{
String valCk = Integer.toString(ck);
QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_clust (pk, ck, val) VALUES ('%s', '%s', '%s')",
- BigFormat.latestVersion, valPk, valCk + longString, randomString));
+ format.getLatestVersion(), valPk, valCk + longString, randomString));
QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_clust_counter SET val = val + 1 WHERE pk = '%s' AND ck='%s'",
- BigFormat.latestVersion, valPk, valCk + longString));
+ format.getLatestVersion(), valPk, valCk + longString));
}
}
StorageService.instance.forceKeyspaceFlush("legacy_tables", ColumnFamilyStore.FlushReason.UNIT_TESTS);
- File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", BigFormat.latestVersion));
+ File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", format.getLatestVersion()));
ksDir.tryCreateDirectories();
- copySstablesFromTestData(String.format("legacy_%s_simple", BigFormat.latestVersion), ksDir);
- copySstablesFromTestData(String.format("legacy_%s_simple_counter", BigFormat.latestVersion), ksDir);
- copySstablesFromTestData(String.format("legacy_%s_clust", BigFormat.latestVersion), ksDir);
- copySstablesFromTestData(String.format("legacy_%s_clust_counter", BigFormat.latestVersion), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_simple", format.getLatestVersion()), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_simple_counter", format.getLatestVersion()), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_clust", format.getLatestVersion()), ksDir);
+ copySstablesFromTestData(String.format("legacy_%s_clust_counter", format.getLatestVersion()), ksDir);
}
public static void copySstablesFromTestData(String table, File ksDir) throws IOException
diff --git a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java
new file mode 100644
index 0000000..2b258ba
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.StorageService;
+
+import static org.junit.Assert.assertEquals;
+
+public class RangeAwareSSTableWriterTest
+{
+ public static final String KEYSPACE1 = "Keyspace1";
+ public static final String CF_STANDARD = "Standard1";
+
+ public static ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void defineSchema() throws Exception
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+ SchemaLoader.cleanupAndLeaveDirs();
+ Keyspace.setInitialized();
+ StorageService.instance.initServer();
+
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
+ .partitioner(Murmur3Partitioner.instance));
+
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
+ cfs.clearUnsafe();
+ cfs.disableAutoCompaction();
+ }
+
+ @Test
+ public void testAccessWriterBeforeAppend() throws IOException
+ {
+
+ SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, 0, 1);
+ Util.flush(cfs);
+
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+
+ RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs,
+ 0,
+ 0,
+ null,
+ false,
+ DatabaseDescriptor.getSelectedSSTableFormat(),
+ 0,
+ 0,
+ txn,
+ SerializationHeader.make(cfs.metadata(),
+ cfs.getLiveSSTables()));
+ assertEquals(cfs.metadata.id, writer.getTableId());
+ assertEquals(0L, writer.getFilePointer());
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
index ec01865..21ac51e 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java
@@ -21,30 +21,40 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-import java.util.*;
-import java.util.function.*;
+import java.util.Random;
+import java.util.function.Consumer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.*;
-import org.apache.cassandra.cache.*;
-import org.apache.cassandra.config.*;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.junit.Assert.assertEquals;
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
new file mode 100644
index 0000000..58ca05b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
+import org.apache.commons.lang3.tuple.Triple;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+
+public class SSTableFlushObserverTest
+{
+ @BeforeClass
+ public static void initDD()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ CommitLog.instance.start();
+ }
+
+ private static final String KS_NAME = "test";
+ private static final String CF_NAME = "flush_observer";
+
+ @Test
+ public void testFlushObserver() throws Exception
+ {
+ TableMetadata cfm =
+ TableMetadata.builder(KS_NAME, CF_NAME)
+ .addPartitionKeyColumn("id", UTF8Type.instance)
+ .addRegularColumn("first_name", UTF8Type.instance)
+ .addRegularColumn("age", Int32Type.instance)
+ .addRegularColumn("height", LongType.instance)
+ .build();
+
+ LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION);
+ FlushObserver observer = new FlushObserver();
+
+ String sstableDirectory = DatabaseDescriptor.getAllDataFileLocations()[0];
+ File directory = new File(sstableDirectory + File.pathSeparator() + KS_NAME + File.pathSeparator() + CF_NAME);
+ directory.deleteOnExit();
+
+ if (!directory.exists() && !directory.tryCreateDirectories())
+ throw new FSWriteError(new IOException("failed to create tmp directory"), directory.absolutePath());
+
+ SSTableFormat<?, ?> sstableFormat = DatabaseDescriptor.getSelectedSSTableFormat();
+ Descriptor descriptor = new Descriptor(sstableFormat.getLatestVersion(),
+ directory,
+ cfm.keyspace,
+ cfm.name,
+ new SequenceBasedSSTableId(0));
+
+ SSTableWriter writer = descriptor.getFormat().getWriterFactory().builder(descriptor)
+ .setKeyCount(10)
+ .setTableMetadataRef(TableMetadataRef.forOfflineTools(cfm))
+ .setMetadataCollector(new MetadataCollector(cfm.comparator).sstableLevel(0))
+ .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS))
+ .setFlushObservers(Collections.singletonList(observer))
+ .addDefaultComponents()
+ .build(transaction, null);
+
+ SSTableReader reader = null;
+ Multimap<ByteBuffer, Cell<?>> expected = ArrayListMultimap.create();
+
+ try
+ {
+ final long now = System.currentTimeMillis();
+
+ ByteBuffer key = UTF8Type.instance.fromString("key1");
+ expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(27)),
+ BufferCell.live(getColumn(cfm, "first_name"), now, UTF8Type.instance.fromString("jack")),
+ BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(183L))));
+
+ writer.append(new RowIterator(cfm, key.duplicate(), Collections.singletonList(buildRow(expected.get(key)))));
+
+ key = UTF8Type.instance.fromString("key2");
+ expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(30)),
+ BufferCell.live(getColumn(cfm, "first_name"), now, UTF8Type.instance.fromString("jim")),
+ BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(180L))));
+
+ writer.append(new RowIterator(cfm, key, Collections.singletonList(buildRow(expected.get(key)))));
+
+ key = UTF8Type.instance.fromString("key3");
+ expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(30)),
+ BufferCell.live(getColumn(cfm, "first_name"), now, UTF8Type.instance.fromString("ken")),
+ BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(178L))));
+
+ writer.append(new RowIterator(cfm, key, Collections.singletonList(buildRow(expected.get(key)))));
+
+ reader = writer.finish(true);
+ }
+ finally
+ {
+ FileUtils.closeQuietly(writer);
+ }
+
+ Assert.assertTrue(observer.isComplete);
+ Assert.assertEquals(expected.size(), observer.rows.size());
+
+ for (Triple<ByteBuffer, Long, Long> e : observer.rows.keySet())
+ {
+ ByteBuffer key = e.getLeft();
+ long dataPosition = e.getMiddle();
+ long indexPosition = e.getRight();
+
+ DecoratedKey indexKey = reader.keyAtPositionFromSecondaryIndex(indexPosition);
+ Assert.assertEquals(0, UTF8Type.instance.compare(key, indexKey.getKey()));
+ Assert.assertEquals(expected.get(key), observer.rows.get(e));
+ }
+ }
+
+ private static class RowIterator extends AbstractUnfilteredRowIterator
+ {
+ private final Iterator<Unfiltered> rows;
+
+ public RowIterator(TableMetadata cfm, ByteBuffer key, Collection<Unfiltered> content)
+ {
+ super(cfm,
+ DatabaseDescriptor.getPartitioner().decorateKey(key),
+ DeletionTime.LIVE,
+ cfm.regularAndStaticColumns(),
+ BTreeRow.emptyRow(Clustering.STATIC_CLUSTERING),
+ false,
+ EncodingStats.NO_STATS);
+
+ rows = content.iterator();
+ }
+
+ @Override
+ protected Unfiltered computeNext()
+ {
+ return rows.hasNext() ? rows.next() : endOfData();
+ }
+ }
+
+ private static class FlushObserver implements SSTableFlushObserver
+ {
+ private final Multimap<Triple<ByteBuffer, Long, Long>, Cell<?>> rows = ArrayListMultimap.create();
+ private final Multimap<Triple<ByteBuffer, Long, Long>, Cell<?>> staticRows = ArrayListMultimap.create();
+
+ private Triple<ByteBuffer, Long, Long> currentKey;
+ private boolean isComplete;
+
+ @Override
+ public void begin()
+ {
+ }
+
+ @Override
+ public void startPartition(DecoratedKey key, long dataPosition, long indexPosition)
+ {
+ currentKey = ImmutableTriple.of(key.getKey(), dataPosition, indexPosition);
+ }
+
+ @Override
+ public void nextUnfilteredCluster(Unfiltered row)
+ {
+ if (row.isRow())
+ ((Row) row).forEach((c) -> rows.put(currentKey, (Cell<?>) c));
+ }
+
+ @Override
+ public void complete()
+ {
+ isComplete = true;
+ }
+
+ @Override
+ public void staticRow(Row staticRow)
+ {
+ staticRow.forEach((c) -> staticRows.put(currentKey, (Cell<?>) c));
+ }
+ }
+
+ private static Row buildRow(Collection<Cell<?>> cells)
+ {
+ Row.Builder rowBuilder = BTreeRow.sortedBuilder();
+ rowBuilder.newRow(Clustering.EMPTY);
+ cells.forEach(rowBuilder::addCell);
+ return rowBuilder.build();
+ }
+
+ private static ColumnMetadata getColumn(TableMetadata cfm, String name)
+ {
+ return cfm.getColumn(UTF8Type.instance.fromString(name));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java
new file mode 100644
index 0000000..30a68dd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.BiFunction;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.Config.SSTableConfig;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.YamlConfigurationLoader;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.mockito.Mockito;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.mockito.Mockito.when;
+
+public class SSTableFormatTest
+{
+ public static abstract class AbstractFormat implements SSTableFormat<SSTableReader, SSTableWriter>
+ {
+ public Map<String, String> options;
+ public String name;
+ public String latestVersion;
+
+ public AbstractFormat(String latestVersion)
+ {
+ this.latestVersion = latestVersion;
+ }
+
+ @Override
+ public Version getVersion(String version)
+ {
+ Version v = Mockito.mock(Version.class);
+ when(v.toString()).thenReturn(version);
+ when(v.isCompatible()).thenReturn(version.charAt(0) == latestVersion.charAt(0));
+ return v;
+ }
+
+ @Override
+ public Version getLatestVersion()
+ {
+ return getVersion(latestVersion);
+ }
+
+ @Override
+ public String name()
+ {
+ return name;
+ }
+
+ static class Factory implements SSTableFormat.Factory
+ {
+ public String name;
+ public BiFunction<Map<String, String>, String, SSTableFormat<?, ?>> provider;
+
+ public Factory(String name, BiFunction<Map<String, String>, String, SSTableFormat<?, ?>> provider)
+ {
+ this.name = name;
+ this.provider = provider;
+ }
+
+ @Override
+ public String name()
+ {
+ return name;
+ }
+
+ @Override
+ public SSTableFormat<?, ?> getInstance(Map<String, String> options)
+ {
+ return provider.apply(options, name);
+ }
+ }
+ }
+
+ public static AbstractFormat.Factory factory(String name, Class<? extends AbstractFormat> clazz)
+ {
+ return new AbstractFormat.Factory(name, (options, version) -> {
+ AbstractFormat format = Mockito.spy(clazz);
+ format.name = name;
+ format.options = options;
+ return format;
+ });
+ }
+
+ public static abstract class Format1 extends AbstractFormat
+ {
+ public Format1()
+ {
+ super("xx");
+ }
+ }
+
+ public static abstract class Format2 extends AbstractFormat
+ {
+ public Format2()
+ {
+ super("yy");
+ }
+ }
+
+ public static abstract class Format3 extends AbstractFormat
+ {
+ public Format3()
+ {
+ super("zz");
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ DatabaseDescriptor.clientInitialization();
+ }
+
+ private static final String yamlContent0 = "";
+ private static final SSTableConfig expected0 = new Config.SSTableConfig();
+
+ private static final String yamlContent1 = "sstable:\n" +
+ " selected_format: aaa\n";
+ private static final SSTableConfig expected1 = new Config.SSTableConfig()
+ {
+ {
+ selected_format = "aaa";
+ }
+ };
+
+ private static final String yamlContent2 = "sstable:\n" +
+ " selected_format: aaa\n" +
+ " format:\n" +
+ " aaa:\n" +
+ " param1: value1\n" +
+ " param2: value2\n" +
+ " bbb:\n" +
+ " param3: value3\n" +
+ " param4: value4\n";
+
+ private static final Config.SSTableConfig expected2 = new SSTableConfig()
+ {
+ {
+ selected_format = "aaa";
+ format = ImmutableMap.of("aaa", ImmutableMap.of("param1", "value1", "param2", "value2"),
+ "bbb", ImmutableMap.of("param3", "value3", "param4", "value4"));
+ }
+ };
+
+ private static final SSTableConfig unexpected = new Config.SSTableConfig()
+ {
+ {
+ selected_format = "aaa";
+ }
+ };
+
+
+ @Test
+ public void testParsingYamlConfig() throws IOException
+ {
+ YamlConfigurationLoader loader = new YamlConfigurationLoader();
+ File f = FileUtils.createTempFile("sstable_format_test_config", ".yaml");
+ URL url = f.toPath().toUri().toURL();
+
+ ImmutableMap.of(yamlContent0, expected0, yamlContent1, expected1, yamlContent2, expected2).forEach((yamlContent, expected) -> {
+ try (FileOutputStreamPlus out = f.newOutputStream(File.WriteMode.OVERWRITE))
+ {
+ out.write(yamlContent.getBytes());
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ Config config = loader.loadConfig(url);
+ assertThat(config.sstable).describedAs("Yaml: \n%s\n", yamlContent).isEqualToComparingFieldByField(expected);
+ });
+ }
+
+ @Before
+ public void before()
+ {
+ }
+
+ public static void configure(SSTableConfig config, SSTableFormat.Factory... factories)
+ {
+ DatabaseDescriptor.resetSSTableFormats(Arrays.asList(factories), config);
+ }
+
+ private void verifyFormat(String name, Map<String, String> options)
+ {
+ AbstractFormat format = (AbstractFormat) DatabaseDescriptor.getSSTableFormats().get(name);
+ assertThat(format.name).isEqualTo(name);
+ assertThat(format.options).isEqualTo(options);
+ }
+
+ private void verifySelectedFormat(String name)
+ {
+ assertThat(DatabaseDescriptor.getSelectedSSTableFormat().name()).isEqualTo(name);
+ }
+
+ @Test
+ public void testValidConfig()
+ {
+ configure(expected1, factory("aaa", Format1.class));
+ assertThat(DatabaseDescriptor.getSSTableFormats()).hasSize(1);
+ verifyFormat("aaa", ImmutableMap.of());
+ verifySelectedFormat("aaa");
+
+ configure(expected2, factory("aaa", Format1.class), factory("bbb", Format2.class), factory("ccc", Format3.class));
+ assertThat(DatabaseDescriptor.getSSTableFormats()).hasSize(3);
+ verifyFormat("aaa", ImmutableMap.of("param1", "value1", "param2", "value2"));
+ verifyFormat("bbb", ImmutableMap.of("param3", "value3", "param4", "value4"));
+ verifyFormat("ccc", ImmutableMap.of());
+ verifySelectedFormat("aaa");
+ }
+
+ @Test
+ public void testConfigValidation()
+ {
+ // invalid name
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("Aa", Format1.class)))
+ .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string");
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("a-a", Format1.class)))
+ .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string");
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("a1", Format1.class)))
+ .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string");
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("", Format1.class)))
+ .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string");
+
+ // duplicate name
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("aaa", Format1.class), factory("aaa", Format2.class)))
+ .withMessageContainingAll("Multiple sstable format implementations with the same name", "aaa");
+
+ // missing name
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory(null, Format1.class)))
+ .withMessageContainingAll("SSTable format name", "cannot be null");
+
+ // Configuration contains options of unknown sstable formats
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected2, factory("aaa", Format1.class)))
+ .withMessageContainingAll("Configuration contains options of unknown sstable formats", "bbb");
+
+ // Selected sstable format '%s' is not available
+ assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("bbb", Format1.class)))
+ .withMessageContainingAll("Selected sstable format", "aaa", "is not available");
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
index 4572e5c..5273daf 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
@@ -33,10 +33,11 @@
import java.util.stream.IntStream;
import com.google.common.collect.Sets;
-import org.apache.cassandra.io.util.File;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -60,16 +61,17 @@
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.SequentialWriter;
import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -101,6 +103,12 @@
return MockSchema.sstableIdGenerators();
}
+ @BeforeClass
+ public static void beforeClass()
+ {
+ Assume.assumeTrue(BigFormat.isSelected());
+ }
+
@Before
public void setup()
{
@@ -178,7 +186,7 @@
return new TupleType(Collections.singletonList(UTF8Type.instance));
}
- private static final Version version = BigFormat.instance.getVersion("mc");
+ private static final Version version = BigFormat.getInstance().getVersion("mc");
private TableMetadata tableMetadata;
private final Set<String> updatedColumns = new HashSet<>();
@@ -386,7 +394,7 @@
ColumnMetadata cd = getColDef(col);
AbstractType<?> dropType = cd.type.expandUserTypes();
cols.removeRegularOrStaticColumn(ci)
- .recordColumnDrop(new ColumnMetadata(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind), FBUtilities.timestampMicros());
+ .recordColumnDrop(new ColumnMetadata(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind, cd.getMask()), FBUtilities.timestampMicros());
}
tableMetadata = cols.build();
@@ -822,11 +830,11 @@
try
{
- Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation), SSTableFormat.Type.BIG);
+ Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation));
// Just create the component files - we don't really need those.
for (Component component : requiredComponents)
- assertTrue(new File(desc.filenameFor(component)).createFileIfNotExists());
+ assertTrue(desc.fileFor(component).createFileIfNotExists());
AbstractType<?> partitionKey = headerMetadata.partitionKeyType;
List<AbstractType<?>> clusteringKey = headerMetadata.clusteringColumns()
@@ -842,7 +850,7 @@
.filter(cd -> cd.kind == ColumnMetadata.Kind.REGULAR)
.collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a));
- File statsFile = new File(desc.filenameFor(Component.STATS));
+ File statsFile = desc.fileFor(Components.STATS);
SerializationHeader.Component header = SerializationHeader.Component.buildComponentForTools(partitionKey,
clusteringKey,
staticColumns,
@@ -855,7 +863,7 @@
out.finish();
}
- return new File(desc.filenameFor(Component.DATA));
+ return desc.fileFor(Components.DATA);
}
catch (Exception e)
{
@@ -988,9 +996,9 @@
private SerializationHeader.Component readHeader(File sstable) throws Exception
{
- Descriptor desc = Descriptor.fromFilename(sstable);
+ Descriptor desc = Descriptor.fromFileWithComponent(sstable, false).left;
return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER);
}
- private static final Component[] requiredComponents = new Component[]{ Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.TOC };
+ private static final Component[] requiredComponents = new Component[]{ Components.DATA, Components.FILTER, Components.PRIMARY_INDEX, Components.TOC };
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java
index c2403dd..d210888 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java
@@ -24,6 +24,7 @@
import com.google.common.io.Files;
+import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.io.util.File;
import org.junit.After;
import org.junit.Before;
@@ -33,6 +34,7 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.schema.Schema;
@@ -40,7 +42,6 @@
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamEvent;
@@ -95,16 +96,33 @@
@After
public void cleanup()
{
- try {
- FileUtils.deleteRecursive(tmpdir);
- } catch (FSWriteError e) {
+ try
+ {
+ tmpdir.deleteRecursive();
+ }
+ catch (FSWriteError e)
+ {
/*
We force a GC here to force buffer deallocation, and then try deleting the directory again.
For more information, see: http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4715154
If this is not the problem, the exception will be rethrown anyway.
*/
System.gc();
- FileUtils.deleteRecursive(tmpdir);
+ tmpdir.deleteRecursive();
+ }
+
+ try
+ {
+ for (String[] keyspaceTable : new String[][] { {KEYSPACE1, CF_STANDARD1},
+ {KEYSPACE1, CF_STANDARD2},
+ {KEYSPACE1, CF_BACKUPS},
+ {KEYSPACE2, CF_STANDARD1},
+ {KEYSPACE2, CF_STANDARD2}})
+ StorageService.instance.truncate(keyspaceTable[0], keyspaceTable[1]);
+ }
+ catch (Exception ex)
+ {
+ throw new RuntimeException("Unable to truncate table!", ex);
}
}
@@ -152,9 +170,11 @@
assertEquals(1, partitions.size());
assertEquals("key1", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey()));
assert metadata != null;
- assertEquals(ByteBufferUtil.bytes("100"), partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")))
- .getCell(metadata.getColumn(ByteBufferUtil.bytes("val")))
- .buffer());
+
+ Row row = partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")));
+ assert row != null;
+
+ assertEquals(ByteBufferUtil.bytes("100"), row.getCell(metadata.getColumn(ByteBufferUtil.bytes("val"))).buffer());
// The stream future is signalled when the work is complete but before releasing references. Wait for release
// before cleanup (CASSANDRA-10118).
@@ -170,7 +190,7 @@
.inDirectory(dataDir)
.forTable(String.format(schema, KEYSPACE1, CF_STANDARD2))
.using(String.format(query, KEYSPACE1, CF_STANDARD2))
- .withBufferSizeInMB(1)
+ .withBufferSizeInMiB(1)
.build();
int NB_PARTITIONS = 5000; // Enough to write >1MiB and get at least one completed sstable before we've closed the writer
@@ -211,10 +231,9 @@
}
@Test
- public void testLoadingSSTableToDifferentKeyspace() throws Exception
+ public void testLoadingSSTableToDifferentKeyspaceAndTable() throws Exception
{
- File dataDir = new File(tmpdir.absolutePath() + File.pathSeparator() + KEYSPACE1 + File.pathSeparator() + CF_STANDARD1);
- assert dataDir.tryCreateDirectories();
+ File dataDir = dataDir(CF_STANDARD1);
TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE1, CF_STANDARD1);
String schema = "CREATE TABLE %s.%s (key ascii, name ascii, val ascii, val1 ascii, PRIMARY KEY (key, name))";
@@ -232,42 +251,60 @@
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
Util.flush(cfs); // wait for sstables to be on disk else we won't be able to stream them
- final CountDownLatch latch = new CountDownLatch(1);
- SSTableLoader loader = new SSTableLoader(dataDir, new TestClient(), new OutputHandler.SystemOutput(false, false), 1, KEYSPACE2);
- loader.stream(Collections.emptySet(), completionStreamListener(latch)).get();
+ for (String table : new String[] { CF_STANDARD2, null })
+ {
+ final CountDownLatch latch = new CountDownLatch(1);
+ SSTableLoader loader = new SSTableLoader(dataDir, new TestClient(), new OutputHandler.SystemOutput(false, false), 1, KEYSPACE2, table);
+ loader.stream(Collections.emptySet(), completionStreamListener(latch)).get();
- cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1);
- Util.flush(cfs);
+ String targetTable = table == null ? CF_STANDARD1 : table;
+ cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(targetTable);
+ Util.flush(cfs);
- List<FilteredPartition> partitions = Util.getAll(Util.cmd(cfs).build());
+ List<FilteredPartition> partitions = Util.getAll(Util.cmd(cfs).build());
- assertEquals(1, partitions.size());
- assertEquals("key1", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey()));
- assert metadata != null;
- assertEquals(ByteBufferUtil.bytes("100"), partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")))
- .getCell(metadata.getColumn(ByteBufferUtil.bytes("val")))
- .buffer());
+ assertEquals(1, partitions.size());
+ assertEquals("key1", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey()));
+ assert metadata != null;
- // The stream future is signalled when the work is complete but before releasing references. Wait for release
- // before cleanup (CASSANDRA-10118).
- latch.await();
+ Row row = partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")));
+ assert row != null;
+
+ assertEquals(ByteBufferUtil.bytes("100"), row.getCell(metadata.getColumn(ByteBufferUtil.bytes("val"))).buffer());
+
+ // The stream future is signalled when the work is complete but before releasing references. Wait for release
+ // before cleanup (CASSANDRA-10118).
+ latch.await();
+ }
}
@Test
public void testLoadingBackupsTable() throws Exception
{
- testLoadingTable(CF_BACKUPS);
+ testLoadingTable(CF_BACKUPS, false);
}
@Test
public void testLoadingSnapshotsTable() throws Exception
{
- testLoadingTable(CF_SNAPSHOTS);
+ testLoadingTable(CF_SNAPSHOTS, false);
}
- private void testLoadingTable(String tableName) throws Exception
+ @Test
+ public void testLoadingLegacyBackupsTable() throws Exception
{
- File dataDir = dataDir(tableName);
+ testLoadingTable(CF_BACKUPS, true);
+ }
+
+ @Test
+ public void testLoadingLegacySnapshotsTable() throws Exception
+ {
+ testLoadingTable(CF_SNAPSHOTS, true);
+ }
+
+ private void testLoadingTable(String tableName, boolean isLegacyTable) throws Exception
+ {
+ File dataDir = dataDir(tableName, isLegacyTable);
TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE1, tableName);
try (CQLSSTableWriter writer = CQLSSTableWriter.builder()
@@ -291,9 +328,11 @@
assertEquals(1, partitions.size());
assertEquals("key", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey()));
assert metadata != null;
- assertEquals(ByteBufferUtil.bytes("100"), partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")))
- .getCell(metadata.getColumn(ByteBufferUtil.bytes("val")))
- .buffer());
+
+ Row row = partitions.get(0).getRow(Clustering.make(ByteBufferUtil.bytes("col1")));
+ assert row != null;
+
+ assertEquals(ByteBufferUtil.bytes("100"), row.getCell(metadata.getColumn(ByteBufferUtil.bytes("val"))).buffer());
// The stream future is signalled when the work is complete but before releasing references. Wait for release
// before cleanup (CASSANDRA-10118).
@@ -302,7 +341,14 @@
private File dataDir(String cf)
{
- File dataDir = new File(tmpdir.absolutePath() + File.pathSeparator() + SSTableLoaderTest.KEYSPACE1 + File.pathSeparator() + cf);
+ return dataDir(cf, false);
+ }
+
+ private File dataDir(String cf, boolean isLegacyTable)
+ {
+ // Add -{tableUuid} suffix to table dir if not a legacy table
+ File dataDir = new File(tmpdir.absolutePath() + File.pathSeparator() + SSTableLoaderTest.KEYSPACE1 + File.pathSeparator() + cf
+ + (isLegacyTable ? "" : String.format("-%s",TableId.generate().toHexString())));
assert dataDir.tryCreateDirectories();
//make sure we have no tables...
assertEquals(Objects.requireNonNull(dataDir.tryList()).length, 0);
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
index 2e5a17a..215f02e 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
@@ -17,8 +17,10 @@
*/
package org.apache.cassandra.io.sstable;
+import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.junit.BeforeClass;
@@ -26,7 +28,6 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
@@ -34,9 +35,11 @@
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class SSTableMetadataTest
@@ -216,11 +219,11 @@
assertEquals(1, store.getLiveSSTables().size());
for (SSTableReader sstable : store.getLiveSSTables())
{
- assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100");
- assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "7col149");
+ assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0col100");
+ assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "7col149");
// make sure the clustering values are minimised
- assertTrue(sstable.getSSTableMetadata().minClusteringValues.get(0).capacity() < 50);
- assertTrue(sstable.getSSTableMetadata().maxClusteringValues.get(0).capacity() < 50);
+ assertTrue(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0).capacity() < 50);
+ assertTrue(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0).capacity() < 50);
}
String key = "row2";
@@ -238,62 +241,42 @@
assertEquals(1, store.getLiveSSTables().size());
for (SSTableReader sstable : store.getLiveSSTables())
{
- assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100");
- assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "9col298");
- // and make sure the clustering values are still minimised after compaction
- assertTrue(sstable.getSSTableMetadata().minClusteringValues.get(0).capacity() < 50);
- assertTrue(sstable.getSSTableMetadata().maxClusteringValues.get(0).capacity() < 50);
+ assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0col100");
+ assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "9col298");
+ // make sure stats don't reference native or off-heap data
+ assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.start().getBufferArray()));
+ assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.end().getBufferArray()));
+ }
+
+ key = "row3";
+ new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), key)
+ .addRangeTombstone("0", "7")
+ .build()
+ .apply();
+
+ store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+ store.forceMajorCompaction();
+ assertEquals(1, store.getLiveSSTables().size());
+ for (SSTableReader sstable : store.getLiveSSTables())
+ {
+ assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0");
+ assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "9col298");
+ // make sure stats don't reference native or off-heap data
+ assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.start().getBufferArray()));
+ assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.end().getBufferArray()));
}
}
- /*@Test
- public void testLegacyCounterShardTracking()
+ public static void assertBuffersAreRetainable(List<ByteBuffer> buffers)
{
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Counter1");
+ for (ByteBuffer b : buffers)
+ {
+ assertFalse(b.isDirect());
+ assertTrue(b.hasArray());
+ assertEquals(b.capacity(), b.remaining());
+ assertEquals(0, b.arrayOffset());
+ assertEquals(b.capacity(), b.array().length);
+ }
+ }
- // A cell with all shards
- CounterContext.ContextState state = CounterContext.ContextState.allocate(1, 1, 1);
- state.writeGlobal(CounterId.fromInt(1), 1L, 1L);
- state.writeLocal(CounterId.fromInt(2), 1L, 1L);
- state.writeRemote(CounterId.fromInt(3), 1L, 1L);
-
- ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata);
- cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE));
- new Mutation(Util.dk("k").getKey(), cells).applyUnsafe();
- Util.flush(cfs);
- assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards);
- cfs.truncateBlocking();
-
- // A cell with global and remote shards
- state = CounterContext.ContextState.allocate(0, 1, 1);
- state.writeLocal(CounterId.fromInt(2), 1L, 1L);
- state.writeRemote(CounterId.fromInt(3), 1L, 1L);
- cells = ArrayBackedSortedColumns.factory.create(cfs.metadata);
- cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE));
- new Mutation(Util.dk("k").getKey(), cells).applyUnsafe();
- Util.flush(cfs);
- assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards);
- cfs.truncateBlocking();
-
- // A cell with global and local shards
- state = CounterContext.ContextState.allocate(1, 1, 0);
- state.writeGlobal(CounterId.fromInt(1), 1L, 1L);
- state.writeLocal(CounterId.fromInt(2), 1L, 1L);
- cells = ArrayBackedSortedColumns.factory.create(cfs.metadata);
- cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE));
- new Mutation(Util.dk("k").getKey(), cells).applyUnsafe();
- Util.flush(cfs);
- assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards);
- cfs.truncateBlocking();
-
- // A cell with global only
- state = CounterContext.ContextState.allocate(1, 0, 0);
- state.writeGlobal(CounterId.fromInt(1), 1L, 1L);
- cells = ArrayBackedSortedColumns.factory.create(cfs.metadata);
- cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE));
- new Mutation(Util.dk("k").getKey(), cells).applyUnsafe();
- Util.flush(cfs);
- assertFalse(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards);
- cfs.truncateBlocking();
- } */
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index f064f19..5a82bc3 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -20,22 +20,38 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
-import org.apache.cassandra.io.util.File;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@ -49,21 +65,35 @@
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.format.big.IndexSummaryComponent;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport;
+import org.apache.cassandra.io.sstable.keycache.KeyCache;
+import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.MmappedRegions;
+import org.apache.cassandra.io.util.PageAware;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FilterFactory;
+import org.mockito.Mockito;
+import static java.lang.String.format;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
public class SSTableReaderTest
{
@@ -81,7 +111,7 @@
Token t(int i)
{
- return partitioner.getToken(ByteBufferUtil.bytes(String.valueOf(i)));
+ return partitioner.getToken(keyFor(i));
}
@BeforeClass
@@ -152,7 +182,7 @@
public void testSpannedIndexPositions() throws IOException
{
int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
- MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments
+ MmappedRegions.MAX_SEGMENT_SIZE = PageAware.PAGE_SIZE;
try
{
@@ -160,7 +190,7 @@
partitioner = store.getPartitioner();
// insert a bunch of data and compact to a single sstable
- for (int j = 0; j < 100; j += 2)
+ for (int j = 0; j < 10000; j += 2)
{
new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
.clustering("0")
@@ -173,19 +203,19 @@
// check that all our keys are found correctly
SSTableReader sstable = store.getLiveSSTables().iterator().next();
- for (int j = 0; j < 100; j += 2)
+ for (int j = 0; j < 10000; j += 2)
{
DecoratedKey dk = Util.dk(String.valueOf(j));
- FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ).position);
+ FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ));
DecoratedKey keyInDisk = sstable.decorateKey(ByteBufferUtil.readWithShortLength(file));
- assert keyInDisk.equals(dk) : String.format("%s != %s in %s", keyInDisk, dk, file.getPath());
+ assert keyInDisk.equals(dk) : format("%s != %s in %s", keyInDisk, dk, file.getPath());
}
// check no false positives
- for (int j = 1; j < 110; j += 2)
+ for (int j = 1; j < 11000; j += 2)
{
DecoratedKey dk = Util.dk(String.valueOf(j));
- assert sstable.getPosition(dk, SSTableReader.Operator.EQ) == null;
+ assert sstable.getPosition(dk, SSTableReader.Operator.EQ) < 0;
}
}
finally
@@ -238,15 +268,41 @@
Util.flush(store);
- SSTableReader sstable = store.getLiveSSTables().iterator().next();
- assertEquals(0, sstable.getReadMeter().count());
+ boolean startState = DatabaseDescriptor.getSStableReadRatePersistenceEnabled();
+ try
+ {
+ DatabaseDescriptor.setSStableReadRatePersistenceEnabled(true);
- DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("4"));
- Util.getAll(Util.cmd(store, key).build());
- assertEquals(1, sstable.getReadMeter().count());
+ SSTableReader sstable = store.getLiveSSTables().iterator().next();
+ assertEquals(0, sstable.getReadMeter().count());
- Util.getAll(Util.cmd(store, key).includeRow("0").build());
- assertEquals(2, sstable.getReadMeter().count());
+ DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("4"));
+ Util.getAll(Util.cmd(store, key).build());
+ assertEquals(1, sstable.getReadMeter().count());
+
+ Util.getAll(Util.cmd(store, key).includeRow("0").build());
+ assertEquals(2, sstable.getReadMeter().count());
+
+ // With persistence enabled, we should be able to retrieve the state of the meter.
+ sstable.maybePersistSSTableReadMeter();
+
+ UntypedResultSet meter = SystemKeyspace.readSSTableActivity(store.keyspace.getName(), store.name, sstable.descriptor.id);
+ assertFalse(meter.isEmpty());
+
+ Util.getAll(Util.cmd(store, key).includeRow("0").build());
+ assertEquals(3, sstable.getReadMeter().count());
+
+ // After cleaning existing state and disabling persistence, there should be no meter state to read.
+ SystemKeyspace.clearSSTableReadMeter(store.keyspace.getName(), store.name, sstable.descriptor.id);
+ DatabaseDescriptor.setSStableReadRatePersistenceEnabled(false);
+ sstable.maybePersistSSTableReadMeter();
+ meter = SystemKeyspace.readSSTableActivity(store.keyspace.getName(), store.name, sstable.descriptor.id);
+ assertTrue(meter.isEmpty());
+ }
+ finally
+ {
+ DatabaseDescriptor.setSStableReadRatePersistenceEnabled(startState);
+ }
}
@Test
@@ -271,10 +327,10 @@
CompactionManager.instance.performMaximal(store, false);
SSTableReader sstable = store.getLiveSSTables().iterator().next();
- long p2 = sstable.getPosition(k(2), SSTableReader.Operator.EQ).position;
- long p3 = sstable.getPosition(k(3), SSTableReader.Operator.EQ).position;
- long p6 = sstable.getPosition(k(6), SSTableReader.Operator.EQ).position;
- long p7 = sstable.getPosition(k(7), SSTableReader.Operator.EQ).position;
+ long p2 = sstable.getPosition(dk(2), SSTableReader.Operator.EQ);
+ long p3 = sstable.getPosition(dk(3), SSTableReader.Operator.EQ);
+ long p6 = sstable.getPosition(dk(6), SSTableReader.Operator.EQ);
+ long p7 = sstable.getPosition(dk(7), SSTableReader.Operator.EQ);
SSTableReader.PartitionPositionBounds p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0);
@@ -307,6 +363,7 @@
@Test
public void testGetPositionsKeyCacheStats()
{
+ Assume.assumeTrue(KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD2);
partitioner = store.getPartitioner();
CacheService.instance.keyCache.setCapacity(1000);
@@ -324,74 +381,208 @@
CompactionManager.instance.performMaximal(store, false);
SSTableReader sstable = store.getLiveSSTables().iterator().next();
+ KeyCache keyCache = ((KeyCacheSupport<?>) sstable).getKeyCache();
+ assumeTrue(keyCache.isEnabled());
// existing, non-cached key
- sstable.getPosition(k(2), SSTableReader.Operator.EQ);
- assertEquals(1, sstable.getKeyCacheRequest());
- assertEquals(0, sstable.getKeyCacheHit());
+ sstable.getPosition(dk(2), SSTableReader.Operator.EQ);
+ assertEquals(1, keyCache.getRequests());
+ assertEquals(0, keyCache.getHits());
// existing, cached key
- sstable.getPosition(k(2), SSTableReader.Operator.EQ);
- assertEquals(2, sstable.getKeyCacheRequest());
- assertEquals(1, sstable.getKeyCacheHit());
+ sstable.getPosition(dk(2), SSTableReader.Operator.EQ);
+ assertEquals(2, keyCache.getRequests());
+ assertEquals(1, keyCache.getHits());
// non-existing key (it is specifically chosen to not be rejected by Bloom Filter check)
- sstable.getPosition(k(14), SSTableReader.Operator.EQ);
- assertEquals(3, sstable.getKeyCacheRequest());
- assertEquals(1, sstable.getKeyCacheHit());
+ sstable.getPosition(dk(14), SSTableReader.Operator.EQ);
+ assertEquals(3, keyCache.getRequests());
+ assertEquals(1, keyCache.getHits());
}
@Test
public void testGetPositionsBloomFilterStats()
{
+ SSTableReaderWithFilter sstable = prepareGetPositions();
+
+ // the keys are specifically chosen to cover certain use cases
+ // existing key is read from index
+ sstable.getPosition(dk(7), SSTableReader.Operator.EQ);
+ assertEquals(1, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(0, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount());
+
+ // existing key is read from Cache Key (if used)
+ sstable.getPosition(dk(7), SSTableReader.Operator.EQ);
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(0, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount());
+
+ // non-existing key is rejected by Bloom Filter check
+ sstable.getPosition(dk(45), SSTableReader.Operator.EQ); // note: 45 falls between 4 and 5
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount());
+
+ // GT should not affect bloom filter counts
+ sstable.getPosition(dk(56), SSTableReader.Operator.GE); // note: 56 falls between 5 and 6
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount());
+ sstable.getPosition(dk(57), SSTableReader.Operator.GT); // note: 57 falls between 5 and 6
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount());
+
+ // non-existing key is rejected by sstable keys range check, if performed, otherwise it's a false positive
+ sstable.getPosition(collisionFor(9), SSTableReader.Operator.EQ);
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ long fpCount = sstable.getFilterTracker().getFalsePositiveCount();
+
+ // existing key filtered out by sstable keys range check, performed because of moved start
+ sstable.getPosition(dk(1), SSTableReader.Operator.EQ);
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(fpCount, sstable.getFilterTracker().getFalsePositiveCount());
+ fpCount = sstable.getFilterTracker().getFalsePositiveCount();
+
+ // non-existing key is rejected by index interval check
+ sstable.getPosition(collisionFor(5), SSTableReader.Operator.EQ);
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(fpCount + 1, sstable.getFilterTracker().getFalsePositiveCount());
+
+ // non-existing key is rejected by index lookup check
+ sstable.getPosition(dk(807), SSTableReader.Operator.EQ);
+ assertEquals(2, sstable.getFilterTracker().getTruePositiveCount());
+ assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount());
+ assertEquals(fpCount + 2, sstable.getFilterTracker().getFalsePositiveCount());
+ }
+
+
+ @Test
+ public void testGetPositionsListenerCalls()
+ {
+ SSTableReaderWithFilter sstable = prepareGetPositions();
+
+ SSTableReadsListener listener = Mockito.mock(SSTableReadsListener.class);
+ // the keys are specifically chosen to cover certain use cases
+ // existing key is read from index
+ sstable.getPosition(dk(7), SSTableReader.Operator.EQ, listener);
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ Mockito.reset(listener);
+
+ // existing key is read from Cache Key (if used)
+ // Note: key cache may fail to cache the partition if it is wide.
+ sstable.getPosition(dk(7), SSTableReader.Operator.EQ, listener);
+ if (sstable instanceof BigTableReader)
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.KEY_CACHE_HIT);
+ else
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ Mockito.reset(listener);
+
+ // As above with other ops
+ sstable.getPosition(dk(7), SSTableReader.Operator.GT, listener); // GT does not engage key cache
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ Mockito.reset(listener);
+
+ sstable.getPosition(dk(7), SSTableReader.Operator.GE, listener); // GE does
+ if (sstable instanceof BigTableReader)
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.KEY_CACHE_HIT);
+ else
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ Mockito.reset(listener);
+
+ // non-existing key is rejected by Bloom Filter check
+ sstable.getPosition(dk(45), SSTableReader.Operator.EQ, listener); // note: 45 falls between 4 and 5
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.BLOOM_FILTER);
+ Mockito.reset(listener);
+
+ // non-existing key is rejected by sstable keys range check, if performed, otherwise it's a false positive
+ sstable.getPosition(collisionFor(9), SSTableReader.Operator.EQ, listener);
+ if (sstable instanceof BigTableReader)
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.MIN_MAX_KEYS);
+ else
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.INDEX_ENTRY_NOT_FOUND);
+ Mockito.reset(listener);
+
+ sstable.getPosition(collisionFor(9), SSTableReader.Operator.GE, listener);
+ if (sstable instanceof BigTableReader)
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.MIN_MAX_KEYS);
+ else
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.INDEX_ENTRY_NOT_FOUND);
+ Mockito.reset(listener);
+
+ sstable.getPosition(dk(9), SSTableReader.Operator.GT, listener);
+ if (sstable instanceof BigTableReader)
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.MIN_MAX_KEYS);
+ else
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.INDEX_ENTRY_NOT_FOUND);
+ Mockito.reset(listener);
+
+ // existing key filtered out by sstable keys range check, performed because of moved start
+ sstable.getPosition(dk(1), SSTableReader.Operator.EQ, listener);
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.MIN_MAX_KEYS);
+ Mockito.reset(listener);
+ long pos = sstable.getPosition(dk(1), SSTableReader.Operator.GT, listener);
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ assertEquals(sstable.getPosition(dk(3), SSTableReader.Operator.EQ), pos);
+ Mockito.reset(listener);
+
+ // non-existing key is rejected by index interval check
+ sstable.getPosition(collisionFor(5), SSTableReader.Operator.EQ, listener);
+ if (sstable instanceof BigTableReader)
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.PARTITION_INDEX_LOOKUP);
+ else
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.INDEX_ENTRY_NOT_FOUND);
+ Mockito.reset(listener);
+
+ // non-existing key is rejected by index lookup
+ sstable.getPosition(dk(807), SSTableReader.Operator.EQ, listener);
+ Mockito.verify(listener).onSSTableSkipped(sstable, SSTableReadsListener.SkippingReason.PARTITION_INDEX_LOOKUP);
+ Mockito.reset(listener);
+
+ // Variations of non-equal match
+ sstable.getPosition(dk(31), SSTableReader.Operator.GE, listener);
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ Mockito.reset(listener);
+
+ sstable.getPosition(dk(81), SSTableReader.Operator.GE, listener);
+ Mockito.verify(listener).onSSTableSelected(sstable, SSTableReadsListener.SelectionReason.INDEX_ENTRY_FOUND);
+ Mockito.reset(listener);
+ }
+
+ private SSTableReaderWithFilter prepareGetPositions()
+ {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD_SMALL_BLOOM_FILTER);
+ store.truncateBlocking();
partitioner = store.getPartitioner();
CacheService.instance.keyCache.setCapacity(1000);
// insert data and compact to a single sstable
for (int j = 0; j < 10; j++)
{
- new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
- .clustering("0")
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
+ if (j == 8) // leave a missing prefix
+ continue;
+
+ int rowCount = j < 5 ? 2000 : 1; // make some of the partitions wide
+ for (int r = 0; r < rowCount; ++r)
+ {
+ new RowUpdateBuilder(store.metadata(), j, String.valueOf(j))
+ .clustering(Integer.toString(r))
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ }
}
Util.flush(store);
CompactionManager.instance.performMaximal(store, false);
- SSTableReader sstable = store.getLiveSSTables().iterator().next();
- // the keys are specifically chosen to cover certain use cases
- // existing key is read from index
- sstable.getPosition(k(2), SSTableReader.Operator.EQ);
- assertEquals(1, sstable.getBloomFilterTruePositiveCount());
- assertEquals(0, sstable.getBloomFilterTrueNegativeCount());
- assertEquals(0, sstable.getBloomFilterFalsePositiveCount());
- // existing key is read from Cache Key
- sstable.getPosition(k(2), SSTableReader.Operator.EQ);
- assertEquals(2, sstable.getBloomFilterTruePositiveCount());
- assertEquals(0, sstable.getBloomFilterTrueNegativeCount());
- assertEquals(0, sstable.getBloomFilterFalsePositiveCount());
- // non-existing key is rejected by Bloom Filter check
- sstable.getPosition(k(10), SSTableReader.Operator.EQ);
- assertEquals(2, sstable.getBloomFilterTruePositiveCount());
- assertEquals(1, sstable.getBloomFilterTrueNegativeCount());
- assertEquals(0, sstable.getBloomFilterFalsePositiveCount());
- // non-existing key is rejected by sstable keys range check
- sstable.getPosition(k(99), SSTableReader.Operator.EQ);
- assertEquals(2, sstable.getBloomFilterTruePositiveCount());
- assertEquals(1, sstable.getBloomFilterTrueNegativeCount());
- assertEquals(1, sstable.getBloomFilterFalsePositiveCount());
- // non-existing key is rejected by index interval check
- sstable.getPosition(k(14), SSTableReader.Operator.EQ);
- assertEquals(2, sstable.getBloomFilterTruePositiveCount());
- assertEquals(1, sstable.getBloomFilterTrueNegativeCount());
- assertEquals(2, sstable.getBloomFilterFalsePositiveCount());
- // non-existing key is rejected by index lookup check
- sstable.getPosition(k(807), SSTableReader.Operator.EQ);
- assertEquals(2, sstable.getBloomFilterTruePositiveCount());
- assertEquals(1, sstable.getBloomFilterTrueNegativeCount());
- assertEquals(3, sstable.getBloomFilterFalsePositiveCount());
+ SSTableReaderWithFilter sstable = (SSTableReaderWithFilter) store.getLiveSSTables().iterator().next();
+ sstable = (SSTableReaderWithFilter) sstable.cloneWithNewStart(dk(3));
+ return sstable;
}
+
@Test
public void testOpeningSSTable() throws Exception
{
@@ -428,92 +619,223 @@
Descriptor desc = sstable.descriptor;
// test to see if sstable can be opened as expected
- SSTableReader target = SSTableReader.open(desc);
- assert target.first.equals(firstKey);
- assert target.last.equals(lastKey);
+ SSTableReader target = SSTableReader.open(store, desc);
+ try
+ {
+ assert target.first.equals(firstKey);
+ assert target.last.equals(lastKey);
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
- executeInternal(String.format("ALTER TABLE \"%s\".\"%s\" WITH bloom_filter_fp_chance = 0.3", ks, cf));
+ if (BigFormat.isSelected())
+ checkOpenedBigTable(ks, cf, store, desc);
+ else if (BtiFormat.isSelected())
+ checkOpenedBtiTable(ks, cf, store, desc);
+ else
+ throw Util.testMustBeImplementedForSSTableFormat();
+ }
- File summaryFile = new File(desc.filenameFor(Component.SUMMARY));
- Path bloomPath = new File(desc.filenameFor(Component.FILTER)).toPath();
- Path summaryPath = summaryFile.toPath();
+ private static void checkOpenedBigTable(String ks, String cf, ColumnFamilyStore store, Descriptor desc) throws Exception
+ {
+ executeInternal(format("ALTER TABLE \"%s\".\"%s\" WITH bloom_filter_fp_chance = 0.3", ks, cf));
- long bloomModified = Files.getLastModifiedTime(bloomPath).toMillis();
- long summaryModified = Files.getLastModifiedTime(summaryPath).toMillis();
+ File bloomFile = desc.fileFor(Components.FILTER);
+ long bloomModified = bloomFile.lastModified();
+
+ File summaryFile = desc.fileFor(Components.SUMMARY);
+ long summaryModified = summaryFile.lastModified();
TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
// Offline tests
// check that bloomfilter/summary ARE NOT regenerated
- target = SSTableReader.openNoValidation(desc, store.metadata);
-
- assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
- assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
-
- target.selfRef().release();
+ SSTableReader target = SSTableReader.openNoValidation(store, desc, store.metadata);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ assertEquals(summaryModified, summaryFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
// check that bloomfilter/summary ARE NOT regenerated and BF=AlwaysPresent when filter component is missing
- Set<Component> components = SSTable.discoverComponentsFor(desc);
- components.remove(Component.FILTER);
+ Set<Component> components = desc.discoverComponents();
+ components.remove(Components.FILTER);
target = SSTableReader.openNoValidation(desc, components, store);
-
- assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
- assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
- assertEquals(FilterFactory.AlwaysPresent, target.getBloomFilter());
-
- target.selfRef().release();
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ assertEquals(summaryModified, summaryFile.lastModified());
+ assertEquals(0, ((SSTableReaderWithFilter) target).getFilterOffHeapSize());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
// #### online tests ####
// check that summary & bloomfilter are not regenerated when SSTable is opened and BFFP has been changed
- target = SSTableReader.open(desc, store.metadata);
-
- assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
- assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
-
- target.selfRef().release();
+ target = SSTableReader.open(store, desc, store.metadata);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ assertEquals(summaryModified, summaryFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
// check that bloomfilter is recreated when it doesn't exist and this causes the summary to be recreated
- components = SSTable.discoverComponentsFor(desc);
- components.remove(Component.FILTER);
+ components = desc.discoverComponents();
+ components.remove(Components.FILTER);
+ components.remove(Components.SUMMARY);
- target = SSTableReader.open(desc, components, store.metadata);
-
- assertTrue("Bloomfilter was not recreated", bloomModified < Files.getLastModifiedTime(bloomPath).toMillis());
- assertTrue("Summary was not recreated", summaryModified < Files.getLastModifiedTime(summaryPath).toMillis());
-
- target.selfRef().release();
+ target = SSTableReader.open(store, desc, components, store.metadata);
+ try {
+ assertTrue("Bloomfilter was not recreated", bloomModified < bloomFile.lastModified());
+ assertTrue("Summary was not recreated", summaryModified < summaryFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
// check that only the summary is regenerated when it is deleted
- components.add(Component.FILTER);
- summaryModified = Files.getLastModifiedTime(summaryPath).toMillis();
+ components.add(Components.FILTER);
+ summaryModified = summaryFile.lastModified();
summaryFile.tryDelete();
TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
- bloomModified = Files.getLastModifiedTime(bloomPath).toMillis();
+ bloomModified = bloomFile.lastModified();
- target = SSTableReader.open(desc, components, store.metadata);
-
- assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
- assertTrue("Summary was not recreated", summaryModified < Files.getLastModifiedTime(summaryPath).toMillis());
-
- target.selfRef().release();
+ target = SSTableReader.open(store, desc, components, store.metadata);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ assertTrue("Summary was not recreated", summaryModified < summaryFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
// check that summary and bloomfilter is not recreated when the INDEX is missing
- components.add(Component.SUMMARY);
- components.remove(Component.PRIMARY_INDEX);
+ components.add(Components.SUMMARY);
+ components.remove(Components.PRIMARY_INDEX);
- summaryModified = Files.getLastModifiedTime(summaryPath).toMillis();
- target = SSTableReader.open(desc, components, store.metadata, false, false);
+ summaryModified = summaryFile.lastModified();
+ target = SSTableReader.open(store, desc, components, store.metadata, false, false);
+ try
+ {
+ TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
+ assertEquals(bloomModified, bloomFile.lastModified());
+ assertEquals(summaryModified, summaryFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
+ }
+
+ private static void checkOpenedBtiTable(String ks, String cf, ColumnFamilyStore store, Descriptor desc) throws Exception
+ {
+ executeInternal(format("ALTER TABLE \"%s\".\"%s\" WITH bloom_filter_fp_chance = 0.3", ks, cf));
+
+ File bloomFile = desc.fileFor(Components.FILTER);
+ long bloomModified = bloomFile.lastModified();
TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
- assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
- assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
- target.selfRef().release();
+ // Offline tests
+ // check that bloomfilter is not regenerated
+ SSTableReader target = SSTableReader.openNoValidation(store, desc, store.metadata);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
+
+ // check that bloomfilter is not regenerated and BF=AlwaysPresent when filter component is missing
+ Set<Component> components = desc.discoverComponents();
+ components.remove(Components.FILTER);
+ target = SSTableReader.openNoValidation(desc, components, store);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ assertEquals(0, ((SSTableReaderWithFilter) target).getFilterOffHeapSize());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
+
+ // #### online tests ####
+ // check that bloomfilter is not regenerated when SSTable is opened and BFFP has been changed
+ TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
+ target = SSTableReader.open(store, desc, store.metadata);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
+
+ // check that bloomfilter is recreated when it doesn't exist
+ components = desc.discoverComponents();
+ components.remove(Components.FILTER);
+
+ target = SSTableReader.open(store, desc, components, store.metadata);
+ try
+ {
+ assertTrue("Bloomfilter was not recreated", bloomModified < bloomFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
+
+ bloomModified = bloomFile.lastModified();
+ TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
+
+ components.add(Components.FILTER);
+ target = SSTableReader.open(store, desc, components, store.metadata);
+ try
+ {
+ assertEquals(bloomModified, bloomFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
+
+ // check that bloomfilter is not recreated when the INDEX is missing
+ components.remove(BtiFormat.Components.PARTITION_INDEX);
+
+ target = SSTableReader.open(store, desc, components, store.metadata, false, false);
+ try
+ {
+ TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different
+ assertEquals(bloomModified, bloomFile.lastModified());
+ }
+ finally
+ {
+ target.selfRef().close();
+ }
}
@Test
- public void testLoadingSummaryUsesCorrectPartitioner()
+ public void testLoadingSummaryUsesCorrectPartitioner() throws Exception
{
ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_INDEXED);
@@ -525,20 +847,25 @@
Util.flush(store);
- for(ColumnFamilyStore indexCfs : store.indexManager.getAllIndexColumnFamilyStores())
+ for (ColumnFamilyStore indexCfs : store.indexManager.getAllIndexColumnFamilyStores())
{
assert indexCfs.isIndex();
SSTableReader sstable = indexCfs.getLiveSSTables().iterator().next();
assert sstable.first.getToken() instanceof LocalToken;
- SSTableReader.saveSummary(sstable.descriptor, sstable.first, sstable.last, sstable.indexSummary);
- SSTableReader reopened = SSTableReader.open(sstable.descriptor);
- assert reopened.first.getToken() instanceof LocalToken;
- reopened.selfRef().release();
+ if (sstable instanceof IndexSummarySupport<?>)
+ {
+ new IndexSummaryComponent(((IndexSummarySupport<?>) sstable).getIndexSummary(), sstable.first, sstable.last).save(sstable.descriptor.fileFor(Components.SUMMARY), true);
+ SSTableReader reopened = SSTableReader.open(store, sstable.descriptor);
+ assert reopened.first.getToken() instanceof LocalToken;
+ reopened.selfRef().release();
+ }
}
}
- /** see CASSANDRA-5407 */
+ /**
+ * see CASSANDRA-5407
+ */
@Test
public void testGetScannerForNoIntersectingRanges() throws Exception
{
@@ -553,11 +880,16 @@
Util.flush(store);
boolean foundScanner = false;
- for (SSTableReader s : store.getLiveSSTables())
+
+ Set<SSTableReader> liveSSTables = store.getLiveSSTables();
+ assertEquals("The table should have only one sstable", 1, liveSSTables.size());
+
+ for (SSTableReader s : liveSSTables)
{
- try (ISSTableScanner scanner = s.getScanner(new Range<Token>(t(0), t(1))))
+ try (ISSTableScanner scanner = s.getScanner(new Range<>(t(0), t(1))))
{
- scanner.next(); // throws exception pre 5407
+ // Make sure no data is returned and nothing fails for non-intersecting range.
+ assertFalse(scanner.hasNext());
foundScanner = true;
}
}
@@ -593,13 +925,13 @@
SSTableReader sstable = store.getLiveSSTables().iterator().next();
List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
- assert sections.size() == 1 : "Expected to find range in sstable" ;
+ assert sections.size() == 1 : "Expected to find range in sstable";
// re-open the same sstable as it would be during bulk loading
- Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
- if (sstable.components.contains(Component.COMPRESSION_INFO))
- components.add(Component.COMPRESSION_INFO);
- SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, store.metadata);
+ Set<Component> components = Sets.newHashSet(sstable.descriptor.getFormat().primaryComponents());
+ if (sstable.components.contains(Components.COMPRESSION_INFO))
+ components.add(Components.COMPRESSION_INFO);
+ SSTableReader bulkLoaded = SSTableReader.openForBatch(store, sstable.descriptor, components, store.metadata);
sections = bulkLoaded.getPositionsForRanges(ranges);
assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading";
bulkLoaded.selfRef().release();
@@ -608,17 +940,17 @@
@Test
public void testIndexSummaryReplacement() throws IOException, ExecutionException, InterruptedException
{
+ assumeTrue(IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD_LOW_INDEX_INTERVAL); // index interval of 8, no key caching
final int NUM_PARTITIONS = 512;
for (int j = 0; j < NUM_PARTITIONS; j++)
{
- new RowUpdateBuilder(store.metadata(), j, String.format("%3d", j))
+ new RowUpdateBuilder(store.metadata(), j, format("%3d", j))
.clustering("0")
- .add("val", String.format("%3d", j))
+ .add("val", format("%3d", j))
.build()
.applyUnsafe();
-
}
Util.flush(store);
CompactionManager.instance.performMaximal(store, false);
@@ -631,7 +963,7 @@
List<Future<?>> futures = new ArrayList<>(NUM_PARTITIONS * 2);
for (int i = 0; i < NUM_PARTITIONS; i++)
{
- final ByteBuffer key = ByteBufferUtil.bytes(String.format("%3d", i));
+ final ByteBuffer key = ByteBufferUtil.bytes(format("%3d", i));
final int index = i;
futures.add(executor.submit(new Runnable()
@@ -639,7 +971,7 @@
public void run()
{
Row row = Util.getOnlyRowUnfiltered(Util.cmd(store, key).build());
- assertEquals(0, ByteBufferUtil.compare(String.format("%3d", index).getBytes(), row.cells().iterator().next().buffer()));
+ assertEquals(0, ByteBufferUtil.compare(format("%3d", index).getBytes(), row.cells().iterator().next().buffer()));
}
}));
@@ -648,7 +980,7 @@
public void run()
{
Iterable<DecoratedKey> results = store.keySamples(
- new Range<>(sstable.getPartitioner().getMinimumToken(), sstable.getPartitioner().getToken(key)));
+ new Range<>(sstable.getPartitioner().getMinimumToken(), sstable.getPartitioner().getToken(key)));
assertTrue(results.iterator().hasNext());
}
}));
@@ -657,7 +989,7 @@
SSTableReader replacement;
try (LifecycleTransaction txn = store.getTracker().tryModify(Collections.singletonList(sstable), OperationType.UNKNOWN))
{
- replacement = sstable.cloneWithNewSummarySamplingLevel(store, 1);
+ replacement = ((IndexSummarySupport<?>) sstable).cloneWithNewSummarySamplingLevel(store, 1);
txn.update(replacement, true);
txn.finish();
}
@@ -670,6 +1002,7 @@
@Test
public void testIndexSummaryUpsampleAndReload() throws Exception
{
+ assumeTrue(IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments
@@ -683,35 +1016,34 @@
}
}
- private void testIndexSummaryUpsampleAndReload0() throws Exception
+ private <R extends SSTableReader & IndexSummarySupport<R>> void testIndexSummaryUpsampleAndReload0() throws Exception
{
ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD_LOW_INDEX_INTERVAL); // index interval of 8, no key caching
final int NUM_PARTITIONS = 512;
for (int j = 0; j < NUM_PARTITIONS; j++)
{
- new RowUpdateBuilder(store.metadata(), j, String.format("%3d", j))
+ new RowUpdateBuilder(store.metadata(), j, format("%3d", j))
.clustering("0")
- .add("val", String.format("%3d", j))
+ .add("val", format("%3d", j))
.build()
.applyUnsafe();
-
}
Util.flush(store);
CompactionManager.instance.performMaximal(store, false);
- Collection<SSTableReader> sstables = store.getLiveSSTables();
+ Collection<R> sstables = ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(store);
assert sstables.size() == 1;
- final SSTableReader sstable = sstables.iterator().next();
+ final R sstable = sstables.iterator().next();
try (LifecycleTransaction txn = store.getTracker().tryModify(Collections.singletonList(sstable), OperationType.UNKNOWN))
{
- SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(store, sstable.getIndexSummarySamplingLevel() + 1);
+ SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(store, sstable.getIndexSummary().getSamplingLevel() + 1);
txn.update(replacement, true);
txn.finish();
}
- SSTableReader reopen = SSTableReader.open(sstable.descriptor);
- assert reopen.getIndexSummarySamplingLevel() == sstable.getIndexSummarySamplingLevel() + 1;
+ R reopen = (R) SSTableReader.open(store, sstable.descriptor);
+ assert reopen.getIndexSummary().getSamplingLevel() == sstable.getIndexSummary().getSamplingLevel() + 1;
}
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS)
@@ -741,9 +1073,19 @@
return Collections.singletonList(new Range<>(left, right));
}
- private DecoratedKey k(int i)
+ private DecoratedKey dk(int i)
{
- return new BufferDecoratedKey(t(i), ByteBufferUtil.bytes(String.valueOf(i)));
+ return partitioner.decorateKey(keyFor(i));
+ }
+
+ private static ByteBuffer keyFor(int i)
+ {
+ return ByteBufferUtil.bytes(String.valueOf(i));
+ }
+
+ private DecoratedKey collisionFor(int i)
+ {
+ return partitioner.decorateKey(Util.generateMurmurCollision(ByteBufferUtil.bytes(String.valueOf(i))));
}
@Test(expected = RuntimeException.class)
@@ -781,19 +1123,19 @@
// make sure the new directory is empty and that the old files exist:
for (Component c : sstable.components)
{
- File f = new File(notLiveDesc.filenameFor(c));
+ File f = notLiveDesc.fileFor(c);
assertFalse(f.exists());
- assertTrue(new File(sstable.descriptor.filenameFor(c)).exists());
+ assertTrue(sstable.descriptor.fileFor(c).exists());
}
SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components, false);
// make sure the files were moved:
for (Component c : sstable.components)
{
- File f = new File(notLiveDesc.filenameFor(c));
+ File f = notLiveDesc.fileFor(c);
assertTrue(f.exists());
- assertTrue(f.toString().contains(String.format("-%s-", id)));
+ assertTrue(f.toString().contains(format("-%s-", id)));
f.deleteOnExit();
- assertFalse(new File(sstable.descriptor.filenameFor(c)).exists());
+ assertFalse(sstable.descriptor.fileFor(c).exists());
}
}
@@ -837,40 +1179,40 @@
Descriptor desc = setUpForTestVerfiyCompressionInfoExistence();
// delete the compression info, so it is corrupted.
- File compressionInfoFile = new File(desc.filenameFor(Component.COMPRESSION_INFO));
+ File compressionInfoFile = desc.fileFor(Components.COMPRESSION_INFO);
compressionInfoFile.tryDelete();
assertFalse("CompressionInfo file should not exist", compressionInfoFile.exists());
// discovert the components on disk after deletion
- Set<Component> components = SSTable.discoverComponentsFor(desc);
+ Set<Component> components = desc.discoverComponents();
expectedException.expect(CorruptSSTableException.class);
expectedException.expectMessage("CompressionInfo.db");
- SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components);
+ CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components);
}
@Test
public void testVerifyCompressionInfoExistenceWhenTOCUnableToOpen()
{
Descriptor desc = setUpForTestVerfiyCompressionInfoExistence();
- Set<Component> components = SSTable.discoverComponentsFor(desc);
- SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components);
+ Set<Component> components = desc.discoverComponents();
+ CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components);
// mark the toc file not readable in order to trigger the FSReadError
- File tocFile = new File(desc.filenameFor(Component.TOC));
+ File tocFile = desc.fileFor(Components.TOC);
tocFile.trySetReadable(false);
expectedException.expect(FSReadError.class);
expectedException.expectMessage("TOC.txt");
- SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components);
+ CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components);
}
@Test
public void testVerifyCompressionInfoExistencePasses()
{
Descriptor desc = setUpForTestVerfiyCompressionInfoExistence();
- Set<Component> components = SSTable.discoverComponentsFor(desc);
- SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components);
+ Set<Component> components = desc.discoverComponents();
+ CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components);
}
private Descriptor setUpForTestVerfiyCompressionInfoExistence()
@@ -881,8 +1223,8 @@
cfs.clearUnsafe();
Descriptor desc = sstable.descriptor;
- File compressionInfoFile = new File(desc.filenameFor(Component.COMPRESSION_INFO));
- File tocFile = new File(desc.filenameFor(Component.TOC));
+ File compressionInfoFile = desc.fileFor(Components.COMPRESSION_INFO);
+ File tocFile = desc.fileFor(Components.TOC);
assertTrue("CompressionInfo file should exist", compressionInfoFile.exists());
assertTrue("TOC file should exist", tocFile.exists());
return desc;
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index c88e0b0..fe3ff27 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -19,7 +19,12 @@
package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -27,8 +32,8 @@
import com.google.common.collect.Sets;
import org.junit.Test;
-import org.apache.cassandra.Util;
import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
@@ -36,24 +41,24 @@
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
import org.apache.cassandra.db.compaction.CompactionIterator;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.compaction.SSTableSplitter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -62,7 +67,10 @@
import static org.apache.cassandra.db.compaction.OperationType.COMPACTION;
import static org.apache.cassandra.utils.FBUtilities.nowInSeconds;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -205,7 +213,9 @@
SSTableReader s = writeFile(cfs, 1000);
cfs.addSSTable(s);
long startStorageMetricsLoad = StorageMetrics.load.getCount();
+ long startUncompressedLoad = StorageMetrics.uncompressedLoad.getCount();
long sBytesOnDisk = s.bytesOnDisk();
+ long sBytesOnDiskUncompressed = s.logicalBytesOnDisk();
Set<SSTableReader> compacting = Sets.newHashSet(s);
List<SSTableReader> sstables;
@@ -236,11 +246,15 @@
LifecycleTransaction.waitForDeletions();
- long sum = 0;
- for (SSTableReader x : cfs.getLiveSSTables())
- sum += x.bytesOnDisk();
+ long sum = cfs.getLiveSSTables().stream().mapToLong(SSTableReader::bytesOnDisk).sum();
assertEquals(sum, cfs.metric.liveDiskSpaceUsed.getCount());
- assertEquals(startStorageMetricsLoad - sBytesOnDisk + sum, StorageMetrics.load.getCount());
+ long endLoad = StorageMetrics.load.getCount();
+ assertEquals(startStorageMetricsLoad - sBytesOnDisk + sum, endLoad);
+
+ long uncompressedSum = cfs.getLiveSSTables().stream().mapToLong(t -> t.logicalBytesOnDisk()).sum();
+ long endUncompressedLoad = StorageMetrics.uncompressedLoad.getCount();
+ assertEquals(startUncompressedLoad - sBytesOnDiskUncompressed + uncompressedSum, endUncompressedLoad);
+
assertEquals(files, sstables.size());
assertEquals(files, cfs.getLiveSSTables().size());
LifecycleTransaction.waitForDeletions();
@@ -878,39 +892,18 @@
try
{
UnfilteredRowIterator uri = mock(UnfilteredRowIterator.class);
- when(uri.partitionLevelDeletion()).thenReturn(new DeletionTime(0,0));
+ when(uri.partitionLevelDeletion()).thenReturn(new DeletionTime(0, 0));
when(uri.partitionKey()).thenReturn(bopKeyFromInt(0));
// should not be able to append after buffer release on switch
firstWriter.append(uri);
fail("Expected AssertionError was not thrown.");
}
- catch(AssertionError ae) {
- if (!ae.getMessage().contains("update is being called after releaseBuffers"))
+ catch (AssertionError ae)
+ {
+ if (!ae.getMessage().contains("update is being called after releaseBuffers") && !ae.getMessage().contains("Attempt to use a closed data output"))
throw ae;
}
}
-
- // Can update a writer that is not eagerly cleared on switch
- eagerWriterMetaRelease = false;
- try (LifecycleTransaction txn = cfs.getTracker().tryModify(new HashSet<>(), OperationType.UNKNOWN);
- SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, 1000000, false, eagerWriterMetaRelease)
- )
- {
- SSTableWriter firstWriter = getWriter(cfs, dir, txn);
- rewriter.switchWriter(firstWriter);
-
- // At least one write so it's not aborted when switched out.
- UnfilteredRowIterator uri = mock(UnfilteredRowIterator.class);
- when(uri.partitionLevelDeletion()).thenReturn(new DeletionTime(0,0));
- when(uri.partitionKey()).thenReturn(bopKeyFromInt(0));
- rewriter.append(uri);
-
- rewriter.switchWriter(getWriter(cfs, dir, txn));
-
- // should be able to append after switch, and assert is not tripped
- when(uri.partitionKey()).thenReturn(bopKeyFromInt(1));
- firstWriter.append(uri);
- }
}
static DecoratedKey bopKeyFromInt(int i)
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
index 922200a..17b8a6c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
@@ -22,16 +22,14 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.function.Consumer;
import com.google.common.collect.Iterables;
-
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DataRange;
import org.apache.cassandra.db.DecoratedKey;
@@ -41,19 +39,22 @@
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.dht.AbstractBounds.isEmpty;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class SSTableScannerTest
{
@@ -239,7 +240,7 @@
// end of range edge conditions
assertScanMatches(sstable, 1, 8, 2, 8);
assertScanMatches(sstable, 1, 9, 2, 9);
- assertScanMatches(sstable, 1, 9, 2, 9);
+ assertScanMatches(sstable, 1, 99, 2, 9);
// single item ranges
assertScanMatches(sstable, 2, 2, 2, 2);
@@ -290,6 +291,93 @@
assertScanMatches(sstable, 1, 0, 2, 9);
}
+ @Test
+ public void testSingleDataRangeWithMovedStart() throws IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(TABLE);
+ store.clearUnsafe();
+
+ // disable compaction while flushing
+ store.disableAutoCompaction();
+
+ for (int i = 2; i < 10; i++)
+ insertRowWithKey(store.metadata(), i);
+ Util.flush(store);
+
+ assertEquals(1, store.getLiveSSTables().size());
+ SSTableReader sstable = store.getLiveSSTables().iterator().next();
+ sstable = sstable.cloneWithNewStart(dk(4));
+
+ // full range scan
+ ISSTableScanner scanner = sstable.getScanner();
+ for (int i = 4; i < 10; i++)
+ assertEquals(toKey(i), new String(scanner.next().partitionKey().getKey().array()));
+
+ scanner.close();
+
+ // a simple read of a chunk in the middle
+ assertScanMatches(sstable, 3, 6, 4, 6);
+
+ // start of range edge conditions
+ assertScanMatches(sstable, 3, 9, 4, 9);
+ assertScanMatches(sstable, 4, 9, 4, 9);
+ assertScanMatches(sstable, 5, 9, 4, 9);
+
+ // end of range edge conditions
+ assertScanMatches(sstable, 1, 8, 4, 8);
+ assertScanMatches(sstable, 1, 9, 4, 9);
+ assertScanMatches(sstable, 1, 99, 4, 9);
+
+ // single item ranges
+ assertScanMatches(sstable, 4, 4, 4, 4);
+ assertScanMatches(sstable, 5, 5, 5, 5);
+ assertScanMatches(sstable, 9, 9, 9, 9);
+
+ // empty ranges
+ assertScanEmpty(sstable, 2, 3);
+ assertScanEmpty(sstable, 10, 11);
+
+ // wrapping, starts in middle
+ assertScanMatches(sstable, 7, 5, 4, 5, 7, 9);
+ assertScanMatches(sstable, 5, 4, 4, 4, 5, 9);
+ assertScanMatches(sstable, 5, 3, 5, 9);
+ assertScanMatches(sstable, 5, Integer.MIN_VALUE, 5, 9);
+ // wrapping, starts at end
+ assertScanMatches(sstable, 9, 8, 4, 8, 9, 9);
+ assertScanMatches(sstable, 9, 5, 4, 5, 9, 9);
+ assertScanMatches(sstable, 9, 4, 4, 4, 9, 9);
+ assertScanMatches(sstable, 9, 3, 9, 9);
+ assertScanMatches(sstable, 9, Integer.MIN_VALUE, 9, 9);
+ assertScanMatches(sstable, 8, 5, 4, 5, 8, 9);
+ assertScanMatches(sstable, 8, 4, 4, 4, 8, 9);
+ assertScanMatches(sstable, 8, 3, 8, 9);
+ assertScanMatches(sstable, 8, Integer.MIN_VALUE, 8, 9);
+ // wrapping, starts past end
+ assertScanMatches(sstable, 10, 9, 4, 9);
+ assertScanMatches(sstable, 10, 5, 4, 5);
+ assertScanMatches(sstable, 10, 4, 4, 4);
+ assertScanEmpty(sstable, 10, 3);
+ assertScanEmpty(sstable, 10, Integer.MIN_VALUE);
+ assertScanMatches(sstable, 11, 10, 4, 9);
+ assertScanMatches(sstable, 11, 9, 4, 9);
+ assertScanMatches(sstable, 11, 5, 4, 5);
+ assertScanMatches(sstable, 11, 4, 4, 4);
+ assertScanEmpty(sstable, 11, 3);
+ assertScanEmpty(sstable, 11, Integer.MIN_VALUE);
+ // wrapping, starts at start
+ assertScanMatches(sstable, 5, 1, 5, 9);
+ assertScanMatches(sstable, 5, Integer.MIN_VALUE, 5, 9);
+ assertScanMatches(sstable, 4, 1, 4, 9);
+ assertScanMatches(sstable, 4, Integer.MIN_VALUE, 4, 9);
+ assertScanMatches(sstable, 1, 0, 4, 9);
+ assertScanMatches(sstable, 1, Integer.MIN_VALUE, 4, 9);
+ // wrapping, starts before
+ assertScanMatches(sstable, 3, -1, 4, 9);
+ assertScanMatches(sstable, 3, Integer.MIN_VALUE, 4, 9);
+ assertScanMatches(sstable, 3, 0, 4, 9);
+ }
+
private static void assertScanContainsRanges(ISSTableScanner scanner, int ... rangePairs) throws IOException
{
assert rangePairs.length % 2 == 0;
@@ -458,4 +546,49 @@
// this will currently fail
assertScanContainsRanges(scanner, 205, 205);
}
+
+ private static void testRequestNextRowIteratorWithoutConsumingPrevious(Consumer<ISSTableScanner> consumer)
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(TABLE);
+ store.clearUnsafe();
+
+ // disable compaction while flushing
+ store.disableAutoCompaction();
+
+ insertRowWithKey(store.metadata(), 0);
+ store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+ assertEquals(1, store.getLiveSSTables().size());
+ SSTableReader sstable = store.getLiveSSTables().iterator().next();
+
+ try (ISSTableScanner scanner = sstable.getScanner();
+ UnfilteredRowIterator currentRowIterator = scanner.next())
+ {
+ assertTrue(currentRowIterator.hasNext());
+ try
+ {
+ consumer.accept(scanner);
+ fail("Should have thrown IllegalStateException");
+ }
+ catch (IllegalStateException e)
+ {
+ assertEquals("The UnfilteredRowIterator returned by the last call to next() was initialized: " +
+ "it must be closed before calling hasNext() or next() again.",
+ e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testHasNextRowIteratorWithoutConsumingPrevious()
+ {
+ testRequestNextRowIteratorWithoutConsumingPrevious(ISSTableScanner::hasNext);
+ }
+
+ @Test
+ public void testNextRowIteratorWithoutConsumingPrevious()
+ {
+ testRequestNextRowIteratorWithoutConsumingPrevious(ISSTableScanner::next);
+ }
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
index cdd3ee0..05964a8 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
@@ -19,23 +19,36 @@
package org.apache.cassandra.io.sstable;
-import org.apache.cassandra.io.util.File;
import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.stream.Stream;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-
import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
-import static org.apache.cassandra.service.ActiveRepairService.*;
+import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.junit.Assert.assertEquals;
public class SSTableUtils
@@ -80,7 +93,7 @@
File cfDir = new File(tempdir, keyspaceName + File.pathSeparator() + cfname);
cfDir.tryCreateDirectories();
cfDir.deleteOnExit();
- File datafile = new File(new Descriptor(cfDir, keyspaceName, cfname, id, SSTableFormat.Type.BIG).filenameFor(Component.DATA));
+ File datafile = new Descriptor(cfDir, keyspaceName, cfname, id, BigFormat.getInstance()).fileFor(Components.DATA);
if (!datafile.createFileIfNotExists())
throw new IOException("unable to create file " + datafile);
datafile.deleteOnExit();
@@ -216,11 +229,11 @@
public Collection<SSTableReader> write(int expectedSize, Appender appender) throws IOException
{
- File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : new File(dest.filenameFor(Component.DATA));
+ File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : dest.fileFor(Components.DATA);
TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname);
ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id);
SerializationHeader header = appender.header();
- SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFilename(datafile.absolutePath()), expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, 0, header);
+ SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFileWithComponent(datafile, false).left, expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, 0, header);
while (appender.append(writer)) { /* pass */ }
Collection<SSTableReader> readers = writer.finish(true);
@@ -228,7 +241,7 @@
if (cleanup)
for (SSTableReader reader: readers)
for (Component component : reader.components)
- new File(reader.descriptor.filenameFor(component)).deleteOnExit();
+ reader.descriptor.fileFor(component).deleteOnExit();
return readers;
}
}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
index 5d20367..ad6bd61 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
@@ -20,27 +20,29 @@
import java.nio.ByteBuffer;
-import org.apache.cassandra.io.util.File;
import org.junit.Test;
-import org.apache.cassandra.*;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.TimeUUID;
-import static org.junit.Assert.fail;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class SSTableWriterTest extends SSTableWriterTestBase
{
@@ -63,34 +65,38 @@
writer.append(builder.build().unfilteredIterator());
}
- SSTableReader s = writer.setMaxDataAge(1000).openEarly();
- assert s != null;
- assertFileCounts(dir.tryListNames());
- for (int i = 10000; i < 20000; i++)
- {
- UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
- for (int j = 0; j < 100; j++)
- builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
- writer.append(builder.build().unfilteredIterator());
- }
- SSTableReader s2 = writer.setMaxDataAge(1000).openEarly();
- assertTrue(s.last.compareTo(s2.last) < 0);
- assertFileCounts(dir.tryListNames());
- s.selfRef().release();
- s2.selfRef().release();
+ writer.setMaxDataAge(1000);
+ writer.openEarly(s -> {
+ assert s != null;
+ assertFileCounts(dir.tryListNames());
+ for (int i = 10000; i < 20000; i++)
+ {
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1);
+ for (int j = 0; j < 100; j++)
+ builder.newRow("" + j).add("val", ByteBuffer.allocate(1000));
+ writer.append(builder.build().unfilteredIterator());
+ }
+ writer.setMaxDataAge(1000);
+ writer.openEarly(s2 -> {
+ assertTrue(s.last.compareTo(s2.last) < 0);
+ assertFileCounts(dir.tryListNames());
+ s.selfRef().release();
+ s2.selfRef().release();
- int datafiles = assertFileCounts(dir.tryListNames());
- assertEquals(datafiles, 1);
+ int datafiles = assertFileCounts(dir.tryListNames());
+ assertEquals(datafiles, 1);
- LifecycleTransaction.waitForDeletions();
- assertFileCounts(dir.tryListNames());
+ LifecycleTransaction.waitForDeletions();
+ assertFileCounts(dir.tryListNames());
- writer.abort();
- txn.abort();
- LifecycleTransaction.waitForDeletions();
- datafiles = assertFileCounts(dir.tryListNames());
- assertEquals(datafiles, 0);
- validateCFS(cfs);
+ writer.abort();
+ txn.abort();
+ LifecycleTransaction.waitForDeletions();
+ datafiles = assertFileCounts(dir.tryListNames());
+ assertEquals(datafiles, 0);
+ validateCFS(cfs);
+ });
+ });
}
}
@@ -293,4 +299,4 @@
assertInvalidRepairMetadata(1, NO_PENDING_REPAIR, true);
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index 83ad136..91b44ed 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@ -25,7 +25,6 @@
import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.io.util.File;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -42,6 +41,8 @@
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.utils.TimeUUID;
@@ -139,7 +140,7 @@
{
if (f.name().contains("Data"))
{
- Descriptor d = Descriptor.fromFilename(f.absolutePath());
+ Descriptor d = Descriptor.fromFileWithComponent(f, false).left;
assertTrue(d.toString(), liveDescriptors.contains(d.id));
}
}
@@ -155,7 +156,17 @@
public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn, long repairedAt, TimeUUID pendingRepair, boolean isTransient)
{
Descriptor desc = cfs.newSSTableDescriptor(directory);
- return SSTableWriter.create(desc, 0, repairedAt, pendingRepair, isTransient, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS), cfs.indexManager.listIndexes(), txn);
+ return desc.getFormat().getWriterFactory().builder(desc)
+ .setTableMetadataRef(cfs.metadata)
+ .setKeyCount(0)
+ .setRepairedAt(repairedAt)
+ .setPendingRepair(pendingRepair)
+ .setTransientSSTable(isTransient)
+ .setSerializationHeader(new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))
+ .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+ .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator))
+ .addDefaultComponents()
+ .build(txn, cfs);
}
public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn)
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java
new file mode 100644
index 0000000..b439b9c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java
@@ -0,0 +1,140 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
+
+public class SSTableWriterTransactionTest extends AbstractTransactionalTest
+{
+ public static final String KEYSPACE1 = "BigTableWriterTest";
+ public static final String CF_STANDARD = "Standard1";
+
+ private static ColumnFamilyStore cfs;
+
+ @BeforeClass
+ public static void defineSchema() throws Exception
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD, 0, Int32Type.instance, AsciiType.instance, Int32Type.instance));
+ cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
+ }
+
+ protected TestableTransaction newTest() throws IOException
+ {
+ return new TestableBTW();
+ }
+
+ private static class TestableBTW extends TestableTransaction
+ {
+ final File file;
+ final Descriptor descriptor;
+ final SSTableTxnWriter writer;
+
+ private TestableBTW()
+ {
+ this(cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables()));
+ }
+
+ private TestableBTW(Descriptor desc)
+ {
+ this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false,
+ new SerializationHeader(true, cfs.metadata(),
+ cfs.metadata().regularAndStaticColumns(),
+ EncodingStats.NO_STATS)));
+ }
+
+ private TestableBTW(Descriptor desc, SSTableTxnWriter sw)
+ {
+ super(sw);
+ this.file = desc.fileFor(Components.DATA);
+ this.descriptor = desc;
+ this.writer = sw;
+
+ for (int i = 0; i < 100; i++)
+ {
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), i);
+ for (int j = 0; j < 10; j++)
+ update.newRow(j).add("val", SSTableRewriterTest.random(0, 1000));
+ writer.append(update.build().unfilteredIterator());
+ }
+ }
+
+ protected void assertInProgress() throws Exception
+ {
+ assertExists(descriptor.version.format.primaryComponents());
+ assertNotExists(descriptor.version.format.generatedOnLoadComponents());
+ Assert.assertTrue(file.length() > 0);
+ }
+
+ protected void assertPrepared() throws Exception
+ {
+ assertExists(descriptor.version.format.primaryComponents());
+ assertExists(descriptor.version.format.generatedOnLoadComponents());
+ }
+
+ protected void assertAborted() throws Exception
+ {
+ assertNotExists(descriptor.version.format.primaryComponents());
+ assertNotExists(descriptor.version.format.generatedOnLoadComponents());
+ Assert.assertFalse(file.exists());
+ }
+
+ protected void assertCommitted() throws Exception
+ {
+ assertPrepared();
+ }
+
+ @Override
+ protected boolean commitCanThrow()
+ {
+ return true;
+ }
+
+ private void assertExists(Collection<Component> components)
+ {
+ for (Component component : components)
+ Assert.assertTrue(descriptor.fileFor(component).exists());
+ }
+
+ private void assertNotExists(Collection<Component> components)
+ {
+ for (Component component : components)
+ Assert.assertFalse(component.toString(), descriptor.fileFor(component).exists());
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java
new file mode 100644
index 0000000..11b92ac
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Function;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.Unpooled;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.net.AsyncStreamingInputPlus;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class SSTableZeroCopyWriterTest
+{
+ public static final String KEYSPACE1 = "BigTableBlockWriterTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_STANDARD2 = "Standard2";
+ public static final String CF_INDEXED = "Indexed1";
+ public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
+
+ public static SSTableReader sstable;
+ public static ColumnFamilyStore store;
+ private static int expectedRowCount;
+
+ @BeforeClass
+ public static void defineSchema() throws Exception
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED, true),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWINDEXINTERVAL)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+
+ String ks = KEYSPACE1;
+ String cf = "Standard1";
+
+ // clear and create just one sstable for this test
+ Keyspace keyspace = Keyspace.open(ks);
+ store = keyspace.getColumnFamilyStore(cf);
+ store.clearUnsafe();
+ store.disableAutoCompaction();
+
+ DecoratedKey firstKey = null, lastKey = null;
+ long timestamp = System.currentTimeMillis();
+ for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
+ {
+ DecoratedKey key = Util.dk(String.valueOf(i));
+ if (firstKey == null)
+ firstKey = key;
+ if (lastKey == null)
+ lastKey = key;
+ if (store.metadata().partitionKeyType.compare(lastKey.getKey(), key.getKey()) < 0)
+ lastKey = key;
+
+ new RowUpdateBuilder(store.metadata(), timestamp, key.getKey())
+ .clustering("col")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
+ expectedRowCount++;
+ }
+ Util.flush(store);
+
+ sstable = store.getLiveSSTables().iterator().next();
+ }
+
+ @Test
+ public void writeDataFile_DataInputPlus()
+ {
+ writeDataTestCycle(buffer -> new DataInputBuffer(buffer.array()));
+ }
+
+ @Test
+ public void writeDataFile_RebufferingByteBufDataInputPlus()
+ {
+ try (AsyncStreamingInputPlus input = new AsyncStreamingInputPlus(new EmbeddedChannel()))
+ {
+ writeDataTestCycle(buffer ->
+ {
+ if (buffer.limit() > 0) { // skip empty files that would cause premature EOF
+ input.append(Unpooled.wrappedBuffer(buffer));
+ }
+ return input;
+ });
+
+ input.requestClosure();
+ }
+ }
+
+
+ private void writeDataTestCycle(Function<ByteBuffer, DataInputPlus> bufferMapper)
+ {
+ File dir = store.getDirectories().getDirectoryForNewSSTables();
+ Descriptor desc = store.newSSTableDescriptor(dir);
+ TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
+
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
+ Set<Component> componentsToWrite = new HashSet<>(desc.getFormat().uploadComponents());
+ if (!metadata.getLocal().params.compression.isEnabled())
+ componentsToWrite.remove(Components.COMPRESSION_INFO);
+
+ SSTableZeroCopyWriter btzcw = desc.getFormat()
+ .getWriterFactory()
+ .builder(desc)
+ .setComponents(componentsToWrite)
+ .setTableMetadataRef(metadata)
+ .createZeroCopyWriter(txn, store);
+
+ for (Component component : componentsToWrite)
+ {
+ if (desc.fileFor(component).exists())
+ {
+ Pair<DataInputPlus, Long> pair = getSSTableComponentData(sstable, component, bufferMapper);
+
+ try
+ {
+ btzcw.writeComponent(component.type, pair.left, pair.right);
+ }
+ catch (ClosedChannelException e)
+ {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+
+ Collection<SSTableReader> readers = btzcw.finish(true);
+
+ SSTableReader reader = readers.toArray(new SSTableReader[0])[0];
+
+ assertNotEquals(sstable.getFilename(), reader.getFilename());
+ assertEquals(sstable.estimatedKeys(), reader.estimatedKeys());
+ assertEquals(sstable.isPendingRepair(), reader.isPendingRepair());
+
+ assertRowCount(expectedRowCount);
+ }
+
+ private void assertRowCount(int expected)
+ {
+ int count = 0;
+ for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
+ {
+ DecoratedKey dk = Util.dk(String.valueOf(i));
+ UnfilteredRowIterator rowIter = sstable.rowIterator(dk,
+ Slices.ALL,
+ ColumnFilter.all(store.metadata()),
+ false,
+ SSTableReadsListener.NOOP_LISTENER);
+ while (rowIter.hasNext())
+ {
+ rowIter.next();
+ count++;
+ }
+ }
+ assertEquals(expected, count);
+ }
+
+ private Pair<DataInputPlus, Long> getSSTableComponentData(SSTableReader sstable, Component component,
+ Function<ByteBuffer, DataInputPlus> bufferMapper)
+ {
+ FileHandle componentFile = new FileHandle.Builder(sstable.descriptor.fileFor(component))
+ .bufferSize(1024).complete();
+ ByteBuffer buffer = ByteBuffer.allocate((int) componentFile.channel.size());
+ componentFile.channel.read(buffer, 0);
+ buffer.flip();
+
+ DataInputPlus inputPlus = bufferMapper.apply(buffer);
+
+ return Pair.create(inputPlus, componentFile.channel.size());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
new file mode 100644
index 0000000..c756d9a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
@@ -0,0 +1,920 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.openhft.chronicle.core.util.ThrowingBiConsumer;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.CounterMutation;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.cassandra.utils.Throwables;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.apache.cassandra.SchemaLoader.counterCFMD;
+import static org.apache.cassandra.SchemaLoader.createKeyspace;
+import static org.apache.cassandra.SchemaLoader.getCompressionParameters;
+import static org.apache.cassandra.SchemaLoader.loadSchema;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_INVALID_LEGACY_SSTABLE_ROOT;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeFalse;
+
+@RunWith(BMUnitRunner.class)
+public class ScrubTest
+{
+ private final static Logger logger = LoggerFactory.getLogger(ScrubTest.class);
+
+ public static final String CF = "Standard1";
+ public static final String COUNTER_CF = "Counter1";
+ public static final String CF_UUID = "UUIDKeys";
+ public static final String CF_INDEX1 = "Indexed1";
+ public static final String CF_INDEX2 = "Indexed2";
+ public static final String CF_INDEX1_BYTEORDERED = "Indexed1_ordered";
+ public static final String CF_INDEX2_BYTEORDERED = "Indexed2_ordered";
+ public static final String COL_INDEX = "birthdate";
+ public static final String COL_NON_INDEX = "notbirthdate";
+
+ public static final Integer COMPRESSION_CHUNK_LENGTH = 4096;
+
+ private static final AtomicInteger seq = new AtomicInteger();
+
+ static WithProperties properties;
+
+ String ksName;
+ Keyspace keyspace;
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setFileCacheEnabled(false);
+ loadSchema();
+ assertNull(ChunkCache.instance);
+ }
+
+ @Before
+ public void setup()
+ {
+ ksName = "scrub_test_" + seq.incrementAndGet();
+ createKeyspace(ksName,
+ KeyspaceParams.simple(1),
+ standardCFMD(ksName, CF),
+ counterCFMD(ksName, COUNTER_CF).compression(getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
+ standardCFMD(ksName, CF_UUID, 0, UUIDType.instance),
+ SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1, true),
+ SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2, true),
+ SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance),
+ SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance));
+ keyspace = Keyspace.open(ksName);
+
+ CompactionManager.instance.disableAutoCompaction();
+ properties = new WithProperties().set(TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST, true);
+ }
+
+ @AfterClass
+ public static void clearClassEnv()
+ {
+ properties.close();
+ }
+
+ @Test
+ public void testScrubOnePartition()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ performScrub(cfs, false, true, false, 2);
+
+ // check data is still there
+ assertOrderedAll(cfs, 1);
+ }
+
+ @Test
+ public void testScrubLastBrokenPartition() throws IOException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ksName, CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ Set<SSTableReader> liveSSTables = cfs.getLiveSSTables();
+ assertThat(liveSSTables).hasSize(1);
+ String fileName = liveSSTables.iterator().next().getFilename();
+ Files.write(Paths.get(fileName), new byte[10], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+
+ performScrub(cfs, true, true, false, 2);
+
+ // check data is still there
+ assertOrderedAll(cfs, 0);
+ }
+
+ @Test
+ public void testScrubCorruptedCounterPartition() throws IOException, WriteTimeoutException
+ {
+ // When compression is enabled, for testing corrupted chunks we need enough partitions to cover
+ // at least 3 chunks of size COMPRESSION_CHUNK_LENGTH
+ int numPartitions = 1000;
+
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
+ cfs.truncateBlocking();
+ fillCounterCF(cfs, numPartitions);
+
+ assertOrderedAll(cfs, numPartitions);
+
+ assertEquals(1, cfs.getLiveSSTables().size());
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ //make sure to override at most 1 chunk when compression is enabled
+ //use 0x00 instead of the usual 0x7A because if by any chance it's able to iterate over the corrupt
+ //section, then we get many out-of-order errors, which we don't want
+ overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x00);
+
+ // with skipCorrupted == false, the scrub is expected to fail
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
+ IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().build()))
+ {
+ scrubber.scrub();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (IOError err)
+ {
+ Throwables.assertAnyCause(err, CorruptSSTableException.class);
+ }
+
+ // with skipCorrupted == true, the corrupt rows will be skipped
+ IScrubber.ScrubResult scrubResult;
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
+ IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().skipCorrupted().checkData().build()))
+ {
+ scrubResult = scrubber.scrubWithResult();
+ }
+
+ assertNotNull(scrubResult);
+
+ boolean compression = sstable.compression;
+ assertEquals(0, scrubResult.emptyPartitions);
+ if (compression)
+ {
+ assertEquals(numPartitions, scrubResult.badPartitions + scrubResult.goodPartitions);
+ //because we only corrupted 1 chunk, and we chose enough partitions to cover at least 3 chunks
+ assertTrue(scrubResult.goodPartitions >= scrubResult.badPartitions * 2);
+ }
+ else
+ {
+ assertEquals(1, scrubResult.badPartitions);
+ assertEquals(numPartitions - 1, scrubResult.goodPartitions);
+ }
+ assertEquals(1, cfs.getLiveSSTables().size());
+
+ assertOrderedAll(cfs, scrubResult.goodPartitions);
+ }
+
+ private List<File> sstableIndexPaths(SSTableReader reader)
+ {
+ if (BigFormat.is(reader.descriptor.getFormat()))
+ return Arrays.asList(reader.descriptor.fileFor(BigFormat.Components.PRIMARY_INDEX));
+ if (BtiFormat.is(reader.descriptor.getFormat()))
+ return Arrays.asList(reader.descriptor.fileFor(BtiFormat.Components.PARTITION_INDEX),
+ reader.descriptor.fileFor(BtiFormat.Components.ROW_INDEX));
+ else
+ throw Util.testMustBeImplementedForSSTableFormat();
+ }
+
+ @Test
+ public void testScrubCorruptedRowInSmallFile() throws Throwable
+ {
+ // overwrite one row with garbage
+ testCorruptionInSmallFile((sstable, keys) ->
+ overrideWithGarbage(sstable,
+ ByteBufferUtil.bytes(keys[0]),
+ ByteBufferUtil.bytes(keys[1]),
+ (byte) 0x7A),
+ false,
+ 4);
+ }
+
+
+ @Test
+ public void testScrubCorruptedIndex() throws Throwable
+ {
+ // overwrite a part of the index with garbage
+ testCorruptionInSmallFile((sstable, keys) ->
+ overrideWithGarbage(sstableIndexPaths(sstable).get(0),
+ 5,
+ 6,
+ (byte) 0x7A),
+ true,
+ 5);
+ }
+
+ @Test
+ public void testScrubCorruptedIndexOnOpen() throws Throwable
+ {
+ // overwrite the whole index with garbage
+ testCorruptionInSmallFile((sstable, keys) ->
+ overrideWithGarbage(sstableIndexPaths(sstable).get(0),
+ 0,
+ 60,
+ (byte) 0x7A),
+ true,
+ 5);
+ }
+
+ @Test
+ public void testScrubCorruptedRowCorruptedIndex() throws Throwable
+ {
+ // overwrite one row, and the index with garbage
+ testCorruptionInSmallFile((sstable, keys) ->
+ {
+ overrideWithGarbage(sstable,
+ ByteBufferUtil.bytes(keys[2]),
+ ByteBufferUtil.bytes(keys[3]),
+ (byte) 0x7A);
+ overrideWithGarbage(sstableIndexPaths(sstable).get(0),
+ 5,
+ 6,
+ (byte) 0x7A);
+ },
+ false,
+ 2); // corrupt after the second partition, no way to resync
+ }
+
+ public void testCorruptionInSmallFile(ThrowingBiConsumer<SSTableReader, String[], IOException> corrupt, boolean isFullyRecoverable, int expectedPartitions) throws IOException, WriteTimeoutException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
+ cfs.clearUnsafe();
+
+ String[] keys = fillCounterCF(cfs, 5);
+
+ assertOrderedAll(cfs, 5);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ // cannot test this with compression
+ assumeFalse(sstable.metadata().params.compression.isEnabled());
+
+ // overwrite one row with garbage
+ corrupt.accept(sstable, keys);
+
+ // with skipCorrupted == false, the scrub is expected to fail
+ if (!isFullyRecoverable)
+ {
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
+ IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().build()))
+ {
+ // with skipCorrupted == true, the corrupt row will be skipped
+ scrubber.scrub();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (IOError expected)
+ {
+ }
+ }
+
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB);
+ IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().skipCorrupted().build()))
+ {
+ // with skipCorrupted == true, the corrupt row will be skipped
+ scrubber.scrub();
+ }
+
+ assertEquals(1, cfs.getLiveSSTables().size());
+ // verify that we can read all the rows, and there is now the expected number of rows
+ assertOrderedAll(cfs, expectedPartitions);
+ }
+
+ @Test
+ public void testScrubOneRowWithCorruptedKey() throws IOException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 4);
+ assertOrderedAll(cfs, 4);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ // cannot test this with compression
+ assumeFalse(sstable.metadata().params.compression.isEnabled());
+
+ overrideWithGarbage(sstable, 0, 2, (byte) 0x7A);
+
+ performScrub(cfs, false, true, false, 2);
+
+ // check data is still there
+ if (BigFormat.is(sstable.descriptor.getFormat()))
+ assertOrderedAll(cfs, 4);
+ else if (BtiFormat.is(sstable.descriptor.getFormat()))
+ // For Trie format we won't be able to recover the damaged partition key (partion index doesn't store the whole key)
+ assertOrderedAll(cfs, 3);
+ else
+ throw Util.testMustBeImplementedForSSTableFormat();
+ }
+
+ @Test
+ public void testScrubCorruptedCounterRowNoEarlyOpen() throws IOException, WriteTimeoutException
+ {
+ boolean oldDisabledVal = SSTableRewriter.disableEarlyOpeningForTests;
+ try
+ {
+ SSTableRewriter.disableEarlyOpeningForTests = true;
+ testScrubCorruptedCounterPartition();
+ }
+ finally
+ {
+ SSTableRewriter.disableEarlyOpeningForTests = oldDisabledVal;
+ }
+ }
+
+ @Test
+ public void testScrubMultiRow()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 10);
+ assertOrderedAll(cfs, 10);
+
+ performScrub(cfs, false, true, false, 2);
+
+ // check data is still there
+ assertOrderedAll(cfs, 10);
+ }
+
+ @Test
+ public void testScrubNoIndex() throws ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 10);
+ assertOrderedAll(cfs, 10);
+
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ sstableIndexPaths(sstable).forEach(File::tryDelete);
+
+ performScrub(cfs, false, true, false, 2);
+
+ // check data is still there
+ assertOrderedAll(cfs, 10);
+ }
+
+ @Test
+ @BMRule(name = "skip partition order verification", targetClass = "SortedTableWriter", targetMethod = "verifyPartition", action = "return true")
+ public void testScrubOutOfOrder()
+ {
+ // Run only for Big Table format because Big Table Format does not complain if partitions are given in invalid
+ // order. Legacy SSTables with out-of-order partitions exist in production systems and must be corrected
+ // by scrubbing. The trie index format does not permit such partitions.
+
+ Assume.assumeTrue(BigFormat.isSelected());
+
+ // This test assumes ByteOrderPartitioner to create out-of-order SSTable
+ IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner();
+ DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner());
+
+ // Create out-of-order SSTable
+ File tempDir = FileUtils.createTempFile("ScrubTest.testScrubOutOfOrder", "").parent();
+ // create ks/cf directory
+ File tempDataDir = new File(tempDir, String.join(File.pathSeparator(), ksName, CF));
+ assertTrue(tempDataDir.tryCreateDirectories());
+ try
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
+ Descriptor desc = cfs.newSSTableDescriptor(tempDataDir);
+
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+ SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, keys.size(), cfs, txn)))
+ {
+ for (String k : keys)
+ {
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), Util.dk(k))
+ .newRow("someName").add("val", "someValue")
+ .build();
+
+ writer.append(update.unfilteredIterator());
+ }
+ writer.finish(false);
+ }
+
+ try
+ {
+ SSTableReader.open(cfs, desc, cfs.metadata);
+ fail("SSTR validation should have caught the out-of-order rows");
+ }
+ catch (CorruptSSTableException ise)
+ { /* this is expected */ }
+
+ // open without validation for scrubbing
+ Set<Component> components = new HashSet<>();
+ if (desc.fileFor(Components.COMPRESSION_INFO).exists())
+ components.add(Components.COMPRESSION_INFO);
+ components.add(Components.DATA);
+ components.add(Components.PRIMARY_INDEX);
+ components.add(Components.FILTER);
+ components.add(Components.STATS);
+ components.add(Components.SUMMARY);
+ components.add(Components.TOC);
+
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+// if (sstable.last.compareTo(sstable.first) < 0)
+// sstable.last = sstable.first;
+
+ try (LifecycleTransaction scrubTxn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
+ IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, scrubTxn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().build()))
+ {
+ scrubber.scrub();
+ }
+ LifecycleTransaction.waitForDeletions();
+ cfs.loadNewSSTables();
+ assertOrderedAll(cfs, 7);
+ }
+ finally
+ {
+ FileUtils.deleteRecursive(tempDataDir);
+ // reset partitioner
+ DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner);
+ }
+ }
+
+ public static void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException
+ {
+ overrideWithGarbage(sstable, key1, key2, (byte) 'z');
+ }
+
+ public static void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2, byte junk) throws IOException
+ {
+ boolean compression = sstable.metadata().params.compression.isEnabled();
+ long startPosition, endPosition;
+
+ if (compression)
+ { // overwrite with garbage the compression chunks from key1 to key2
+ CompressionMetadata compData = CompressionInfoComponent.load(sstable.descriptor);
+
+ CompressionMetadata.Chunk chunk1 = compData.chunkFor(
+ sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ));
+ CompressionMetadata.Chunk chunk2 = compData.chunkFor(
+ sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ));
+
+ startPosition = Math.min(chunk1.offset, chunk2.offset);
+ endPosition = Math.max(chunk1.offset + chunk1.length, chunk2.offset + chunk2.length);
+
+ compData.close();
+ }
+ else
+ { // overwrite with garbage from key1 to key2
+ long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ);
+ long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ);
+ startPosition = Math.min(row0Start, row1Start);
+ endPosition = Math.max(row0Start, row1Start);
+ }
+
+ overrideWithGarbage(sstable, startPosition, endPosition, junk);
+ }
+
+ private static void overrideWithGarbage(SSTableReader sstable, long startPosition, long endPosition) throws IOException
+ {
+ overrideWithGarbage(sstable, startPosition, endPosition, (byte) 'z');
+ }
+
+ private static void overrideWithGarbage(SSTableReader sstable, long startPosition, long endPosition, byte junk) throws IOException
+ {
+ overrideWithGarbage(sstable.getDataChannel().file(), startPosition, endPosition, junk);
+ }
+
+ private static void overrideWithGarbage(File path, long startPosition, long endPosition, byte junk) throws IOException
+ {
+ try (RandomAccessFile file = new RandomAccessFile(path.toJavaIOFile(), "rw"))
+ {
+ file.seek(startPosition);
+ int length = (int) (endPosition - startPosition);
+ byte[] buff = new byte[length];
+ Arrays.fill(buff, junk);
+ file.write(buff, 0, length);
+ }
+ if (ChunkCache.instance != null)
+ ChunkCache.instance.invalidateFile(path.toString());
+ }
+
+ public static void assertOrderedAll(ColumnFamilyStore cfs, int expectedSize)
+ {
+ assertOrdered(Util.cmd(cfs).build(), expectedSize);
+ }
+
+ private static void assertOrdered(ReadCommand cmd, int expectedSize)
+ {
+ int size = 0;
+ DecoratedKey prev = null;
+ logger.info("Reading data from " + cmd);
+ for (Partition partition : Util.getAllUnfiltered(cmd))
+ {
+ DecoratedKey current = partition.partitionKey();
+ logger.info("Read " + current.toString());
+ if (!(prev == null || prev.compareTo(current) < 0))
+ logger.error("key " + current + " does not sort after previous key " + prev);
+ assertTrue("key " + current + " does not sort after previous key " + prev, prev == null || prev.compareTo(current) < 0);
+ prev = current;
+ ++size;
+ }
+ assertEquals(expectedSize, size);
+ }
+
+ public static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
+ {
+ for (int i = 0; i < partitionsPerSSTable; i++)
+ {
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+ .newRow("r1").add("val", "1")
+ .newRow("r1").add("val", "1")
+ .build();
+
+ new Mutation(update).applyUnsafe();
+ }
+
+ Util.flush(cfs);
+ }
+
+ public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long... values)
+ {
+ assertEquals(0, values.length % 2);
+ for (int i = 0; i < values.length; i += 2)
+ {
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), String.valueOf(i));
+ if (composite)
+ {
+ builder.newRow("c" + i)
+ .add(COL_INDEX, values[i])
+ .add(COL_NON_INDEX, values[i + 1]);
+ }
+ else
+ {
+ builder.newRow()
+ .add(COL_INDEX, values[i])
+ .add(COL_NON_INDEX, values[i + 1]);
+ }
+ new Mutation(builder.build()).applyUnsafe();
+ }
+
+ Util.flush(cfs);
+ }
+
+ public static String[] fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException
+ {
+ SortedSet<String> tokenSorted = Sets.newTreeSet(Comparator.comparing(a -> cfs.getPartitioner()
+ .decorateKey(ByteBufferUtil.bytes(a))));
+ for (int i = 0; i < partitionsPerSSTable; i++)
+ {
+ if (i < 10)
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+ .newRow("r1").add("val", 100L)
+ .build();
+ tokenSorted.add(String.valueOf(i));
+ new CounterMutation(new Mutation(update), ConsistencyLevel.ONE).apply();
+ }
+
+ Util.flush(cfs);
+
+ return tokenSorted.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
+ }
+
+ @Test
+ public void testScrubColumnValidation() throws RequestExecutionException
+ {
+ QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_static_columns (a bigint, b timeuuid, c boolean static, d text, PRIMARY KEY (a, b))", ksName), ConsistencyLevel.ONE);
+
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_static_columns");
+
+ QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_static_columns (a, b, c, d) VALUES (123, c3db07e8-b602-11e3-bc6b-e0b9a54a6d93, true, 'foobar')", ksName));
+ Util.flush(cfs);
+ performScrub(cfs, false, true, false, 2);
+
+ QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_scrub_validation (a text primary key, b int)", ksName), ConsistencyLevel.ONE);
+ ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore("test_scrub_validation");
+
+ new Mutation(UpdateBuilder.create(cfs2.metadata(), "key").newRow().add("b", Int32Type.instance.decompose(1)).build()).apply();
+ Util.flush(cfs2);
+
+ performScrub(cfs2, false, false, false, 2);
+ }
+
+ /**
+ * For CASSANDRA-6892 too, check that for a compact table with one cluster column, we can insert whatever
+ * we want as value for the clustering column, including something that would conflict with a CQL column definition.
+ */
+ @Test
+ public void testValidationCompactStorage()
+ {
+ QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_dynamic_columns (a int, b text, c text, PRIMARY KEY (a, b)) WITH COMPACT STORAGE", ksName), ConsistencyLevel.ONE);
+
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_dynamic_columns");
+
+ QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'a', 'foo')", ksName));
+ QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'b', 'bar')", ksName));
+ QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'c', 'boo')", ksName));
+ Util.flush(cfs);
+ performScrub(cfs, true, true, false, 2);
+
+ // Scrub is silent, but it will remove broken records. So reading everything back to make sure nothing to "scrubbed away"
+ UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".test_compact_dynamic_columns", ksName));
+ assertNotNull(rs);
+ assertEquals(3, rs.size());
+
+ Iterator<UntypedResultSet.Row> iter = rs.iterator();
+ assertEquals("foo", iter.next().getString("c"));
+ assertEquals("bar", iter.next().getString("c"));
+ assertEquals("boo", iter.next().getString("c"));
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testScrubKeysIndex_preserveOrder() throws IOException, ExecutionException, InterruptedException
+ {
+ //If the partitioner preserves the order then SecondaryIndex uses BytesType comparator,
+ // otherwise it uses LocalByPartitionerType
+ testScrubIndex(CF_INDEX1_BYTEORDERED, COL_INDEX, false, true);
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testScrubCompositeIndex_preserveOrder() throws IOException, ExecutionException, InterruptedException
+ {
+ testScrubIndex(CF_INDEX2_BYTEORDERED, COL_INDEX, true, true);
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testScrubKeysIndex() throws IOException, ExecutionException, InterruptedException
+ {
+ testScrubIndex(CF_INDEX1, COL_INDEX, false, true);
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testScrubCompositeIndex() throws IOException, ExecutionException, InterruptedException
+ {
+ testScrubIndex(CF_INDEX2, COL_INDEX, true, true);
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testFailScrubKeysIndex() throws IOException, ExecutionException, InterruptedException
+ {
+ testScrubIndex(CF_INDEX1, COL_INDEX, false, false);
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testFailScrubCompositeIndex() throws IOException, ExecutionException, InterruptedException
+ {
+ testScrubIndex(CF_INDEX2, COL_INDEX, true, false);
+ }
+
+ @Test /* CASSANDRA-5174 */
+ public void testScrubTwice() throws IOException, ExecutionException, InterruptedException
+ {
+ testScrubIndex(CF_INDEX2, COL_INDEX, true, true, true);
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private void testScrubIndex(String cfName, String colName, boolean composite, boolean... scrubs)
+ throws IOException, ExecutionException, InterruptedException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
+
+ int numRows = 1000;
+ long[] colValues = new long[numRows * 2]; // each row has two columns
+ for (int i = 0; i < colValues.length; i += 2)
+ {
+ colValues[i] = (i % 4 == 0 ? 1L : 2L); // index column
+ colValues[i + 1] = 3L; //other column
+ }
+ fillIndexCF(cfs, composite, colValues);
+
+ // check index
+
+ assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2);
+
+ // scrub index
+ Set<ColumnFamilyStore> indexCfss = cfs.indexManager.getAllIndexColumnFamilyStores();
+ assertEquals(1, indexCfss.size());
+ for (ColumnFamilyStore indexCfs : indexCfss)
+ {
+ for (int i = 0; i < scrubs.length; i++)
+ {
+ boolean failure = !scrubs[i];
+ if (failure)
+ { //make sure the next scrub fails
+ overrideWithGarbage(indexCfs.getLiveSSTables().iterator().next(), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes(2L), (byte) 0x7A);
+ }
+ CompactionManager.AllSSTableOpStatus result = indexCfs.scrub(false, true, IScrubber.options().build(), 0);
+ assertEquals(failure ?
+ CompactionManager.AllSSTableOpStatus.ABORTED :
+ CompactionManager.AllSSTableOpStatus.SUCCESSFUL,
+ result);
+ }
+ }
+
+
+ // check index is still working
+ assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2);
+ }
+
+ private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, ColumnFamilyStore cfs, LifecycleTransaction txn)
+ {
+ SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
+ MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator).sstableLevel(0);
+ SSTableWriter writer = descriptor.getFormat()
+ .getWriterFactory()
+ .builder(descriptor)
+ .setKeyCount(keyCount)
+ .setRepairedAt(0)
+ .setPendingRepair(null)
+ .setTransientSSTable(false)
+ .setTableMetadataRef(cfs.metadata)
+ .setMetadataCollector(collector)
+ .setSerializationHeader(header)
+ .setFlushObservers(Collections.emptyList())
+ .addDefaultComponents()
+ .build(txn, cfs);
+
+ return new TestMultiWriter(writer, txn);
+ }
+
+ private static class TestMultiWriter extends SimpleSSTableMultiWriter
+ {
+ TestMultiWriter(SSTableWriter writer, LifecycleNewTracker lifecycleNewTracker)
+ {
+ super(writer, lifecycleNewTracker);
+ }
+ }
+
+ /**
+ * Tests with invalid sstables (containing duplicate entries in 2.0 and 3.0 storage format),
+ * that were caused by upgrading from 2.x with duplicate range tombstones.
+ * <p>
+ * See CASSANDRA-12144 for details.
+ */
+ @Test
+ public void testFilterOutDuplicates() throws Exception
+ {
+ Assume.assumeTrue(BigFormat.isSelected());
+
+ IPartitioner oldPart = DatabaseDescriptor.getPartitioner();
+ try
+ {
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+ QueryProcessor.process(String.format("CREATE TABLE \"%s\".cf_with_duplicates_3_0 (a int, b int, c int, PRIMARY KEY (a, b))", ksName), ConsistencyLevel.ONE);
+
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("cf_with_duplicates_3_0");
+
+ Path legacySSTableRoot = Paths.get(TEST_INVALID_LEGACY_SSTABLE_ROOT.getString(),
+ "Keyspace1",
+ "cf_with_duplicates_3_0");
+
+ for (String filename : new String[]{ "mb-3-big-CompressionInfo.db",
+ "mb-3-big-Digest.crc32",
+ "mb-3-big-Index.db",
+ "mb-3-big-Summary.db",
+ "mb-3-big-Data.db",
+ "mb-3-big-Filter.db",
+ "mb-3-big-Statistics.db",
+ "mb-3-big-TOC.txt" })
+ {
+ Files.copy(Paths.get(legacySSTableRoot.toString(), filename), cfs.getDirectories().getDirectoryForNewSSTables().toPath().resolve(filename));
+ }
+
+ cfs.loadNewSSTables();
+
+ cfs.scrub(true, false, IScrubber.options().skipCorrupted().build(), 1);
+
+ UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".cf_with_duplicates_3_0", ksName));
+ assertNotNull(rs);
+ assertEquals(1, rs.size());
+
+ QueryProcessor.executeInternal(String.format("DELETE FROM \"%s\".cf_with_duplicates_3_0 WHERE a=1 AND b =2", ksName));
+ rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".cf_with_duplicates_3_0", ksName));
+ assertNotNull(rs);
+ assertEquals(0, rs.size());
+ }
+ finally
+ {
+ DatabaseDescriptor.setPartitionerUnsafe(oldPart);
+ }
+ }
+
+ private static CompactionManager.AllSSTableOpStatus performScrub(ColumnFamilyStore cfs, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs)
+ {
+ IScrubber.Options options = IScrubber.options()
+ .skipCorrupted(skipCorrupted)
+ .checkData(checkData)
+ .reinsertOverflowedTTLRows(reinsertOverflowedTTL)
+ .build();
+ return CompactionManager.instance.performScrub(cfs, options, jobs);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
new file mode 100644
index 0000000..40bbe08
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
@@ -0,0 +1,847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.zip.CRC32;
+import java.util.zip.CheckedInputStream;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.batchlog.Batch;
+import org.apache.cassandra.batchlog.BatchlogManager;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter;
+import org.apache.cassandra.io.sstable.format.SortedTableVerifier.RangeOwnHelper;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.OutputHandler;
+
+import static org.apache.cassandra.SchemaLoader.counterCFMD;
+import static org.apache.cassandra.SchemaLoader.createKeyspace;
+import static org.apache.cassandra.SchemaLoader.loadSchema;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test for {@link IVerifier}.
+ * <p>
+ * Note: the complete coverage is composed of:
+ * - {@link org.apache.cassandra.tools.StandaloneVerifierOnSSTablesTest}
+ * - {@link org.apache.cassandra.tools.StandaloneVerifierTest}
+ * - {@link VerifyTest}
+ */
+public class VerifyTest
+{
+ private final static Logger logger = LoggerFactory.getLogger(VerifyTest.class);
+
+ public static final String KEYSPACE = "Keyspace1";
+ public static final String CF = "Standard1";
+ public static final String CF2 = "Standard2";
+ public static final String CF3 = "Standard3";
+ public static final String CF4 = "Standard4";
+ public static final String COUNTER_CF = "Counter1";
+ public static final String COUNTER_CF2 = "Counter2";
+ public static final String COUNTER_CF3 = "Counter3";
+ public static final String COUNTER_CF4 = "Counter4";
+ public static final String CORRUPT_CF = "Corrupt1";
+ public static final String CORRUPT_CF2 = "Corrupt2";
+ public static final String CORRUPTCOUNTER_CF = "CounterCorrupt1";
+ public static final String CORRUPTCOUNTER_CF2 = "CounterCorrupt2";
+
+ public static final String CF_UUID = "UUIDKeys";
+ public static final String BF_ALWAYS_PRESENT = "BfAlwaysPresent";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ CompressionParams compressionParameters = CompressionParams.snappy(32768);
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setColumnIndexSizeInKiB(0);
+
+ loadSchema();
+ createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ standardCFMD(KEYSPACE, CF).compression(compressionParameters),
+ standardCFMD(KEYSPACE, CF2).compression(compressionParameters),
+ standardCFMD(KEYSPACE, CF3),
+ standardCFMD(KEYSPACE, CF4),
+ standardCFMD(KEYSPACE, CORRUPT_CF),
+ standardCFMD(KEYSPACE, CORRUPT_CF2),
+ counterCFMD(KEYSPACE, COUNTER_CF).compression(compressionParameters),
+ counterCFMD(KEYSPACE, COUNTER_CF2).compression(compressionParameters),
+ counterCFMD(KEYSPACE, COUNTER_CF3),
+ counterCFMD(KEYSPACE, COUNTER_CF4),
+ counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF),
+ counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF2),
+ standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance),
+ standardCFMD(KEYSPACE, BF_ALWAYS_PRESENT).bloomFilterFpChance(1.0));
+ }
+
+
+ @Test
+ public void testVerifyCorrect()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testVerifyCounterCorrect()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
+
+ fillCounterCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testExtendedVerifyCorrect()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testExtendedVerifyCounterCorrect()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF2);
+
+ fillCounterCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testVerifyCorrectUncompressed()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testVerifyCounterCorrectUncompressed()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF3);
+
+ fillCounterCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testExtendedVerifyCorrectUncompressed()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF4);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+ @Test
+ public void testExtendedVerifyCounterCorrectUncompressed()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF4);
+
+ fillCounterCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ fail("Unexpected CorruptSSTableException");
+ }
+ }
+
+
+ @Test
+ public void testVerifyIncorrectDigest() throws IOException, WriteTimeoutException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF);
+
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+
+ try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST)))
+ {
+ long correctChecksum = file.readLong();
+
+ writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
+ }
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(false).build()))
+ {
+ verifier.verify();
+ fail("Expected a RuntimeException to be thrown");
+ }
+ catch (RuntimeException expected)
+ {
+ }
+ }
+
+
+ @Test
+ public void testVerifyCorruptRowCorrectDigest() throws IOException, WriteTimeoutException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ // overwrite one row with garbage
+ long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ);
+ long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ);
+ long startPosition = Math.min(row0Start, row1Start);
+ long endPosition = Math.max(row0Start, row1Start);
+
+ try (FileChannel file = new File(sstable.getFilename()).newReadWriteChannel()) {
+ file.position(startPosition);
+ file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
+ }
+ if (ChunkCache.instance != null)
+ ChunkCache.instance.invalidateFile(sstable.getFilename());
+
+ // Update the Digest to have the right Checksum
+ writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.fileFor(Components.DIGEST));
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ // First a simple verify checking digest, which should succeed
+ try
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ logger.error("Unexpected exception", err);
+ fail("Simple verify should have succeeded as digest matched");
+ }
+ }
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
+ {
+ // Now try extended verify
+ try
+ {
+ verifier.verify();
+ }
+ catch (CorruptSSTableException err)
+ {
+ return;
+ }
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ }
+
+ @Test
+ public void testVerifyBrokenSSTableMetadata() throws IOException, WriteTimeoutException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+ cfs.truncateBlocking();
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ FileChannel file = sstable.descriptor.fileFor(Components.STATS).newReadWriteChannel();
+ file.position(0);
+ file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
+ file.close();
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(false).build()))
+ {
+ verifier.verify();
+ fail("Expected a RuntimeException to be thrown");
+ }
+ catch (CorruptSSTableException unexpected)
+ {
+ fail("wrong exception thrown");
+ }
+ catch (RuntimeException expected)
+ {
+ }
+ }
+
+ @Test
+ public void testVerifyMutateRepairStatus() throws IOException, WriteTimeoutException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+ cfs.truncateBlocking();
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ // make the sstable repaired:
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), sstable.getPendingRepair(), sstable.isTransient());
+ sstable.reloadSSTableMetadata();
+
+ // break the sstable:
+ long correctChecksum;
+ try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST)))
+ {
+ correctChecksum = file.readLong();
+ }
+ writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+
+ assertTrue(sstable.isRepaired());
+
+ // now the repair status should be changed:
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+ assertFalse(sstable.isRepaired());
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testOutOfRangeTokens() throws IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ fillCF(cfs, 100);
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ byte[] tk1 = new byte[1], tk2 = new byte[1];
+ tk1[0] = 2;
+ tk2[0] = 1;
+ tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkOwnsTokens(true).extendedVerification(true).build()))
+ {
+ verifier.verify();
+ }
+ finally
+ {
+ StorageService.instance.getTokenMetadata().clearUnsafe();
+ }
+ }
+
+ @Test
+ public void testMutateRepair() throws IOException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, sstable.getPendingRepair(), sstable.isTransient());
+ sstable.reloadSSTableMetadata();
+ cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
+ assertTrue(sstable.isRepaired());
+ cfs.forceMajorCompaction();
+
+ sstable = cfs.getLiveSSTables().iterator().next();
+ long correctChecksum;
+ try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST)))
+ {
+ correctChecksum = file.readLong();
+ }
+ writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true).build()))
+ {
+ verifier.verify();
+ fail("should be corrupt");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+ assertFalse(sstable.isRepaired());
+ }
+
+ @Test
+ public void testVerifyIndex() throws IOException
+ {
+ if (BigFormat.isSelected())
+ testBrokenComponentHelper(BigFormat.Components.PRIMARY_INDEX);
+ else if (BtiFormat.isSelected())
+ testBrokenComponentHelper(BtiFormat.Components.PARTITION_INDEX);
+ else
+ throw Util.testMustBeImplementedForSSTableFormat();
+ }
+
+ @Test
+ public void testVerifyBf() throws IOException
+ {
+ Assume.assumeTrue(SSTableReaderWithFilter.class.isAssignableFrom(DatabaseDescriptor.getSelectedSSTableFormat().getReaderFactory().getReaderClass()));
+ testBrokenComponentHelper(Components.FILTER);
+ }
+
+ @Test
+ public void testVerifyIndexSummary() throws IOException
+ {
+ Assume.assumeTrue(BigFormat.isSelected());
+ testBrokenComponentHelper(Components.SUMMARY);
+ }
+
+ private void testBrokenComponentHelper(Component componentToBreak) throws IOException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+
+ fillCF(cfs, 2);
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().build()))
+ {
+ verifier.verify(); //still not corrupt, should pass
+ }
+ try (FileChannel fileChannel = sstable.descriptor.fileFor(componentToBreak).newReadWriteChannel())
+ {
+ fileChannel.truncate(3);
+ }
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("should throw exception");
+ }
+ catch (CorruptSSTableException e)
+ {
+ //expected
+ }
+ }
+
+ @Test
+ public void testQuick() throws IOException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF);
+
+ fillCF(cfs, 2);
+
+ Util.getAll(Util.cmd(cfs).build());
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+
+ try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST)))
+ {
+ long correctChecksum = file.readLong();
+
+ writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
+ }
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).quick(true).build())) // with quick = true we don't verify the digest
+ {
+ verifier.verify();
+ }
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ {
+ verifier.verify();
+ fail("Expected a RuntimeException to be thrown");
+ }
+ catch (CorruptSSTableException expected)
+ {
+ }
+ }
+
+ @Test
+ public void testRangeOwnHelper()
+ {
+ List<Range<Token>> normalized = new ArrayList<>();
+ normalized.add(r(Long.MIN_VALUE, Long.MIN_VALUE + 1));
+ normalized.add(r(Long.MIN_VALUE + 5, Long.MIN_VALUE + 6));
+ normalized.add(r(Long.MIN_VALUE + 10, Long.MIN_VALUE + 11));
+ normalized.add(r(0, 10));
+ normalized.add(r(10, 11));
+ normalized.add(r(20, 25));
+ normalized.add(r(26, 200));
+
+ RangeOwnHelper roh = new RangeOwnHelper(normalized);
+
+ roh.validate(dk(1));
+ roh.validate(dk(10));
+ roh.validate(dk(11));
+ roh.validate(dk(21));
+ roh.validate(dk(25));
+ boolean gotException = false;
+ try
+ {
+ roh.validate(dk(26));
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testRangeOwnHelperBadToken()
+ {
+ List<Range<Token>> normalized = new ArrayList<>();
+ normalized.add(r(0, 10));
+ RangeOwnHelper roh = new RangeOwnHelper(normalized);
+ roh.validate(dk(1));
+ // call with smaller token to get exception
+ roh.validate(dk(0));
+ }
+
+
+ @Test
+ public void testRangeOwnHelperNormalize()
+ {
+ List<Range<Token>> normalized = Range.normalize(Collections.singletonList(r(0, 0)));
+ RangeOwnHelper roh = new RangeOwnHelper(normalized);
+ roh.validate(dk(Long.MIN_VALUE));
+ roh.validate(dk(0));
+ roh.validate(dk(Long.MAX_VALUE));
+ }
+
+ @Test
+ public void testRangeOwnHelperNormalizeWrap()
+ {
+ List<Range<Token>> normalized = Range.normalize(Collections.singletonList(r(Long.MAX_VALUE - 1000, Long.MIN_VALUE + 1000)));
+ RangeOwnHelper roh = new RangeOwnHelper(normalized);
+ roh.validate(dk(Long.MIN_VALUE));
+ roh.validate(dk(Long.MAX_VALUE));
+ boolean gotException = false;
+ try
+ {
+ roh.validate(dk(26));
+ }
+ catch (Throwable t)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ @Test
+ public void testEmptyRanges()
+ {
+ new RangeOwnHelper(Collections.emptyList()).validate(dk(1));
+ }
+
+ @Test
+ public void testVerifyLocalPartitioner() throws UnknownHostException
+ {
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ byte[] tk1 = new byte[1], tk2 = new byte[1];
+ tk1[0] = 2;
+ tk2[0] = 1;
+ tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk1), InetAddressAndPort.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2"));
+ // write some bogus to a localpartitioner table
+ Batch bogus = Batch.createLocal(nextTimeUUID(), 0, Collections.emptyList());
+ BatchlogManager.store(bogus);
+ ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("batches");
+ Util.flush(cfs);
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkOwnsTokens(true).build()))
+ {
+ verifier.verify();
+ }
+ }
+ }
+
+ @Test
+ public void testNoFilterFile()
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(BF_ALWAYS_PRESENT);
+ fillCF(cfs, 100);
+ assertEquals(1.0, cfs.metadata().params.bloomFilterFpChance, 0.0);
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ File f = sstable.descriptor.fileFor(Components.FILTER);
+ assertFalse(f.exists());
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().build()))
+ {
+ verifier.verify();
+ }
+ }
+ }
+
+
+ private DecoratedKey dk(long l)
+ {
+ return new BufferDecoratedKey(t(l), ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ }
+
+ private Range<Token> r(long s, long e)
+ {
+ return new Range<>(t(s), t(e));
+ }
+
+ private Token t(long t)
+ {
+ return new Murmur3Partitioner.LongToken(t);
+ }
+
+
+ protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
+ {
+ for (int i = 0; i < partitionsPerSSTable; i++)
+ {
+ UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+ .newRow("c1").add("val", "1")
+ .newRow("c2").add("val", "2")
+ .apply();
+ }
+
+ Util.flush(cfs);
+ }
+
+ protected void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException
+ {
+ for (int i = 0; i < partitionsPerSSTable; i++)
+ {
+ UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+ .newRow("c1").add("val", 100L)
+ .apply();
+ }
+
+ Util.flush(cfs);
+ }
+
+ protected long simpleFullChecksum(String filename) throws IOException
+ {
+ try (FileInputStreamPlus inputStream = new FileInputStreamPlus(filename);
+ CheckedInputStream cinStream = new CheckedInputStream(inputStream, new CRC32()))
+ {
+ byte[] b = new byte[128];
+ //noinspection StatementWithEmptyBody
+ while (cinStream.read(b) >= 0)
+ {
+ }
+ return cinStream.getChecksum().getValue();
+ }
+ }
+
+ public static void writeChecksum(long checksum, File file)
+ {
+ BufferedWriter out = null;
+ try
+ {
+ out = Files.newBufferedWriter(file.toPath(), Charsets.UTF_8);
+ out.write(String.valueOf(checksum));
+ out.flush();
+ out.close();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, file);
+ }
+ finally
+ {
+ FileUtils.closeQuietly(out);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java b/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java
new file mode 100644
index 0000000..3c95455
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.filter;
+
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class BloomFilterTrackerTest
+{
+ @Test
+ public void testAddingFalsePositives()
+ {
+ BloomFilterTracker bft = new BloomFilterTracker();
+ assertEquals(0L, bft.getFalsePositiveCount());
+ assertEquals(0L, bft.getRecentFalsePositiveCount());
+ bft.addFalsePositive();
+ bft.addFalsePositive();
+ assertEquals(2L, bft.getFalsePositiveCount());
+ assertEquals(2L, bft.getRecentFalsePositiveCount());
+ assertEquals(0L, bft.getRecentFalsePositiveCount());
+ assertEquals(2L, bft.getFalsePositiveCount()); // sanity check
+ }
+
+ @Test
+ public void testAddingTruePositives()
+ {
+ BloomFilterTracker bft = new BloomFilterTracker();
+ assertEquals(0L, bft.getTruePositiveCount());
+ assertEquals(0L, bft.getRecentTruePositiveCount());
+ bft.addTruePositive();
+ bft.addTruePositive();
+ assertEquals(2L, bft.getTruePositiveCount());
+ assertEquals(2L, bft.getRecentTruePositiveCount());
+ assertEquals(0L, bft.getRecentTruePositiveCount());
+ assertEquals(2L, bft.getTruePositiveCount()); // sanity check
+ }
+
+ @Test
+ public void testAddingToOneLeavesTheOtherAlone()
+ {
+ BloomFilterTracker bft = new BloomFilterTracker();
+ bft.addFalsePositive();
+ assertEquals(0L, bft.getTruePositiveCount());
+ assertEquals(0L, bft.getRecentTruePositiveCount());
+ bft.addTruePositive();
+ assertEquals(1L, bft.getFalsePositiveCount());
+ assertEquals(1L, bft.getRecentFalsePositiveCount());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java b/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java
new file mode 100644
index 0000000..a6c4c06
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.assertj.core.api.Assertions;
+
+public abstract class AbstractTestVersionSupportedFeatures
+{
+ protected static final List<String> ALL_VERSIONS = IntStream.rangeClosed('a', 'z')
+ .mapToObj(i -> String.valueOf((char) i))
+ .flatMap(first -> IntStream.rangeClosed('a', 'z').mapToObj(second -> first + (char) second))
+ .collect(Collectors.toList());
+
+ protected abstract Version getVersion(String v);
+
+ protected abstract Stream<String> getPendingRepairSupportedVersions();
+
+ protected abstract Stream<String> getPartitionLevelDeletionPresenceMarkerSupportedVersions();
+
+ protected abstract Stream<String> getLegacyMinMaxSupportedVersions();
+
+ protected abstract Stream<String> getImprovedMinMaxSupportedVersions();
+
+ protected abstract Stream<String> getKeyRangeSupportedVersions();
+
+ protected abstract Stream<String> getOriginatingHostIdSupportedVersions();
+
+ @BeforeClass
+ public static void initDD()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void testCompatibility()
+ {
+ checkPredicateAgainstVersions(Version::hasPendingRepair, getPendingRepairSupportedVersions());
+ checkPredicateAgainstVersions(Version::hasImprovedMinMax, getImprovedMinMaxSupportedVersions());
+ checkPredicateAgainstVersions(Version::hasLegacyMinMax, getLegacyMinMaxSupportedVersions());
+ checkPredicateAgainstVersions(Version::hasPartitionLevelDeletionsPresenceMarker, getPartitionLevelDeletionPresenceMarkerSupportedVersions());
+ checkPredicateAgainstVersions(Version::hasKeyRange, getKeyRangeSupportedVersions());
+ checkPredicateAgainstVersions(Version::hasOriginatingHostId, getOriginatingHostIdSupportedVersions());
+ }
+
+ public static Stream<String> range(String fromIncl, String toIncl)
+ {
+ int fromIdx = ALL_VERSIONS.indexOf(fromIncl);
+ int toIdx = ALL_VERSIONS.indexOf(toIncl);
+ assert fromIdx >= 0 && toIdx >= 0;
+ return ALL_VERSIONS.subList(fromIdx, toIdx + 1).stream();
+ }
+
+ /**
+ * Check the version predicate against the provided versions.
+ *
+ * @param predicate predicate to check against version
+ * @param versionBounds a stream of versions for which the predicate should return true
+ */
+ private void checkPredicateAgainstVersions(Predicate<Version> predicate, Stream<String> versionBounds)
+ {
+ List<String> expected = versionBounds.collect(Collectors.toList());
+ List<String> actual = ALL_VERSIONS.stream().filter(v -> predicate.test(getVersion(v))).collect(Collectors.toList());
+ Assertions.assertThat(actual).isEqualTo(expected);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java
deleted file mode 100644
index 14b48b7..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.io.sstable.format;
-
-import java.io.IOException;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.StorageService;
-
-import static org.junit.Assert.assertEquals;
-
-public class RangeAwareSSTableWriterTest
-{
- public static final String KEYSPACE1 = "Keyspace1";
- public static final String CF_STANDARD = "Standard1";
-
- public static ColumnFamilyStore cfs;
-
- @BeforeClass
- public static void defineSchema() throws Exception
- {
- DatabaseDescriptor.daemonInitialization();
- DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
- SchemaLoader.cleanupAndLeaveDirs();
- Keyspace.setInitialized();
- StorageService.instance.initServer();
-
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
- .partitioner(Murmur3Partitioner.instance));
-
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
- cfs.clearUnsafe();
- cfs.disableAutoCompaction();
- }
-
- @Test
- public void testAccessWriterBeforeAppend() throws IOException
- {
-
- SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, 0, 1);
- Util.flush(cfs);
-
- LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
-
- RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs,
- 0,
- 0,
- null,
- false,
- SSTableFormat.Type.BIG,
- 0,
- 0,
- txn,
- SerializationHeader.make(cfs.metadata(),
- cfs.getLiveSSTables()));
- assertEquals(cfs.metadata.id, writer.getTableId());
- assertEquals(0L, writer.getFilePointer());
-
- }
-}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java
deleted file mode 100644
index b5aaf8e..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.io.FSReadError;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-
-import org.junit.Assert;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class SSTableFlushObserverTest
-{
- @BeforeClass
- public static void initDD()
- {
- DatabaseDescriptor.daemonInitialization();
- CommitLog.instance.start();
- }
-
- private static final String KS_NAME = "test";
- private static final String CF_NAME = "flush_observer";
-
- @Test
- public void testFlushObserver()
- {
- TableMetadata cfm =
- TableMetadata.builder(KS_NAME, CF_NAME)
- .addPartitionKeyColumn("id", UTF8Type.instance)
- .addRegularColumn("first_name", UTF8Type.instance)
- .addRegularColumn("age", Int32Type.instance)
- .addRegularColumn("height", LongType.instance)
- .build();
-
- LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION);
- FlushObserver observer = new FlushObserver();
-
- String sstableDirectory = DatabaseDescriptor.getAllDataFileLocations()[0];
- File directory = new File(sstableDirectory + File.pathSeparator() + KS_NAME + File.pathSeparator() + CF_NAME);
- directory.deleteOnExit();
-
- if (!directory.exists() && !directory.tryCreateDirectories())
- throw new FSWriteError(new IOException("failed to create tmp directory"), directory.absolutePath());
-
- SSTableFormat.Type sstableFormat = SSTableFormat.Type.current();
- Descriptor descriptor = new Descriptor(sstableFormat.info.getLatestVersion(),
- directory,
- cfm.keyspace,
- cfm.name,
- new SequenceBasedSSTableId(0),
- sstableFormat);
-
- BigTableWriter writer = new BigTableWriter(descriptor,
- 10L, 0L, null, false, TableMetadataRef.forOfflineTools(cfm),
- new MetadataCollector(cfm.comparator).sstableLevel(0),
- new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS),
- Collections.singletonList(observer),
- transaction);
-
- SSTableReader reader = null;
- Multimap<ByteBuffer, Cell<?>> expected = ArrayListMultimap.create();
-
- try
- {
- final long now = System.currentTimeMillis();
-
- ByteBuffer key = UTF8Type.instance.fromString("key1");
- expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(27)),
- BufferCell.live(getColumn(cfm, "first_name"), now,UTF8Type.instance.fromString("jack")),
- BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(183L))));
-
- writer.append(new RowIterator(cfm, key.duplicate(), Collections.singletonList(buildRow(expected.get(key)))));
-
- key = UTF8Type.instance.fromString("key2");
- expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(30)),
- BufferCell.live(getColumn(cfm, "first_name"), now,UTF8Type.instance.fromString("jim")),
- BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(180L))));
-
- writer.append(new RowIterator(cfm, key, Collections.singletonList(buildRow(expected.get(key)))));
-
- key = UTF8Type.instance.fromString("key3");
- expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(30)),
- BufferCell.live(getColumn(cfm, "first_name"), now,UTF8Type.instance.fromString("ken")),
- BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(178L))));
-
- writer.append(new RowIterator(cfm, key, Collections.singletonList(buildRow(expected.get(key)))));
-
- reader = writer.finish(true);
- }
- finally
- {
- FileUtils.closeQuietly(writer);
- }
-
- Assert.assertTrue(observer.isComplete);
- Assert.assertEquals(expected.size(), observer.rows.size());
-
- for (Pair<ByteBuffer, Long> e : observer.rows.keySet())
- {
- ByteBuffer key = e.left;
- Long indexPosition = e.right;
-
- try (FileDataInput index = reader.ifile.createReader(indexPosition))
- {
- ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(index);
- Assert.assertEquals(0, UTF8Type.instance.compare(key, indexKey));
- }
- catch (IOException ex)
- {
- throw new FSReadError(ex, reader.getIndexFilename());
- }
-
- Assert.assertEquals(expected.get(key), observer.rows.get(e));
- }
- }
-
- private static class RowIterator extends AbstractUnfilteredRowIterator
- {
- private final Iterator<Unfiltered> rows;
-
- public RowIterator(TableMetadata cfm, ByteBuffer key, Collection<Unfiltered> content)
- {
- super(cfm,
- DatabaseDescriptor.getPartitioner().decorateKey(key),
- DeletionTime.LIVE,
- cfm.regularAndStaticColumns(),
- BTreeRow.emptyRow(Clustering.STATIC_CLUSTERING),
- false,
- EncodingStats.NO_STATS);
-
- rows = content.iterator();
- }
-
- @Override
- protected Unfiltered computeNext()
- {
- return rows.hasNext() ? rows.next() : endOfData();
- }
- }
-
- private static class FlushObserver implements SSTableFlushObserver
- {
- private final Multimap<Pair<ByteBuffer, Long>, Cell<?>> rows = ArrayListMultimap.create();
- private Pair<ByteBuffer, Long> currentKey;
- private boolean isComplete;
-
- @Override
- public void begin()
- {}
-
- @Override
- public void startPartition(DecoratedKey key, long indexPosition)
- {
- currentKey = Pair.create(key.getKey(), indexPosition);
- }
-
- @Override
- public void nextUnfilteredCluster(Unfiltered row)
- {
- if (row.isRow())
- ((Row) row).forEach((c) -> rows.put(currentKey, (Cell<?>) c));
- }
-
- @Override
- public void complete()
- {
- isComplete = true;
- }
- }
-
- private static Row buildRow(Collection<Cell<?>> cells)
- {
- Row.Builder rowBuilder = BTreeRow.sortedBuilder();
- rowBuilder.newRow(Clustering.EMPTY);
- cells.forEach(rowBuilder::addCell);
- return rowBuilder.build();
- }
-
- private static ColumnMetadata getColumn(TableMetadata cfm, String name)
- {
- return cfm.getColumn(UTF8Type.instance.fromString(name));
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java b/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java
deleted file mode 100644
index 4e62b9c..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable.format;
-
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.Assert.assertEquals;
-
-public class VersionAndTypeTest
-{
- @Test
- public void testValidInput()
- {
- assertEquals(VersionAndType.fromString("big-bc").toString(), "big-bc");
- }
-
- @Test
- public void testInvalidInputs()
- {
- assertThatThrownBy(() -> VersionAndType.fromString(" ")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("should be of the form 'big-bc'");
- assertThatThrownBy(() -> VersionAndType.fromString("mcc-")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("should be of the form 'big-bc'");
- assertThatThrownBy(() -> VersionAndType.fromString("mcc-d")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("No Type constant mcc");
- assertThatThrownBy(() -> VersionAndType.fromString("mcc-d-")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("No Type constant mcc");
- assertThatThrownBy(() -> VersionAndType.fromString("mcc")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("should be of the form 'big-bc'");
- assertThatThrownBy(() -> VersionAndType.fromString("-")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("should be of the form 'big-bc'");
- assertThatThrownBy(() -> VersionAndType.fromString("--")).isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("should be of the form 'big-bc'");
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java
deleted file mode 100644
index e6d2020..0000000
--- a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.io.sstable.format.big;
-
-import java.io.ByteArrayInputStream;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedChannelException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.Collection;
-import java.util.Set;
-import java.util.function.Function;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.cassandra.io.util.File;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import io.netty.buffer.Unpooled;
-import io.netty.channel.embedded.EmbeddedChannel;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.Slices;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.FileHandle;
-import org.apache.cassandra.net.AsyncStreamingInputPlus;
-import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.Pair;
-
-import static org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-public class BigTableZeroCopyWriterTest
-{
- public static final String KEYSPACE1 = "BigTableBlockWriterTest";
- public static final String CF_STANDARD = "Standard1";
- public static final String CF_STANDARD2 = "Standard2";
- public static final String CF_INDEXED = "Indexed1";
- public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
-
- public static SSTableReader sstable;
- public static ColumnFamilyStore store;
- private static int expectedRowCount;
-
- @BeforeClass
- public static void defineSchema() throws Exception
- {
- SchemaLoader.prepareServer();
- SchemaLoader.createKeyspace(KEYSPACE1,
- KeyspaceParams.simple(1),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
- SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED, true),
- SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWINDEXINTERVAL)
- .minIndexInterval(8)
- .maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING));
-
- String ks = KEYSPACE1;
- String cf = "Standard1";
-
- // clear and create just one sstable for this test
- Keyspace keyspace = Keyspace.open(ks);
- store = keyspace.getColumnFamilyStore(cf);
- store.clearUnsafe();
- store.disableAutoCompaction();
-
- DecoratedKey firstKey = null, lastKey = null;
- long timestamp = System.currentTimeMillis();
- for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- if (firstKey == null)
- firstKey = key;
- if (lastKey == null)
- lastKey = key;
- if (store.metadata().partitionKeyType.compare(lastKey.getKey(), key.getKey()) < 0)
- lastKey = key;
-
- new RowUpdateBuilder(store.metadata(), timestamp, key.getKey())
- .clustering("col")
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
- expectedRowCount++;
- }
- Util.flush(store);
-
- sstable = store.getLiveSSTables().iterator().next();
- }
-
- @Test
- public void writeDataFile_DataInputPlus()
- {
- writeDataTestCycle(buffer -> new DataInputStreamPlus(new ByteArrayInputStream(buffer.array())));
- }
-
- @Test
- public void writeDataFile_RebufferingByteBufDataInputPlus()
- {
- try (AsyncStreamingInputPlus input = new AsyncStreamingInputPlus(new EmbeddedChannel()))
- {
- writeDataTestCycle(buffer ->
- {
- input.append(Unpooled.wrappedBuffer(buffer));
- return input;
- });
-
- input.requestClosure();
- }
- }
-
-
- private void writeDataTestCycle(Function<ByteBuffer, DataInputPlus> bufferMapper)
- {
- File dir = store.getDirectories().getDirectoryForNewSSTables();
- Descriptor desc = store.newSSTableDescriptor(dir);
- TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
-
- LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM);
- Set<Component> componentsToWrite = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX,
- Component.STATS);
-
- BigTableZeroCopyWriter btzcw = new BigTableZeroCopyWriter(desc, metadata, txn, componentsToWrite);
-
- for (Component component : componentsToWrite)
- {
- if (Files.exists(Paths.get(desc.filenameFor(component))))
- {
- Pair<DataInputPlus, Long> pair = getSSTableComponentData(sstable, component, bufferMapper);
-
- try
- {
- btzcw.writeComponent(component.type, pair.left, pair.right);
- }
- catch (ClosedChannelException e)
- {
- throw new UncheckedIOException(e);
- }
- }
- }
-
- Collection<SSTableReader> readers = btzcw.finish(true);
-
- SSTableReader reader = readers.toArray(new SSTableReader[0])[0];
-
- assertNotEquals(sstable.getFilename(), reader.getFilename());
- assertEquals(sstable.estimatedKeys(), reader.estimatedKeys());
- assertEquals(sstable.isPendingRepair(), reader.isPendingRepair());
-
- assertRowCount(expectedRowCount);
- }
-
- private void assertRowCount(int expected)
- {
- int count = 0;
- for (int i = 0; i < store.metadata().params.minIndexInterval; i++)
- {
- DecoratedKey dk = Util.dk(String.valueOf(i));
- UnfilteredRowIterator rowIter = sstable.rowIterator(dk,
- Slices.ALL,
- ColumnFilter.all(store.metadata()),
- false,
- SSTableReadsListener.NOOP_LISTENER);
- while (rowIter.hasNext())
- {
- rowIter.next();
- count++;
- }
- }
- assertEquals(expected, count);
- }
-
- private Pair<DataInputPlus, Long> getSSTableComponentData(SSTableReader sstable, Component component,
- Function<ByteBuffer, DataInputPlus> bufferMapper)
- {
- FileHandle componentFile = new FileHandle.Builder(sstable.descriptor.filenameFor(component))
- .bufferSize(1024).complete();
- ByteBuffer buffer = ByteBuffer.allocate((int) componentFile.channel.size());
- componentFile.channel.read(buffer, 0);
- buffer.flip();
-
- DataInputPlus inputPlus = bufferMapper.apply(buffer);
-
- return Pair.create(inputPlus, componentFile.channel.size());
- }
-}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java
new file mode 100644
index 0000000..ca2a4c5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java
@@ -0,0 +1,859 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.primitives.Ints;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.IMeasurableMemory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.serializers.LongSerializer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.btree.BTree;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class RowIndexEntryTest extends CQLTester
+{
+ private static final List<AbstractType<?>> clusterTypes = Collections.singletonList(LongType.instance);
+ private static final ClusteringComparator comp = new ClusteringComparator(clusterTypes);
+
+ private static final byte[] dummy_100k = new byte[100000];
+
+ private static Clustering<?> cn(long l)
+ {
+ return Util.clustering(comp, l);
+ }
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ Assume.assumeTrue(BigFormat.isSelected());
+ }
+
+ @Test
+ public void testC11206AgainstPreviousArray() throws Exception
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(99999);
+ testC11206AgainstPrevious();
+ }
+
+ @Test
+ public void testC11206AgainstPreviousShallow() throws Exception
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(0);
+ testC11206AgainstPrevious();
+ }
+
+ private static void testC11206AgainstPrevious() throws Exception
+ {
+ // partition without IndexInfo
+ try (DoubleSerializer doubleSerializer = new DoubleSerializer())
+ {
+ doubleSerializer.build(null, partitionKey(42L),
+ Collections.singletonList(cn(42)),
+ 0L);
+ assertEquals(doubleSerializer.rieOldSerialized, doubleSerializer.rieNewSerialized);
+ }
+
+ // partition with multiple IndexInfo
+ try (DoubleSerializer doubleSerializer = new DoubleSerializer())
+ {
+ doubleSerializer.build(null, partitionKey(42L),
+ Arrays.asList(cn(42), cn(43), cn(44)),
+ 0L);
+ assertEquals(doubleSerializer.rieOldSerialized, doubleSerializer.rieNewSerialized);
+ }
+
+ // partition with multiple IndexInfo
+ try (DoubleSerializer doubleSerializer = new DoubleSerializer())
+ {
+ doubleSerializer.build(null, partitionKey(42L),
+ Arrays.asList(cn(42), cn(43), cn(44), cn(45), cn(46), cn(47), cn(48), cn(49), cn(50), cn(51)),
+ 0L);
+ assertEquals(doubleSerializer.rieOldSerialized, doubleSerializer.rieNewSerialized);
+ }
+ }
+
+ private static DecoratedKey partitionKey(long l)
+ {
+ ByteBuffer key = LongSerializer.instance.serialize(l);
+ Token token = Murmur3Partitioner.instance.getToken(key);
+ return new BufferDecoratedKey(token, key);
+ }
+
+ private static class DoubleSerializer implements AutoCloseable
+ {
+ TableMetadata metadata =
+ CreateTableStatement.parse("CREATE TABLE pipe.dev_null (pk bigint, ck bigint, val text, PRIMARY KEY(pk, ck))", "foo")
+ .build();
+
+ Version version = BigFormat.getInstance().getLatestVersion();
+
+ DeletionTime deletionInfo = new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds());
+ LivenessInfo primaryKeyLivenessInfo = LivenessInfo.EMPTY;
+ Row.Deletion deletion = Row.Deletion.LIVE;
+
+ SerializationHeader header = new SerializationHeader(true, metadata, metadata.regularAndStaticColumns(), EncodingStats.NO_STATS);
+
+ // create C-11206 + old serializer instances
+ RowIndexEntry.IndexSerializer rieSerializer = new RowIndexEntry.Serializer(version, header, null);
+ Pre_C_11206_RowIndexEntry.Serializer oldSerializer = new Pre_C_11206_RowIndexEntry.Serializer(metadata, version, header);
+
+ @SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" })
+ final DataOutputBuffer rieOutput = new DataOutputBuffer(1024);
+ @SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" })
+ final DataOutputBuffer oldOutput = new DataOutputBuffer(1024);
+
+ final SequentialWriter dataWriterNew;
+ final SequentialWriter dataWriterOld;
+ final BigFormatPartitionWriter partitionWriter;
+
+ RowIndexEntry rieNew;
+ ByteBuffer rieNewSerialized;
+ Pre_C_11206_RowIndexEntry rieOld;
+ ByteBuffer rieOldSerialized;
+
+ DoubleSerializer() throws IOException
+ {
+ SequentialWriterOption option = SequentialWriterOption.newBuilder().bufferSize(1024).build();
+ File f = FileUtils.createTempFile("RowIndexEntryTest-", "db");
+ dataWriterNew = new SequentialWriter(f, option);
+ partitionWriter = new BigFormatPartitionWriter(header, dataWriterNew, version, rieSerializer.indexInfoSerializer());
+
+ f = FileUtils.createTempFile("RowIndexEntryTest-", "db");
+ dataWriterOld = new SequentialWriter(f, option);
+ }
+
+ public void close() throws Exception
+ {
+ dataWriterNew.close();
+ dataWriterOld.close();
+ }
+
+ void build(Row staticRow, DecoratedKey partitionKey,
+ Collection<Clustering<?>> clusterings, long startPosition) throws IOException
+ {
+
+ Iterator<Clustering<?>> clusteringIter = clusterings.iterator();
+ partitionWriter.start(partitionKey, DeletionTime.LIVE);
+ if (staticRow != null)
+ partitionWriter.addStaticRow(staticRow);
+ AbstractUnfilteredRowIterator rowIter = makeRowIter(staticRow, partitionKey, clusteringIter, dataWriterNew);
+ while (rowIter.hasNext())
+ partitionWriter.addUnfiltered(rowIter.next());
+ partitionWriter.finish();
+
+ rieNew = RowIndexEntry.create(startPosition, 0L,
+ deletionInfo, partitionWriter.getHeaderLength(), partitionWriter.getColumnIndexCount(),
+ partitionWriter.indexInfoSerializedSize(),
+ partitionWriter.indexSamples(), partitionWriter.offsets(),
+ rieSerializer.indexInfoSerializer());
+ rieSerializer.serialize(rieNew, rieOutput, partitionWriter.buffer());
+ rieNewSerialized = rieOutput.buffer().duplicate();
+
+ Iterator<Clustering<?>> clusteringIter2 = clusterings.iterator();
+ ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(makeRowIter(staticRow, partitionKey, clusteringIter2, dataWriterOld),
+ dataWriterOld, header, Collections.emptySet(), BigFormat.getInstance().getLatestVersion());
+ rieOld = Pre_C_11206_RowIndexEntry.create(startPosition, deletionInfo, columnIndex);
+ oldSerializer.serialize(rieOld, oldOutput);
+ rieOldSerialized = oldOutput.buffer().duplicate();
+ }
+
+ private AbstractUnfilteredRowIterator makeRowIter(Row staticRow, DecoratedKey partitionKey,
+ Iterator<Clustering<?>> clusteringIter, SequentialWriter dataWriter)
+ {
+ return new AbstractUnfilteredRowIterator(metadata, partitionKey, deletionInfo, metadata.regularAndStaticColumns(),
+ staticRow, false, new EncodingStats(0, 0, 0))
+ {
+ protected Unfiltered computeNext()
+ {
+ if (!clusteringIter.hasNext())
+ return endOfData();
+ try
+ {
+ // write some fake bytes to the data file to force writing the IndexInfo object
+ dataWriter.write(dummy_100k);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return buildRow(clusteringIter.next());
+ }
+ };
+ }
+
+ private Unfiltered buildRow(Clustering<?> clustering)
+ {
+ BTree.Builder<ColumnData> builder = BTree.builder(ColumnData.comparator);
+ builder.add(BufferCell.live(metadata.regularAndStaticColumns().iterator().next(),
+ 1L,
+ ByteBuffer.allocate(0)));
+ return BTreeRow.create(clustering, primaryKeyLivenessInfo, deletion, builder.build());
+ }
+ }
+
+ /**
+ * Pre C-11206 code.
+ */
+ static final class ColumnIndex
+ {
+ final long partitionHeaderLength;
+ final List<IndexInfo> columnsIndex;
+
+ private static final ColumnIndex EMPTY = new ColumnIndex(-1, Collections.emptyList());
+
+ private ColumnIndex(long partitionHeaderLength, List<IndexInfo> columnsIndex)
+ {
+ assert columnsIndex != null;
+
+ this.partitionHeaderLength = partitionHeaderLength;
+ this.columnsIndex = columnsIndex;
+ }
+
+ static ColumnIndex writeAndBuildIndex(UnfilteredRowIterator iterator,
+ SequentialWriter output,
+ SerializationHeader header,
+ Collection<SSTableFlushObserver> observers,
+ Version version) throws IOException
+ {
+ assert !iterator.isEmpty();
+
+ Builder builder = new Builder(iterator, output, header, observers, version.correspondingMessagingVersion());
+ return builder.build();
+ }
+
+ public static ColumnIndex nothing()
+ {
+ return EMPTY;
+ }
+
+ /**
+ * Help to create an index for a column family based on size of columns,
+ * and write said columns to disk.
+ */
+ private static class Builder
+ {
+ private final UnfilteredRowIterator iterator;
+ private final SequentialWriter writer;
+ private final SerializationHelper helper;
+ private final SerializationHeader header;
+ private final int version;
+
+ private final List<IndexInfo> columnsIndex = new ArrayList<>();
+ private final long initialPosition;
+ private long headerLength = -1;
+
+ private long startPosition = -1;
+
+ private int written;
+ private long previousRowStart;
+
+ private ClusteringPrefix<?> firstClustering;
+ private ClusteringPrefix<?> lastClustering;
+
+ private DeletionTime openMarker;
+
+ private final Collection<SSTableFlushObserver> observers;
+
+ Builder(UnfilteredRowIterator iterator,
+ SequentialWriter writer,
+ SerializationHeader header,
+ Collection<SSTableFlushObserver> observers,
+ int version)
+ {
+ this.iterator = iterator;
+ this.writer = writer;
+ this.helper = new SerializationHelper(header);
+ this.header = header;
+ this.version = version;
+ this.observers = observers == null ? Collections.emptyList() : observers;
+ this.initialPosition = writer.position();
+ }
+
+ private void writePartitionHeader(UnfilteredRowIterator iterator) throws IOException
+ {
+ ByteBufferUtil.writeWithShortLength(iterator.partitionKey().getKey(), writer);
+ DeletionTime.serializer.serialize(iterator.partitionLevelDeletion(), writer);
+ if (header.hasStatic())
+ UnfilteredSerializer.serializer.serializeStaticRow(iterator.staticRow(), helper, writer, version);
+ }
+
+ public ColumnIndex build() throws IOException
+ {
+ writePartitionHeader(iterator);
+ this.headerLength = writer.position() - initialPosition;
+
+ while (iterator.hasNext())
+ add(iterator.next());
+
+ return close();
+ }
+
+ private long currentPosition()
+ {
+ return writer.position() - initialPosition;
+ }
+
+ private void addIndexBlock()
+ {
+ IndexInfo cIndexInfo = new IndexInfo(firstClustering,
+ lastClustering,
+ startPosition,
+ currentPosition() - startPosition,
+ openMarker);
+ columnsIndex.add(cIndexInfo);
+ firstClustering = null;
+ }
+
+ private void add(Unfiltered unfiltered) throws IOException
+ {
+ long pos = currentPosition();
+
+ if (firstClustering == null)
+ {
+ // Beginning of an index block. Remember the start and position
+ firstClustering = unfiltered.clustering();
+ startPosition = pos;
+ }
+
+ UnfilteredSerializer.serializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version);
+
+ // notify observers about each new row
+ if (!observers.isEmpty())
+ observers.forEach((o) -> o.nextUnfilteredCluster(unfiltered));
+
+ lastClustering = unfiltered.clustering();
+ previousRowStart = pos;
+ ++written;
+
+ if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
+ {
+ RangeTombstoneMarker marker = (RangeTombstoneMarker)unfiltered;
+ openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null;
+ }
+
+ // if we hit the column index size that we have to index after, go ahead and index it.
+ if (currentPosition() - startPosition >= DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY))
+ addIndexBlock();
+
+ }
+
+ private ColumnIndex close() throws IOException
+ {
+ UnfilteredSerializer.serializer.writeEndOfPartition(writer);
+
+ // It's possible we add no rows, just a top level deletion
+ if (written == 0)
+ return RowIndexEntryTest.ColumnIndex.EMPTY;
+
+ // the last column may have fallen on an index boundary already. if not, index it explicitly.
+ if (firstClustering != null)
+ addIndexBlock();
+
+ // we should always have at least one computed index block, but we only write it out if there is more than that.
+ assert !columnsIndex.isEmpty() && headerLength >= 0;
+ return new ColumnIndex(headerLength, columnsIndex);
+ }
+ }
+ }
+
+ @Test
+ public void testSerializedSize() throws Throwable
+ {
+ String tableName = createTable("CREATE TABLE %s (a int, b text, c int, PRIMARY KEY(a, b))");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+
+ Pre_C_11206_RowIndexEntry simple = new Pre_C_11206_RowIndexEntry(123);
+
+ DataOutputBuffer buffer = new DataOutputBuffer();
+ SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
+ Pre_C_11206_RowIndexEntry.Serializer serializer = new Pre_C_11206_RowIndexEntry.Serializer(cfs.metadata(), BigFormat.getInstance().getLatestVersion(), header);
+
+ serializer.serialize(simple, buffer);
+
+ assertEquals(buffer.getLength(), serializer.serializedSize(simple));
+
+ // write enough rows to ensure we get a few column index entries
+ for (int i = 0; i <= DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY) / 4; i++)
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, String.valueOf(i), i);
+
+ ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build());
+
+ File tempFile = FileUtils.createTempFile("row_index_entry_test", null);
+ tempFile.deleteOnExit();
+ SequentialWriter writer = new SequentialWriter(tempFile);
+ ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(partition.unfilteredIterator(), writer, header, Collections.emptySet(), BigFormat.getInstance().getLatestVersion());
+ Pre_C_11206_RowIndexEntry withIndex = Pre_C_11206_RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex);
+ IndexInfo.Serializer indexSerializer = IndexInfo.serializer(BigFormat.getInstance().getLatestVersion(), header);
+
+ // sanity check
+ assertTrue(columnIndex.columnsIndex.size() >= 3);
+
+ buffer = new DataOutputBuffer();
+ serializer.serialize(withIndex, buffer);
+ assertEquals(buffer.getLength(), serializer.serializedSize(withIndex));
+
+ // serialization check
+
+ ByteBuffer bb = buffer.buffer();
+ DataInputBuffer input = new DataInputBuffer(bb, false);
+ serializationCheck(withIndex, indexSerializer, bb, input);
+
+ // test with an output stream that doesn't support a file-pointer
+ buffer = new DataOutputBuffer()
+ {
+ public boolean hasPosition()
+ {
+ return false;
+ }
+
+ public long position()
+ {
+ throw new UnsupportedOperationException();
+ }
+ };
+ serializer.serialize(withIndex, buffer);
+ bb = buffer.buffer();
+ input = new DataInputBuffer(bb, false);
+ serializationCheck(withIndex, indexSerializer, bb, input);
+
+ //
+
+ bb = buffer.buffer();
+ input = new DataInputBuffer(bb, false);
+ Pre_C_11206_RowIndexEntry.Serializer.skip(input, BigFormat.getInstance().getLatestVersion());
+ Assert.assertEquals(0, bb.remaining());
+ }
+
+ private static void serializationCheck(Pre_C_11206_RowIndexEntry withIndex, IndexInfo.Serializer indexSerializer, ByteBuffer bb, DataInputBuffer input) throws IOException
+ {
+ Assert.assertEquals(0xdeadbeef, input.readUnsignedVInt());
+ Assert.assertEquals(withIndex.promotedSize(indexSerializer), input.readUnsignedVInt());
+
+ Assert.assertEquals(withIndex.headerLength(), input.readUnsignedVInt());
+ Assert.assertEquals(withIndex.deletionTime(), DeletionTime.serializer.deserialize(input));
+ Assert.assertEquals(withIndex.columnsIndex().size(), input.readUnsignedVInt());
+
+ int offset = bb.position();
+ int[] offsets = new int[withIndex.columnsIndex().size()];
+ for (int i = 0; i < withIndex.columnsIndex().size(); i++)
+ {
+ int pos = bb.position();
+ offsets[i] = pos - offset;
+ IndexInfo info = indexSerializer.deserialize(input);
+ int end = bb.position();
+
+ Assert.assertEquals(indexSerializer.serializedSize(info), end - pos);
+
+ Assert.assertEquals(withIndex.columnsIndex().get(i).offset, info.offset);
+ Assert.assertEquals(withIndex.columnsIndex().get(i).width, info.width);
+ Assert.assertEquals(withIndex.columnsIndex().get(i).endOpenMarker, info.endOpenMarker);
+ Assert.assertEquals(withIndex.columnsIndex().get(i).firstName, info.firstName);
+ Assert.assertEquals(withIndex.columnsIndex().get(i).lastName, info.lastName);
+ }
+
+ for (int i = 0; i < withIndex.columnsIndex().size(); i++)
+ Assert.assertEquals(offsets[i], input.readInt());
+
+ Assert.assertEquals(0, bb.remaining());
+ }
+
+ static class Pre_C_11206_RowIndexEntry implements IMeasurableMemory
+ {
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new Pre_C_11206_RowIndexEntry(0));
+
+ public final long position;
+
+ Pre_C_11206_RowIndexEntry(long position)
+ {
+ this.position = position;
+ }
+
+ protected int promotedSize(IndexInfo.Serializer idxSerializer)
+ {
+ return 0;
+ }
+
+ public static Pre_C_11206_RowIndexEntry create(long position, DeletionTime deletionTime, ColumnIndex index)
+ {
+ assert index != null;
+ assert deletionTime != null;
+
+ // we only consider the columns summary when determining whether to create an IndexedEntry,
+ // since if there are insufficient columns to be worth indexing we're going to seek to
+ // the beginning of the row anyway, so we might as well read the tombstone there as well.
+ if (index.columnsIndex.size() > 1)
+ return new Pre_C_11206_RowIndexEntry.IndexedEntry(position, deletionTime, index.partitionHeaderLength, index.columnsIndex);
+ else
+ return new Pre_C_11206_RowIndexEntry(position);
+ }
+
+ /**
+ * @return true if this index entry contains the row-level tombstone and column summary. Otherwise,
+ * caller should fetch these from the row header.
+ */
+ public boolean isIndexed()
+ {
+ return !columnsIndex().isEmpty();
+ }
+
+ public DeletionTime deletionTime()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * The length of the row header (partition key, partition deletion and static row).
+ * This value is only provided for indexed entries and this method will throw
+ * {@code UnsupportedOperationException} if {@code !isIndexed()}.
+ */
+ public long headerLength()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public List<IndexInfo> columnsIndex()
+ {
+ return Collections.emptyList();
+ }
+
+ public long unsharedHeapSize()
+ {
+ return EMPTY_SIZE;
+ }
+
+ public static class Serializer
+ {
+ private final IndexInfo.Serializer idxSerializer;
+ private final Version version;
+
+ Serializer(TableMetadata metadata, Version version, SerializationHeader header)
+ {
+ this.idxSerializer = IndexInfo.serializer(version, header);
+ this.version = version;
+ }
+
+ public void serialize(Pre_C_11206_RowIndexEntry rie, DataOutputPlus out) throws IOException
+ {
+ out.writeUnsignedVInt(rie.position);
+ out.writeUnsignedVInt32(rie.promotedSize(idxSerializer));
+
+ if (rie.isIndexed())
+ {
+ out.writeUnsignedVInt(rie.headerLength());
+ DeletionTime.serializer.serialize(rie.deletionTime(), out);
+ out.writeUnsignedVInt32(rie.columnsIndex().size());
+
+ // Calculate and write the offsets to the IndexInfo objects.
+
+ int[] offsets = new int[rie.columnsIndex().size()];
+
+ if (out.hasPosition())
+ {
+ // Out is usually a SequentialWriter, so using the file-pointer is fine to generate the offsets.
+ // A DataOutputBuffer also works.
+ long start = out.position();
+ int i = 0;
+ for (IndexInfo info : rie.columnsIndex())
+ {
+ offsets[i] = i == 0 ? 0 : (int)(out.position() - start);
+ i++;
+ idxSerializer.serialize(info, out);
+ }
+ }
+ else
+ {
+ // Not sure this branch will ever be needed, but if it is called, it has to calculate the
+ // serialized sizes instead of simply using the file-pointer.
+ int i = 0;
+ int offset = 0;
+ for (IndexInfo info : rie.columnsIndex())
+ {
+ offsets[i++] = offset;
+ idxSerializer.serialize(info, out);
+ offset += idxSerializer.serializedSize(info);
+ }
+ }
+
+ for (int off : offsets)
+ out.writeInt(off);
+ }
+ }
+
+ public Pre_C_11206_RowIndexEntry deserialize(DataInputPlus in) throws IOException
+ {
+ long position = in.readUnsignedVInt();
+
+ int size = in.readUnsignedVInt32();
+ if (size > 0)
+ {
+ long headerLength = in.readUnsignedVInt();
+ DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
+ int entries = in.readUnsignedVInt32();
+ List<IndexInfo> columnsIndex = new ArrayList<>(entries);
+ for (int i = 0; i < entries; i++)
+ columnsIndex.add(idxSerializer.deserialize(in));
+
+ in.skipBytesFully(entries * TypeSizes.sizeof(0));
+
+ return new Pre_C_11206_RowIndexEntry.IndexedEntry(position, deletionTime, headerLength, columnsIndex);
+ }
+ else
+ {
+ return new Pre_C_11206_RowIndexEntry(position);
+ }
+ }
+
+ // Reads only the data 'position' of the index entry and returns it. Note that this left 'in' in the middle
+ // of reading an entry, so this is only useful if you know what you are doing and in most case 'deserialize'
+ // should be used instead.
+ static long readPosition(DataInputPlus in, Version version) throws IOException
+ {
+ return in.readUnsignedVInt();
+ }
+
+ public static void skip(DataInputPlus in, Version version) throws IOException
+ {
+ readPosition(in, version);
+ skipPromotedIndex(in, version);
+ }
+
+ private static void skipPromotedIndex(DataInputPlus in, Version version) throws IOException
+ {
+ int size = in.readUnsignedVInt32();
+ if (size <= 0)
+ return;
+
+ in.skipBytesFully(size);
+ }
+
+ public int serializedSize(Pre_C_11206_RowIndexEntry rie)
+ {
+ int indexedSize = 0;
+ if (rie.isIndexed())
+ {
+ List<IndexInfo> index = rie.columnsIndex();
+
+ indexedSize += TypeSizes.sizeofUnsignedVInt(rie.headerLength());
+ indexedSize += DeletionTime.serializer.serializedSize(rie.deletionTime());
+ indexedSize += TypeSizes.sizeofUnsignedVInt(index.size());
+
+ for (IndexInfo info : index)
+ indexedSize += idxSerializer.serializedSize(info);
+
+ indexedSize += index.size() * TypeSizes.sizeof(0);
+ }
+
+ return TypeSizes.sizeofUnsignedVInt(rie.position) + TypeSizes.sizeofUnsignedVInt(indexedSize) + indexedSize;
+ }
+ }
+
+ /**
+ * An entry in the row index for a row whose columns are indexed.
+ */
+ private static final class IndexedEntry extends Pre_C_11206_RowIndexEntry
+ {
+ private final DeletionTime deletionTime;
+
+ // The offset in the file when the index entry end
+ private final long headerLength;
+ private final List<IndexInfo> columnsIndex;
+ private static final long BASE_SIZE =
+ ObjectSizes.measure(new IndexedEntry(0, DeletionTime.LIVE, 0, Arrays.asList(null, null)))
+ + ObjectSizes.measure(new ArrayList<>(1));
+
+ private IndexedEntry(long position, DeletionTime deletionTime, long headerLength, List<IndexInfo> columnsIndex)
+ {
+ super(position);
+ assert deletionTime != null;
+ assert columnsIndex != null && columnsIndex.size() > 1;
+ this.deletionTime = deletionTime;
+ this.headerLength = headerLength;
+ this.columnsIndex = columnsIndex;
+ }
+
+ @Override
+ public DeletionTime deletionTime()
+ {
+ return deletionTime;
+ }
+
+ @Override
+ public long headerLength()
+ {
+ return headerLength;
+ }
+
+ @Override
+ public List<IndexInfo> columnsIndex()
+ {
+ return columnsIndex;
+ }
+
+ @Override
+ protected int promotedSize(IndexInfo.Serializer idxSerializer)
+ {
+ long size = TypeSizes.sizeofUnsignedVInt(headerLength)
+ + DeletionTime.serializer.serializedSize(deletionTime)
+ + TypeSizes.sizeofUnsignedVInt(columnsIndex.size()); // number of entries
+ for (IndexInfo info : columnsIndex)
+ size += idxSerializer.serializedSize(info);
+
+ size += columnsIndex.size() * TypeSizes.sizeof(0);
+
+ return Ints.checkedCast(size);
+ }
+
+ @Override
+ public long unsharedHeapSize()
+ {
+ long entrySize = 0;
+ for (IndexInfo idx : columnsIndex)
+ entrySize += idx.unsharedHeapSize();
+
+ return BASE_SIZE
+ + entrySize
+ + deletionTime.unsharedHeapSize()
+ + ObjectSizes.sizeOfReferenceArray(columnsIndex.size());
+ }
+ }
+ }
+
+ @Test
+ public void testIndexFor() throws IOException
+ {
+ DeletionTime deletionInfo = new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds());
+
+ List<IndexInfo> indexes = new ArrayList<>();
+ indexes.add(new IndexInfo(cn(0L), cn(5L), 0, 0, deletionInfo));
+ indexes.add(new IndexInfo(cn(10L), cn(15L), 0, 0, deletionInfo));
+ indexes.add(new IndexInfo(cn(20L), cn(25L), 0, 0, deletionInfo));
+
+ RowIndexEntry rie = new RowIndexEntry(0L)
+ {
+ public IndexInfoRetriever openWithIndex(FileHandle indexFile)
+ {
+ return new IndexInfoRetriever()
+ {
+ public IndexInfo columnsIndex(int index)
+ {
+ return indexes.get(index);
+ }
+
+ public void close()
+ {
+ }
+ };
+ }
+
+ public int blockCount()
+ {
+ return indexes.size();
+ }
+ };
+
+ IndexState indexState = new IndexState(
+ null, comp, rie, false, null
+ );
+
+ assertEquals(0, indexState.indexFor(cn(-1L), -1));
+ assertEquals(0, indexState.indexFor(cn(5L), -1));
+ assertEquals(1, indexState.indexFor(cn(12L), -1));
+ assertEquals(2, indexState.indexFor(cn(17L), -1));
+ assertEquals(3, indexState.indexFor(cn(100L), -1));
+ assertEquals(3, indexState.indexFor(cn(100L), 0));
+ assertEquals(3, indexState.indexFor(cn(100L), 1));
+ assertEquals(3, indexState.indexFor(cn(100L), 2));
+ assertEquals(3, indexState.indexFor(cn(100L), 3));
+
+ indexState = new IndexState(
+ null, comp, rie, true, null
+ );
+
+ assertEquals(-1, indexState.indexFor(cn(-1L), -1));
+ assertEquals(0, indexState.indexFor(cn(5L), 3));
+ assertEquals(0, indexState.indexFor(cn(5L), 2));
+ assertEquals(1, indexState.indexFor(cn(17L), 3));
+ assertEquals(2, indexState.indexFor(cn(100L), 3));
+ assertEquals(2, indexState.indexFor(cn(100L), 4));
+ assertEquals(1, indexState.indexFor(cn(12L), 3));
+ assertEquals(1, indexState.indexFor(cn(12L), 2));
+ assertEquals(1, indexState.indexFor(cn(100L), 1));
+ assertEquals(2, indexState.indexFor(cn(100L), 2));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java
new file mode 100644
index 0000000..a74b99e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.util.stream.Stream;
+
+import org.apache.cassandra.io.sstable.format.AbstractTestVersionSupportedFeatures;
+import org.apache.cassandra.io.sstable.format.Version;
+
+public class VersionSupportedFeaturesTest extends AbstractTestVersionSupportedFeatures
+{
+ @Override
+ protected Version getVersion(String v)
+ {
+ return BigFormat.getInstance().getVersion(v);
+ }
+
+ @Override
+ protected Stream<String> getPendingRepairSupportedVersions()
+ {
+ return range("na", "zz");
+ }
+
+ @Override
+ protected Stream<String> getPartitionLevelDeletionPresenceMarkerSupportedVersions()
+ {
+ return range("nc", "zz");
+ }
+
+ @Override
+ protected Stream<String> getLegacyMinMaxSupportedVersions()
+ {
+ return range("ma", "nz");
+ }
+
+ @Override
+ protected Stream<String> getImprovedMinMaxSupportedVersions()
+ {
+ return range("nc", "zz");
+ }
+
+ @Override
+ protected Stream<String> getKeyRangeSupportedVersions()
+ {
+ return range("nc", "zz");
+ }
+
+ @Override
+ protected Stream<String> getOriginatingHostIdSupportedVersions()
+ {
+ return Stream.concat(range("me", "mz"), range("nb", "zz"));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
new file mode 100644
index 0000000..8f24ced
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
@@ -0,0 +1,888 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multiset;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.ChunkCache;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.io.tries.TrieNode;
+import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.io.util.WrappingRebufferer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class PartitionIndexTest
+{
+ private final static Logger logger = LoggerFactory.getLogger(PartitionIndexTest.class);
+
+ private final static long SEED = System.nanoTime();
+ private final static Random random = new Random(SEED);
+
+ static final ByteComparable.Version VERSION = Walker.BYTE_COMPARABLE_VERSION;
+
+ static
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ static final IPartitioner partitioner = Util.testPartitioner();
+ //Lower the size of the indexes when running without the chunk cache, otherwise the test times out on Jenkins
+ static final int COUNT = ChunkCache.instance != null ? 245256 : 24525;
+
+ @Parameterized.Parameters()
+ public static Collection<Object[]> generateData()
+ {
+ return Arrays.asList(new Object[]{ Config.DiskAccessMode.standard },
+ new Object[]{ Config.DiskAccessMode.mmap });
+ }
+
+ @Parameterized.Parameter(value = 0)
+ public static Config.DiskAccessMode accessMode = Config.DiskAccessMode.standard;
+
+ @BeforeClass
+ public static void beforeClass()
+ {
+ logger.info("Using random seed: {}", SEED);
+ }
+
+ /**
+ * Tests last-nodes-sizing failure uncovered during code review.
+ */
+ @Test
+ public void testSizingBug() throws IOException
+ {
+ for (int i = 1; i < COUNT; i *= 10)
+ {
+ testGetEq(generateRandomIndex(i));
+ testGetEq(generateSequentialIndex(i));
+ }
+ }
+
+ @Test
+ public void testGetEq() throws IOException
+ {
+ testGetEq(generateRandomIndex(COUNT));
+ testGetEq(generateSequentialIndex(COUNT));
+ }
+
+ @Test
+ public void testBrokenFile() throws IOException
+ {
+ // put some garbage in the file
+ final Pair<List<DecoratedKey>, PartitionIndex> data = generateRandomIndex(COUNT);
+ File f = new File(data.right.getFileHandle().path());
+ try (FileChannel ch = FileChannel.open(f.toPath(), StandardOpenOption.WRITE))
+ {
+ ch.write(generateRandomKey().getKey(), f.length() * 2 / 3);
+ }
+
+ assertThatThrownBy(() -> testGetEq(data)).isInstanceOfAny(AssertionError.class, IndexOutOfBoundsException.class, IllegalArgumentException.class);
+ }
+
+ @Test
+ public void testLongKeys() throws IOException
+ {
+ testGetEq(generateLongKeysIndex(COUNT / 10));
+ }
+
+ void testGetEq(Pair<List<DecoratedKey>, PartitionIndex> data)
+ {
+ List<DecoratedKey> keys = data.left;
+ try (PartitionIndex summary = data.right;
+ PartitionIndex.Reader reader = summary.openReader())
+ {
+ for (int i = 0; i < data.left.size(); i++)
+ {
+ assertEquals(i, reader.exactCandidate(keys.get(i)));
+ DecoratedKey key = generateRandomKey();
+ assertEquals(eq(keys, key), eq(keys, key, reader.exactCandidate(key)));
+ }
+ }
+ }
+
+ @Test
+ public void testGetGt() throws IOException
+ {
+ testGetGt(generateRandomIndex(COUNT));
+ testGetGt(generateSequentialIndex(COUNT));
+ }
+
+ private void testGetGt(Pair<List<DecoratedKey>, PartitionIndex> data) throws IOException
+ {
+ List<DecoratedKey> keys = data.left;
+ try (PartitionIndex summary = data.right;
+ PartitionIndex.Reader reader = summary.openReader())
+ {
+ for (int i = 0; i < data.left.size(); i++)
+ {
+ assertEquals(i < data.left.size() - 1 ? i + 1 : -1, gt(keys, keys.get(i), reader));
+ DecoratedKey key = generateRandomKey();
+ assertEquals(gt(keys, key), gt(keys, key, reader));
+ }
+ }
+ }
+
+ @Test
+ public void testGetGe() throws IOException
+ {
+ testGetGe(generateRandomIndex(COUNT));
+ testGetGe(generateSequentialIndex(COUNT));
+ }
+
+ public void testGetGe(Pair<List<DecoratedKey>, PartitionIndex> data) throws IOException
+ {
+ List<DecoratedKey> keys = data.left;
+ try (PartitionIndex summary = data.right;
+ PartitionIndex.Reader reader = summary.openReader())
+ {
+ for (int i = 0; i < data.left.size(); i++)
+ {
+ assertEquals(i, ge(keys, keys.get(i), reader));
+ DecoratedKey key = generateRandomKey();
+ assertEquals(ge(keys, key), ge(keys, key, reader));
+ }
+ }
+ }
+
+
+ @Test
+ public void testGetLt() throws IOException
+ {
+ testGetLt(generateRandomIndex(COUNT));
+ testGetLt(generateSequentialIndex(COUNT));
+ }
+
+ public void testGetLt(Pair<List<DecoratedKey>, PartitionIndex> data) throws IOException
+ {
+ List<DecoratedKey> keys = data.left;
+ try (PartitionIndex summary = data.right;
+ PartitionIndex.Reader reader = summary.openReader())
+ {
+ for (int i = 0; i < data.left.size(); i++)
+ {
+ assertEquals(i - 1, lt(keys, keys.get(i), reader));
+ DecoratedKey key = generateRandomKey();
+ assertEquals(lt(keys, key), lt(keys, key, reader));
+ }
+ }
+ }
+
+ private long gt(List<DecoratedKey> keys, DecoratedKey key, PartitionIndex.Reader summary) throws IOException
+ {
+ return Optional.ofNullable(summary.ceiling(key, (pos, assumeNoMatch, sk) -> (assumeNoMatch || keys.get((int) pos).compareTo(sk) > 0) ? pos : null)).orElse(-1L);
+ }
+
+ private long ge(List<DecoratedKey> keys, DecoratedKey key, PartitionIndex.Reader summary) throws IOException
+ {
+ return Optional.ofNullable(summary.ceiling(key, (pos, assumeNoMatch, sk) -> (assumeNoMatch || keys.get((int) pos).compareTo(sk) >= 0) ? pos : null)).orElse(-1L);
+ }
+
+
+ private long lt(List<DecoratedKey> keys, DecoratedKey key, PartitionIndex.Reader summary) throws IOException
+ {
+ return Optional.ofNullable(summary.floor(key, (pos, assumeNoMatch, sk) -> (assumeNoMatch || keys.get((int) pos).compareTo(sk) < 0) ? pos : null)).orElse(-1L);
+ }
+
+ private long eq(List<DecoratedKey> keys, DecoratedKey key, long exactCandidate)
+ {
+ int idx = (int) exactCandidate;
+ if (exactCandidate == PartitionIndex.NOT_FOUND)
+ return -1;
+ return (keys.get(idx).equals(key)) ? idx : -1;
+ }
+
+ private long gt(List<DecoratedKey> keys, DecoratedKey key)
+ {
+ int index = Collections.binarySearch(keys, key);
+ if (index < 0)
+ index = -1 - index;
+ else
+ ++index;
+ return index < keys.size() ? index : -1;
+ }
+
+ private long lt(List<DecoratedKey> keys, DecoratedKey key)
+ {
+ int index = Collections.binarySearch(keys, key);
+
+ if (index < 0)
+ index = -index - 2;
+
+ return index >= 0 ? index : -1;
+ }
+
+ private long ge(List<DecoratedKey> keys, DecoratedKey key)
+ {
+ int index = Collections.binarySearch(keys, key);
+ if (index < 0)
+ index = -1 - index;
+ return index < keys.size() ? index : -1;
+ }
+
+ private long eq(List<DecoratedKey> keys, DecoratedKey key)
+ {
+ int index = Collections.binarySearch(keys, key);
+ return index >= 0 ? index : -1;
+ }
+
+ @Test
+ public void testAddEmptyKey() throws Exception
+ {
+ IPartitioner p = new RandomPartitioner();
+ File file = FileUtils.createTempFile("ColumnTrieReaderTest", "");
+
+ FileHandle.Builder fhBuilder = makeHandle(file);
+ try (SequentialWriter writer = makeWriter(file);
+ PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+ )
+ {
+ DecoratedKey key = p.decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ builder.addEntry(key, 42);
+ builder.complete();
+ try (PartitionIndex summary = loadPartitionIndex(fhBuilder, writer);
+ PartitionIndex.Reader reader = summary.openReader())
+ {
+ assertEquals(1, summary.size());
+ assertEquals(42, reader.getLastIndexPosition());
+ assertEquals(42, reader.exactCandidate(key));
+ }
+ }
+ }
+
+ @Test
+ public void testIteration() throws IOException
+ {
+ Pair<List<DecoratedKey>, PartitionIndex> random = generateRandomIndex(COUNT);
+ checkIteration(random.left.size(), random.right);
+ random.right.close();
+ }
+
+ public void checkIteration(int keysSize, PartitionIndex index)
+ {
+ try (PartitionIndex enforceIndexClosing = index;
+ PartitionIndex.IndexPosIterator iter = index.allKeysIterator())
+ {
+ int i = 0;
+ while (true)
+ {
+ long pos = iter.nextIndexPos();
+ if (pos == PartitionIndex.NOT_FOUND)
+ break;
+ assertEquals(i, pos);
+ ++i;
+ }
+ assertEquals(keysSize, i);
+ }
+ }
+
+ @Test
+ public void testConstrainedIteration() throws IOException
+ {
+ Pair<List<DecoratedKey>, PartitionIndex> random = generateRandomIndex(COUNT);
+ try (PartitionIndex summary = random.right)
+ {
+ List<DecoratedKey> keys = random.left;
+ Random rand = new Random();
+
+ for (int i = 0; i < 1000; ++i)
+ {
+ boolean exactLeft = rand.nextBoolean();
+ boolean exactRight = rand.nextBoolean();
+ DecoratedKey left = exactLeft ? keys.get(rand.nextInt(keys.size())) : generateRandomKey();
+ DecoratedKey right = exactRight ? keys.get(rand.nextInt(keys.size())) : generateRandomKey();
+ if (right.compareTo(left) < 0)
+ {
+ DecoratedKey t = left;
+ left = right;
+ right = t;
+ boolean b = exactLeft;
+ exactLeft = exactRight;
+ exactRight = b;
+ }
+
+ try (PartitionIndex.IndexPosIterator iter = new PartitionIndex.IndexPosIterator(summary, left, right))
+ {
+ long p = iter.nextIndexPos();
+ if (p == PartitionIndex.NOT_FOUND)
+ {
+ int idx = (int) ge(keys, left); // first greater key
+ if (idx == -1)
+ continue;
+ assertTrue(left + " <= " + keys.get(idx) + " <= " + right + " but " + idx + " wasn't iterated.", right.compareTo(keys.get(idx)) < 0);
+ continue;
+ }
+
+ int idx = (int) p;
+ if (p > 0)
+ assertTrue(left.compareTo(keys.get(idx - 1)) > 0);
+ if (p < keys.size() - 1)
+ assertTrue(left.compareTo(keys.get(idx + 1)) < 0);
+ if (exactLeft) // must be precise on exact, otherwise could be in any relation
+ assertSame(left, keys.get(idx));
+ while (true)
+ {
+ ++idx;
+ long pos = iter.nextIndexPos();
+ if (pos == PartitionIndex.NOT_FOUND)
+ break;
+ assertEquals(idx, pos);
+ }
+ --idx; // seek at last returned
+ if (idx < keys.size() - 1)
+ assertTrue(right.compareTo(keys.get(idx + 1)) < 0);
+ if (idx > 0)
+ assertTrue(right.compareTo(keys.get(idx - 1)) > 0);
+ if (exactRight) // must be precise on exact, otherwise could be in any relation
+ assertSame(right, keys.get(idx));
+ }
+ catch (AssertionError e)
+ {
+ StringBuilder buf = new StringBuilder();
+ buf.append(String.format("Left %s%s Right %s%s%n", left.byteComparableAsString(VERSION), exactLeft ? "#" : "", right.byteComparableAsString(VERSION), exactRight ? "#" : ""));
+ try (PartitionIndex.IndexPosIterator iter2 = new PartitionIndex.IndexPosIterator(summary, left, right))
+ {
+ long pos;
+ while ((pos = iter2.nextIndexPos()) != PartitionIndex.NOT_FOUND)
+ buf.append(keys.get((int) pos).byteComparableAsString(VERSION)).append("\n");
+ buf.append(String.format("Left %s%s Right %s%s%n", left.byteComparableAsString(VERSION), exactLeft ? "#" : "", right.byteComparableAsString(VERSION), exactRight ? "#" : ""));
+ }
+ logger.error(buf.toString(), e);
+ throw e;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testPartialIndex() throws IOException
+ {
+ for (int reps = 0; reps < 10; ++reps)
+ {
+ File file = FileUtils.createTempFile("ColumnTrieReaderTest", "");
+ List<DecoratedKey> list = Lists.newArrayList();
+ int parts = 15;
+ FileHandle.Builder fhBuilder = makeHandle(file);
+ try (SequentialWriter writer = makeWriter(file);
+ PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+ )
+ {
+ writer.setPostFlushListener(() -> builder.markPartitionIndexSynced(writer.getLastFlushOffset()));
+ for (int i = 0; i < COUNT; i++)
+ {
+ DecoratedKey key = generateRandomLengthKey();
+ list.add(key);
+ }
+ Collections.sort(list);
+ AtomicInteger callCount = new AtomicInteger();
+
+ int i = 0;
+ for (int part = 1; part <= parts; ++part)
+ {
+ for (; i < COUNT * part / parts; i++)
+ builder.addEntry(list.get(i), i);
+
+ final long addedSize = i;
+ builder.buildPartial(index ->
+ {
+ int indexSize = Collections.binarySearch(list, index.lastKey()) + 1;
+ assert indexSize >= addedSize - 1;
+ checkIteration(indexSize, index);
+ callCount.incrementAndGet();
+ }, 0, i * 1024L);
+ builder.markDataSynced(i * 1024L);
+ // verifier will be called when the sequentialWriter finishes a chunk
+ }
+
+ for (; i < COUNT; ++i)
+ builder.addEntry(list.get(i), i);
+ builder.complete();
+ try (PartitionIndex index = loadPartitionIndex(fhBuilder, writer))
+ {
+ checkIteration(list.size(), index);
+ }
+ if (COUNT / parts > 16000)
+ {
+ assertTrue(String.format("Expected %d or %d calls, got %d", parts, parts - 1, callCount.get()),
+ callCount.get() == parts - 1 || callCount.get() == parts);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDeepRecursion()
+ {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ // Check that long repeated strings don't cause stack overflow
+ // Test both normal completion and partial construction.
+ Thread t = new Thread(null, () ->
+ {
+ try
+ {
+ File file = FileUtils.createTempFile("ColumnTrieReaderTest", "");
+ SequentialWriter writer = new SequentialWriter(file, SequentialWriterOption.newBuilder().finishOnClose(true).build());
+ List<DecoratedKey> list = Lists.newArrayList();
+ String longString = "";
+ for (int i = 0; i < PageAware.PAGE_SIZE + 99; ++i)
+ {
+ longString += i;
+ }
+ IPartitioner partitioner = ByteOrderedPartitioner.instance;
+ list.add(partitioner.decorateKey(ByteBufferUtil.bytes(longString + "A")));
+ list.add(partitioner.decorateKey(ByteBufferUtil.bytes(longString + "B")));
+ list.add(partitioner.decorateKey(ByteBufferUtil.bytes(longString + "C")));
+ list.add(partitioner.decorateKey(ByteBufferUtil.bytes(longString + "D")));
+ list.add(partitioner.decorateKey(ByteBufferUtil.bytes(longString + "E")));
+
+ FileHandle.Builder fhBuilder = new FileHandle.Builder(file)
+ .bufferSize(PageAware.PAGE_SIZE)
+ .withChunkCache(ChunkCache.instance);
+ try (PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder))
+ {
+ int i = 0;
+ for (i = 0; i < 3; ++i)
+ builder.addEntry(list.get(i), i);
+
+ writer.setPostFlushListener(() -> builder.markPartitionIndexSynced(writer.getLastFlushOffset()));
+ AtomicInteger callCount = new AtomicInteger();
+
+ final int addedSize = i;
+ builder.buildPartial(index ->
+ {
+ int indexSize = Collections.binarySearch(list, index.lastKey()) + 1;
+ assert indexSize >= addedSize - 1;
+ checkIteration(indexSize, index);
+ index.close();
+ callCount.incrementAndGet();
+ }, 0, i * 1024L);
+
+ for (; i < list.size(); ++i)
+ builder.addEntry(list.get(i), i);
+ builder.complete();
+
+ try (PartitionIndex index = PartitionIndex.load(fhBuilder, partitioner, true))
+ {
+ checkIteration(list.size(), index);
+ }
+ }
+ future.complete(null);
+ }
+ catch (Throwable err)
+ {
+ future.completeExceptionally(err);
+ }
+ }, "testThread", 32 * 1024);
+
+ t.start();
+ future.join();
+ }
+
+ class JumpingFile extends SequentialWriter
+ {
+ long[] cutoffs;
+ long[] offsets;
+
+ JumpingFile(File file, SequentialWriterOption option, long... cutoffsAndOffsets)
+ {
+ super(file, option);
+ assert (cutoffsAndOffsets.length & 1) == 0;
+ cutoffs = new long[cutoffsAndOffsets.length / 2];
+ offsets = new long[cutoffs.length];
+ for (int i = 0; i < cutoffs.length; ++i)
+ {
+ cutoffs[i] = cutoffsAndOffsets[i * 2];
+ offsets[i] = cutoffsAndOffsets[i * 2 + 1];
+ }
+ }
+
+ @Override
+ public long position()
+ {
+ return jumped(super.position(), cutoffs, offsets);
+ }
+ }
+
+ class JumpingRebufferer extends WrappingRebufferer
+ {
+ long[] cutoffs;
+ long[] offsets;
+
+ JumpingRebufferer(Rebufferer source, long... cutoffsAndOffsets)
+ {
+ super(source);
+ assert (cutoffsAndOffsets.length & 1) == 0;
+ cutoffs = new long[cutoffsAndOffsets.length / 2];
+ offsets = new long[cutoffs.length];
+ for (int i = 0; i < cutoffs.length; ++i)
+ {
+ cutoffs[i] = cutoffsAndOffsets[i * 2];
+ offsets[i] = cutoffsAndOffsets[i * 2 + 1];
+ }
+ }
+
+ @Override
+ public BufferHolder rebuffer(long position)
+ {
+ long pos;
+
+ int idx = Arrays.binarySearch(offsets, position);
+ if (idx < 0)
+ idx = -2 - idx;
+ pos = position;
+ if (idx >= 0)
+ pos = pos - offsets[idx] + cutoffs[idx];
+
+ super.rebuffer(pos);
+
+ if (idx < cutoffs.length - 1 && buffer.limit() + offset > cutoffs[idx + 1])
+ buffer.limit((int) (cutoffs[idx + 1] - offset));
+ if (idx >= 0)
+ offset = offset - cutoffs[idx] + offsets[idx];
+
+ return this;
+ }
+
+ @Override
+ public long fileLength()
+ {
+ return jumped(wrapped.fileLength(), cutoffs, offsets);
+ }
+
+ @Override
+ public String toString()
+ {
+ return Arrays.toString(cutoffs) + Arrays.toString(offsets);
+ }
+ }
+
+ public class PartitionIndexJumping extends PartitionIndex
+ {
+ final long[] cutoffsAndOffsets;
+
+ public PartitionIndexJumping(FileHandle fh, long trieRoot, long keyCount, DecoratedKey first, DecoratedKey last,
+ long... cutoffsAndOffsets)
+ {
+ super(fh, trieRoot, keyCount, first, last);
+ this.cutoffsAndOffsets = cutoffsAndOffsets;
+ }
+
+ @Override
+ protected Rebufferer instantiateRebufferer()
+ {
+ return new JumpingRebufferer(super.instantiateRebufferer(), cutoffsAndOffsets);
+ }
+ }
+
+ long jumped(long pos, long[] cutoffs, long[] offsets)
+ {
+ int idx = Arrays.binarySearch(cutoffs, pos);
+ if (idx < 0)
+ idx = -2 - idx;
+ if (idx < 0)
+ return pos;
+ return pos - cutoffs[idx] + offsets[idx];
+ }
+
+ @Test
+ public void testPointerGrowth() throws IOException
+ {
+ for (int reps = 0; reps < 10; ++reps)
+ {
+ File file = FileUtils.createTempFile("ColumnTrieReaderTest", "");
+ long[] cutoffsAndOffsets = new long[]{
+ 2 * 4096, 1L << 16,
+ 4 * 4096, 1L << 24,
+ 6 * 4096, 1L << 31,
+ 8 * 4096, 1L << 32,
+ 10 * 4096, 1L << 33,
+ 12 * 4096, 1L << 34,
+ 14 * 4096, 1L << 40,
+ 16 * 4096, 1L << 42
+ };
+
+ List<DecoratedKey> list = Lists.newArrayList();
+ FileHandle.Builder fhBuilder = makeHandle(file);
+ try (SequentialWriter writer = makeJumpingWriter(file, cutoffsAndOffsets);
+ PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+ )
+ {
+ writer.setPostFlushListener(() -> builder.markPartitionIndexSynced(writer.getLastFlushOffset()));
+ for (int i = 0; i < COUNT; i++)
+ {
+ DecoratedKey key = generateRandomKey();
+ list.add(key);
+ }
+ Collections.sort(list);
+
+ for (int i = 0; i < COUNT; ++i)
+ builder.addEntry(list.get(i), i);
+ long root = builder.complete();
+
+ try (FileHandle fh = fhBuilder.complete();
+ PartitionIndex index = new PartitionIndexJumping(fh, root, COUNT, null, null, cutoffsAndOffsets);
+ Analyzer analyzer = new Analyzer(index))
+ {
+ checkIteration(list.size(), index);
+
+ analyzer.run();
+ if (analyzer.countPerType.elementSet().size() < 7)
+ {
+ Assert.fail("Expecting at least 7 different node types, got " + analyzer.countPerType.elementSet().size() + "\n" + analyzer.countPerType);
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDumpTrieToFile() throws IOException
+ {
+ File file = FileUtils.createTempFile("testDumpTrieToFile", "index");
+
+ ArrayList<DecoratedKey> list = Lists.newArrayList();
+ FileHandle.Builder fhBuilder = makeHandle(file);
+ try (SequentialWriter writer = new SequentialWriter(file, SequentialWriterOption.DEFAULT);
+ PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+ )
+ {
+ writer.setPostFlushListener(() -> builder.markPartitionIndexSynced(writer.getLastFlushOffset()));
+ for (int i = 0; i < 1000; i++)
+ {
+ DecoratedKey key = generateRandomKey();
+ list.add(key);
+ }
+ Collections.sort(list);
+
+ for (int i = 0; i < 1000; ++i)
+ builder.addEntry(list.get(i), i);
+ long root = builder.complete();
+
+ try (FileHandle fh = fhBuilder.complete();
+ PartitionIndex index = new PartitionIndex(fh, root, 1000, null, null))
+ {
+ File dump = FileUtils.createTempFile("testDumpTrieToFile", "dumpedTrie");
+ index.dumpTrie(dump.toString());
+ String dumpContent = String.join("\n", Files.readAllLines(dump.toPath()));
+ logger.info("Dumped trie: \n{}", dumpContent);
+ assertFalse(dumpContent.isEmpty());
+ }
+ }
+ }
+
+ public static class Analyzer extends PartitionIndex.Reader
+ {
+ Multiset<TrieNode> countPerType = HashMultiset.create();
+
+ public Analyzer(PartitionIndex index)
+ {
+ super(index);
+ }
+
+ public void run()
+ {
+ run(root);
+ }
+
+ void run(long node)
+ {
+ go(node);
+
+ countPerType.add(nodeType);
+
+ int tr = transitionRange();
+ for (int i = 0; i < tr; ++i)
+ {
+ long child = transition(i);
+ if (child == NONE)
+ continue;
+ run(child);
+ go(node);
+ }
+ }
+ }
+
+
+ private Pair<List<DecoratedKey>, PartitionIndex> generateRandomIndex(int size) throws IOException
+ {
+ return generateIndex(size, this::generateRandomKey);
+ }
+
+ Pair<List<DecoratedKey>, PartitionIndex> generateLongKeysIndex(int size) throws IOException
+ {
+ return generateIndex(size, this::generateLongKey);
+ }
+
+ private Pair<List<DecoratedKey>, PartitionIndex> generateSequentialIndex(int size) throws IOException
+ {
+ return generateIndex(size, new Supplier<DecoratedKey>()
+ {
+ long i = 0;
+
+ public DecoratedKey get()
+ {
+ return sequentialKey(i++);
+ }
+ });
+ }
+
+ Pair<List<DecoratedKey>, PartitionIndex> generateIndex(int size, Supplier<DecoratedKey> keyGenerator) throws IOException
+ {
+ File file = FileUtils.createTempFile("ColumnTrieReaderTest", "");
+ List<DecoratedKey> list = Lists.newArrayList();
+ FileHandle.Builder fhBuilder = makeHandle(file);
+ try (SequentialWriter writer = makeWriter(file);
+ PartitionIndexBuilder builder = new PartitionIndexBuilder(writer, fhBuilder)
+ )
+ {
+ for (int i = 0; i < size; i++)
+ {
+ DecoratedKey key = keyGenerator.get();
+ list.add(key);
+ }
+ Collections.sort(list);
+
+ for (int i = 0; i < size; i++)
+ builder.addEntry(list.get(i), i);
+ builder.complete();
+
+ PartitionIndex summary = loadPartitionIndex(fhBuilder, writer);
+
+ return Pair.create(list, summary);
+ }
+ }
+
+ DecoratedKey generateRandomKey()
+ {
+ UUID uuid = new UUID(random.nextLong(), random.nextLong());
+ return partitioner.decorateKey(ByteBufferUtil.bytes(uuid));
+ }
+
+ DecoratedKey generateRandomLengthKey()
+ {
+ Random rand = ThreadLocalRandom.current();
+ int length = nextPowerRandom(rand, 100, 10, 2); // favor long strings
+ StringBuilder s = new StringBuilder();
+ for (int i = 0; i < length; ++i)
+ s.append(alphabet.charAt(nextPowerRandom(rand, 0, alphabet.length(), 2))); // favor clashes at a
+
+ return partitioner.decorateKey(ByteBufferUtil.bytes(s.toString()));
+ }
+
+ DecoratedKey generateLongKey()
+ {
+ Random rand = ThreadLocalRandom.current();
+ int length = nextPowerRandom(rand, 10000, 2000, 2); // favor long strings
+ StringBuilder s = new StringBuilder();
+ for (int i = 0; i < length; ++i)
+ s.append(alphabet.charAt(nextPowerRandom(rand, 0, alphabet.length(), 2))); // favor clashes at a
+
+ return partitioner.decorateKey(ByteBufferUtil.bytes(s.toString()));
+ }
+
+ int nextPowerRandom(Random rand, int x0, int x1, double power)
+ {
+ double r = Math.pow(rand.nextDouble(), power);
+ return x0 + (int) ((x1 - x0) * r);
+ }
+
+ private static final String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+ DecoratedKey sequentialKey(long i)
+ {
+ String s = "";
+ for (int j = 50; j >= 0; j--)
+ {
+ int p = (int) Math.pow(10, j);
+ int idx = (int) ((j + i) / p);
+ s += alphabet.charAt(idx % alphabet.length());
+ }
+ return partitioner.decorateKey(ByteBufferUtil.bytes(s));
+ }
+
+
+ protected SequentialWriter makeWriter(File file)
+ {
+ return new SequentialWriter(file, SequentialWriterOption.newBuilder().finishOnClose(false).build());
+ }
+
+ public SequentialWriter makeJumpingWriter(File file, long[] cutoffsAndOffsets)
+ {
+ return new JumpingFile(file, SequentialWriterOption.newBuilder().finishOnClose(true).build(), cutoffsAndOffsets);
+ }
+
+ protected FileHandle.Builder makeHandle(File file)
+ {
+ return new FileHandle.Builder(file)
+ .bufferSize(PageAware.PAGE_SIZE)
+ .mmapped(accessMode == Config.DiskAccessMode.mmap)
+ .withChunkCache(ChunkCache.instance);
+ }
+
+ protected PartitionIndex loadPartitionIndex(FileHandle.Builder fhBuilder, SequentialWriter writer) throws IOException
+ {
+ return PartitionIndex.load(fhBuilder, partitioner, false);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java
new file mode 100644
index 0000000..059f6e9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java
@@ -0,0 +1,517 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.io.sstable.format.bti.RowIndexReader.IndexInfo;
+import org.apache.cassandra.io.tries.Walker;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class RowIndexTest
+{
+ private final static Logger logger = LoggerFactory.getLogger(RowIndexTest.class);
+
+ static final ByteComparable.Version VERSION = Walker.BYTE_COMPARABLE_VERSION;
+
+ static final Random RANDOM;
+
+ static
+ {
+ long seed = System.currentTimeMillis();
+ logger.info("seed = " + seed);
+ RANDOM = new Random(seed);
+
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ static final ClusteringComparator comparator = new ClusteringComparator(UUIDType.instance);
+ static final long END_MARKER = 1L << 40;
+ static final int COUNT = 8192;
+
+ @Parameterized.Parameters()
+ public static Collection<Object[]> generateData()
+ {
+ return Arrays.asList(new Object[]{ Config.DiskAccessMode.standard },
+ new Object[]{ Config.DiskAccessMode.mmap });
+ }
+
+ @Parameterized.Parameter(value = 0)
+ public static Config.DiskAccessMode accessMode = Config.DiskAccessMode.standard;
+
+ @Test
+ public void testSingletons() throws IOException
+ {
+ Pair<List<ClusteringPrefix<?>>, RowIndexReader> random = generateRandomIndexSingletons(COUNT);
+ RowIndexReader summary = random.right;
+ List<ClusteringPrefix<?>> keys = random.left;
+ for (int i = 0; i < COUNT; i++)
+ {
+ assertEquals(i, summary.separatorFloor(comparator.asByteComparable(keys.get(i))).offset);
+ }
+ summary.close();
+ }
+
+ @Test
+ public void testSpans() throws IOException
+ {
+ Pair<List<ClusteringPrefix<?>>, RowIndexReader> random = generateRandomIndexQuads(COUNT);
+ RowIndexReader summary = random.right;
+ List<ClusteringPrefix<?>> keys = random.left;
+ int missCount = 0;
+ IndexInfo ii;
+ for (int i = 0; i < COUNT; i++)
+ {
+ // These need to all be within the span
+ assertEquals(i, (ii = summary.separatorFloor(comparator.asByteComparable(keys.get(4 * i + 1)))).offset);
+ assertEquals(i, summary.separatorFloor(comparator.asByteComparable(keys.get(4 * i + 2))).offset);
+ assertEquals(i, summary.separatorFloor(comparator.asByteComparable(keys.get(4 * i + 3))).offset);
+
+ // check other data
+ assertEquals(i + 2, ii.openDeletion.markedForDeleteAt());
+ assertEquals(i - 3, ii.openDeletion.localDeletionTime());
+
+ // before entry. hopefully here, but could end up in prev if matches prevMax too well
+ ii = summary.separatorFloor(comparator.asByteComparable(keys.get(4 * i)));
+ if (ii.offset != i)
+ {
+ ++missCount;
+ assertEquals(i - 1, ii.offset);
+ }
+ }
+ ii = summary.separatorFloor(comparator.asByteComparable(keys.get(4 * COUNT)));
+ if (ii.offset != END_MARKER)
+ {
+ ++missCount;
+ assertEquals(COUNT - 1, ii.offset);
+ }
+ ii = summary.separatorFloor(comparator.asByteComparable(ClusteringBound.BOTTOM));
+ assertEquals(0, ii.offset);
+
+ ii = summary.separatorFloor(comparator.asByteComparable(ClusteringBound.TOP));
+ assertEquals(END_MARKER, ii.offset);
+
+ summary.close();
+ if (missCount > COUNT / 5)
+ logger.error("Unexpectedly high miss count: {}/{}", missCount, COUNT);
+ }
+
+ File file;
+ DataOutputStreamPlus dos;
+ RowIndexWriter writer;
+ FileHandle fh;
+ long root;
+
+ @After
+ public void cleanUp()
+ {
+ FileUtils.closeQuietly(dos);
+ FileUtils.closeQuietly(writer);
+ FileUtils.closeQuietly(fh);
+ }
+
+ public RowIndexTest() throws IOException
+ {
+ this(FileUtils.createTempFile("ColumnTrieReaderTest", ""));
+ }
+
+ RowIndexTest(File file) throws IOException
+ {
+ this(file, new SequentialWriter(file, SequentialWriterOption.newBuilder().finishOnClose(true).build()));
+ }
+
+ RowIndexTest(File file, DataOutputStreamPlus dos) throws IOException
+ {
+ this.file = file;
+ this.dos = dos;
+
+ // write some junk
+ dos.writeUTF("JUNK");
+ dos.writeUTF("JUNK");
+
+ writer = new RowIndexWriter(comparator, dos);
+ }
+
+ public void complete() throws IOException
+ {
+ root = writer.complete(END_MARKER);
+ dos.writeUTF("JUNK");
+ dos.writeUTF("JUNK");
+ dos.close();
+ dos = null;
+ }
+
+ public RowIndexReader completeAndRead() throws IOException
+ {
+ complete();
+
+ FileHandle.Builder builder = new FileHandle.Builder(file).mmapped(accessMode == Config.DiskAccessMode.mmap);
+ fh = builder.complete();
+ try (RandomAccessReader rdr = fh.createReader())
+ {
+ assertEquals("JUNK", rdr.readUTF());
+ assertEquals("JUNK", rdr.readUTF());
+ }
+ return new RowIndexReader(fh, root);
+ }
+
+ @Test
+ public void testAddEmptyKey() throws Exception
+ {
+ ClusteringPrefix<?> key = Clustering.EMPTY;
+ writer.add(key, key, new IndexInfo(42, DeletionTime.LIVE));
+ try (RowIndexReader summary = completeAndRead())
+ {
+ IndexInfo i = summary.min();
+ assertEquals(42, i.offset);
+
+ i = summary.separatorFloor(comparator.asByteComparable(ClusteringBound.BOTTOM));
+ assertEquals(42, i.offset);
+
+ i = summary.separatorFloor(comparator.asByteComparable(ClusteringBound.TOP));
+ assertEquals(END_MARKER, i.offset);
+
+ i = summary.separatorFloor(comparator.asByteComparable(key));
+ assertEquals(42, i.offset);
+ }
+ }
+
+ @Test
+ public void testAddDuplicateEmptyThrow() throws Exception
+ {
+ ClusteringPrefix<?> key = Clustering.EMPTY;
+ Throwable t = null;
+ writer.add(key, key, new IndexInfo(42, DeletionTime.LIVE));
+ try
+ {
+ writer.add(key, key, new IndexInfo(43, DeletionTime.LIVE));
+ try (RowIndexReader summary = completeAndRead())
+ {
+ // failing path
+ }
+ }
+ catch (AssertionError e)
+ {
+ // correct path
+ t = e;
+ logger.info("Got " + e.getMessage());
+ }
+ Assert.assertNotNull("Should throw an assertion error.", t);
+ }
+
+ @Test
+ public void testAddDuplicateThrow() throws Exception
+ {
+ ClusteringPrefix<?> key = generateRandomKey();
+ Throwable t = null;
+ writer.add(key, key, new IndexInfo(42, DeletionTime.LIVE));
+ try
+ {
+ writer.add(key, key, new IndexInfo(43, DeletionTime.LIVE));
+ try (RowIndexReader summary = completeAndRead())
+ {
+ // failing path
+ }
+ }
+ catch (AssertionError e)
+ {
+ // correct path
+ t = e;
+ logger.info("Got " + e.getMessage());
+ }
+ Assert.assertNotNull("Should throw an assertion error.", t);
+ }
+
+ @Test
+ public void testAddOutOfOrderThrow() throws Exception
+ {
+ ClusteringPrefix<?> key1 = generateRandomKey();
+ ClusteringPrefix<?> key2 = generateRandomKey();
+ while (comparator.compare(key1, key2) <= 0) // make key2 smaller than 1
+ key2 = generateRandomKey();
+
+ Throwable t = null;
+ writer.add(key1, key1, new IndexInfo(42, DeletionTime.LIVE));
+ try
+ {
+ writer.add(key2, key2, new IndexInfo(43, DeletionTime.LIVE));
+ try (RowIndexReader summary = completeAndRead())
+ {
+ // failing path
+ }
+ }
+ catch (AssertionError e)
+ {
+ // correct path
+ t = e;
+ logger.info("Got " + e.getMessage());
+ }
+ Assert.assertNotNull("Should throw an assertion error.", t);
+ }
+
+ @Test
+ public void testConstrainedIteration() throws IOException
+ {
+ // This is not too relevant: due to the way we construct separators we can't be good enough on the left side.
+ Pair<List<ClusteringPrefix<?>>, RowIndexReader> random = generateRandomIndexSingletons(COUNT);
+ List<ClusteringPrefix<?>> keys = random.left;
+
+ for (int i = 0; i < 500; ++i)
+ {
+ boolean exactLeft = RANDOM.nextBoolean();
+ boolean exactRight = RANDOM.nextBoolean();
+ ClusteringPrefix<?> left = exactLeft ? keys.get(RANDOM.nextInt(keys.size())) : generateRandomKey();
+ ClusteringPrefix<?> right = exactRight ? keys.get(RANDOM.nextInt(keys.size())) : generateRandomKey();
+ if (comparator.compare(right, left) < 0)
+ {
+ ClusteringPrefix<?> t = left;
+ left = right;
+ right = t;
+ boolean b = exactLeft;
+ exactLeft = exactRight;
+ exactRight = b;
+ }
+
+ try (RowIndexReverseIterator iter = new RowIndexReverseIterator(fh, root, comparator.asByteComparable(left), comparator.asByteComparable(right)))
+ {
+ IndexInfo indexInfo = iter.nextIndexInfo();
+ if (indexInfo == null)
+ {
+ int idx = Collections.binarySearch(keys, right, comparator);
+ if (idx < 0)
+ idx = -2 - idx; // less than or equal
+ if (idx <= 0)
+ continue;
+ assertTrue(comparator.asByteComparable(left) + " <= "
+ + comparator.asByteComparable(keys.get(idx)) + " <= "
+ + comparator.asByteComparable(right) + " but " + idx + " wasn't iterated.",
+ comparator.compare(left, keys.get(idx - 1)) > 0);
+ continue;
+ }
+
+ int idx = (int) indexInfo.offset;
+ if (indexInfo.offset == END_MARKER)
+ idx = keys.size();
+ if (idx > 0)
+ assertTrue(comparator.compare(right, keys.get(idx - 1)) > 0);
+ if (idx < keys.size() - 1)
+ assertTrue(comparator.compare(right, keys.get(idx + 1)) < 0);
+ if (exactRight) // must be precise on exact, otherwise could be in any relation
+ assertEquals(right, keys.get(idx));
+ while (true)
+ {
+ --idx;
+ IndexInfo ii = iter.nextIndexInfo();
+ if (ii == null)
+ break;
+ assertEquals(idx, (int) ii.offset);
+ }
+ ++idx; // seek at last returned
+ if (idx < keys.size() - 1)
+ assertTrue(comparator.compare(left, keys.get(idx + 1)) < 0);
+ // Because of the way we build the index (using non-prefix separator) we are usually going to miss the last item.
+ if (idx >= 2)
+ assertTrue(comparator.compare(left, keys.get(idx - 2)) > 0);
+ }
+ catch (AssertionError e)
+ {
+ logger.error(e.getMessage(), e);
+ ClusteringPrefix<?> ll = left;
+ ClusteringPrefix<?> rr = right;
+ logger.info(keys.stream()
+ .filter(x -> comparator.compare(ll, x) <= 0 && comparator.compare(x, rr) <= 0)
+ .map(clustering -> comparator.asByteComparable(clustering))
+ .map(bc -> bc.byteComparableAsString(VERSION))
+ .collect(Collectors.joining(", ")));
+ logger.info("Left {}{} Right {}{}", comparator.asByteComparable(left), exactLeft ? "#" : "", comparator.asByteComparable(right), exactRight ? "#" : "");
+ try (RowIndexReverseIterator iter2 = new RowIndexReverseIterator(fh, root, comparator.asByteComparable(left), comparator.asByteComparable(right)))
+ {
+ IndexInfo ii;
+ while ((ii = iter2.nextIndexInfo()) != null)
+ {
+ logger.info(comparator.asByteComparable(keys.get((int) ii.offset)).toString());
+ }
+ logger.info("Left {}{} Right {}{}", comparator.asByteComparable(left), exactLeft ? "#" : "", comparator.asByteComparable(right), exactRight ? "#" : "");
+ }
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testReverseIteration() throws IOException
+ {
+ Pair<List<ClusteringPrefix<?>>, RowIndexReader> random = generateRandomIndexSingletons(COUNT);
+ List<ClusteringPrefix<?>> keys = random.left;
+
+ for (int i = 0; i < 1000; ++i)
+ {
+ boolean exactRight = RANDOM.nextBoolean();
+ ClusteringPrefix<?> right = exactRight ? keys.get(RANDOM.nextInt(keys.size())) : generateRandomKey();
+
+ int idx = 0;
+ try (RowIndexReverseIterator iter = new RowIndexReverseIterator(fh, root, ByteComparable.EMPTY, comparator.asByteComparable(right)))
+ {
+ IndexInfo indexInfo = iter.nextIndexInfo();
+ if (indexInfo == null)
+ {
+ idx = Collections.binarySearch(keys, right, comparator);
+ if (idx < 0)
+ idx = -2 - idx; // less than or equal
+ assertTrue(comparator.asByteComparable(keys.get(idx)) + " <= "
+ + comparator.asByteComparable(right) + " but " + idx + " wasn't iterated.",
+ idx < 0);
+ continue;
+ }
+
+ idx = (int) indexInfo.offset;
+ if (indexInfo.offset == END_MARKER)
+ idx = keys.size();
+ if (idx > 0)
+ assertTrue(comparator.compare(right, keys.get(idx - 1)) > 0);
+ if (idx < keys.size() - 1)
+ assertTrue(comparator.compare(right, keys.get(idx + 1)) < 0);
+ if (exactRight) // must be precise on exact, otherwise could be in any relation
+ assertEquals(right, keys.get(idx));
+ while (true)
+ {
+ --idx;
+ IndexInfo ii = iter.nextIndexInfo();
+ if (ii == null)
+ break;
+ assertEquals(idx, (int) ii.offset);
+ }
+ assertEquals(-1, idx);
+ }
+ catch (AssertionError e)
+ {
+ logger.error(e.getMessage(), e);
+ ClusteringPrefix<?> rr = right;
+ logger.info(keys.stream()
+ .filter(x -> comparator.compare(x, rr) <= 0)
+ .map(comparator::asByteComparable)
+ .map(bc -> bc.byteComparableAsString(VERSION))
+ .collect(Collectors.joining(", ")));
+ logger.info("Right {}{}", comparator.asByteComparable(right), exactRight ? "#" : "");
+ try (RowIndexReverseIterator iter2 = new RowIndexReverseIterator(fh, root, ByteComparable.EMPTY, comparator.asByteComparable(right)))
+ {
+ IndexInfo ii;
+ while ((ii = iter2.nextIndexInfo()) != null)
+ {
+ logger.info(comparator.asByteComparable(keys.get((int) ii.offset)).toString());
+ }
+ }
+ logger.info("Right {}{}", comparator.asByteComparable(right), exactRight ? "#" : "");
+ throw e;
+ }
+ }
+ }
+
+ private Pair<List<ClusteringPrefix<?>>, RowIndexReader> generateRandomIndexSingletons(int size) throws IOException
+ {
+ List<ClusteringPrefix<?>> list = generateList(size);
+ for (int i = 0; i < size; i++)
+ {
+ assert i == 0 || comparator.compare(list.get(i - 1), list.get(i)) < 0;
+ assert i == 0 || ByteComparable.compare(comparator.asByteComparable(list.get(i - 1)), comparator.asByteComparable(list.get(i)), VERSION) < 0 :
+ String.format("%s bs %s versus %s bs %s", list.get(i - 1).clustering().clusteringString(comparator.subtypes()), comparator.asByteComparable(list.get(i - 1)), list.get(i).clustering().clusteringString(comparator.subtypes()), comparator.asByteComparable(list.get(i)));
+ writer.add(list.get(i), list.get(i), new IndexInfo(i, DeletionTime.LIVE));
+ }
+
+ RowIndexReader summary = completeAndRead();
+ return Pair.create(list, summary);
+ }
+
+ List<ClusteringPrefix<?>> generateList(int size)
+ {
+ List<ClusteringPrefix<?>> list = Lists.newArrayList();
+
+ Set<ClusteringPrefix<?>> set = Sets.newHashSet();
+ for (int i = 0; i < size; i++)
+ {
+ ClusteringPrefix<?> key = generateRandomKey(); // keys must be unique
+ while (!set.add(key))
+ key = generateRandomKey();
+ list.add(key);
+ }
+ list.sort(comparator);
+ return list;
+ }
+
+ private Pair<List<ClusteringPrefix<?>>, RowIndexReader> generateRandomIndexQuads(int size) throws IOException
+ {
+ List<ClusteringPrefix<?>> list = generateList(4 * size + 1);
+ for (int i = 0; i < size; i++)
+ writer.add(list.get(i * 4 + 1), list.get(i * 4 + 3), new IndexInfo(i, new DeletionTime(i + 2, i - 3)));
+
+ RowIndexReader summary = completeAndRead();
+ return Pair.create(list, summary);
+ }
+
+ ClusteringPrefix<?> generateRandomKey()
+ {
+ UUID uuid = randomSeededUUID();
+ ClusteringPrefix<?> key = comparator.make(uuid);
+ return key;
+ }
+
+ private static UUID randomSeededUUID()
+ {
+ byte[] randomBytes = new byte[16];
+ RANDOM.nextBytes(randomBytes);
+ return UUID.nameUUIDFromBytes(randomBytes);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java
new file mode 100644
index 0000000..3f408c9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format.bti;
+
+import java.util.stream.Stream;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.sstable.format.AbstractTestVersionSupportedFeatures;
+import org.apache.cassandra.io.sstable.format.Version;
+
+public class VersionSupportedFeaturesTest extends AbstractTestVersionSupportedFeatures
+{
+ @Override
+ protected Version getVersion(String v)
+ {
+ return DatabaseDescriptor.getSSTableFormats().get(BtiFormat.NAME).getVersion(v);
+ }
+
+ @Override
+ protected Stream<String> getPendingRepairSupportedVersions()
+ {
+ return ALL_VERSIONS.stream();
+ }
+
+ @Override
+ protected Stream<String> getPartitionLevelDeletionPresenceMarkerSupportedVersions()
+ {
+ return ALL_VERSIONS.stream();
+ }
+
+ @Override
+ protected Stream<String> getLegacyMinMaxSupportedVersions()
+ {
+ return Stream.empty();
+ }
+
+ @Override
+ protected Stream<String> getImprovedMinMaxSupportedVersions()
+ {
+ return ALL_VERSIONS.stream();
+ }
+
+ @Override
+ protected Stream<String> getKeyRangeSupportedVersions()
+ {
+ return ALL_VERSIONS.stream();
+ }
+
+ @Override
+ protected Stream<String> getOriginatingHostIdSupportedVersions()
+ {
+ return ALL_VERSIONS.stream();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java
new file mode 100644
index 0000000..40d04a4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java
@@ -0,0 +1,797 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.metrics.RestorableMeter;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static com.google.common.collect.ImmutableMap.of;
+import static org.apache.cassandra.Util.assertOnDiskState;
+import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution.DOWNSAMPLE_THESHOLD;
+import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution.UPSAMPLE_THRESHOLD;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class IndexSummaryManagerTest<R extends SSTableReader & IndexSummarySupport<R>>
+{
+ private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManagerTest.class);
+
+ int originalMinIndexInterval;
+ int originalMaxIndexInterval;
+ long originalCapacity;
+
+ private static final String KEYSPACE1 = "IndexSummaryManagerTest";
+ // index interval of 8, no key caching
+ private static final String CF_STANDARDLOWiINTERVAL = "StandardLowIndexInterval";
+ private static final String CF_STANDARDRACE = "StandardRace";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ Assume.assumeTrue("This test make sense only if the default SSTable format support index summary",
+ IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWiINTERVAL)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDRACE)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+ }
+
+ @Before
+ public void beforeTest()
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ originalMinIndexInterval = cfs.metadata().params.minIndexInterval;
+ originalMaxIndexInterval = cfs.metadata().params.maxIndexInterval;
+ originalCapacity = IndexSummaryManager.instance.getMemoryPoolCapacityInMB();
+ }
+
+ @After
+ public void afterTest()
+ {
+ for (CompactionInfo.Holder holder : CompactionManager.instance.active.getCompactions())
+ {
+ holder.stop();
+ }
+
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build());
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMaxIndexInterval).build());
+
+ IndexSummaryManager.instance.setMemoryPoolCapacityInMB(originalCapacity);
+ }
+
+ private long totalOffHeapSize(List<R> sstables)
+ {
+ long total = 0;
+ for (R sstable : sstables)
+ total += sstable.getIndexSummary().getOffHeapSize();
+ return total;
+ }
+
+ private List<R> resetSummaries(ColumnFamilyStore cfs, List<R> sstables, long originalOffHeapSize) throws IOException
+ {
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), originalOffHeapSize * sstables.size());
+ }
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel());
+
+ return sstables;
+ }
+
+ private void validateData(ColumnFamilyStore cfs, int numPartition)
+ {
+ for (int i = 0; i < numPartition; i++)
+ {
+ Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, String.format("%3d", i)).build());
+ Cell<?> cell = row.getCell(cfs.metadata().getColumn(ByteBufferUtil.bytes("val")));
+ assertNotNull(cell);
+ assertEquals(100, cell.buffer().array().length);
+
+ }
+ }
+
+ private final Comparator<SSTableReader> hotnessComparator = Comparator.comparingDouble(o -> o.getReadMeter().fifteenMinuteRate());
+
+ private void createSSTables(String ksname, String cfname, int numSSTables, int numPartition)
+ {
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ ArrayList<Future> futures = new ArrayList<>(numSSTables);
+ ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+ for (int sstable = 0; sstable < numSSTables; sstable++)
+ {
+ for (int p = 0; p < numPartition; p++)
+ {
+
+ String key = String.format("%3d", p);
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
+ .clustering("column")
+ .add("val", value)
+ .build()
+ .applyUnsafe();
+ }
+ futures.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
+ }
+ for (Future future : futures)
+ {
+ try
+ {
+ future.get();
+ }
+ catch (InterruptedException | ExecutionException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ assertEquals(numSSTables, ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).size());
+ validateData(cfs, numPartition);
+ }
+
+ @Test
+ public <R extends SSTableReader & IndexSummarySupport<R>> void testChangeMinIndexInterval() throws IOException
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ int numSSTables = 1;
+ int numRows = 256;
+ createSSTables(ksname, cfname, numSSTables, numRows);
+
+ List<R> sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ for (R sstable : sstables)
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+
+ // double the min_index_interval
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build());
+ IndexSummaryManager.instance.redistributeSummaries();
+ for (R sstable : ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs))
+ {
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().size());
+ }
+
+ // return min_index_interval to its original value
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build());
+ IndexSummaryManager.instance.redistributeSummaries();
+ for (R sstable : ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs))
+ {
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().size());
+ }
+
+ // halve the min_index_interval, but constrain the available space to exactly what we have now; as a result,
+ // the summary shouldn't change
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval / 2).build());
+ R sstable = ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs).iterator().next();
+ long summarySpace = sstable.getIndexSummary().getOffHeapSize();
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), summarySpace);
+ }
+
+ sstable = ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs).iterator().next();
+ assertEquals(originalMinIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / originalMinIndexInterval, sstable.getIndexSummary().size());
+
+ // keep the min_index_interval the same, but now give the summary enough space to grow by 50%
+ double previousInterval = sstable.getIndexSummary().getEffectiveIndexInterval();
+ int previousSize = sstable.getIndexSummary().size();
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace * 1.5));
+ }
+ sstable = ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs).iterator().next();
+ assertEquals(previousSize * 1.5, (double) sstable.getIndexSummary().size(), 1);
+ assertEquals(previousInterval * (1.0 / 1.5), sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+
+ // return min_index_interval to it's original value (double it), but only give the summary enough space
+ // to have an effective index interval of twice the new min
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace / 2.0));
+ }
+ sstable = ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs).iterator().next();
+ assertEquals(originalMinIndexInterval * 2, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / (originalMinIndexInterval * 2), sstable.getIndexSummary().size());
+
+ // raise the min_index_interval above our current effective interval, but set the max_index_interval lower
+ // than what we actually have space for (meaning the index summary would ideally be smaller, but this would
+ // result in an effective interval above the new max)
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 4).build());
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMinIndexInterval * 4).build());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 10);
+ }
+ sstable = ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs).iterator().next();
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ }
+
+ @Test
+ public void testChangeMaxIndexInterval() throws IOException
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ int numSSTables = 1;
+ int numRows = 256;
+ createSSTables(ksname, cfname, numSSTables, numRows);
+
+ List<R> sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 10);
+ }
+ sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.01);
+
+ // halve the max_index_interval
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval / 2).build());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 1);
+ }
+ sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ {
+ assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.01);
+ assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().size());
+ }
+
+ // return max_index_interval to its original value
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval * 2).build());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 1);
+ }
+ for (R sstable : ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs))
+ {
+ assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.01);
+ assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().size());
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testRedistributeSummaries() throws IOException
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ int numSSTables = 4;
+ int numRows = 256;
+ createSSTables(ksname, cfname, numSSTables, numRows);
+
+ int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata().params.minIndexInterval) / cfs.metadata().params.maxIndexInterval;
+
+ List<R> sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummary().getOffHeapSize();
+
+ // there should be enough space to not downsample anything
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables));
+ }
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel());
+ assertEquals(singleSummaryOffHeapSpace * numSSTables, totalOffHeapSize(sstables));
+ validateData(cfs, numRows);
+
+ // everything should get cut in half
+ assert sstables.size() == 4;
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
+ }
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // everything should get cut to a quarter
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 4)));
+ }
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL / 4, sstable.getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // upsample back up to half
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2) + 4));
+ }
+ assert sstables.size() == 4;
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // upsample back up to the original index summary
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables));
+ }
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // make two of the four sstables cold, only leave enough space for three full index summaries,
+ // so the two cold sstables should get downsampled to be half of their original size
+ sstables.get(0).overrideReadMeter(new RestorableMeter(50.0, 50.0));
+ sstables.get(1).overrideReadMeter(new RestorableMeter(50.0, 50.0));
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3));
+ }
+ Collections.sort(sstables, hotnessComparator);
+ assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // small increases or decreases in the read rate don't result in downsampling or upsampling
+ double lowerRate = 50.0 * (DOWNSAMPLE_THESHOLD + (DOWNSAMPLE_THESHOLD * 0.10));
+ double higherRate = 50.0 * (UPSAMPLE_THRESHOLD - (UPSAMPLE_THRESHOLD * 0.10));
+ sstables.get(0).overrideReadMeter(new RestorableMeter(lowerRate, lowerRate));
+ sstables.get(1).overrideReadMeter(new RestorableMeter(higherRate, higherRate));
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3));
+ }
+ Collections.sort(sstables, hotnessComparator);
+ assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // reset, and then this time, leave enough space for one of the cold sstables to not get downsampled
+ sstables = resetSummaries(cfs, sstables, singleSummaryOffHeapSpace);
+ sstables.get(0).overrideReadMeter(new RestorableMeter(1.0, 1.0));
+ sstables.get(1).overrideReadMeter(new RestorableMeter(2.0, 2.0));
+ sstables.get(2).overrideReadMeter(new RestorableMeter(1000.0, 1000.0));
+ sstables.get(3).overrideReadMeter(new RestorableMeter(1000.0, 1000.0));
+
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3) + 50);
+ }
+ Collections.sort(sstables, hotnessComparator);
+
+ if (sstables.get(0).getIndexSummary().getSamplingLevel() == minSamplingLevel)
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(1).getIndexSummary().getSamplingLevel());
+ else
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(0).getIndexSummary().getSamplingLevel());
+
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummary().getSamplingLevel());
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+
+ // Cause a mix of upsampling and downsampling. We'll leave enough space for two full index summaries. The two
+ // coldest sstables will get downsampled to 4/128 of their size, leaving us with 1 and 92/128th index
+ // summaries worth of space. The hottest sstable should get a full index summary, and the one in the middle
+ // should get the remainder.
+ sstables.get(0).overrideReadMeter(new RestorableMeter(0.0, 0.0));
+ sstables.get(1).overrideReadMeter(new RestorableMeter(0.0, 0.0));
+ sstables.get(2).overrideReadMeter(new RestorableMeter(92, 92));
+ sstables.get(3).overrideReadMeter(new RestorableMeter(128.0, 128.0));
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL))));
+ }
+ Collections.sort(sstables, hotnessComparator);
+ assertEquals(1, sstables.get(0).getIndexSummary().size()); // at the min sampling level
+ assertEquals(1, sstables.get(0).getIndexSummary().size()); // at the min sampling level
+ assertTrue(sstables.get(2).getIndexSummary().getSamplingLevel() > minSamplingLevel);
+ assertTrue(sstables.get(2).getIndexSummary().getSamplingLevel() < BASE_SAMPLING_LEVEL);
+ assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+
+ // Don't leave enough space for even the minimal index summaries
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 10);
+ }
+ for (R sstable : sstables)
+ assertEquals(1, sstable.getIndexSummary().size()); // at the min sampling level
+ validateData(cfs, numRows);
+ }
+
+ @Test
+ public void testRebuildAtSamplingLevel() throws IOException
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL;
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+
+ int numRows = 256;
+ for (int row = 0; row < numRows; row++)
+ {
+ String key = String.format("%3d", row);
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
+ .clustering("column")
+ .add("val", value)
+ .build()
+ .applyUnsafe();
+ }
+
+ Util.flush(cfs);
+
+ List<R> sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ assertEquals(1, sstables.size());
+ R original = sstables.get(0);
+
+ R sstable = original;
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN))
+ {
+ for (int samplingLevel = 1; samplingLevel < BASE_SAMPLING_LEVEL; samplingLevel++)
+ {
+ sstable = sstable.cloneWithNewSummarySamplingLevel(cfs, samplingLevel);
+ assertEquals(samplingLevel, sstable.getIndexSummary().getSamplingLevel());
+ int expectedSize = (numRows * samplingLevel) / (cfs.metadata().params.minIndexInterval * BASE_SAMPLING_LEVEL);
+ assertEquals(expectedSize, sstable.getIndexSummary().size(), 1);
+ txn.update(sstable, true);
+ txn.checkpoint();
+ }
+ txn.finish();
+ }
+ }
+
+ @Test
+ public void testJMXFunctions() throws IOException
+ {
+ IndexSummaryManager manager = IndexSummaryManager.instance;
+
+ // resize interval
+ manager.setResizeIntervalInMinutes(-1);
+ assertEquals(-1, DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
+ assertNull(manager.getTimeToNextResize(TimeUnit.MINUTES));
+
+ manager.setResizeIntervalInMinutes(10);
+ assertEquals(10, manager.getResizeIntervalInMinutes());
+ assertEquals(10, DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
+ assertEquals(10, manager.getTimeToNextResize(TimeUnit.MINUTES), 1);
+ manager.setResizeIntervalInMinutes(15);
+ assertEquals(15, manager.getResizeIntervalInMinutes());
+ assertEquals(15, DatabaseDescriptor.getIndexSummaryResizeIntervalInMinutes());
+ assertEquals(15, manager.getTimeToNextResize(TimeUnit.MINUTES), 2);
+
+ // memory pool capacity
+ assertTrue(manager.getMemoryPoolCapacityInMB() >= 0);
+ manager.setMemoryPoolCapacityInMB(10);
+ assertEquals(10, manager.getMemoryPoolCapacityInMB());
+
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+
+ int numSSTables = 2;
+ int numRows = 10;
+ for (int sstable = 0; sstable < numSSTables; sstable++)
+ {
+ for (int row = 0; row < numRows; row++)
+ {
+ String key = String.format("%3d", row);
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
+ .clustering("column")
+ .add("val", value)
+ .build()
+ .applyUnsafe();
+ }
+ Util.flush(cfs);
+ }
+
+ assertTrue(manager.getAverageIndexInterval() >= cfs.metadata().params.minIndexInterval);
+ Map<String, Integer> intervals = manager.getIndexIntervals();
+ for (Map.Entry<String, Integer> entry : intervals.entrySet())
+ if (entry.getKey().contains(CF_STANDARDLOWiINTERVAL))
+ assertEquals(cfs.metadata().params.minIndexInterval, entry.getValue(), 0.001);
+
+ manager.setMemoryPoolCapacityInMB(0);
+ manager.redistributeSummaries();
+ assertTrue(manager.getAverageIndexInterval() > cfs.metadata().params.minIndexInterval);
+ intervals = manager.getIndexIntervals();
+ for (Map.Entry<String, Integer> entry : intervals.entrySet())
+ {
+ if (entry.getKey().contains(CF_STANDARDLOWiINTERVAL))
+ assertTrue(entry.getValue() >= cfs.metadata().params.minIndexInterval);
+ }
+ }
+
+ @Test
+ public void testCancelIndex() throws Exception
+ {
+ testCancelIndexHelper((cfs) -> CompactionManager.instance.stopCompaction("INDEX_SUMMARY"));
+ }
+
+ @Test
+ public void testCancelIndexInterrupt() throws Exception
+ {
+ testCancelIndexHelper((cfs) -> CompactionManager.instance.interruptCompactionFor(Collections.singleton(cfs.metadata()), (sstable) -> true, false));
+ }
+
+ public void testCancelIndexHelper(Consumer<ColumnFamilyStore> cancelFunction) throws Exception
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ cfs.disableAutoCompaction();
+ final int numSSTables = 8;
+ int numRows = 256;
+ createSSTables(ksname, cfname, numSSTables, numRows);
+
+ List<R> allSSTables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ List<R> sstables = allSSTables.subList(0, 4);
+ List<R> compacting = allSSTables.subList(4, 8);
+
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ final long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummary().getOffHeapSize();
+
+ // everything should get cut in half
+ final AtomicReference<CompactionInterruptedException> exception = new AtomicReference<>();
+ // barrier to control when redistribution runs
+ final CountDownLatch barrier = new CountDownLatch(1);
+ CompactionInfo.Holder ongoingCompaction = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting);
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+ try (LifecycleTransaction ignored = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN))
+ {
+ CompactionManager.instance.active.beginCompaction(ongoingCompaction);
+
+ Thread t = NamedThreadFactory.createAnonymousThread(new Runnable()
+ {
+ public void run()
+ {
+ try
+ {
+ // Don't leave enough space for even the minimal index summaries
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ IndexSummaryManager.redistributeSummaries(new ObservableRedistribution(of(cfs.metadata.id, txn),
+ 0,
+ singleSummaryOffHeapSpace,
+ barrier));
+ }
+ }
+ catch (CompactionInterruptedException ex)
+ {
+ exception.set(ex);
+ }
+ catch (IOException ignored)
+ {
+ }
+ }
+ });
+
+ t.start();
+ while (CompactionManager.instance.getActiveCompactions() < 2 && t.isAlive())
+ Thread.sleep(1);
+ // to ensure that the stop condition check in IndexSummaryRedistribution::redistributeSummaries
+ // is made *after* the halt request is made to the CompactionManager, don't allow the redistribution
+ // to proceed until stopCompaction has been called.
+ cancelFunction.accept(cfs);
+ // allows the redistribution to proceed
+ barrier.countDown();
+ t.join();
+ }
+ finally
+ {
+ CompactionManager.instance.active.finishCompaction(ongoingCompaction);
+ }
+
+ assertNotNull("Expected compaction interrupted exception", exception.get());
+ assertTrue("Expected no active compactions", CompactionManager.instance.active.getCompactions().isEmpty());
+
+ Set<R> beforeRedistributionSSTables = new HashSet<>(allSSTables);
+ Set<R> afterCancelSSTables = Sets.newHashSet(ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs));
+ Set<R> disjoint = Sets.symmetricDifference(beforeRedistributionSSTables, afterCancelSSTables);
+ assertTrue(String.format("Mismatched files before and after cancelling redistribution: %s",
+ Joiner.on(",").join(disjoint)),
+ disjoint.isEmpty());
+ assertOnDiskState(cfs, 8);
+ validateData(cfs, numRows);
+ }
+
+ @Test
+ public void testPauseIndexSummaryManager() throws Exception
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ int numSSTables = 4;
+ int numRows = 256;
+ createSSTables(ksname, cfname, numSSTables, numRows);
+
+ List<R> sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummary().getOffHeapSize();
+
+ // everything should get cut in half
+ assert sstables.size() == numSSTables;
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN))
+ {
+ try (AutoCloseable toresume = CompactionManager.instance.pauseGlobalCompaction())
+ {
+ sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata().id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2)));
+ fail("The redistribution should fail - we got paused before adding to active compactions, but after marking compacting");
+ }
+ }
+ catch (CompactionInterruptedException e)
+ {
+ // expected
+ }
+ for (R sstable : sstables)
+ assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel());
+ validateData(cfs, numRows);
+ assertOnDiskState(cfs, numSSTables);
+ }
+
+ private List<R> redistributeSummaries(List<R> compacting,
+ Map<TableId, LifecycleTransaction> transactions,
+ long memoryPoolBytes)
+ throws IOException
+ {
+ long nonRedistributingOffHeapSize = compacting.stream().mapToLong(t -> t.getIndexSummary().getOffHeapSize()).sum();
+ return IndexSummaryManager.redistributeSummaries(new IndexSummaryRedistribution(transactions,
+ nonRedistributingOffHeapSize,
+ memoryPoolBytes));
+ }
+
+ private static class ObservableRedistribution extends IndexSummaryRedistribution
+ {
+ CountDownLatch barrier;
+
+ ObservableRedistribution(Map<TableId, LifecycleTransaction> transactions,
+ long nonRedistributingOffHeapSize,
+ long memoryPoolBytes,
+ CountDownLatch barrier)
+ {
+ super(transactions, nonRedistributingOffHeapSize, memoryPoolBytes);
+ this.barrier = barrier;
+ }
+
+ public <R extends SSTableReader & IndexSummarySupport<R>> List<R> redistributeSummaries() throws IOException
+ {
+ try
+ {
+ barrier.await();
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException("Interrupted waiting on test barrier");
+ }
+ return super.redistributeSummaries();
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistributionTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistributionTest.java
new file mode 100644
index 0000000..0c103ab
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistributionTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.metrics.RestorableMeter;
+import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
+
+import static org.junit.Assert.assertEquals;
+
+public class IndexSummaryRedistributionTest<R extends SSTableReader & IndexSummarySupport<R>>
+{
+ private static final String KEYSPACE1 = "IndexSummaryRedistributionTest";
+ private static final String CF_STANDARD = "Standard";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ Assume.assumeTrue("This test make sense only if the default SSTable format support index summary",
+ IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()));
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
+ .minIndexInterval(8)
+ .maxIndexInterval(256)
+ .caching(CachingParams.CACHE_NOTHING));
+ }
+
+ @Test
+ public void testMetricsLoadAfterRedistribution() throws IOException
+ {
+ String ksname = KEYSPACE1;
+ String cfname = CF_STANDARD;
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ int numSSTables = 1;
+ int numRows = 1024 * 10;
+
+ long load = StorageMetrics.load.getCount();
+ StorageMetrics.load.dec(load); // reset the load metric
+ long uncompressedLoad = StorageMetrics.uncompressedLoad.getCount();
+ StorageMetrics.uncompressedLoad.dec(uncompressedLoad); // reset the uncompressed load metric
+
+ createSSTables(ksname, cfname, numSSTables, numRows);
+
+ List<R> sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs);
+ for (R sstable : sstables)
+ sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+
+ long oldSize = 0;
+ long oldSizeUncompressed = 0;
+
+ for (R sstable : sstables)
+ {
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ oldSize += sstable.bytesOnDisk();
+ oldSizeUncompressed += sstable.logicalBytesOnDisk();
+ }
+
+ load = StorageMetrics.load.getCount();
+ long others = load - oldSize; // Other SSTables size, e.g. schema and other system SSTables
+
+ uncompressedLoad = StorageMetrics.uncompressedLoad.getCount();
+ long othersUncompressed = uncompressedLoad - oldSizeUncompressed;
+
+ int originalMinIndexInterval = cfs.metadata().params.minIndexInterval;
+ // double the min_index_interval
+ SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build());
+ IndexSummaryManager.instance.redistributeSummaries();
+
+ long newSize = 0;
+ long newSizeUncompressed = 0;
+
+ for (R sstable : ServerTestUtils.<R>getLiveIndexSummarySupportingReaders(cfs))
+ {
+ assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001);
+ assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().size());
+ newSize += sstable.bytesOnDisk();
+ newSizeUncompressed += sstable.logicalBytesOnDisk();
+ }
+
+ newSize += others;
+ load = StorageMetrics.load.getCount();
+ // new size we calculate should be almost the same as the load in metrics
+ assertEquals(newSize, load, newSize / 10.0);
+
+ newSizeUncompressed += othersUncompressed;
+ uncompressedLoad = StorageMetrics.uncompressedLoad.getCount();
+ assertEquals(newSizeUncompressed, uncompressedLoad, newSizeUncompressed / 10.0);
+ }
+
+ private void createSSTables(String ksname, String cfname, int numSSTables, int numRows)
+ {
+ Keyspace keyspace = Keyspace.open(ksname);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+
+ ArrayList<Future<CommitLogPosition>> futures = new ArrayList<>(numSSTables);
+ ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+ for (int sstable = 0; sstable < numSSTables; sstable++)
+ {
+ for (int row = 0; row < numRows; row++)
+ {
+ String key = String.format("%3d", row);
+ new RowUpdateBuilder(cfs.metadata(), 0, key)
+ .clustering("column")
+ .add("val", value)
+ .build()
+ .applyUnsafe();
+ }
+ futures.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
+ }
+ for (Future<CommitLogPosition> future : futures)
+ {
+ try
+ {
+ future.get();
+ }
+ catch (InterruptedException | ExecutionException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ assertEquals(numSSTables, ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).size());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
new file mode 100644
index 0000000..aea166a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.indexsummary;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.io.sstable.Downsampling;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+
+import static org.apache.cassandra.config.CassandraRelevantEnv.CIRCLECI;
+import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder.downsample;
+import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder.entriesAtSamplingLevel;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class IndexSummaryTest
+{
+ private final static Random random = new Random();
+
+ @BeforeClass
+ public static void initDD()
+ {
+ DatabaseDescriptor.daemonInitialization();
+
+ final long seed = nanoTime();
+ System.out.println("Using seed: " + seed);
+ random.setSeed(seed);
+ }
+
+ IPartitioner partitioner = Util.testPartitioner();
+
+ @BeforeClass
+ public static void setup()
+ {
+ final long seed = nanoTime();
+ System.out.println("Using seed: " + seed);
+ random.setSeed(seed);
+ }
+
+ @Test
+ public void testIndexSummaryKeySizes() throws IOException
+ {
+ // On Circle CI we normally don't have enough off-heap memory for this test so ignore it
+ Assume.assumeTrue(CIRCLECI.getString() == null);
+
+ testIndexSummaryProperties(32, 100);
+ testIndexSummaryProperties(64, 100);
+ testIndexSummaryProperties(100, 100);
+ testIndexSummaryProperties(1000, 100);
+ testIndexSummaryProperties(10000, 100);
+ }
+
+ private void testIndexSummaryProperties(int keySize, int numKeys) throws IOException
+ {
+ final int minIndexInterval = 1;
+ final List<DecoratedKey> keys = new ArrayList<>(numKeys);
+
+ try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
+ {
+ for (int i = 0; i < numKeys; i++)
+ {
+ byte[] randomBytes = new byte[keySize];
+ random.nextBytes(randomBytes);
+ DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
+ keys.add(key);
+ builder.maybeAddEntry(key, i);
+ }
+
+ try(IndexSummary indexSummary = builder.build(partitioner))
+ {
+ assertEquals(numKeys, keys.size());
+ assertEquals(minIndexInterval, indexSummary.getMinIndexInterval());
+ assertEquals(numKeys, indexSummary.getMaxNumberOfEntries());
+ assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
+
+ for (int i = 0; i < numKeys; i++)
+ assertEquals(keys.get(i).getKey(), ByteBuffer.wrap(indexSummary.getKey(i)));
+ }
+ }
+ }
+
+ /**
+ * Test an index summary whose total size is bigger than 2GiB,
+ * the index summary builder should log an error but it should still
+ * create an index summary, albeit one that does not cover the entire sstable.
+ */
+ @Test
+ public void testLargeIndexSummary() throws IOException
+ {
+ // On Circle CI we normally don't have enough off-heap memory for this test so ignore it
+ Assume.assumeTrue(CIRCLECI.getString() == null);
+
+ final int numKeys = 1000000;
+ final int keySize = 3000;
+ final int minIndexInterval = 1;
+
+ try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
+ {
+ for (int i = 0; i < numKeys; i++)
+ {
+ byte[] randomBytes = new byte[keySize];
+ random.nextBytes(randomBytes);
+ DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
+ builder.maybeAddEntry(key, i);
+ }
+
+ try (IndexSummary indexSummary = builder.build(partitioner))
+ {
+ assertNotNull(indexSummary);
+ assertEquals(numKeys, indexSummary.getMaxNumberOfEntries());
+ assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
+ }
+ }
+ }
+
+ /**
+ * Test an index summary whose total size is bigger than 2GiB,
+ * having updated IndexSummaryBuilder.defaultExpectedKeySize to match the size,
+ * the index summary should be downsampled automatically.
+ */
+ @Test
+ public void testLargeIndexSummaryWithExpectedSizeMatching() throws IOException
+ {
+ // On Circle CI we normally don't have enough off-heap memory for this test so ignore it
+ Assume.assumeTrue(CIRCLECI.getString() == null);
+
+ final int numKeys = 1000000;
+ final int keySize = 3000;
+ final int minIndexInterval = 1;
+
+ long oldExpectedKeySize = IndexSummaryBuilder.defaultExpectedKeySize;
+ IndexSummaryBuilder.defaultExpectedKeySize = 3000;
+
+ try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
+ {
+ for (int i = 0; i < numKeys; i++)
+ {
+ byte[] randomBytes = new byte[keySize];
+ random.nextBytes(randomBytes);
+ DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
+ builder.maybeAddEntry(key, i);
+ }
+
+ try (IndexSummary indexSummary = builder.build(partitioner))
+ {
+ assertNotNull(indexSummary);
+ assertEquals(minIndexInterval * 2, indexSummary.getMinIndexInterval());
+ assertEquals(numKeys / 2, indexSummary.getMaxNumberOfEntries());
+ assertEquals(numKeys + 2, indexSummary.getEstimatedKeyCount());
+ }
+ }
+ finally
+ {
+ IndexSummaryBuilder.defaultExpectedKeySize = oldExpectedKeySize;
+ }
+ }
+
+ @Test
+ public void testGetKey()
+ {
+ Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
+ for (int i = 0; i < 100; i++)
+ assertEquals(random.left.get(i).getKey(), ByteBuffer.wrap(random.right.getKey(i)));
+ random.right.close();
+ }
+
+ @Test
+ public void testBinarySearch()
+ {
+ Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
+ for (int i = 0; i < 100; i++)
+ assertEquals(i, random.right.binarySearch(random.left.get(i)));
+ random.right.close();
+ }
+
+ @Test
+ public void testGetPosition()
+ {
+ Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 2);
+ for (int i = 0; i < 50; i++)
+ assertEquals(i*2, random.right.getPosition(i));
+ random.right.close();
+ }
+
+ @Test
+ public void testSerialization() throws IOException
+ {
+ Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
+ DataOutputBuffer dos = new DataOutputBuffer();
+ IndexSummary.serializer.serialize(random.right, dos);
+ // write junk
+ dos.writeUTF("JUNK");
+ dos.writeUTF("JUNK");
+ FileUtils.closeQuietly(dos);
+ DataInputStreamPlus dis = new DataInputBuffer(dos.toByteArray());
+ IndexSummary is = IndexSummary.serializer.deserialize(dis, partitioner, 1, 1);
+ for (int i = 0; i < 100; i++)
+ assertEquals(i, is.binarySearch(random.left.get(i)));
+ // read the junk
+ assertEquals(dis.readUTF(), "JUNK");
+ assertEquals(dis.readUTF(), "JUNK");
+ is.close();
+ FileUtils.closeQuietly(dis);
+ random.right.close();
+ }
+
+ @Test
+ public void testAddEmptyKey() throws Exception
+ {
+ IPartitioner p = new RandomPartitioner();
+ try (IndexSummaryBuilder builder = new IndexSummaryBuilder(1, 1, BASE_SAMPLING_LEVEL))
+ {
+ builder.maybeAddEntry(p.decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER), 0);
+ IndexSummary summary = builder.build(p);
+ assertEquals(1, summary.size());
+ assertEquals(0, summary.getPosition(0));
+ assertArrayEquals(new byte[0], summary.getKey(0));
+
+ DataOutputBuffer dos = new DataOutputBuffer();
+ IndexSummary.serializer.serialize(summary, dos);
+ DataInputStreamPlus dis = new DataInputBuffer(dos.toByteArray());
+ IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, 1, 1);
+
+ assertEquals(1, loaded.size());
+ assertEquals(summary.getPosition(0), loaded.getPosition(0));
+ assertArrayEquals(summary.getKey(0), summary.getKey(0));
+ summary.close();
+ loaded.close();
+ }
+ }
+
+ private Pair<List<DecoratedKey>, IndexSummary> generateRandomIndex(int size, int interval)
+ {
+ List<DecoratedKey> list = Lists.newArrayList();
+ try (IndexSummaryBuilder builder = new IndexSummaryBuilder(list.size(), interval, BASE_SAMPLING_LEVEL))
+ {
+ for (int i = 0; i < size; i++)
+ {
+ UUID uuid = UUID.randomUUID();
+ DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.bytes(uuid));
+ list.add(key);
+ }
+ Collections.sort(list);
+ for (int i = 0; i < size; i++)
+ builder.maybeAddEntry(list.get(i), i);
+ IndexSummary summary = builder.build(partitioner);
+ return Pair.create(list, summary);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testDownsamplePatterns()
+ {
+ Assert.assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(0));
+ assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(1));
+
+ assertEquals(Arrays.asList(1, 0), Downsampling.getSamplingPattern(2));
+ assertEquals(Arrays.asList(3, 1, 2, 0), Downsampling.getSamplingPattern(4));
+ assertEquals(Arrays.asList(7, 3, 5, 1, 6, 2, 4, 0), Downsampling.getSamplingPattern(8));
+ assertEquals(Arrays.asList(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0), Downsampling.getSamplingPattern(16));
+ }
+
+ private static boolean shouldSkip(int index, List<Integer> startPoints)
+ {
+ for (int start : startPoints)
+ {
+ if ((index - start) % BASE_SAMPLING_LEVEL == 0)
+ return true;
+ }
+ return false;
+ }
+
+ @Test
+ public void testDownsample()
+ {
+ final int NUM_KEYS = 4096;
+ final int INDEX_INTERVAL = 128;
+ final int ORIGINAL_NUM_ENTRIES = NUM_KEYS / INDEX_INTERVAL;
+
+
+ Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(NUM_KEYS, INDEX_INTERVAL);
+ List<DecoratedKey> keys = random.left;
+ IndexSummary original = random.right;
+
+ // sanity check on the original index summary
+ for (int i = 0; i < ORIGINAL_NUM_ENTRIES; i++)
+ assertEquals(keys.get(i * INDEX_INTERVAL).getKey(), ByteBuffer.wrap(original.getKey(i)));
+
+ List<Integer> samplePattern = Downsampling.getSamplingPattern(BASE_SAMPLING_LEVEL);
+
+ // downsample by one level, then two levels, then three levels...
+ int downsamplingRound = 1;
+ for (int samplingLevel = BASE_SAMPLING_LEVEL - 1; samplingLevel >= 1; samplingLevel--)
+ {
+ try (IndexSummary downsampled = downsample(original, samplingLevel, 128, partitioner))
+ {
+ assertEquals(entriesAtSamplingLevel(samplingLevel, original.getMaxNumberOfEntries()), downsampled.size());
+
+ int sampledCount = 0;
+ List<Integer> skipStartPoints = samplePattern.subList(0, downsamplingRound);
+ for (int i = 0; i < ORIGINAL_NUM_ENTRIES; i++)
+ {
+ if (!shouldSkip(i, skipStartPoints))
+ {
+ assertEquals(keys.get(i * INDEX_INTERVAL).getKey(), ByteBuffer.wrap(downsampled.getKey(sampledCount)));
+ sampledCount++;
+ }
+ }
+
+ testPosition(original, downsampled, keys);
+ downsamplingRound++;
+ }
+ }
+
+ // downsample one level each time
+ IndexSummary previous = original;
+ downsamplingRound = 1;
+ for (int downsampleLevel = BASE_SAMPLING_LEVEL - 1; downsampleLevel >= 1; downsampleLevel--)
+ {
+ IndexSummary downsampled = downsample(previous, downsampleLevel, 128, partitioner);
+ if (previous != original)
+ previous.close();
+ assertEquals(entriesAtSamplingLevel(downsampleLevel, original.getMaxNumberOfEntries()), downsampled.size());
+
+ int sampledCount = 0;
+ List<Integer> skipStartPoints = samplePattern.subList(0, downsamplingRound);
+ for (int i = 0; i < ORIGINAL_NUM_ENTRIES; i++)
+ {
+ if (!shouldSkip(i, skipStartPoints))
+ {
+ assertEquals(keys.get(i * INDEX_INTERVAL).getKey(), ByteBuffer.wrap(downsampled.getKey(sampledCount)));
+ sampledCount++;
+ }
+ }
+
+ testPosition(original, downsampled, keys);
+ previous = downsampled;
+ downsamplingRound++;
+ }
+ previous.close();
+ original.close();
+ }
+
+ private void testPosition(IndexSummary original, IndexSummary downsampled, List<DecoratedKey> keys)
+ {
+ for (DecoratedKey key : keys)
+ {
+ long orig = original.getScanPosition(key);
+ int binarySearch = downsampled.binarySearch(key);
+ int index = IndexSummary.getIndexFromBinarySearchResult(binarySearch);
+ int scanFrom = (int) downsampled.getScanPositionFromBinarySearchResult(index);
+ assert scanFrom <= orig;
+ int effectiveInterval = downsampled.getEffectiveIndexIntervalAfterIndex(index);
+ DecoratedKey k = null;
+ for (int i = 0 ; k != key && i < effectiveInterval && scanFrom < keys.size() ; i++, scanFrom ++)
+ k = keys.get(scanFrom);
+ assert k == key;
+ }
+ }
+
+ @Test
+ public void testOriginalIndexLookup()
+ {
+ for (int i = BASE_SAMPLING_LEVEL; i >= 1; i--)
+ assertEquals(i, Downsampling.getOriginalIndexes(i).size());
+
+ ArrayList<Integer> full = new ArrayList<>();
+ for (int i = 0; i < BASE_SAMPLING_LEVEL; i++)
+ full.add(i);
+
+ assertEquals(full, Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL));
+ // the entry at index 127 is the first to go
+ assertEquals(full.subList(0, full.size() - 1), Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL - 1));
+
+ // spot check a few values (these depend on BASE_SAMPLING_LEVEL being 128)
+ assertEquals(128, BASE_SAMPLING_LEVEL);
+ assertEquals(Arrays.asList(0, 32, 64, 96), Downsampling.getOriginalIndexes(4));
+ assertEquals(Arrays.asList(0, 64), Downsampling.getOriginalIndexes(2));
+ assertEquals(Arrays.asList(0), Downsampling.getOriginalIndexes(1));
+ }
+
+ @Test
+ public void testGetNumberOfSkippedEntriesAfterIndex()
+ {
+ int indexInterval = 128;
+ for (int i = 0; i < BASE_SAMPLING_LEVEL; i++)
+ assertEquals(indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(i, BASE_SAMPLING_LEVEL, indexInterval));
+
+ // with one round of downsampling, only the last summary entry has been removed, so only the last index will have
+ // double the gap until the next sample
+ for (int i = 0; i < BASE_SAMPLING_LEVEL - 2; i++)
+ assertEquals(indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(i, BASE_SAMPLING_LEVEL - 1, indexInterval));
+ assertEquals(indexInterval * 2, Downsampling.getEffectiveIndexIntervalAfterIndex(BASE_SAMPLING_LEVEL - 2, BASE_SAMPLING_LEVEL - 1, indexInterval));
+
+ // at samplingLevel=2, the retained summary points are [0, 64] (assumes BASE_SAMPLING_LEVEL is 128)
+ assertEquals(128, BASE_SAMPLING_LEVEL);
+ assertEquals(64 * indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(0, 2, indexInterval));
+ assertEquals(64 * indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(1, 2, indexInterval));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java b/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java
new file mode 100644
index 0000000..5ef70c4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.keycache;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.AutoSavingCache;
+import org.apache.cassandra.cache.ICache;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Refs;
+import org.assertj.core.api.Assertions;
+import org.hamcrest.Matchers;
+import org.mockito.Mockito;
+import org.mockito.internal.stubbing.answers.AnswersWithDelay;
+import org.mockito.invocation.InvocationOnMock;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+public class KeyCacheTest
+{
+ private static boolean sstableImplCachesKeys;
+
+ private static final String KEYSPACE1 = "KeyCacheTest1";
+ private static final String KEYSPACE2 = "KeyCacheTest2";
+ private static final String COLUMN_FAMILY1 = "Standard1";
+ private static final String COLUMN_FAMILY2 = "Standard2";
+ private static final String COLUMN_FAMILY3 = "Standard3";
+ private static final String COLUMN_FAMILY4 = "Standard4";
+ private static final String COLUMN_FAMILY5 = "Standard5";
+ private static final String COLUMN_FAMILY6 = "Standard6";
+ private static final String COLUMN_FAMILY7 = "Standard7";
+ private static final String COLUMN_FAMILY8 = "Standard8";
+ private static final String COLUMN_FAMILY9 = "Standard9";
+ private static final String COLUMN_FAMILY10 = "Standard10";
+
+ private static final String COLUMN_FAMILY_K2_1 = "Standard1";
+
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ sstableImplCachesKeys = KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat());
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY1),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY2),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY3),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY4),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY5),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY6),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY7),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY8),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY9),
+ SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY10));
+
+ SchemaLoader.createKeyspace(KEYSPACE2,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE2, COLUMN_FAMILY_K2_1));
+
+ }
+
+ @AfterClass
+ public static void cleanup()
+ {
+ SchemaLoader.cleanupSavedCaches();
+ }
+
+ @Test
+ public void testKeyCacheLoadShallowIndexEntry() throws Exception
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(0);
+ testKeyCacheLoad(COLUMN_FAMILY2);
+ }
+
+ @Test
+ public void testKeyCacheLoadIndexInfoOnHeap() throws Exception
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(8);
+ testKeyCacheLoad(COLUMN_FAMILY5);
+ }
+
+ private void testKeyCacheLoad(String cf) throws Exception
+ {
+ CompactionManager.instance.disableAutoCompaction();
+
+ ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
+
+ // empty the cache
+ CacheService.instance.invalidateKeyCache();
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+
+ // insert data and force to disk
+ SchemaLoader.insertData(KEYSPACE1, cf, 0, 100);
+ Util.flush(store);
+
+ // populate the cache
+ readData(KEYSPACE1, cf, 0, 100);
+ assertKeyCacheSize(100, KEYSPACE1, cf);
+
+ // really? our caches don't implement the map interface? (hence no .addAll)
+ Map<KeyCacheKey, AbstractRowIndexEntry> savedMap = new HashMap<>();
+ Map<KeyCacheKey, RowIndexEntry.IndexInfoRetriever> savedInfoMap = new HashMap<>();
+ for (Iterator<KeyCacheKey> iter = CacheService.instance.keyCache.keyIterator();
+ iter.hasNext();)
+ {
+ KeyCacheKey k = iter.next();
+ if (k.desc.ksname.equals(KEYSPACE1) && k.desc.cfname.equals(cf))
+ {
+ AbstractRowIndexEntry rie = CacheService.instance.keyCache.get(k);
+ savedMap.put(k, rie);
+ if (rie instanceof RowIndexEntry)
+ {
+ BigTableReader sstr = (BigTableReader) readerForKey(k);
+ savedInfoMap.put(k, ((RowIndexEntry) rie).openWithIndex(sstr.getIndexFile()));
+ }
+ }
+ }
+
+ // force the cache to disk
+ CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
+
+ CacheService.instance.invalidateKeyCache();
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+
+ CacheService.instance.keyCache.loadSaved();
+ assertKeyCacheSize(savedMap.size(), KEYSPACE1, cf);
+
+ // probably it's better to add equals/hashCode to RowIndexEntry...
+ for (Map.Entry<KeyCacheKey, AbstractRowIndexEntry> entry : savedMap.entrySet())
+ {
+ AbstractRowIndexEntry expected = entry.getValue();
+ AbstractRowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey());
+ assertEquals(expected.position, actual.position);
+ assertEquals(expected.blockCount(), actual.blockCount());
+ assertEquals(expected.getSSTableFormat(), actual.getSSTableFormat());
+ for (int i = 0; i < expected.blockCount(); i++)
+ {
+ SSTableReader actualSstr = readerForKey(entry.getKey());
+ Assertions.assertThat(actualSstr.descriptor.version.format).isEqualTo(expected.getSSTableFormat());
+ if (actual instanceof RowIndexEntry)
+ {
+ try (RowIndexEntry.IndexInfoRetriever actualIir = ((RowIndexEntry) actual).openWithIndex(((BigTableReader) actualSstr).getIndexFile()))
+ {
+ RowIndexEntry.IndexInfoRetriever expectedIir = savedInfoMap.get(entry.getKey());
+ assertEquals(expectedIir.columnsIndex(i), actualIir.columnsIndex(i));
+ }
+ }
+ }
+ if (expected.isIndexed())
+ {
+ assertEquals(expected.deletionTime(), actual.deletionTime());
+ }
+ }
+
+ savedInfoMap.values().forEach(iir -> {
+ try
+ {
+ if (iir != null)
+ iir.close();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private static SSTableReader readerForKey(KeyCacheKey k)
+ {
+ return ColumnFamilyStore.getIfExists(k.desc.ksname, k.desc.cfname).getLiveSSTables()
+ .stream()
+ .filter(sstreader -> sstreader.descriptor.id == k.desc.id)
+ .findFirst().get();
+ }
+
+ @Test
+ public void testKeyCacheLoadWithLostTableShallowIndexEntry() throws Exception
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(0);
+ testKeyCacheLoadWithLostTable(COLUMN_FAMILY3);
+ }
+
+ @Test
+ public void testKeyCacheLoadWithLostTableIndexInfoOnHeap() throws Exception
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(8);
+ testKeyCacheLoadWithLostTable(COLUMN_FAMILY6);
+ }
+
+ private void testKeyCacheLoadWithLostTable(String cf) throws Exception
+ {
+ CompactionManager.instance.disableAutoCompaction();
+
+ ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
+
+ // empty the cache
+ CacheService.instance.invalidateKeyCache();
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+
+ // insert data and force to disk
+ SchemaLoader.insertData(KEYSPACE1, cf, 0, 100);
+ Util.flush(store);
+
+ Collection<SSTableReader> firstFlushTables = ImmutableList.copyOf(store.getLiveSSTables());
+
+ // populate the cache
+ readData(KEYSPACE1, cf, 0, 100);
+ assertKeyCacheSize(100, KEYSPACE1, cf);
+
+ // insert some new data and force to disk
+ SchemaLoader.insertData(KEYSPACE1, cf, 100, 50);
+ Util.flush(store);
+
+ // check that it's fine
+ readData(KEYSPACE1, cf, 100, 50);
+ assertKeyCacheSize(150, KEYSPACE1, cf);
+
+ // force the cache to disk
+ CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
+
+ CacheService.instance.invalidateKeyCache();
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+
+ // check that the content is written correctly
+ CacheService.instance.keyCache.loadSaved();
+ assertKeyCacheSize(150, KEYSPACE1, cf);
+
+ CacheService.instance.invalidateKeyCache();
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+
+ // now remove the first sstable from the store to simulate losing the file
+ store.markObsolete(firstFlushTables, OperationType.UNKNOWN);
+
+ // check that reading now correctly skips over lost table and reads the rest (CASSANDRA-10219)
+ CacheService.instance.keyCache.loadSaved();
+ assertKeyCacheSize(50, KEYSPACE1, cf);
+ }
+
+ @Test
+ public void testKeyCacheShallowIndexEntry() throws ExecutionException, InterruptedException
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(0);
+ testKeyCache(COLUMN_FAMILY1);
+ }
+
+ @Test
+ public void testKeyCacheIndexInfoOnHeap() throws ExecutionException, InterruptedException
+ {
+ DatabaseDescriptor.setColumnIndexCacheSize(8);
+ testKeyCache(COLUMN_FAMILY4);
+ }
+
+ private void testKeyCache(String cf) throws ExecutionException, InterruptedException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cf);
+
+ // just to make sure that everything is clean
+ CacheService.instance.invalidateKeyCache();
+
+ // KeyCache should start at size 0 if we're caching X% of zero data.
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+
+ Mutation rm;
+
+ // inserts
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("1").build().applyUnsafe();
+ new RowUpdateBuilder(cfs.metadata(), 0, "key2").clustering("2").build().applyUnsafe();
+
+ // to make sure we have SSTable
+ Util.flush(cfs);
+
+ // reads to cache key position
+ Util.getAll(Util.cmd(cfs, "key1").build());
+ Util.getAll(Util.cmd(cfs, "key2").build());
+
+ assertKeyCacheSize(2, KEYSPACE1, cf);
+
+ Set<SSTableReader> readers = cfs.getLiveSSTables();
+ Refs<SSTableReader> refs = Refs.tryRef(readers);
+ if (refs == null)
+ throw new IllegalStateException();
+
+ Util.compactAll(cfs, Integer.MAX_VALUE).get();
+ boolean noEarlyOpen = DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMiB() < 0;
+
+ // after compaction cache should have entries for new SSTables,
+ // but since we have kept a reference to the old sstables,
+ // if we had 2 keys in cache previously it should become 4
+ assertKeyCacheSize(noEarlyOpen ? 2 : 4, KEYSPACE1, cf);
+
+ refs.release();
+
+ LifecycleTransaction.waitForDeletions();
+
+ // after releasing the reference this should drop to 2
+ assertKeyCacheSize(2, KEYSPACE1, cf);
+
+ // re-read same keys to verify that key cache didn't grow further
+ Util.getAll(Util.cmd(cfs, "key1").build());
+ Util.getAll(Util.cmd(cfs, "key2").build());
+
+ assertKeyCacheSize(noEarlyOpen ? 4 : 2, KEYSPACE1, cf);
+ }
+
+ @Test
+ public void testKeyCacheLoadZeroCacheLoadTime() throws Exception
+ {
+ DatabaseDescriptor.setCacheLoadTimeout(0);
+ String cf = COLUMN_FAMILY7;
+
+ createAndInvalidateCache(Collections.singletonList(Pair.create(KEYSPACE1, cf)), 100);
+
+ CacheService.instance.keyCache.loadSaved();
+
+ // Here max time to load cache is zero which means no time left to load cache. So the keyCache size should
+ // be zero after loadSaved().
+ assertKeyCacheSize(0, KEYSPACE1, cf);
+ }
+
+ @Test
+ public void testKeyCacheLoadTwoTablesTime() throws Exception
+ {
+ DatabaseDescriptor.setCacheLoadTimeout(60);
+ String columnFamily1 = COLUMN_FAMILY8;
+ String columnFamily2 = COLUMN_FAMILY_K2_1;
+ int numberOfRows = 100;
+ List<Pair<String, String>> tables = new ArrayList<>(2);
+ tables.add(Pair.create(KEYSPACE1, columnFamily1));
+ tables.add(Pair.create(KEYSPACE2, columnFamily2));
+
+ createAndInvalidateCache(tables, numberOfRows);
+
+ CacheService.instance.keyCache.loadSaved();
+
+ // Here max time to load cache is negative which means no time left to load cache. So the keyCache size should
+ // be zero after load.
+ assertKeyCacheSize(numberOfRows, KEYSPACE1, columnFamily1);
+ assertKeyCacheSize(numberOfRows, KEYSPACE2, columnFamily2);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void testKeyCacheLoadCacheLoadTimeExceedingLimit() throws Exception
+ {
+ DatabaseDescriptor.setCacheLoadTimeout(2);
+ int delayMillis = 1000;
+ int numberOfRows = 100;
+
+ String cf = COLUMN_FAMILY9;
+
+ createAndInvalidateCache(Collections.singletonList(Pair.create(KEYSPACE1, cf)), numberOfRows);
+
+ // Testing cache load. Here using custom built AutoSavingCache instance as simulating delay is not possible with
+ // 'CacheService.instance.keyCache'. 'AutoSavingCache.loadSaved()' is returning no.of entries loaded so we don't need
+ // to instantiate ICache.class.
+ CacheService.KeyCacheSerializer keyCacheSerializer = new CacheService.KeyCacheSerializer();
+ CacheService.KeyCacheSerializer keyCacheSerializerSpy = Mockito.spy(keyCacheSerializer);
+ AutoSavingCache autoSavingCache = new AutoSavingCache(mock(ICache.class),
+ CacheService.CacheType.KEY_CACHE,
+ keyCacheSerializerSpy);
+
+ doAnswer(new AnswersWithDelay(delayMillis, InvocationOnMock::callRealMethod)).when(keyCacheSerializerSpy)
+ .deserialize(any(DataInputPlus.class));
+
+ long maxExpectedKeyCache = Math.min(numberOfRows,
+ 1 + TimeUnit.SECONDS.toMillis(DatabaseDescriptor.getCacheLoadTimeout()) / delayMillis);
+
+ long keysLoaded = autoSavingCache.loadSaved();
+ assertThat(keysLoaded, Matchers.lessThanOrEqualTo(maxExpectedKeyCache));
+ if (sstableImplCachesKeys)
+ {
+ assertNotEquals(0, keysLoaded);
+ Mockito.verify(keyCacheSerializerSpy, Mockito.times(1)).cleanupAfterDeserialize();
+ }
+ else
+ {
+ assertEquals(0, keysLoaded);
+ Mockito.verify(keyCacheSerializerSpy, Mockito.never()).cleanupAfterDeserialize();
+ }
+ }
+
+ private void createAndInvalidateCache(List<Pair<String, String>> tables, int numberOfRows) throws ExecutionException, InterruptedException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+
+ // empty the cache
+ CacheService.instance.invalidateKeyCache();
+ assertEquals(0, CacheService.instance.keyCache.size());
+
+ for(Pair<String, String> entry : tables)
+ {
+ String keyspace = entry.left();
+ String cf = entry.right();
+ ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(cf);
+
+ // insert data and force to disk
+ SchemaLoader.insertData(keyspace, cf, 0, numberOfRows);
+ Util.flush(store);
+ }
+ for(Pair<String, String> entry : tables)
+ {
+ String keyspace = entry.left();
+ String cf = entry.right();
+ // populate the cache
+ readData(keyspace, cf, 0, numberOfRows);
+ assertKeyCacheSize(numberOfRows, keyspace, cf);
+ }
+
+ // force the cache to disk
+ CacheService.instance.keyCache.submitWrite(CacheService.instance.keyCache.size()).get();
+
+ CacheService.instance.invalidateKeyCache();
+ assertEquals(0, CacheService.instance.keyCache.size());
+ }
+
+ private static void readData(String keyspace, String columnFamily, int startRow, int numberOfRows)
+ {
+ ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily);
+ for (int i = 0; i < numberOfRows; i++)
+ Util.getAll(Util.cmd(store, "key" + (i + startRow)).includeRow("col" + (i + startRow)).build());
+ }
+
+
+ private void assertKeyCacheSize(int expected, String keyspace, String columnFamily)
+ {
+ int size = 0;
+ for (Iterator<KeyCacheKey> iter = CacheService.instance.keyCache.keyIterator();
+ iter.hasNext();)
+ {
+ KeyCacheKey k = iter.next();
+ if (k.desc.ksname.equals(keyspace) && k.desc.cfname.equals(columnFamily))
+ size++;
+ }
+ assertEquals(sstableImplCachesKeys ? expected : 0, size);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 9b5ff62..0566eb8 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -17,29 +17,40 @@
*/
package org.apache.cassandra.io.sstable.metadata;
-import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
-import org.apache.cassandra.io.util.*;
import java.io.IOException;
-import java.util.Arrays;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.RandomPartitioner;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.Throwables;
import static org.junit.Assert.assertEquals;
@@ -48,21 +59,26 @@
public class MetadataSerializerTest
{
+ private final static Logger logger = LoggerFactory.getLogger(MetadataSerializerTest.class);
+
+ private static SSTableFormat<?, ?> format;
+
@BeforeClass
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
+ format = DatabaseDescriptor.getSelectedSSTableFormat();
}
@Test
public void testSerialization() throws IOException
{
- Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata(false);
MetadataSerializer serializer = new MetadataSerializer();
- File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
+ File statsFile = serialize(originalMetadata, serializer, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion());
- Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.BIG);
+ Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
try (RandomAccessReader in = RandomAccessReader.open(statsFile))
{
Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
@@ -77,7 +93,7 @@
@Test
public void testHistogramSterilization() throws IOException
{
- Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata(false);
// Modify the histograms to overflow:
StatsMetadata originalStats = (StatsMetadata) originalMetadata.get(MetadataType.STATS);
@@ -88,8 +104,8 @@
// Serialize w/ overflowed histograms:
MetadataSerializer serializer = new MetadataSerializer();
- File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
- Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.BIG);
+ File statsFile = serialize(originalMetadata, serializer, format.getLatestVersion());
+ Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), format);
try (RandomAccessReader in = RandomAccessReader.open(statsFile))
{
@@ -105,7 +121,7 @@
throws IOException
{
// Serialize to tmp file
- File statsFile = FileUtils.createTempFile(Component.STATS.name, null);
+ File statsFile = FileUtils.createTempFile(Components.STATS.name, null);
try (DataOutputStreamPlus out = new FileOutputStreamPlus(statsFile))
{
serializer.serialize(metadata, out, version);
@@ -113,34 +129,44 @@
return statsFile;
}
- public Map<MetadataType, MetadataComponent> constructMetadata()
+ public Map<MetadataType, MetadataComponent> constructMetadata(boolean withNulls)
{
CommitLogPosition club = new CommitLogPosition(11L, 12);
CommitLogPosition cllb = new CommitLogPosition(9L, 12);
- TableMetadata cfm = SchemaLoader.standardCFMD("ks1", "cf1").build();
+ TableMetadata cfm = TableMetadata.builder("ks1", "cf1")
+ .addPartitionKeyColumn("k", AsciiType.instance)
+ .addClusteringColumn("c1", UTF8Type.instance)
+ .addClusteringColumn("c2", Int32Type.instance)
+ .addRegularColumn("v", Int32Type.instance)
+ .build();
MetadataCollector collector = new MetadataCollector(cfm.comparator)
.commitLogIntervals(new IntervalSet<>(cllb, club));
String partitioner = RandomPartitioner.class.getCanonicalName();
double bfFpChance = 0.1;
- return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()));
+ collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("abc"), Int32Type.instance.decompose(123)));
+ collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("cba"), withNulls ? null : Int32Type.instance.decompose(234)));
+ ByteBuffer first = AsciiType.instance.decompose("a");
+ ByteBuffer last = AsciiType.instance.decompose("b");
+ return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()), first, last);
}
- private void testVersions(String... versions) throws Throwable
+ private void testVersions(List<String> versions) throws Throwable
{
+ logger.info("Testing minor versions {} compatibility for sstable format {}", versions, format.getClass().getName());
Throwable t = null;
- for (int oldIdx = 0; oldIdx < versions.length; oldIdx++)
+ for (int oldIdx = 0; oldIdx < versions.size(); oldIdx++)
{
- for (int newIdx = oldIdx; newIdx < versions.length; newIdx++)
+ for (int newIdx = oldIdx; newIdx < versions.size(); newIdx++)
{
try
{
- testOldReadsNew(versions[oldIdx], versions[newIdx]);
+ testOldReadsNew(versions.get(oldIdx), versions.get(newIdx));
}
catch (Exception | AssertionError e)
{
- t = Throwables.merge(t, new AssertionError("Failed to test " + versions[oldIdx] + " -> " + versions[newIdx], e));
+ t = Throwables.merge(t, new AssertionError("Failed to test " + versions.get(oldIdx) + " -> " + versions.get(newIdx), e));
}
}
}
@@ -151,28 +177,32 @@
}
@Test
- public void testMVersions() throws Throwable
+ public void testMinorVersionsCompatibilty() throws Throwable
{
- testVersions("ma", "mb", "mc", "md", "me");
- }
+ Map<Character, List<String>> supportedVersions = new LinkedHashMap<>();
- @Test
- public void testNVersions() throws Throwable
- {
- testVersions("na", "nb");
+ for (char major = 'a'; major <= 'z'; major++){
+ for (char minor = 'a'; minor <= 'z'; minor++){
+ Version version = format.getVersion(String.format("%s%s", major, minor));
+ if (version.isCompatible())
+ supportedVersions.computeIfAbsent(major, ignored -> new ArrayList<>()).add(version.version);
+ }
+ }
+
+ for (List<String> minorVersions : supportedVersions.values())
+ testVersions(minorVersions);
}
public void testOldReadsNew(String oldV, String newV) throws IOException
{
- Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata(true);
MetadataSerializer serializer = new MetadataSerializer();
// Write metadata in two minor formats.
- File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV));
- File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
+ File statsFileLb = serialize(originalMetadata, serializer, format.getVersion(newV));
+ File statsFileLa = serialize(originalMetadata, serializer, format.getVersion(oldV));
// Reading both as earlier version should yield identical results.
- SSTableFormat.Type stype = SSTableFormat.Type.current();
- Descriptor desc = new Descriptor(stype.info.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0), stype);
+ Descriptor desc = new Descriptor(format.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0));
try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
@@ -189,17 +219,4 @@
}
}
- @Test
- public void pendingRepairCompatibility()
- {
- Arrays.asList("ma", "mb", "mc", "md", "me").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPendingRepair()));
- Arrays.asList("na", "nb").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPendingRepair()));
- }
-
- @Test
- public void originatingHostCompatibility()
- {
- Arrays.asList("ma", "mb", "mc", "md", "na").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasOriginatingHostId()));
- Arrays.asList("me", "nb").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasOriginatingHostId()));
- }
}
diff --git a/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java b/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java
new file mode 100644
index 0000000..2eae20c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/tries/AbstractTrieTestBase.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.function.BiFunction;
+
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.PageAware;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+@RunWith(Parameterized.class)
+abstract public class AbstractTrieTestBase
+{
+ @Parameterized.Parameter(0)
+ public TestClass writerClass;
+
+ enum TestClass
+ {
+ SIMPLE(IncrementalTrieWriterSimple::new),
+ PAGE_AWARE(IncrementalTrieWriterPageAware::new),
+ PAGE_AWARE_DEEP_ON_STACK((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 256)),
+ PAGE_AWARE_DEEP_ON_HEAP((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 0)),
+ PAGE_AWARE_DEEP_MIXED((serializer, dest) -> new IncrementalDeepTrieWriterPageAware<>(serializer, dest, 2));
+
+ final BiFunction<TrieSerializer<Integer, DataOutputPlus>, DataOutputPlus, IncrementalTrieWriter<Integer>> constructor;
+ TestClass(BiFunction<TrieSerializer<Integer, DataOutputPlus>, DataOutputPlus, IncrementalTrieWriter<Integer>> constructor)
+ {
+ this.constructor = constructor;
+ }
+ }
+
+ @Parameterized.Parameters(name = "{index}: trie writer class={0}")
+ public static Collection<Object[]> data()
+ {
+ return Arrays.asList(new Object[]{ TestClass.SIMPLE },
+ new Object[]{ TestClass.PAGE_AWARE },
+ new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_STACK },
+ new Object[]{ TestClass.PAGE_AWARE_DEEP_ON_HEAP },
+ new Object[]{ TestClass.PAGE_AWARE_DEEP_MIXED });
+ }
+
+ protected final static Logger logger = LoggerFactory.getLogger(TrieBuilderTest.class);
+ protected final static int BASE = 80;
+
+ protected boolean dump = false;
+ protected int payloadSize = 0;
+
+ @Before
+ public void beforeTest()
+ {
+ dump = false;
+ payloadSize = 0;
+ }
+
+ IncrementalTrieWriter<Integer> newTrieWriter(TrieSerializer<Integer, DataOutputPlus> serializer, DataOutputPlus out)
+ {
+ return writerClass.constructor.apply(serializer, out);
+ }
+
+ protected final TrieSerializer<Integer, DataOutputPlus> serializer = new TrieSerializer<Integer, DataOutputPlus>()
+ {
+ public int sizeofNode(SerializationNode<Integer> node, long nodePosition)
+ {
+ return TrieNode.typeFor(node, nodePosition).sizeofNode(node) + payloadSize;
+ }
+
+ public void write(DataOutputPlus dataOutput, SerializationNode<Integer> node, long nodePosition) throws IOException
+ {
+ if (dump)
+ logger.info("Writing at {} type {} size {}: {}", Long.toHexString(nodePosition), TrieNode.typeFor(node, nodePosition), TrieNode.typeFor(node, nodePosition).sizeofNode(node), node);
+ // Our payload value is an integer of four bits.
+ // We use the payload bits in the trie node header to fully store it.
+ TrieNode.typeFor(node, nodePosition).serialize(dataOutput, node, node.payload() != null ? node.payload() : 0, nodePosition);
+ // and we also add some padding if a test needs it
+ dataOutput.write(new byte[payloadSize]);
+ }
+ };
+
+
+ protected int valueFor(long found)
+ {
+ return Long.bitCount(found + 1) & 0xF;
+ }
+
+ protected ByteComparable source(String s)
+ {
+ if (s == null)
+ return null;
+ ByteBuffer buf = ByteBuffer.allocate(s.length());
+ for (int i = 0; i < s.length(); ++i)
+ buf.put((byte) s.charAt(i));
+ buf.rewind();
+ return ByteComparable.fixedLength(buf);
+ }
+
+ protected String toBase(long v)
+ {
+ return BigInteger.valueOf(v).toString(BASE);
+ }
+
+ // In-memory buffer with added paging parameters, to make sure the code below does the proper layout
+ protected static class DataOutputBufferPaged extends DataOutputBuffer
+ {
+ public int maxBytesInPage()
+ {
+ return PageAware.PAGE_SIZE;
+ }
+
+ public void padToPageBoundary() throws IOException
+ {
+ PageAware.pad(this);
+ }
+
+ public int bytesLeftInPage()
+ {
+ long position = position();
+ long bytesLeft = PageAware.pageLimit(position) - position;
+ return (int) bytesLeft;
+ }
+
+ public long paddedPosition()
+ {
+ return PageAware.padded(position());
+ }
+ }
+
+ protected static class ByteBufRebufferer implements Rebufferer, Rebufferer.BufferHolder
+ {
+ final ByteBuffer buffer;
+
+ ByteBufRebufferer(ByteBuffer buffer)
+ {
+ this.buffer = buffer;
+ }
+
+ @Override
+ public ChannelProxy channel()
+ {
+ return null;
+ }
+
+ @Override
+ public ByteBuffer buffer()
+ {
+ return buffer;
+ }
+
+ @Override
+ public long fileLength()
+ {
+ return buffer.remaining();
+ }
+
+ @Override
+ public double getCrcCheckChance()
+ {
+ return 0;
+ }
+
+ @Override
+ public BufferHolder rebuffer(long position)
+ {
+ return this;
+ }
+
+ @Override
+ public long offset()
+ {
+ return 0;
+ }
+
+ @Override
+ public void release()
+ {
+ // nothing
+ }
+
+ @Override
+ public void close()
+ {
+ // nothing
+ }
+
+ @Override
+ public void closeReader()
+ {
+ // nothing
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java b/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java
new file mode 100644
index 0000000..e484033
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/tries/TrieBuilderTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.TailOverridingRebufferer;
+
+import static org.junit.Assert.assertEquals;
+
+public class TrieBuilderTest extends AbstractTrieTestBase
+{
+ @Test
+ public void testPartialBuildRecalculationBug() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = newTrieWriter(serializer, buf);
+ long count = 0;
+
+ count += addUntilBytesWritten(buf, builder, "a", 1); // Make a node whose children are written
+ long reset = count;
+ count += addUntilBytesWritten(buf, builder, "c", 64 * 1024); // Finalize it and write long enough to grow its pointer size
+
+ dump = true;
+ IncrementalTrieWriter.PartialTail tail = builder.makePartialRoot();
+ // The line above hit an assertion as that node's parent had a pre-calculated branch size which was no longer
+ // correct, and we didn't bother to reset it.
+ dump = false;
+
+ // Check that partial representation has the right content.
+ Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
+ source = new TailOverridingRebufferer(source, tail.cutoff(), tail.tail());
+ verifyContent(count, source, tail.root(), reset);
+
+ long reset2 = count;
+
+ // Also check the completed trie.
+ count += addUntilBytesWritten(buf, builder, "e", 16 * 1024);
+ dump = true;
+ long root = builder.complete();
+ // The line above hit another size assertion as the size of a node's branch growing caused it to need to switch
+ // format, but we didn't bother to recalculate its size.
+ dump = false;
+
+ source = new ByteBufRebufferer(buf.asNewBuffer());
+ verifyContent(count, source, root, reset, reset2);
+ }
+
+ public void verifyContent(long count, Rebufferer source, long root, long... resets)
+ {
+ ValueIterator<?> iter = new ValueIterator<>(source, root);
+ long found = 0;
+ long ofs = 0;
+ int rpos = 0;
+ long pos;
+ while ((pos = iter.nextPayloadedNode()) != -1)
+ {
+ iter.go(pos);
+ assertEquals(valueFor(found - ofs), iter.payloadFlags());
+ ++found;
+ if (rpos < resets.length && found >= resets[rpos])
+ {
+ ofs = resets[rpos];
+ ++rpos;
+ }
+ }
+ assertEquals(count, found);
+ }
+
+ private long addUntilBytesWritten(DataOutputBuffer buf,
+ IncrementalTrieWriter<Integer> builder,
+ String prefix,
+ long howMany) throws IOException
+ {
+ long pos = buf.position();
+ long idx = 0;
+ while (pos + howMany > buf.position())
+ {
+ builder.add(source(String.format("%s%8s", prefix, toBase(idx))), valueFor(idx));
+ logger.info("Adding {} : {}", String.format("%s%8s", prefix, toBase(idx)), valueFor(idx));
+ ++idx;
+ }
+ logger.info(String.format("%s%8s", prefix, toBase(idx - 1)));
+ return idx;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/tries/TrieNodeTest.java b/test/unit/org/apache/cassandra/io/tries/TrieNodeTest.java
new file mode 100644
index 0000000..20ad1f4
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/tries/TrieNodeTest.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.mockito.Mockito;
+import org.quicktheories.api.Pair;
+import org.quicktheories.generators.Generate;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.when;
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.longs;
+
+@SuppressWarnings("unchecked")
+public class TrieNodeTest
+{
+ private final SerializationNode<Integer> sn = Mockito.mock(SerializationNode.class);
+ private final DataOutputBuffer out = new DataOutputBuffer();
+
+ @After
+ public void after()
+ {
+ reset(sn);
+ out.clear();
+ }
+
+ @Test
+ public void testTypeFor0Children()
+ {
+ when(sn.childCount()).thenReturn(0);
+ assertSame(TrieNode.Types.PAYLOAD_ONLY, TrieNode.typeFor(sn, 0));
+ }
+
+ @Test
+ public void testTypeFor1ChildNoPayload()
+ {
+ when(sn.childCount()).thenReturn(1);
+ when(sn.payload()).thenReturn(null);
+
+ qt().forAll(Generate.pick(Arrays.asList(
+ Pair.of(Ranges.BITS_4, TrieNode.Types.SINGLE_NOPAYLOAD_4),
+ Pair.of(Ranges.BITS_8, TrieNode.Types.SINGLE_8),
+ Pair.of(Ranges.BITS_12, TrieNode.Types.SINGLE_NOPAYLOAD_12),
+ Pair.of(Ranges.BITS_16, TrieNode.Types.SINGLE_16),
+ Pair.of(Ranges.BITS_24, TrieNode.Types.DENSE_24),
+ Pair.of(Ranges.BITS_32, TrieNode.Types.DENSE_32),
+ Pair.of(Ranges.BITS_40, TrieNode.Types.DENSE_40),
+ Pair.of(Ranges.BITS_GT40, TrieNode.Types.LONG_DENSE)
+ )).flatMap(p -> longs().between(p._1.min, p._1.max).map(v -> Pair.of(v, p._2)))).check(p -> {
+ when(sn.maxPositionDelta(0)).thenReturn(-p._1);
+ return p._2 == TrieNode.typeFor(sn, 0);
+ });
+ }
+
+ @Test
+ public void testTypeFor1ChildAndPayload()
+ {
+ when(sn.childCount()).thenReturn(1);
+ when(sn.payload()).thenReturn(1);
+
+ qt().forAll(Generate.pick(Arrays.asList(
+ Pair.of(Ranges.BITS_4, TrieNode.Types.SINGLE_8),
+ Pair.of(Ranges.BITS_8, TrieNode.Types.SINGLE_8),
+ Pair.of(Ranges.BITS_12, TrieNode.Types.SINGLE_16),
+ Pair.of(Ranges.BITS_16, TrieNode.Types.SINGLE_16),
+ Pair.of(Ranges.BITS_24, TrieNode.Types.DENSE_24),
+ Pair.of(Ranges.BITS_32, TrieNode.Types.DENSE_32),
+ Pair.of(Ranges.BITS_40, TrieNode.Types.DENSE_40),
+ Pair.of(Ranges.BITS_GT40, TrieNode.Types.LONG_DENSE)
+ )).flatMap(p -> longs().between(p._1.min, p._1.max).map(v -> Pair.of(v, p._2)))).check(p -> {
+ when(sn.maxPositionDelta(0)).thenReturn(-p._1);
+ return p._2 == TrieNode.typeFor(sn, 0);
+ });
+ }
+
+ @Test
+ public void testTypeForMoreChildrenAndNoPayload()
+ {
+ when(sn.childCount()).thenReturn(2);
+ when(sn.payload()).thenReturn(null);
+
+ qt().forAll(Generate.pick(Arrays.asList(
+ Pair.of(Ranges.BITS_4, TrieNode.Types.DENSE_12),
+ Pair.of(Ranges.BITS_8, TrieNode.Types.DENSE_12),
+ Pair.of(Ranges.BITS_12, TrieNode.Types.DENSE_12),
+ Pair.of(Ranges.BITS_16, TrieNode.Types.DENSE_16),
+ Pair.of(Ranges.BITS_24, TrieNode.Types.DENSE_24),
+ Pair.of(Ranges.BITS_32, TrieNode.Types.DENSE_32),
+ Pair.of(Ranges.BITS_40, TrieNode.Types.DENSE_40),
+ Pair.of(Ranges.BITS_GT40, TrieNode.Types.LONG_DENSE)
+ )).flatMap(p -> longs().between(p._1.min, p._1.max).map(v -> Pair.of(v, p._2)))).check(p -> {
+ when(sn.maxPositionDelta(0)).thenReturn(-p._1);
+ return p._2 == TrieNode.typeFor(sn, 0);
+ });
+ }
+
+ @Test
+ public void testPayloadOnlyNode() throws IOException
+ {
+ TrieNode.Types.PAYLOAD_ONLY.serialize(out, null, 1 | 4, 0);
+ out.flush();
+
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 0);
+ assertEquals(TrieNode.Types.PAYLOAD_ONLY, node);
+ assertEquals(1 | 4, node.payloadFlags(out.asNewBuffer(), 0));
+ assertEquals(1, node.payloadPosition(out.asNewBuffer(), 0));
+ assertEquals(1, node.sizeofNode(null));
+ assertEquals(-1, node.search(null, 0, 0));
+ assertEquals(Integer.MAX_VALUE, node.transitionByte(null, 0, 0));
+ assertEquals(123, node.lesserTransition(null, 0, 0, 0, 123));
+ assertEquals(123, node.greaterTransition(null, 0, 0, 0, 123));
+ assertEquals(-1, node.lastTransition(null, 0, 0));
+ assertEquals(-1, node.transition(null, 0, 0, 0));
+ assertEquals(0, node.transitionRange(null, 0));
+ assertEquals(0, node.transitionDelta(null, 0, 0));
+ }
+
+ private void prepareSingleNode(long delta)
+ {
+ when(sn.childCount()).thenReturn(1);
+ when(sn.transition(0)).thenReturn(123);
+ when(sn.serializedPositionDelta(0, 0)).thenReturn(delta);
+ }
+
+ private void singleNodeAssertions(TrieNode node, int payloadFlags, int size, long pos)
+ {
+ assertEquals(payloadFlags, node.payloadFlags(out.asNewBuffer(), 0));
+ assertEquals(size, node.sizeofNode(null));
+ assertEquals(0, node.search(out.asNewBuffer(), 0, 123));
+ assertEquals(-1, node.search(out.asNewBuffer(), 0, 122));
+ assertEquals(-2, node.search(out.asNewBuffer(), 0, 124));
+ assertEquals(123, node.transitionByte(out.asNewBuffer(), 0, 0));
+ assertEquals(Integer.MAX_VALUE, node.transitionByte(out.asNewBuffer(), 0, 1));
+ assertEquals(100, node.lesserTransition(out.asNewBuffer(), 0, 100 - pos, -2, 123));
+ assertEquals(234, node.greaterTransition(null, 0, 0, 0, 234));
+ assertEquals(234, node.greaterTransition(out.asNewBuffer(), 0, 100 - pos, -2, 234));
+ assertEquals(100, node.greaterTransition(out.asNewBuffer(), 0, 100 - pos, -1, 234));
+ assertEquals(234, node.greaterTransition(out.asNewBuffer(), 0, 100 - pos, 0, 234));
+ assertEquals(100, node.lastTransition(out.asNewBuffer(), 0, 100 - pos));
+ assertEquals(100, node.transition(out.asNewBuffer(), 0, 100 - pos, 0));
+ assertEquals(1, node.transitionRange(out.asNewBuffer(), 0));
+ assertEquals(pos, node.transitionDelta(out.asNewBuffer(), 0, 0));
+ }
+
+ @Test
+ public void testSingle16Node() throws IOException
+ {
+ prepareSingleNode(-43210L);
+ TrieNode.Types.SINGLE_16.serialize(out, sn, 1 | 4, 0);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 0);
+ assertEquals(TrieNode.Types.SINGLE_16, node);
+ singleNodeAssertions(node, 1 | 4, 4, -43210);
+ }
+
+ @Test
+ public void testSingleNoPayload4Node() throws IOException
+ {
+ prepareSingleNode(-7L);
+ TrieNode.Types.SINGLE_NOPAYLOAD_4.serialize(out, sn, 0, 0);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 0);
+ assertEquals(TrieNode.Types.SINGLE_NOPAYLOAD_4, node);
+ singleNodeAssertions(node, 0, 2, -7);
+ }
+
+ @Test
+ public void testSingleNoPayload12Node() throws IOException
+ {
+ prepareSingleNode(-1234L);
+ TrieNode.Types.SINGLE_NOPAYLOAD_12.serialize(out, sn, 0, 0);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 0);
+ assertEquals(TrieNode.Types.SINGLE_NOPAYLOAD_12, node);
+ singleNodeAssertions(node, 0, 3, -1234L);
+ }
+
+ private void prepareSparseNode(long delta) throws IOException
+ {
+ out.write(new byte[6]);
+ when(sn.childCount()).thenReturn(3);
+ when(sn.transition(0)).thenReturn(10);
+ when(sn.transition(1)).thenReturn(20);
+ when(sn.transition(2)).thenReturn(30);
+ when(sn.serializedPositionDelta(0, 6)).thenReturn(delta);
+ when(sn.serializedPositionDelta(1, 6)).thenReturn(delta + 2);
+ when(sn.serializedPositionDelta(2, 6)).thenReturn(delta + 4);
+ }
+
+ private void sparseOrDenseNodeAssertions(TrieNode node, int payloadFlags, int size, long pos)
+ {
+ assertEquals(size, node.sizeofNode(sn));
+ assertEquals(payloadFlags, node.payloadFlags(out.asNewBuffer(), 6));
+ assertEquals(3, node.transitionRange(out.asNewBuffer(), 6));
+ assertEquals(6 + size, node.payloadPosition(out.asNewBuffer(), 6));
+
+ assertEquals(Integer.MAX_VALUE, node.transitionByte(out.asNewBuffer(), 6, 3));
+
+ assertEquals(10, node.lesserTransition(out.asNewBuffer(), 6, 10 - pos, 1, 123));
+ assertEquals(14, node.lesserTransition(out.asNewBuffer(), 6, 10 - pos, -4, 123));
+
+ assertEquals(14, node.greaterTransition(out.asNewBuffer(), 6, 10 - pos, 1, 234));
+ assertEquals(234, node.greaterTransition(out.asNewBuffer(), 6, 10 - pos, 2, 234));
+ assertEquals(10, node.greaterTransition(out.asNewBuffer(), 6, 10 - pos, -1, 234));
+
+ assertEquals(14, node.lastTransition(out.asNewBuffer(), 6, 10 - pos));
+
+ assertEquals(pos, node.transitionDelta(out.asNewBuffer(), 6, 0));
+ assertEquals(pos + 2, node.transitionDelta(out.asNewBuffer(), 6, 1));
+ assertEquals(pos + 4, node.transitionDelta(out.asNewBuffer(), 6, 2));
+
+ assertEquals(10, node.transition(out.asNewBuffer(), 6, 10 - pos, 0));
+ assertEquals(12, node.transition(out.asNewBuffer(), 6, 10 - pos, 1));
+ assertEquals(14, node.transition(out.asNewBuffer(), 6, 10 - pos, 2));
+ }
+
+ private void sparseNodeAssertions(TrieNode node, int payloadFlags, int size, long pos)
+ {
+ sparseOrDenseNodeAssertions(node, payloadFlags, size, pos);
+ assertEquals(-1, node.search(out.asNewBuffer(), 6, 5));
+ assertEquals(0, node.search(out.asNewBuffer(), 6, 10));
+ assertEquals(-2, node.search(out.asNewBuffer(), 6, 15));
+ assertEquals(-4, node.search(out.asNewBuffer(), 6, 35));
+
+ assertEquals(10, node.transitionByte(out.asNewBuffer(), 6, 0));
+ assertEquals(20, node.transitionByte(out.asNewBuffer(), 6, 1));
+ assertEquals(30, node.transitionByte(out.asNewBuffer(), 6, 2));
+ }
+
+ @Test
+ public void testSparse16Node() throws IOException
+ {
+ prepareSparseNode(-43210L);
+ TrieNode.Types.SPARSE_16.serialize(out, sn, 1 | 4, 6);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 6);
+ assertEquals(TrieNode.Types.SPARSE_16, node);
+ sparseNodeAssertions(node, 1 | 4, 11, -43210L);
+ }
+
+ @Test
+ public void testSparse12Node() throws IOException
+ {
+ prepareSparseNode(-1234L);
+ TrieNode.Types.SPARSE_12.serialize(out, sn, 1 | 4, 6);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 6);
+ assertEquals(TrieNode.Types.SPARSE_12, node);
+ sparseNodeAssertions(node, 1 | 4, 10, -1234L);
+ }
+
+ private void prepareDenseNode(long delta) throws IOException
+ {
+ out.write(new byte[6]);
+ when(sn.childCount()).thenReturn(3);
+ when(sn.transition(0)).thenReturn(11);
+ when(sn.transition(1)).thenReturn(12);
+ when(sn.transition(2)).thenReturn(13);
+ when(sn.serializedPositionDelta(0, 6)).thenReturn(delta);
+ when(sn.serializedPositionDelta(1, 6)).thenReturn(delta + 2);
+ when(sn.serializedPositionDelta(2, 6)).thenReturn(delta + 4);
+ }
+
+ private void denseNodeAssertions(TrieNode node, int payload, int size, long pos)
+ {
+ sparseOrDenseNodeAssertions(node, payload, size, pos);
+ assertEquals(-1, node.search(out.asNewBuffer(), 6, 10));
+ assertEquals(0, node.search(out.asNewBuffer(), 6, 11));
+ assertEquals(-4, node.search(out.asNewBuffer(), 6, 14));
+
+ assertEquals(11, node.transitionByte(out.asNewBuffer(), 6, 0));
+ assertEquals(12, node.transitionByte(out.asNewBuffer(), 6, 1));
+ assertEquals(13, node.transitionByte(out.asNewBuffer(), 6, 2));
+ }
+
+ @Test
+ public void testDense16Node() throws IOException
+ {
+ prepareDenseNode(-43210L);
+ TrieNode.Types.DENSE_16.serialize(out, sn, 1 | 4, 6);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 6);
+ assertEquals(TrieNode.Types.DENSE_16, node);
+ denseNodeAssertions(node, 1 | 4, 9, -43210L);
+ }
+
+ @Test
+ public void testDense12Node() throws IOException
+ {
+ prepareDenseNode(-1234L);
+ TrieNode.Types.DENSE_12.serialize(out, sn, 1 | 4, 6);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 6);
+ assertEquals(TrieNode.Types.DENSE_12, node);
+ denseNodeAssertions(node, 1 | 4, 8, -1234L);
+ }
+
+ @Test
+ public void testLongDenseNode() throws IOException
+ {
+ prepareDenseNode(-0x7ffffffffffffffL);
+ TrieNode.Types.LONG_DENSE.serialize(out, sn, 1 | 4, 6);
+ TrieNode node = TrieNode.at(out.asNewBuffer(), 6);
+ assertEquals(TrieNode.Types.LONG_DENSE, node);
+ denseNodeAssertions(node, 1 | 4, 27, -0x7ffffffffffffffL);
+ }
+
+ private enum Ranges
+ {
+ BITS_4(0x1L, 0xfL),
+ BITS_8(0x10L, 0xffL),
+ BITS_12(0x100L, 0xfffL),
+ BITS_16(0x1000L, 0xffffL),
+ BITS_24(0x100000L, 0xffffffL),
+ BITS_32(0x10000000L, 0xffffffffL),
+ BITS_40(0x1000000000L, 0xffffffffffL),
+ BITS_GT40(0x100000000000L, 0x7fffffffffffffffL);
+
+ private final long min, max;
+
+ Ranges(long min, long max)
+ {
+ this.min = min;
+ this.max = max;
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/tries/WalkerTest.java b/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
new file mode 100644
index 0000000..abd6c2e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.tries;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.function.LongSupplier;
+import java.util.function.LongToIntFunction;
+import java.util.stream.IntStream;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.Rebufferer;
+import org.apache.cassandra.io.util.TailOverridingRebufferer;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+
+@SuppressWarnings("unchecked")
+public class WalkerTest extends AbstractTrieTestBase
+{
+ @Test
+ public void testWalker() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+ long rootPos = builder.complete();
+
+ Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
+
+ Walker<?> it = new Walker<>(source, rootPos);
+
+ DataOutputBuffer dumpBuf = new DataOutputBuffer();
+ it.dumpTrie(new PrintStream(dumpBuf), (buf1, payloadPos, payloadFlags) -> String.format("%d/%d", payloadPos, payloadFlags));
+ logger.info("Trie dump: \n{}", new String(dumpBuf.getData()));
+ logger.info("Trie toString: {}", it);
+
+ it.goMax(rootPos);
+ assertEquals(12, it.payloadFlags());
+ assertEquals(TrieNode.Types.PAYLOAD_ONLY.ordinal, it.nodeTypeOrdinal());
+ assertEquals(1, it.nodeSize());
+ assertFalse(it.hasChildren());
+
+ it.goMin(rootPos);
+ assertEquals(1, it.payloadFlags());
+ assertEquals(TrieNode.Types.PAYLOAD_ONLY.ordinal, it.nodeTypeOrdinal());
+ assertEquals(1, it.nodeSize());
+ assertFalse(it.hasChildren());
+
+ assertEquals(-1, it.follow(source("151")));
+ assertEquals(2, it.payloadFlags());
+
+ assertEquals('3', it.follow(source("135")));
+
+ assertEquals('3', it.followWithGreater(source("135")));
+ it.goMin(it.greaterBranch);
+ assertEquals(2, it.payloadFlags());
+
+ assertEquals('3', it.followWithLesser(source("135")));
+ it.goMax(it.lesserBranch);
+ assertEquals(1, it.payloadFlags());
+
+ assertEquals(3, (Object) it.prefix(source("155"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ assertNull(it.prefix(source("516"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ assertEquals(5, (Object) it.prefix(source("5151"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ assertEquals(1, (Object) it.prefix(source("1151"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+
+ assertEquals(3, (Object) it.prefixAndNeighbours(source("155"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ assertNull(it.prefixAndNeighbours(source("516"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ assertEquals(5, (Object) it.prefixAndNeighbours(source("5151"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ assertEquals(1, (Object) it.prefixAndNeighbours(source("1151"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+
+ assertEquals(3, (Object) it.prefixAndNeighbours(source("1555"), (walker, payloadPosition, payloadFlags) -> payloadFlags));
+ it.goMax(it.lesserBranch);
+ assertEquals(2, it.payloadFlags());
+ it.goMin(it.greaterBranch);
+ assertEquals(4, it.payloadFlags());
+ }
+
+ @Test
+ public void testIteratorWithoutBounds() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+ long rootPos = builder.complete();
+
+ int[] expected = IntStream.range(1, 13).toArray();
+ checkIterates(buf.asNewBuffer(), rootPos, expected);
+
+ checkIterates(buf.asNewBuffer(), rootPos, null, null, false, expected);
+ checkIterates(buf.asNewBuffer(), rootPos, null, null, true, expected);
+ }
+
+ @Test
+ public void testIteratorWithBounds() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+ long rootPos = builder.complete();
+
+ checkIterates(buf.asNewBuffer(), rootPos, "151", "515", false, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", false, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, "705", "73", false, 9, 10, 11);
+ checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", false, 10, 11);
+ checkIterates(buf.asNewBuffer(), rootPos, "70", "737", false, 9, 10, 11, 12);
+ checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", false, 10, 11);
+
+ checkIterates(buf.asNewBuffer(), rootPos, null, "515", false, 1, 2, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, null, "51515", false, 1, 2, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, "151", null, false, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+ checkIterates(buf.asNewBuffer(), rootPos, "15151", null, false, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+ }
+
+ @Test
+ public void testIteratorWithBoundsAndAdmitPrefix() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+ long rootPos = builder.complete();
+
+ checkIterates(buf.asNewBuffer(), rootPos, "151", "515", true, 2, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, "15151", "51515", true, 2, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, "705", "73", true, 8, 9, 10, 11);
+ checkIterates(buf.asNewBuffer(), rootPos, "7051", "735", true, 9, 10, 11);
+ checkIterates(buf.asNewBuffer(), rootPos, "70", "737", true, 8, 9, 10, 11, 12);
+ // Note: 7054 has 70 as prefix, but we don't include 70 because a clearly smaller non-prefix entry 7051
+ // exists between the bound and the prefix
+ checkIterates(buf.asNewBuffer(), rootPos, "7054", "735", true, 10, 11);
+
+
+ checkIterates(buf.asNewBuffer(), rootPos, null, "515", true, 1, 2, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, null, "51515", true, 1, 2, 3, 4, 5);
+ checkIterates(buf.asNewBuffer(), rootPos, "151", null, true, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+ checkIterates(buf.asNewBuffer(), rootPos, "15151", null, true, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+ checkIterates(buf.asNewBuffer(), rootPos, "7054", null, true, 10, 11, 12);
+ }
+
+ private void checkIterates(ByteBuffer buffer, long rootPos, String from, String to, boolean admitPrefix, int... expected)
+ {
+ Rebufferer source = new ByteBufRebufferer(buffer);
+ ValueIterator<?> it = new ValueIterator<>(source, rootPos, source(from), source(to), admitPrefix);
+ checkReturns(from + "-->" + to, it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+
+ ReverseValueIterator<?> rit = new ReverseValueIterator<>(source, rootPos, source(from), source(to), admitPrefix);
+ reverse(expected);
+ checkReturns(from + "<--" + to, rit::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+ reverse(expected); // return array in its original form if reused
+ }
+
+ private void checkIterates(ByteBuffer buffer, long rootPos, int... expected)
+ {
+ Rebufferer source = new ByteBufRebufferer(buffer);
+ ValueIterator<?> it = new ValueIterator<>(source, rootPos);
+ checkReturns("Forward", it::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+
+ ReverseValueIterator<?> rit = new ReverseValueIterator<>(source, rootPos);
+ reverse(expected);
+ checkReturns("Reverse", rit::nextPayloadedNode, pos -> getPayloadFlags(buffer, (int) pos), expected);
+ reverse(expected); // return array in its original form if reused
+ }
+
+ private void reverse(int[] expected)
+ {
+ final int size = expected.length;
+ for (int i = 0; i < size / 2; ++i)
+ {
+ int t = expected[i];
+ expected[i] = expected[size - 1 - i];
+ expected[size - i - 1] = t;
+ }
+ }
+
+ private int getPayloadFlags(ByteBuffer buffer, int pos)
+ {
+ return TrieNode.at(buffer, pos).payloadFlags(buffer, pos);
+ }
+
+ private void checkReturns(String testCase, LongSupplier supplier, LongToIntFunction mapper, int... expected)
+ {
+ IntArrayList list = new IntArrayList();
+ while (true)
+ {
+ long pos = supplier.getAsLong();
+ if (pos == Walker.NONE)
+ break;
+ list.add(mapper.applyAsInt(pos));
+ }
+ assertArrayEquals(testCase + ": " + list + " != " + Arrays.toString(expected), expected, list.toIntArray());
+ }
+
+ @Test
+ public void testPartialTail() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = makeTrie(buf);
+ IncrementalTrieWriter.PartialTail ptail = builder.makePartialRoot();
+ long rootPos = builder.complete();
+ try (Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
+ Rebufferer partialSource = new TailOverridingRebufferer(new ByteBufRebufferer(buf.asNewBuffer()), ptail.cutoff(), ptail.tail());
+ ValueIterator<?> it = new ValueIterator<>(new ByteBufRebufferer(buf.asNewBuffer()), rootPos, source("151"), source("515"), true);
+ ValueIterator<?> tailIt = new ValueIterator<>(new TailOverridingRebufferer(new ByteBufRebufferer(buf.asNewBuffer()), ptail.cutoff(), ptail.tail()), ptail.root(), source("151"), source("515"), true))
+ {
+ while (true)
+ {
+ long i1 = it.nextPayloadedNode();
+ long i2 = tailIt.nextPayloadedNode();
+ if (i1 == -1 || i2 == -1)
+ break;
+
+ Rebufferer.BufferHolder bh1 = source.rebuffer(i1);
+ Rebufferer.BufferHolder bh2 = partialSource.rebuffer(i2);
+
+ int f1 = TrieNode.at(bh1.buffer(), (int) (i1 - bh1.offset())).payloadFlags(bh1.buffer(), (int) (i1 - bh1.offset()));
+ int f2 = TrieNode.at(bh2.buffer(), (int) (i2 - bh2.offset())).payloadFlags(bh2.buffer(), (int) (i2 - bh2.offset()));
+ assertEquals(f1, f2);
+
+ bh2.release();
+ bh1.release();
+ }
+ }
+ }
+
+ @Test
+ public void testBigTrie() throws IOException
+ {
+ DataOutputBuffer buf = new DataOutputBufferPaged();
+ IncrementalTrieWriter<Integer> builder = newTrieWriter(serializer, buf);
+ payloadSize = 0;
+ makeBigTrie(builder);
+ builder.reset();
+ payloadSize = 200;
+ makeBigTrie(builder);
+
+ long rootPos = builder.complete();
+ Rebufferer source = new ByteBufRebufferer(buf.asNewBuffer());
+ ValueIterator<?> it = new ValueIterator<>(source, rootPos);
+
+ while (true)
+ {
+ long i1 = it.nextPayloadedNode();
+ if (i1 == -1)
+ break;
+
+ TrieNode node = TrieNode.at(buf.asNewBuffer(), (int) i1);
+ assertNotEquals(0, node.payloadFlags(buf.asNewBuffer(), (int) i1));
+ }
+ }
+
+
+ private IncrementalTrieWriter<Integer> makeTrie(DataOutputBuffer out) throws IOException
+ {
+ IncrementalTrieWriter<Integer> builder = newTrieWriter(serializer, out);
+ dump = true;
+ builder.add(source("115"), 1);
+ builder.add(source("151"), 2);
+ builder.add(source("155"), 3);
+ builder.add(source("511"), 4);
+ builder.add(source("515"), 5);
+ builder.add(source("551"), 6);
+ builder.add(source("555555555555555555555555555555555555555555555555555555555555555555"), 7);
+
+ builder.add(source("70"), 8);
+ builder.add(source("7051"), 9);
+ builder.add(source("717"), 10);
+ builder.add(source("73"), 11);
+ builder.add(source("737"), 12);
+ return builder;
+ }
+
+ private void makeBigTrie(IncrementalTrieWriter<Integer> builder) throws IOException
+ {
+ dump = false;
+ for (int shift = 0; shift < 8; shift++)
+ for (long i = 1; i < 80; i++)
+ builder.add(longSource(i, shift * 8, 100), (int) (i % 7) + 1);
+ }
+
+ private ByteComparable longSource(long l, int shift, int size)
+ {
+ String s = StringUtils.leftPad(toBase(l), 8, '0');
+ s = StringUtils.rightPad(s, 8 + shift, '0');
+ s = StringUtils.leftPad(s, size, '0');
+ return source(s);
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java b/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java
index 040a080..ee8bb38 100644
--- a/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java
+++ b/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java
@@ -20,15 +20,14 @@
*/
package org.apache.cassandra.io.util;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.UTFDataFormatException;
+import com.google.common.primitives.UnsignedBytes;
+import com.google.common.primitives.UnsignedInteger;
+import com.google.common.primitives.UnsignedLong;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.vint.VIntCoding;
+import org.junit.Test;
+
+import java.io.*;
import java.lang.reflect.Field;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
@@ -37,14 +36,6 @@
import java.util.Arrays;
import java.util.Random;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.vint.VIntCoding;
-import org.junit.Test;
-
-import com.google.common.primitives.UnsignedBytes;
-import com.google.common.primitives.UnsignedInteger;
-import com.google.common.primitives.UnsignedLong;
-
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.FBUtilities.preventIllegalAccessWarnings;
import static org.junit.Assert.*;
@@ -628,7 +619,7 @@
int size = r.nextInt(9);
byte[] bytes = ByteBufferUtil.bytes(val).array();
canonical.write(bytes, 0, size);
- dosp.writeBytes(val, size);
+ dosp.writeMostSignificantBytes(val, size);
}
dosp.flush();
assertArrayEquals(canonical.toByteArray(), generated.toByteArray());
diff --git a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
index bb54f25..9f98fb0 100644
--- a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
+++ b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
@@ -19,9 +19,6 @@
*/
package org.apache.cassandra.io.util;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -29,6 +26,9 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
import static org.apache.cassandra.Util.expectEOF;
import static org.apache.cassandra.Util.expectException;
import static org.junit.Assert.assertEquals;
@@ -56,84 +56,82 @@
w.sync();
// reading small amount of data from file, this is handled by initial buffer
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()))
+ FileHandle.Builder builder = new FileHandle.Builder(w.getFile());
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader r = fh.createReader())
{
- try (FileHandle fh = builder.complete();
- RandomAccessReader r = fh.createReader())
- {
- byte[] buffer = new byte[data.length];
- assertEquals(data.length, r.read(buffer));
- assertTrue(Arrays.equals(buffer, data)); // we read exactly what we wrote
- assertEquals(r.read(), -1); // nothing more to read EOF
- assert r.bytesRemaining() == 0 && r.isEOF();
+ byte[] buffer = new byte[data.length];
+ assertEquals(data.length, r.read(buffer));
+ assertTrue(Arrays.equals(buffer, data)); // we read exactly what we wrote
+ assertEquals(r.read(), -1); // nothing more to read EOF
+ assert r.bytesRemaining() == 0 && r.isEOF();
+ }
+
+ // writing buffer bigger than page size, which will trigger reBuffer()
+ byte[] bigData = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE + 10];
+
+ for (int i = 0; i < bigData.length; i++)
+ bigData[i] = 'd';
+
+ long initialPosition = w.position();
+ w.write(bigData); // writing data
+ assertEquals(w.position(), initialPosition + bigData.length);
+ assertEquals(w.length(), initialPosition + bigData.length); // file size should equals to last position
+
+ w.sync();
+
+ // re-opening file in read-only mode
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader r = fh.createReader())
+ {
+
+ // reading written buffer
+ r.seek(initialPosition); // back to initial (before write) position
+ data = new byte[bigData.length];
+ long sizeRead = 0;
+ for (int i = 0; i < data.length; i++)
+ {
+ data[i] = (byte) r.read();
+ sizeRead++;
}
- // writing buffer bigger than page size, which will trigger reBuffer()
- byte[] bigData = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE + 10];
+ assertEquals(sizeRead, data.length); // read exactly data.length bytes
+ assertEquals(r.getFilePointer(), initialPosition + data.length);
+ assertEquals(r.length(), initialPosition + bigData.length);
+ assertTrue(Arrays.equals(bigData, data));
+ assertTrue(r.bytesRemaining() == 0 && r.isEOF()); // we are at the of the file
- for (int i = 0; i < bigData.length; i++)
- bigData[i] = 'd';
+ // test readBytes(int) method
+ r.seek(0);
+ ByteBuffer fileContent = ByteBufferUtil.read(r, (int) w.length());
+ assertEquals(fileContent.limit(), w.length());
+ assert ByteBufferUtil.string(fileContent).equals("Hello" + new String(bigData));
- long initialPosition = w.position();
- w.write(bigData); // writing data
- assertEquals(w.position(), initialPosition + bigData.length);
- assertEquals(w.length(), initialPosition + bigData.length); // file size should equals to last position
+ // read the same buffer but using readFully(int)
+ data = new byte[bigData.length];
+ r.seek(initialPosition);
+ r.readFully(data);
+ assert r.bytesRemaining() == 0 && r.isEOF(); // we should be at EOF
+ assertTrue(Arrays.equals(bigData, data));
- w.sync();
+ // try to read past mark (all methods should return -1)
+ data = new byte[10];
+ assertEquals(r.read(), -1);
+ assertEquals(r.read(data), -1);
+ assertEquals(r.read(data, 0, data.length), -1);
- // re-opening file in read-only mode
- try (FileHandle fh = builder.complete();
- RandomAccessReader r = fh.createReader())
+ // test read(byte[], int, int)
+ r.seek(0);
+ data = new byte[20];
+ assertEquals(15, r.read(data, 0, 15));
+ assertTrue(new String(data).contains("Hellodddddddddd"));
+ for (int i = 16; i < data.length; i++)
{
-
- // reading written buffer
- r.seek(initialPosition); // back to initial (before write) position
- data = new byte[bigData.length];
- long sizeRead = 0;
- for (int i = 0; i < data.length; i++)
- {
- data[i] = (byte) r.read();
- sizeRead++;
- }
-
- assertEquals(sizeRead, data.length); // read exactly data.length bytes
- assertEquals(r.getFilePointer(), initialPosition + data.length);
- assertEquals(r.length(), initialPosition + bigData.length);
- assertTrue(Arrays.equals(bigData, data));
- assertTrue(r.bytesRemaining() == 0 && r.isEOF()); // we are at the of the file
-
- // test readBytes(int) method
- r.seek(0);
- ByteBuffer fileContent = ByteBufferUtil.read(r, (int) w.length());
- assertEquals(fileContent.limit(), w.length());
- assert ByteBufferUtil.string(fileContent).equals("Hello" + new String(bigData));
-
- // read the same buffer but using readFully(int)
- data = new byte[bigData.length];
- r.seek(initialPosition);
- r.readFully(data);
- assert r.bytesRemaining() == 0 && r.isEOF(); // we should be at EOF
- assertTrue(Arrays.equals(bigData, data));
-
- // try to read past mark (all methods should return -1)
- data = new byte[10];
- assertEquals(r.read(), -1);
- assertEquals(r.read(data), -1);
- assertEquals(r.read(data, 0, data.length), -1);
-
- // test read(byte[], int, int)
- r.seek(0);
- data = new byte[20];
- assertEquals(15, r.read(data, 0, 15));
- assertTrue(new String(data).contains("Hellodddddddddd"));
- for (int i = 16; i < data.length; i++)
- {
- assert data[i] == 0;
- }
-
- w.finish();
+ assert data[i] == 0;
}
+
+ w.finish();
}
}
@@ -147,8 +145,7 @@
byte[] in = generateByteArray(RandomAccessReader.DEFAULT_BUFFER_SIZE);
w.write(in);
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader r = fh.createReader())
{
// Read it into a same size array.
@@ -190,8 +187,7 @@
w.finish();
// will use cachedlength
- try (FileHandle.Builder builder = new FileHandle.Builder(tmpFile.path());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(tmpFile).complete();
RandomAccessReader r = fh.createReader())
{
assertEquals(lessThenBuffer.length + biggerThenBuffer.length, r.length());
@@ -214,8 +210,7 @@
w.write(data);
w.sync();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader r = fh.createReader())
{
@@ -245,8 +240,7 @@
w.write(data);
w.finish();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader file = fh.createReader())
{
file.seek(0);
@@ -277,8 +271,7 @@
w.write(generateByteArray(RandomAccessReader.DEFAULT_BUFFER_SIZE * 2));
w.finish();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader file = fh.createReader())
{
@@ -313,8 +306,7 @@
w.sync();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader r = fh.createReader())
{
@@ -351,8 +343,7 @@
for (final int offset : Arrays.asList(0, 8))
{
File file1 = writeTemporaryFile(new byte[16]);
- try (FileHandle.Builder builder = new FileHandle.Builder(file1.path()).bufferSize(bufferSize);
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(file1).bufferSize(bufferSize).complete();
RandomAccessReader file = fh.createReader())
{
expectEOF(() -> { file.readFully(target, offset, 17); return null; });
@@ -363,8 +354,7 @@
for (final int n : Arrays.asList(1, 2, 4, 8))
{
File file1 = writeTemporaryFile(new byte[16]);
- try (FileHandle.Builder builder = new FileHandle.Builder(file1.path()).bufferSize(bufferSize);
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(file1).bufferSize(bufferSize).complete();
RandomAccessReader file = fh.createReader())
{
expectEOF(() -> {
@@ -396,8 +386,7 @@
w.sync();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader r = fh.createReader())
{
@@ -424,8 +413,7 @@
tmpFile.deleteOnExit();
// Create the BRAF by filename instead of by file.
- try (FileHandle.Builder builder = new FileHandle.Builder(tmpFile.path());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(tmpFile).complete();
RandomAccessReader r = fh.createReader())
{
assert tmpFile.path().equals(r.getPath());
@@ -459,7 +447,7 @@
//Writing to a BufferedOutputStream that is closed generates no error
//Going to allow the NPE to throw to catch as a bug any use after close. Notably it won't throw NPE for a
//write of a 0 length, but that is kind of a corner case
- expectException(() -> { w.write(generateByteArray(1)); return null; }, NullPointerException.class);
+ expectException(() -> { w.write(generateByteArray(1)); return null; }, AssertionError.class);
try (RandomAccessReader copy = RandomAccessReader.open(new File(r.getPath())))
{
@@ -478,8 +466,7 @@
w.finish();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader file = fh.createReader())
{
file.seek(10);
@@ -514,8 +501,7 @@
w.write(new byte[30]);
w.flush();
- try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete();
RandomAccessReader r = fh.createReader())
{
r.seek(10);
@@ -620,4 +606,4 @@
return arr;
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/util/DataOutputTest.java b/test/unit/org/apache/cassandra/io/util/DataOutputTest.java
index 41631af..83e2cbd 100644
--- a/test/unit/org/apache/cassandra/io/util/DataOutputTest.java
+++ b/test/unit/org/apache/cassandra/io/util/DataOutputTest.java
@@ -469,7 +469,7 @@
// keep only first i random bytes
Arrays.fill(rndBytes, i, rndBytes.length, (byte) 0);
long val = ByteBufferUtil.toLong(ByteBuffer.wrap(rndBytes));
- test.writeBytes(val, i);
+ test.writeMostSignificantBytes(val, i);
byte[] arr = new byte[i];
System.arraycopy(rndBytes, 0, arr, 0, i);
canon.write(arr);
diff --git a/test/unit/org/apache/cassandra/io/util/FileSystems.java b/test/unit/org/apache/cassandra/io/util/FileSystems.java
new file mode 100644
index 0000000..fcd5db7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/FileSystems.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+
+import com.google.common.base.StandardSystemProperty;
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+
+import org.apache.cassandra.io.filesystem.ForwardingFileSystem;
+import org.apache.cassandra.io.filesystem.ListenableFileSystem;
+
+public class FileSystems
+{
+ public static ListenableFileSystem newGlobalInMemoryFileSystem()
+ {
+ return global(jimfs());
+ }
+
+ public static ListenableFileSystem global()
+ {
+ return global(File.unsafeGetFilesystem());
+ }
+
+ public static ListenableFileSystem global(FileSystem real)
+ {
+ FileSystem current = File.unsafeGetFilesystem();
+ ListenableFileSystem fs = new ListenableFileSystem(new ForwardingFileSystem(real)
+ {
+ @Override
+ public void close() throws IOException
+ {
+ try
+ {
+ super.close();
+ }
+ finally
+ {
+ File.unsafeSetFilesystem(current);
+ }
+ }
+ });
+ File.unsafeSetFilesystem(fs);
+ return fs;
+ }
+
+ public static FileSystem jimfs()
+ {
+ return Jimfs.newFileSystem(jimfsConfig());
+ }
+
+ public static FileSystem jimfs(String name)
+ {
+ return Jimfs.newFileSystem(name, jimfsConfig());
+ }
+
+ private static Configuration jimfsConfig()
+ {
+ return Configuration.unix().toBuilder()
+ .setMaxSize(4L << 30).setBlockSize(512)
+ .build();
+ }
+
+ public static File maybeCreateTmp()
+ {
+ File dir = new File(StandardSystemProperty.JAVA_IO_TMPDIR.value());
+ if (!dir.exists())
+ dir.tryCreateDirectories();
+ return dir;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
index 7194d30..e6b5dd0 100644
--- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
@@ -20,12 +20,13 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Random;
import com.google.common.primitives.Ints;
+import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,27 +35,37 @@
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.io.compress.CompressedSequentialWriter;
import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.compress.CompressionMetadata.Chunk;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
-import static org.junit.Assert.assertNull;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
public class MmappedRegionsTest
{
private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class);
+ private final int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+
@BeforeClass
public static void setupDD()
{
DatabaseDescriptor.daemonInitialization();
}
+ @After
+ public void resetMaxSegmentSize()
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+
private static ByteBuffer allocateBuffer(int size)
{
ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size));
@@ -63,6 +74,11 @@
logger.info("Seed {}", seed);
new Random(seed).nextBytes(ret.array());
+ byte[] arr = ret.array();
+ for (int i = 0; i < arr.length; i++)
+ {
+ arr[i] = (byte) (arr[i] & 0xf);
+ }
return ret;
}
@@ -80,15 +96,14 @@
assert ret.exists();
assert ret.length() >= buffer.capacity();
return ret;
-
}
@Test
public void testEmpty() throws Exception
{
ByteBuffer buffer = allocateBuffer(1024);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
assertTrue(regions.isEmpty());
assertTrue(regions.isValid(channel));
@@ -99,8 +114,8 @@
public void testTwoSegments() throws Exception
{
ByteBuffer buffer = allocateBuffer(2048);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
regions.extend(1024);
for (int i = 0; i < 1024; i++)
@@ -133,12 +148,11 @@
@Test
public void testSmallSegmentSize() throws Exception
{
- int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
MmappedRegions.MAX_SEGMENT_SIZE = 1024;
ByteBuffer buffer = allocateBuffer(4096);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
regions.extend(1024);
regions.extend(2048);
@@ -153,22 +167,17 @@
assertEquals(SIZE + (SIZE * (i / SIZE)), region.end());
}
}
- finally
- {
- MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
- }
}
@Test
public void testAllocRegions() throws Exception
{
- int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
MmappedRegions.MAX_SEGMENT_SIZE = 1024;
ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
regions.extend(buffer.capacity());
@@ -181,10 +190,6 @@
assertEquals(SIZE + (SIZE * (i / SIZE)), region.end());
}
}
- finally
- {
- MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
- }
}
@Test
@@ -195,8 +200,8 @@
MmappedRegions snapshot;
ChannelProxy channelCopy;
- try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
- MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
{
// create 3 more segments, one per quater capacity
regions.extend(buffer.capacity() / 2);
@@ -237,8 +242,8 @@
MmappedRegions snapshot;
ChannelProxy channelCopy;
- try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
regions.extend(buffer.capacity() / 2);
@@ -264,8 +269,8 @@
public void testExtendOutOfOrder() throws Exception
{
ByteBuffer buffer = allocateBuffer(4096);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
regions.extend(4096);
regions.extend(1024);
@@ -285,8 +290,8 @@
public void testNegativeExtend() throws Exception
{
ByteBuffer buffer = allocateBuffer(1024);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
- MmappedRegions regions = MmappedRegions.empty(channel))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
{
regions.extend(-1);
}
@@ -295,7 +300,6 @@
@Test
public void testMapForCompressionMetadata() throws Exception
{
- int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
MmappedRegions.MAX_SEGMENT_SIZE = 1024;
ByteBuffer buffer = allocateBuffer(128 * 1024);
@@ -306,41 +310,29 @@
cf.deleteOnExit();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
- try(SequentialWriter writer = new CompressedSequentialWriter(f, cf.absolutePath(),
- null, SequentialWriterOption.DEFAULT,
- CompressionParams.snappy(), sstableMetadataCollector))
+ try (SequentialWriter writer = new CompressedSequentialWriter(f, cf,
+ null, SequentialWriterOption.DEFAULT,
+ CompressionParams.snappy(), sstableMetadataCollector))
{
writer.write(buffer);
writer.finish();
}
- CompressionMetadata metadata = new CompressionMetadata(cf.absolutePath(), f.length(), true);
- try(ChannelProxy channel = new ChannelProxy(f);
- MmappedRegions regions = MmappedRegions.map(channel, metadata))
+ CompressionMetadata metadata = CompressionMetadata.open(cf, f.length(), true);
+ try (ChannelProxy channel = new ChannelProxy(f);
+ MmappedRegions regions = MmappedRegions.map(channel, metadata))
{
assertFalse(regions.isEmpty());
- int i = 0;
- while(i < buffer.capacity())
+ int dataOffset = 0;
+ while (dataOffset < buffer.capacity())
{
- CompressionMetadata.Chunk chunk = metadata.chunkFor(i);
-
- MmappedRegions.Region region = regions.floor(chunk.offset);
- assertNotNull(region);
-
- ByteBuffer compressedChunk = region.buffer.duplicate();
- assertNotNull(compressedChunk);
- assertEquals(chunk.length + 4, compressedChunk.capacity());
-
- assertEquals(chunk.offset, region.offset());
- assertEquals(chunk.offset + chunk.length + 4, region.end());
-
- i += metadata.chunkLength();
+ verifyChunks(f, metadata, dataOffset, regions);
+ dataOffset += metadata.chunkLength();
}
}
finally
{
- MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
metadata.close();
}
}
@@ -349,8 +341,8 @@
public void testIllegalArgForMap1() throws Exception
{
ByteBuffer buffer = allocateBuffer(1024);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
- MmappedRegions regions = MmappedRegions.map(channel, 0))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, 0))
{
assertTrue(regions.isEmpty());
}
@@ -360,8 +352,8 @@
public void testIllegalArgForMap2() throws Exception
{
ByteBuffer buffer = allocateBuffer(1024);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
- MmappedRegions regions = MmappedRegions.map(channel, -1L))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, -1L))
{
assertTrue(regions.isEmpty());
}
@@ -371,11 +363,131 @@
public void testIllegalArgForMap3() throws Exception
{
ByteBuffer buffer = allocateBuffer(1024);
- try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
- MmappedRegions regions = MmappedRegions.map(channel, null))
+ try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, null))
{
assertTrue(regions.isEmpty());
}
}
+ @Test
+ public void testExtendForCompressionMetadata() throws Exception
+ {
+ testExtendForCompressionMetadata(8, 4, 4, 8, 12);
+ testExtendForCompressionMetadata(4, 4, 4, 8, 12);
+ testExtendForCompressionMetadata(2, 4, 4, 8, 12);
+ }
+
+ public void testExtendForCompressionMetadata(int maxSegmentSize, int chunkSize, int... writeSizes) throws Exception
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = maxSegmentSize << 10;
+ int size = Arrays.stream(writeSizes).sum() << 10;
+
+ ByteBuffer buffer = allocateBuffer(size);
+ File f = FileUtils.createTempFile("testMapForCompressionMetadata", "1");
+ f.deleteOnExit();
+
+ File cf = FileUtils.createTempFile(f.name() + ".metadata", "1");
+ cf.deleteOnExit();
+
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, cf, null,
+ SequentialWriterOption.DEFAULT,
+ CompressionParams.deflate(chunkSize << 10),
+ sstableMetadataCollector))
+ {
+ ByteBuffer slice = buffer.slice();
+ slice.limit(writeSizes[0] << 10);
+ writer.write(slice);
+ writer.sync();
+
+ try (ChannelProxy channel = new ChannelProxy(f);
+ CompressionMetadata metadata = writer.open(writer.getLastFlushOffset());
+ MmappedRegions regions = MmappedRegions.map(channel, metadata))
+ {
+ assertFalse(regions.isEmpty());
+ int dataOffset = 0;
+ while (dataOffset < metadata.dataLength)
+ {
+ verifyChunks(f, metadata, dataOffset, regions);
+ dataOffset += metadata.chunkLength();
+ }
+
+ int idx = 1;
+ int pos = writeSizes[0] << 10;
+ while (idx < writeSizes.length)
+ {
+ slice = buffer.slice();
+ slice.position(pos).limit(pos + (writeSizes[idx] << 10));
+ writer.write(slice);
+ writer.sync();
+
+ // verify that calling extend for the same (first iteration) or some previous metadata (further iterations) has no effect
+ assertFalse(regions.extend(metadata));
+
+ logger.info("Checking extend on compressed chunk for range={} {}..{} / {}", idx, pos, pos + (writeSizes[idx] << 10), size);
+ checkExtendOnCompressedChunks(f, writer, regions);
+ pos += writeSizes[idx] << 10;
+ idx++;
+ }
+ }
+ }
+ }
+
+ private void checkExtendOnCompressedChunks(File f, CompressedSequentialWriter writer, MmappedRegions regions)
+ {
+ int dataOffset;
+ try (CompressionMetadata metadata = writer.open(writer.getLastFlushOffset()))
+ {
+ regions.extend(metadata);
+ assertFalse(regions.isEmpty());
+ dataOffset = 0;
+ while (dataOffset < metadata.dataLength)
+ {
+ logger.info("Checking chunk {}..{}", dataOffset, dataOffset + metadata.chunkLength());
+ verifyChunks(f, metadata, dataOffset, regions);
+ dataOffset += metadata.chunkLength();
+ }
+ }
+ }
+
+ private ByteBuffer fromRegions(MmappedRegions regions, int offset, int size)
+ {
+ ByteBuffer buf = ByteBuffer.allocate(size);
+
+ while (buf.remaining() > 0)
+ {
+ MmappedRegions.Region region = regions.floor(offset);
+ ByteBuffer regBuf = region.buffer.slice();
+ int regBufOffset = (int) (offset - region.offset);
+ regBuf.position(regBufOffset);
+ regBuf.limit(regBufOffset + Math.min(buf.remaining(), regBuf.remaining()));
+ offset += regBuf.remaining();
+ buf.put(regBuf);
+ }
+
+ buf.flip();
+ return buf;
+ }
+
+ private Chunk verifyChunks(File f, CompressionMetadata metadata, long dataOffset, MmappedRegions regions)
+ {
+ Chunk chunk = metadata.chunkFor(dataOffset);
+
+ ByteBuffer compressedChunk = fromRegions(regions, (int) chunk.offset, chunk.length + 4);
+ assertThat(compressedChunk.capacity()).isEqualTo(chunk.length + 4);
+
+ try (ChannelProxy channel = new ChannelProxy(f))
+ {
+ ByteBuffer buf = ByteBuffer.allocate(compressedChunk.remaining());
+ long len = channel.read(buf, chunk.offset);
+ assertThat(len).isEqualTo(chunk.length + 4);
+ buf.flip();
+ String mmappedHex = ByteBufferUtil.bytesToHex(compressedChunk);
+ String fileHex = ByteBufferUtil.bytesToHex(buf);
+ assertThat(fileHex).isEqualTo(mmappedHex);
+ }
+
+ return chunk;
+ }
}
diff --git a/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java b/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java
index 31fa53c..3f28cd4 100644
--- a/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java
+++ b/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java
@@ -20,12 +20,13 @@
*/
package org.apache.cassandra.io.util;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
+import com.google.common.base.Charsets;
+import com.google.common.primitives.UnsignedBytes;
+import com.google.common.primitives.UnsignedInteger;
+import com.google.common.primitives.UnsignedLong;
+import org.junit.Test;
+
+import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
@@ -33,13 +34,6 @@
import java.util.Queue;
import java.util.Random;
-import org.junit.Test;
-
-import com.google.common.base.Charsets;
-import com.google.common.primitives.UnsignedBytes;
-import com.google.common.primitives.UnsignedInteger;
-import com.google.common.primitives.UnsignedLong;
-
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.junit.Assert.*;
diff --git a/test/unit/org/apache/cassandra/io/util/PageAwareTest.java b/test/unit/org/apache/cassandra/io/util/PageAwareTest.java
new file mode 100644
index 0000000..32b0f88
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/PageAwareTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.junit.Test;
+
+import static org.apache.cassandra.io.util.PageAware.PAGE_SIZE;
+import static org.junit.Assert.assertEquals;
+
+public class PageAwareTest
+{
+ @Test
+ public void pageLimit()
+ {
+ assertEquals(PAGE_SIZE, PageAware.pageLimit(0));
+ assertEquals(2 * PAGE_SIZE, PageAware.pageLimit(PAGE_SIZE));
+ assertEquals(3 * PAGE_SIZE, PageAware.pageLimit(2 * PAGE_SIZE));
+
+ assertEquals(PAGE_SIZE, PageAware.pageLimit(PAGE_SIZE / 3));
+ assertEquals(PAGE_SIZE, PageAware.pageLimit(PAGE_SIZE - 1));
+ assertEquals(PAGE_SIZE, PageAware.pageLimit(1));
+
+ assertEquals(2 * PAGE_SIZE, PageAware.pageLimit(PAGE_SIZE + PAGE_SIZE / 3));
+ assertEquals(2 * PAGE_SIZE, PageAware.pageLimit(PAGE_SIZE + PAGE_SIZE - 1));
+ assertEquals(2 * PAGE_SIZE, PageAware.pageLimit(PAGE_SIZE + 1));
+ }
+
+ @Test
+ public void bytesLeftInPage()
+ {
+ assertEquals(PAGE_SIZE, PageAware.bytesLeftInPage(0));
+ assertEquals(PAGE_SIZE, PageAware.bytesLeftInPage(PAGE_SIZE));
+ assertEquals(PAGE_SIZE, PageAware.bytesLeftInPage(2 * PAGE_SIZE));
+
+ assertEquals(PAGE_SIZE - (PAGE_SIZE / 3), PageAware.bytesLeftInPage(PAGE_SIZE / 3));
+ assertEquals(1, PageAware.bytesLeftInPage(PAGE_SIZE - 1));
+ assertEquals(PAGE_SIZE - 1, PageAware.bytesLeftInPage(1));
+
+ assertEquals(PAGE_SIZE - (PAGE_SIZE / 3), PageAware.bytesLeftInPage(PAGE_SIZE + PAGE_SIZE / 3));
+ assertEquals(1, PageAware.bytesLeftInPage(PAGE_SIZE + PAGE_SIZE - 1));
+ assertEquals(PAGE_SIZE - 1, PageAware.bytesLeftInPage(PAGE_SIZE + 1));
+ }
+
+ @Test
+ public void pageStart()
+ {
+ assertEquals(0, PageAware.pageStart(0));
+ assertEquals(PAGE_SIZE, PageAware.pageStart(PAGE_SIZE));
+ assertEquals(2 * PAGE_SIZE, PageAware.pageStart(2 * PAGE_SIZE));
+
+ assertEquals(0, PageAware.pageStart(PAGE_SIZE / 3));
+ assertEquals(0, PageAware.pageStart(PAGE_SIZE - 1));
+ assertEquals(0, PageAware.pageStart(1));
+
+ assertEquals(PAGE_SIZE, PageAware.pageStart(PAGE_SIZE + PAGE_SIZE / 3));
+ assertEquals(PAGE_SIZE, PageAware.pageStart(PAGE_SIZE + PAGE_SIZE - 1));
+ assertEquals(PAGE_SIZE, PageAware.pageStart(PAGE_SIZE + 1));
+ }
+
+ @Test
+ public void padded()
+ {
+ assertEquals(0, PageAware.padded(0));
+ assertEquals(PAGE_SIZE, PageAware.padded(PAGE_SIZE));
+ assertEquals(2 * PAGE_SIZE, PageAware.padded(2 * PAGE_SIZE));
+
+ assertEquals(PAGE_SIZE, PageAware.padded(PAGE_SIZE / 3));
+ assertEquals(PAGE_SIZE, PageAware.padded(PAGE_SIZE - 1));
+ assertEquals(PAGE_SIZE, PageAware.padded(1));
+
+ assertEquals(2 * PAGE_SIZE, PageAware.padded(PAGE_SIZE + PAGE_SIZE / 3));
+ assertEquals(2 * PAGE_SIZE, PageAware.padded(PAGE_SIZE + PAGE_SIZE - 1));
+ assertEquals(2 * PAGE_SIZE, PageAware.padded(PAGE_SIZE + 1));
+ }
+
+ @Test
+ public void numPages()
+ {
+ assertEquals(0, PageAware.numPages(0));
+ assertEquals(1, PageAware.numPages(PAGE_SIZE));
+ assertEquals(2, PageAware.numPages(2 * PAGE_SIZE));
+
+ assertEquals(1, PageAware.numPages(PAGE_SIZE / 3));
+ assertEquals(1, PageAware.numPages(PAGE_SIZE - 1));
+ assertEquals(1, PageAware.numPages(1));
+
+ assertEquals(2, PageAware.numPages(PAGE_SIZE + PAGE_SIZE / 3));
+ assertEquals(2, PageAware.numPages(PAGE_SIZE + PAGE_SIZE - 1));
+ assertEquals(2, PageAware.numPages(PAGE_SIZE + 1));
+ }
+
+ @Test
+ public void pageNum()
+ {
+ assertEquals(0, PageAware.pageNum(0));
+ assertEquals(1, PageAware.pageNum(PAGE_SIZE));
+ assertEquals(2, PageAware.pageNum(2 * PAGE_SIZE));
+
+ assertEquals(0, PageAware.pageNum(PAGE_SIZE / 3));
+ assertEquals(0, PageAware.pageNum(PAGE_SIZE - 1));
+ assertEquals(0, PageAware.pageNum(1));
+
+ assertEquals(1, PageAware.pageNum(PAGE_SIZE + PAGE_SIZE / 3));
+ assertEquals(1, PageAware.pageNum(PAGE_SIZE + PAGE_SIZE - 1));
+ assertEquals(1, PageAware.pageNum(PAGE_SIZE + 1));
+ }
+
+ @Test
+ public void pad() throws IOException
+ {
+ testPad(0, 0);
+ testPad(PAGE_SIZE, PAGE_SIZE);
+ testPad(2 * PAGE_SIZE, 2 * PAGE_SIZE);
+
+ testPad(PAGE_SIZE, PAGE_SIZE / 3);
+ testPad(PAGE_SIZE, PAGE_SIZE - 1);
+ testPad(PAGE_SIZE, 1);
+
+ testPad(2 * PAGE_SIZE, PAGE_SIZE + PAGE_SIZE / 3);
+ testPad(2 * PAGE_SIZE, PAGE_SIZE + PAGE_SIZE - 1);
+ testPad(2 * PAGE_SIZE, PAGE_SIZE + 1);
+ }
+
+ private void testPad(int expectedSize, int currentSize) throws IOException
+ {
+ ByteBuffer expectedBuf = ByteBuffer.allocate(expectedSize);
+ try (DataOutputBuffer out = new DataOutputBuffer())
+ {
+ for (int i = 0; i < currentSize; i++)
+ {
+ expectedBuf.put((byte) 1);
+ out.write(1);
+ }
+ for (int i = currentSize; i < expectedSize; i++)
+ {
+ expectedBuf.put((byte) 0);
+ }
+
+ PageAware.pad(out);
+ out.flush();
+
+ assertEquals(expectedBuf.rewind(), out.asNewBuffer());
+ }
+ }
+
+ @Test
+ public void randomizedTest()
+ {
+ Random rand = new Random();
+ for (int i = 0; i < 100000; ++i)
+ {
+ long pos = rand.nextLong() & ((1L << rand.nextInt(64)) - 1); // positive long with random length
+ long pageStart = (pos / PAGE_SIZE) * PAGE_SIZE;
+ long pageLimit = pageStart + PAGE_SIZE;
+ assertEquals(pageLimit, PageAware.pageLimit(pos));
+ assertEquals(pageStart, PageAware.pageStart(pos));
+ assertEquals(pageLimit - pos, PageAware.bytesLeftInPage(pos));
+ assertEquals(pos == pageStart ? pageStart : pageLimit, PageAware.padded(pos));
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
index f933cf1..8535646 100644
--- a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java
@@ -38,17 +38,19 @@
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.junit.Assert.*;
-
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class RandomAccessReaderTest
{
private static final Logger logger = LoggerFactory.getLogger(RandomAccessReaderTest.class);
@@ -142,17 +144,16 @@
Parameters params = new Parameters(SIZE, 1 << 20); // 1MiB
- try (ChannelProxy channel = new ChannelProxy("abc", new FakeFileChannel(SIZE));
- FileHandle.Builder builder = new FileHandle.Builder(channel)
- .bufferType(params.bufferType).bufferSize(params.bufferSize);
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(new File("abc")).bufferType(params.bufferType)
+ .bufferSize(params.bufferSize)
+ .complete(f -> new ChannelProxy(f, new FakeFileChannel(SIZE)));
RandomAccessReader reader = fh.createReader())
{
- assertEquals(channel.size(), reader.length());
- assertEquals(channel.size(), reader.bytesRemaining());
+ assertEquals(fh.channel.size(), reader.length());
+ assertEquals(fh.channel.size(), reader.bytesRemaining());
assertEquals(Integer.MAX_VALUE, reader.available());
- assertEquals(channel.size(), reader.skip(channel.size()));
+ assertEquals(fh.channel.size(), reader.skip(fh.channel.size()));
assertTrue(reader.isEOF());
assertEquals(0, reader.bytesRemaining());
@@ -290,30 +291,28 @@
private static void testReadFully(Parameters params) throws IOException
{
final File f = writeFile(params);
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path())
- .bufferType(params.bufferType).bufferSize(params.bufferSize))
+ FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType)
+ .bufferSize(params.bufferSize);
+ builder.mmapped(params.mmappedRegions);
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
{
- builder.mmapped(params.mmappedRegions);
- try (FileHandle fh = builder.complete();
- RandomAccessReader reader = fh.createReader())
+ assertEquals(f.absolutePath(), reader.getPath());
+ assertEquals(f.length(), reader.length());
+ assertEquals(f.length(), reader.bytesRemaining());
+ assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available());
+
+ byte[] b = new byte[params.expected.length];
+ long numRead = 0;
+ while (numRead < params.fileLength)
{
- assertEquals(f.absolutePath(), reader.getPath());
- assertEquals(f.length(), reader.length());
- assertEquals(f.length(), reader.bytesRemaining());
- assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available());
-
- byte[] b = new byte[params.expected.length];
- long numRead = 0;
- while (numRead < params.fileLength)
- {
- reader.readFully(b);
- assertTrue(Arrays.equals(params.expected, b));
- numRead += b.length;
- }
-
- assertTrue(reader.isEOF());
- assertEquals(0, reader.bytesRemaining());
+ reader.readFully(b);
+ assertTrue(Arrays.equals(params.expected, b));
+ numRead += b.length;
}
+
+ assertTrue(reader.isEOF());
+ assertEquals(0, reader.bytesRemaining());
}
}
@@ -331,8 +330,7 @@
assert f.exists();
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(f).complete();
RandomAccessReader reader = fh.createReader())
{
assertEquals(f.absolutePath(), reader.getPath());
@@ -362,8 +360,7 @@
assert f.exists();
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path());
- FileHandle fh = builder.complete();
+ try (FileHandle fh = new FileHandle.Builder(f).complete();
RandomAccessReader reader = fh.createReader())
{
assertEquals(expected.length() * numIterations, reader.length());
@@ -442,61 +439,59 @@
assert f.exists();
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path()))
+ FileHandle.Builder builder = new FileHandle.Builder(f);
+ final Runnable worker = () ->
{
- final Runnable worker = () ->
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
{
- try (FileHandle fh = builder.complete();
- RandomAccessReader reader = fh.createReader())
+ assertEquals(expected.length, reader.length());
+
+ ByteBuffer b = ByteBufferUtil.read(reader, expected.length);
+ assertTrue(Arrays.equals(expected, b.array()));
+ assertTrue(reader.isEOF());
+ assertEquals(0, reader.bytesRemaining());
+
+ reader.seek(0);
+ b = ByteBufferUtil.read(reader, expected.length);
+ assertTrue(Arrays.equals(expected, b.array()));
+ assertTrue(reader.isEOF());
+ assertEquals(0, reader.bytesRemaining());
+
+ for (int i = 0; i < 10; i++)
{
- assertEquals(expected.length, reader.length());
+ int pos = r.nextInt(expected.length);
+ reader.seek(pos);
+ assertEquals(pos, reader.getPosition());
- ByteBuffer b = ByteBufferUtil.read(reader, expected.length);
- assertTrue(Arrays.equals(expected, b.array()));
- assertTrue(reader.isEOF());
- assertEquals(0, reader.bytesRemaining());
+ ByteBuffer buf = ByteBuffer.wrap(expected, pos, expected.length - pos)
+ .order(ByteOrder.BIG_ENDIAN);
- reader.seek(0);
- b = ByteBufferUtil.read(reader, expected.length);
- assertTrue(Arrays.equals(expected, b.array()));
- assertTrue(reader.isEOF());
- assertEquals(0, reader.bytesRemaining());
-
- for (int i = 0; i < 10; i++)
- {
- int pos = r.nextInt(expected.length);
- reader.seek(pos);
- assertEquals(pos, reader.getPosition());
-
- ByteBuffer buf = ByteBuffer.wrap(expected, pos, expected.length - pos)
- .order(ByteOrder.BIG_ENDIAN);
-
- while (reader.bytesRemaining() > 4)
- assertEquals(buf.getInt(), reader.readInt());
- }
-
- reader.close();
+ while (reader.bytesRemaining() > 4)
+ assertEquals(buf.getInt(), reader.readInt());
}
- catch (Exception ex)
- {
- ex.printStackTrace();
- fail(ex.getMessage());
- }
- };
- if (numThreads == 1)
- {
- worker.run();
+ reader.close();
}
- else
+ catch (Exception ex)
{
- ExecutorService executor = Executors.newFixedThreadPool(numThreads);
- for (int i = 0; i < numThreads; i++)
- executor.submit(worker);
+ ex.printStackTrace();
+ fail(ex.getMessage());
+ }
+ };
- executor.shutdown();
- Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
- }
+ if (numThreads == 1)
+ {
+ worker.run();
+ }
+ else
+ {
+ ExecutorService executor = Executors.newFixedThreadPool(numThreads);
+ for (int i = 0; i < numThreads; i++)
+ executor.submit(worker);
+
+ executor.shutdown();
+ Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
}
}
@@ -519,16 +514,14 @@
{
Parameters params = new Parameters(8192, 4096);
final File f = writeFile(params);
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path())
- .bufferType(params.bufferType).bufferSize(params.bufferSize))
+ FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType)
+ .bufferSize(params.bufferSize)
+ .mmapped(params.mmappedRegions);
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
{
- builder.mmapped(params.mmappedRegions);
- try (FileHandle fh = builder.complete();
- RandomAccessReader reader = fh.createReader())
- {
- assertEquals(0, reader.skipBytes(0));
- assertEquals(0, reader.skipBytes(-1));
- }
+ assertEquals(0, reader.skipBytes(0));
+ assertEquals(0, reader.skipBytes(-1));
}
}
@@ -537,42 +530,38 @@
{
Parameters params = new Parameters(8192, 4096);
final File f = writeFile(params);
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path())
- .bufferType(params.bufferType).bufferSize(params.bufferSize))
+ FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType)
+ .bufferSize(params.bufferSize);
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
{
- try (FileHandle fh = builder.complete();
- RandomAccessReader reader = fh.createReader())
- {
- reader.close();
- reader.skipBytes(31415);
- }
+ reader.close();
+ reader.skipBytes(31415);
}
}
private static void testSkipBytes(Parameters params, int expectationMultiples) throws IOException
{
final File f = writeFile(params);
- try (FileHandle.Builder builder = new FileHandle.Builder(f.path())
- .bufferType(params.bufferType).bufferSize(params.bufferSize))
+ FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType)
+ .bufferSize(params.bufferSize)
+ .mmapped(params.mmappedRegions);
+ try (FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
{
- builder.mmapped(params.mmappedRegions);
- try (FileHandle fh = builder.complete();
- RandomAccessReader reader = fh.createReader())
- {
- int toSkip = expectationMultiples * params.expected.length;
- byte[] b = new byte[params.expected.length];
- long numRead = 0;
+ int toSkip = expectationMultiples * params.expected.length;
+ byte[] b = new byte[params.expected.length];
+ long numRead = 0;
- while (numRead < params.fileLength)
- {
- reader.readFully(b);
- assertTrue(Arrays.equals(params.expected, b));
- numRead += b.length;
- int skipped = reader.skipBytes(toSkip);
- long expectedSkipped = Math.max(Math.min(toSkip, params.fileLength - numRead), 0);
- assertEquals(expectedSkipped, skipped);
- numRead += skipped;
- }
+ while (numRead < params.fileLength)
+ {
+ reader.readFully(b);
+ assertTrue(Arrays.equals(params.expected, b));
+ numRead += b.length;
+ int skipped = reader.skipBytes(toSkip);
+ long expectedSkipped = Math.max(Math.min(toSkip, params.fileLength - numRead), 0);
+ assertEquals(expectedSkipped, skipped);
+ numRead += skipped;
}
}
}
diff --git a/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java b/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java
new file mode 100644
index 0000000..9a3a953
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.apache.commons.io.EndianUtils;
+import org.junit.Test;
+
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Generators;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.junit.Assert.assertEquals;
+import static org.quicktheories.QuickTheory.qt;
+import static org.quicktheories.generators.SourceDSL.longs;
+
+public class SizedIntsTest
+{
+
+ @Test
+ public void nonZeroSize()
+ {
+ assertEquals(1, SizedInts.nonZeroSize(0));
+ assertEquals(1, SizedInts.nonZeroSize(1));
+ assertEquals(1, SizedInts.nonZeroSize(127));
+ assertEquals(1, SizedInts.nonZeroSize(-127));
+ assertEquals(2, SizedInts.nonZeroSize(128));
+ assertEquals(1, SizedInts.nonZeroSize(-128));
+ assertEquals(2, SizedInts.nonZeroSize(-129));
+ assertEquals(8, SizedInts.nonZeroSize(0x7fffffffffffffffL));
+ assertEquals(8, SizedInts.nonZeroSize(0x7ffffffffffffffL));
+ assertEquals(7, SizedInts.nonZeroSize(0x7fffffffffffffL));
+ assertEquals(7, SizedInts.nonZeroSize(0x7ffffffffffffL));
+ assertEquals(6, SizedInts.nonZeroSize(0x7fffffffffffL));
+ assertEquals(6, SizedInts.nonZeroSize(0x7ffffffffffL));
+ assertEquals(5, SizedInts.nonZeroSize(0x7fffffffffL));
+ assertEquals(5, SizedInts.nonZeroSize(0x7ffffffffL));
+ assertEquals(4, SizedInts.nonZeroSize(0x7fffffffL));
+ assertEquals(4, SizedInts.nonZeroSize(0x7ffffffL));
+ assertEquals(3, SizedInts.nonZeroSize(0x7fffffL));
+ assertEquals(3, SizedInts.nonZeroSize(0x7ffffL));
+ assertEquals(2, SizedInts.nonZeroSize(0x7fffL));
+ assertEquals(2, SizedInts.nonZeroSize(0x7ffL));
+ assertEquals(1, SizedInts.nonZeroSize(0x7fL));
+ assertEquals(1, SizedInts.nonZeroSize(0x7L));
+ }
+
+ @Test
+ public void sizeAllowingZero()
+ {
+ assertEquals(0, SizedInts.sizeAllowingZero(0));
+ assertEquals(1, SizedInts.sizeAllowingZero(1));
+ assertEquals(1, SizedInts.sizeAllowingZero(127));
+ assertEquals(1, SizedInts.sizeAllowingZero(-127));
+ assertEquals(2, SizedInts.sizeAllowingZero(128));
+ assertEquals(1, SizedInts.sizeAllowingZero(-128));
+ assertEquals(2, SizedInts.sizeAllowingZero(-129));
+ }
+
+
+ @Test
+ public void readWrite()
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer(8))
+ {
+ qt().forAll(Generators.bytes(1, 8).map(bb -> new BigInteger(ByteBufferUtil.getArray(bb)).longValue()).mix(longs().all())).check(v -> {
+ out.clear();
+ try
+ {
+ SizedInts.write(out, v, SizedInts.sizeAllowingZero(v));
+ out.flush();
+ long r = SizedInts.read(out.asNewBuffer(), 0, SizedInts.sizeAllowingZero(v));
+ return v == r;
+ }
+ catch (IOException e)
+ {
+ throw Throwables.cleaned(e);
+ }
+ });
+ }
+ }
+
+ @Test
+ public void readWriteUnsigned()
+ {
+ try (DataOutputBuffer out = new DataOutputBuffer(8))
+ {
+ byte[] buf1 = new byte[8];
+ byte[] buf2 = new byte[8];
+ qt().forAll(Generators.bytes(1, 8).map(bb -> new BigInteger(ByteBufferUtil.getArray(bb)).longValue()).mix(longs().all())).check(v -> {
+ out.clear();
+ try
+ {
+ int size = SizedInts.sizeAllowingZero(v);
+ EndianUtils.writeSwappedLong(buf1, 0, v);
+ SizedInts.write(out, v, size);
+ out.flush();
+ long r = SizedInts.readUnsigned(out.asNewBuffer(), 0, size);
+ EndianUtils.writeSwappedLong(buf2, 0, r);
+ return ByteArrayUtil.compareUnsigned(buf1, 0, buf2, 0, size) == 0;
+ }
+ catch (IOException e)
+ {
+ throw Throwables.cleaned(e);
+ }
+ });
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java b/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java
new file mode 100644
index 0000000..7bb65a0
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class TailOverridingRebuffererTest
+{
+ ByteBuffer head = ByteBuffer.wrap(new byte[]{ 1, 2, 3, 4, 5, 6, 7, 8 });
+ ByteBuffer tail = ByteBuffer.wrap(new byte[]{ 9, 10 });
+
+ Rebufferer r = Mockito.mock(Rebufferer.class);
+ Rebufferer.BufferHolder bh = Mockito.mock(Rebufferer.BufferHolder.class);
+
+ public void before()
+ {
+ reset(r, bh);
+ }
+
+ @Test
+ public void testAccessLeftToTailFully()
+ {
+ when(r.rebuffer(anyLong())).thenReturn(bh);
+ when(bh.buffer()).thenReturn(head.duplicate());
+ when(bh.offset()).thenReturn(0L);
+ Rebufferer tor = new TailOverridingRebufferer(r, 8, tail.duplicate());
+
+ for (int i = 0; i < 8; i++)
+ {
+ Rebufferer.BufferHolder bh = tor.rebuffer(i);
+ assertEquals(head, bh.buffer());
+ bh.release();
+ }
+
+ assertEquals(10, tor.fileLength());
+ }
+
+ @Test
+ public void testAccessLeftToTailPartial()
+ {
+ when(r.rebuffer(anyLong())).thenReturn(bh);
+ when(bh.buffer()).thenReturn(head.duplicate());
+ when(bh.offset()).thenReturn(2L);
+ Rebufferer tor = new TailOverridingRebufferer(r, 8, tail.duplicate());
+
+ for (int i = 2; i < 8; i++)
+ {
+ Rebufferer.BufferHolder bh = tor.rebuffer(i);
+ assertEquals(head.limit(6), bh.buffer());
+ bh.release();
+ }
+
+ assertEquals(10, tor.fileLength());
+ }
+
+ @Test
+ public void testAccessRightToTail()
+ {
+ when(r.rebuffer(anyLong())).thenReturn(bh);
+ when(bh.buffer()).thenReturn(head.duplicate());
+ when(bh.offset()).thenReturn(0L);
+ Rebufferer tor = new TailOverridingRebufferer(r, 8, tail.duplicate());
+
+ for (int i = 8; i < 10; i++)
+ {
+ Rebufferer.BufferHolder bh = tor.rebuffer(i);
+ assertEquals(tail, bh.buffer());
+ bh.release();
+ }
+
+ assertEquals(10, tor.fileLength());
+ }
+
+ @Test
+ public void testOtherMethods() throws IOException
+ {
+ Rebufferer tor = new TailOverridingRebufferer(r, 8, tail.duplicate());
+
+ File tmp = FileUtils.createTempFile("fakeChannelProxy", "");
+ try (ChannelProxy channelProxy = new ChannelProxy(tmp))
+ {
+ when(r.channel()).thenReturn(channelProxy);
+ assertSame(channelProxy, tor.channel());
+ verify(r).channel();
+ reset(r);
+ }
+
+ tor.closeReader();
+ verify(r).closeReader();
+ reset(r);
+
+ tor.close();
+ verify(r).close();
+ reset(r);
+
+ when(r.getCrcCheckChance()).thenReturn(0.123d);
+ assertEquals(0.123d, tor.getCrcCheckChance(), 0);
+ verify(r).getCrcCheckChance();
+ reset(r);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java b/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java
new file mode 100644
index 0000000..0d448cd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class WrappingRebuffererTest
+{
+ @Test
+ public void testRebufferRelease()
+ {
+ TestRebufferer mock = new TestRebufferer();
+ try (WrappingRebufferer rebufferer = new WrappingRebufferer(mock)
+ {
+ })
+ {
+ Rebufferer.BufferHolder ret = rebufferer.rebuffer(0);
+ assertNotNull(ret);
+ assertEquals(mock.buffer(), ret.buffer());
+ assertEquals(mock.offset(), ret.offset());
+
+ ret.release();
+ assertTrue(mock.released);
+ }
+ }
+
+ @Test
+ public void testRebufferReleaseFailingContract()
+ {
+ TestRebufferer mock = new TestRebufferer();
+ try (WrappingRebufferer rebufferer = new WrappingRebufferer(mock)
+ {
+ })
+ {
+ Rebufferer.BufferHolder ret1 = rebufferer.rebuffer(0);
+ assertNotNull(ret1);
+ assertEquals(mock.buffer(), ret1.buffer());
+ assertEquals(mock.offset(), ret1.offset());
+
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> rebufferer.rebuffer(1));
+ ret1.release();
+
+ assertTrue(mock.released);
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(ret1::buffer);
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(ret1::offset);
+ }
+ }
+
+ @Test
+ public void testRebufferReleaseFailingContractWhenClosing()
+ {
+ assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> {
+ TestRebufferer mock = new TestRebufferer();
+ try (WrappingRebufferer rebufferer = new WrappingRebufferer(mock)
+ {
+ })
+ {
+ Rebufferer.BufferHolder ret1 = rebufferer.rebuffer(0);
+ assertNotNull(ret1);
+ assertEquals(mock.buffer(), ret1.buffer());
+ assertEquals(mock.offset(), ret1.offset());
+ }
+ });
+ }
+
+
+ private static class TestRebufferer implements Rebufferer, Rebufferer.BufferHolder
+ {
+ final ByteBuffer buffer;
+ boolean released;
+ long offset;
+
+ TestRebufferer()
+ {
+ this.buffer = ByteBuffer.allocate(0);
+ this.released = false;
+ this.offset = 0;
+ }
+
+ @Override
+ public ChannelProxy channel()
+ {
+ return null;
+ }
+
+ @Override
+ public ByteBuffer buffer()
+ {
+ return buffer;
+ }
+
+ public long fileLength()
+ {
+ return buffer.remaining();
+ }
+
+ public double getCrcCheckChance()
+ {
+ return 0;
+ }
+
+ public BufferHolder rebuffer(long position)
+ {
+ offset = position;
+ return this;
+ }
+
+ public long offset()
+ {
+ return offset;
+ }
+
+ public void release()
+ {
+ released = true;
+ }
+
+ public long adjustExternal(long position)
+ {
+ return position;
+ }
+
+ public long adjustInternal(long position)
+ {
+ return position;
+ }
+
+ public void close()
+ {
+ // nothing
+ }
+
+ public void closeReader()
+ {
+ // nothing
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
index 04540cf..82108f2 100644
--- a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java
@@ -36,6 +36,7 @@
import static org.apache.cassandra.ServerTestUtils.cleanup;
import static org.apache.cassandra.ServerTestUtils.mkdirs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.assertEquals;
public class AlibabaCloudSnitchTest
@@ -45,7 +46,7 @@
@BeforeClass
public static void setup() throws Exception
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
diff --git a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
index 51e8371..e167e99 100644
--- a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
@@ -37,6 +37,7 @@
import static org.apache.cassandra.ServerTestUtils.cleanup;
import static org.apache.cassandra.ServerTestUtils.mkdirs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.assertEquals;
public class CloudstackSnitchTest
@@ -46,7 +47,7 @@
@BeforeClass
public static void setup() throws Exception
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
diff --git a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
index 69c1287..78c5d40 100644
--- a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
@@ -42,6 +42,7 @@
import static org.apache.cassandra.ServerTestUtils.cleanup;
import static org.apache.cassandra.ServerTestUtils.mkdirs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.apache.cassandra.locator.Ec2Snitch.EC2_NAMING_LEGACY;
import static org.junit.Assert.assertEquals;
@@ -60,7 +61,7 @@
@BeforeClass
public static void setup() throws Exception
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
diff --git a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
index 67192bd..1fdfd1c 100644
--- a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
@@ -38,6 +38,7 @@
import static org.apache.cassandra.ServerTestUtils.cleanup;
import static org.apache.cassandra.ServerTestUtils.mkdirs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.assertEquals;
public class GoogleCloudSnitchTest
@@ -47,7 +48,7 @@
@BeforeClass
public static void setup() throws Exception
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
CommitLog.instance.segmentManager.awaitManagementTasksCompletion();
diff --git a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
index 23bcf1b..af9e970 100644
--- a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java
@@ -49,6 +49,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.*;
/**
@@ -71,7 +72,7 @@
@Before
public void setup() throws ConfigurationException, IOException
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
String confFile = FBUtilities.resourceToFile(PropertyFileSnitch.SNITCH_PROPERTIES_FILENAME);
effectiveFile = Paths.get(confFile);
backupFile = Paths.get(confFile + ".bak");
diff --git a/test/unit/org/apache/cassandra/locator/SimpleSeedProviderTest.java b/test/unit/org/apache/cassandra/locator/SimpleSeedProviderTest.java
new file mode 100644
index 0000000..eba216d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/locator/SimpleSeedProviderTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.utils.FBUtilities;
+import org.mockito.MockedStatic;
+
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static org.apache.cassandra.locator.SimpleSeedProvider.RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY;
+import static org.apache.cassandra.locator.SimpleSeedProvider.SEEDS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mockStatic;
+
+public class SimpleSeedProviderTest
+{
+ private static final String dnsName1 = "dns-name-1";
+ private static final String dnsName2 = "dns-name-2";
+
+ @Test
+ public void testSeedsResolution() throws Throwable
+ {
+ MockedStatic<InetAddressAndPort> inetAddressAndPortMock = null;
+ MockedStatic<DatabaseDescriptor> descriptorMock = null;
+ MockedStatic<FBUtilities> fbUtilitiesMock = null;
+
+ try
+ {
+ byte[] addressBytes1 = new byte[]{ 127, 0, 0, 1 };
+ byte[] addressBytes2 = new byte[]{ 127, 0, 0, 2 };
+ byte[] addressBytes3 = new byte[]{ 127, 0, 0, 3 };
+ InetAddressAndPort address1 = new InetAddressAndPort(InetAddress.getByAddress(addressBytes1), addressBytes1, 7000);
+ InetAddressAndPort address2 = new InetAddressAndPort(InetAddress.getByAddress(addressBytes2), addressBytes2, 7000);
+ InetAddressAndPort address3 = new InetAddressAndPort(InetAddress.getByAddress(addressBytes3), addressBytes3, 7000);
+
+ inetAddressAndPortMock = mockStatic(InetAddressAndPort.class);
+ inetAddressAndPortMock.when(() -> InetAddressAndPort.getByName(dnsName1)).thenReturn(address1);
+ inetAddressAndPortMock.when(() -> InetAddressAndPort.getAllByName(dnsName1)).thenReturn(singletonList(address1));
+ inetAddressAndPortMock.when(() -> InetAddressAndPort.getAllByName(dnsName2)).thenReturn(asList(address2, address3));
+ inetAddressAndPortMock.when(() -> InetAddressAndPort.getByName(dnsName2)).thenReturn(address2);
+
+ fbUtilitiesMock = mockStatic(FBUtilities.class);
+ fbUtilitiesMock.when(FBUtilities::getLocalAddressAndPort).thenReturn(address1);
+
+ descriptorMock = mockStatic(DatabaseDescriptor.class);
+
+ Map<String, String> seedProviderArgs = new HashMap<>();
+
+ //
+ // dns 1 without multiple ips per record
+ //
+ seedProviderArgs.put(SEEDS_KEY, dnsName1);
+ // resolve_multiple_ip_addresses_per_dns_record is implicitly false here
+
+ descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
+ SimpleSeedProvider provider = new SimpleSeedProvider(null);
+ List<InetAddressAndPort> seeds = provider.getSeeds();
+
+ assertEquals(1, seeds.size());
+ assertEquals(address1, seeds.get(0));
+
+ //
+ // dns 2 without multiple ips per record
+ //
+ seedProviderArgs.put(SEEDS_KEY, dnsName2);
+ // resolve_multiple_ip_addresses_per_dns_record is implicitly false here
+
+ descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
+ provider = new SimpleSeedProvider(null);
+ seeds = provider.getSeeds();
+
+ assertEquals(1, seeds.size());
+ assertTrue(seeds.contains(address2));
+
+ //
+ // dns 1 with multiple ips per record
+ //
+ seedProviderArgs.put(SEEDS_KEY, dnsName1);
+ seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "true");
+
+ descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
+ provider = new SimpleSeedProvider(null);
+ seeds = provider.getSeeds();
+
+ assertEquals(1, seeds.size());
+ assertEquals(address1, seeds.get(0));
+
+ //
+ // dns 2 with multiple ips per record
+ //
+ seedProviderArgs.put(SEEDS_KEY, dnsName2);
+ seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "true");
+
+ descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
+ provider = new SimpleSeedProvider(null);
+ seeds = provider.getSeeds();
+
+ assertEquals(2, seeds.size());
+ assertTrue(seeds.containsAll(asList(address2, address3)));
+
+ //
+ // dns 1 and dns 2 without multiple ips per record
+ //
+ seedProviderArgs.put(SEEDS_KEY, format("%s,%s", dnsName1, dnsName2));
+ seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "false");
+
+ descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
+ provider = new SimpleSeedProvider(null);
+ seeds = provider.getSeeds();
+
+ assertEquals(2, seeds.size());
+ assertTrue(seeds.containsAll(asList(address1, address2)));
+
+ //
+ // dns 1 and dns 2 with multiple ips per record
+ //
+ seedProviderArgs.put(SEEDS_KEY, format("%s,%s", dnsName1, dnsName2));
+ seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "true");
+
+ descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
+ provider = new SimpleSeedProvider(null);
+ seeds = provider.getSeeds();
+
+ assertEquals(3, seeds.size());
+ assertTrue(seeds.containsAll(asList(address1, address2, address3)));
+ }
+ finally
+ {
+ if (inetAddressAndPortMock != null)
+ inetAddressAndPortMock.close();
+
+ if (descriptorMock != null)
+ descriptorMock.close();
+
+ if (fbUtilitiesMock != null)
+ fbUtilitiesMock.close();
+ }
+ }
+
+ private static Config getConfig(Map<String, String> parameters)
+ {
+ Config config = new Config();
+ config.seed_provider = new ParameterizedClass();
+ config.seed_provider.class_name = SimpleSeedProvider.class.getName();
+ config.seed_provider.parameters = parameters;
+ return config;
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/ClientRequestRowAndColumnMetricsTest.java b/test/unit/org/apache/cassandra/metrics/ClientRequestRowAndColumnMetricsTest.java
new file mode 100644
index 0000000..cb4b388
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/ClientRequestRowAndColumnMetricsTest.java
@@ -0,0 +1,625 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.paxos.Paxos;
+import org.apache.cassandra.transport.SimpleClient;
+import org.apache.cassandra.transport.messages.BatchMessage;
+import org.apache.cassandra.transport.messages.QueryMessage;
+
+import static org.junit.Assert.assertEquals;
+
+import static org.apache.cassandra.transport.ProtocolVersion.CURRENT;
+
+public class ClientRequestRowAndColumnMetricsTest extends CQLTester
+{
+ @BeforeClass
+ public static void setup()
+ {
+ requireNetwork();
+ }
+
+ @Before
+ public void clearMetrics()
+ {
+ ClientRequestSizeMetrics.totalRowsRead.dec(ClientRequestSizeMetrics.totalRowsRead.getCount());
+ ClientRequestSizeMetrics.totalColumnsRead.dec(ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ ClientRequestSizeMetrics.totalRowsWritten.dec(ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ ClientRequestSizeMetrics.totalColumnsWritten.dec(ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+
+ StorageProxy.instance.setClientRequestSizeMetricsEnabled(true);
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForMultiRowPartitionSelection() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1");
+
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key is provided by the client in the request, so we don't consider those columns as read.
+ assertEquals(4, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsWithOnlyPartitionKeyInSelect() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 2)");
+ executeNet(CURRENT, "SELECT pk FROM %s WHERE pk = 1");
+
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key is provided by the client in the request, so we don't consider that column read.
+ assertEquals(0, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsWithOnlyClusteringKeyInSelect() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 2)");
+ executeNet(CURRENT, "SELECT ck FROM %s WHERE pk = 1");
+
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key is provided by the client in the request, so we don't consider that column read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldNotRecordReadMetricsWhenDisabled() throws Throwable
+ {
+ StorageProxy.instance.setClientRequestSizeMetricsEnabled(false);
+
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1");
+
+ assertEquals(0, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ assertEquals(0, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsWithSingleRowSelection() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1 AND ck = 1");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // Both the partition key and clustering key are provided by the client in the request.
+ assertEquals(1, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsWithSliceRestriction() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1 AND ck > 0");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key is selected, but the restriction over the clustering key is a slice.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsWithINRestrictionSinglePartition() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1 AND ck IN (0, 1)");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key and clustering key are both selected.
+ assertEquals(1, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsWithINRestrictionMultiplePartitions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 3)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (4, 5, 6)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk IN (1, 4)");
+
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key is selected, but there is no clustering restriction.
+ assertEquals(4, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForMultiColumnClusteringRestriction() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, ck3 int, v int, PRIMARY KEY (pk, ck1, ck2, ck3))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck1, ck2, ck3, v) VALUES (1, 2, 3, 4, 6)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1 AND ck1 = 2 AND (ck2, ck3) = (3, 4)");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The entire primary key is selected, so only one value is actually read.
+ assertEquals(1, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForClusteringSlice() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, ck3 int, v int, PRIMARY KEY (pk, ck1, ck2, ck3))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck1, ck2, ck3, v) VALUES (1, 2, 3, 4, 6)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1 AND ck1 = 2 AND ck2 = 3 AND ck3 >= 4");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The last clustering key element isn't bound, so count it as being read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForTokenAndClusteringSlice() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck1 int, ck2 int, ck3 int, v int, PRIMARY KEY (pk, ck1, ck2, ck3))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck1, ck2, ck3, v) VALUES (1, 2, 3, 4, 6)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE token(pk) = token(1) AND ck1 = 2 AND ck2 = 3 AND ck3 >= 4 ALLOW FILTERING");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // Last clustering is a slice, and the partition key is restricted on token, so count them as read.
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForSingleValueRow() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(1, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldNotRecordWriteMetricsWhenDisabled() throws Throwable
+ {
+ StorageProxy.instance.setClientRequestSizeMetricsEnabled(false);
+
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+
+ assertEquals(0, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(0, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForMultiValueRow() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int, v3 int)");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, v1, v2, v3) VALUES (1, 2, 3, 4)");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForBatch() throws Exception
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int)");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+
+ String first = String.format("INSERT INTO %s.%s (pk, v1, v2) VALUES (1, 10, 100)", KEYSPACE, currentTable());
+ String second = String.format("INSERT INTO %s.%s (pk, v1, v2) VALUES (2, 20, 200)", KEYSPACE, currentTable());
+
+ List<List<ByteBuffer>> values = ImmutableList.of(Collections.emptyList(), Collections.emptyList());
+ BatchMessage batch = new BatchMessage(BatchStatement.Type.LOGGED, ImmutableList.of(first, second), values, QueryOptions.DEFAULT);
+ client.execute(batch);
+
+ // The metrics should reflect the batch as a single write operation with multiple rows and columns.
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(4, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForCellDeletes() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int, v3 int)");
+
+ executeNet(CURRENT, "DELETE v1, v2, v3 FROM %s WHERE pk = 1");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForCellNulls() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int, v3 int)");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, v1, v2, v3) VALUES (1, null, null, null)");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForSingleStaticInsert() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v0 int static, v1 int, v2 int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v0, v1, v2) VALUES (0, 1, 2, 3, 4)");
+
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForBatchedStaticInserts() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v0 int static, v1 int, v2 int, PRIMARY KEY (pk, ck))");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+
+ String first = String.format("INSERT INTO %s.%s (pk, ck, v0, v1, v2) VALUES (0, 1, 2, 3, 4)", KEYSPACE, currentTable());
+ String second = String.format("INSERT INTO %s.%s (pk, ck, v0, v1, v2) VALUES (0, 2, 3, 5, 6)", KEYSPACE, currentTable());
+
+ List<List<ByteBuffer>> values = ImmutableList.of(Collections.emptyList(), Collections.emptyList());
+ BatchMessage batch = new BatchMessage(BatchStatement.Type.LOGGED, ImmutableList.of(first, second), values, QueryOptions.DEFAULT);
+ client.execute(batch);
+
+ // Two normal rows and the single static row:
+ assertEquals(3, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ // Two normal columns per insert, and then one columns for the static row:
+ assertEquals(5, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForRowDelete() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v0 int static, v1 int, v2 int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "DELETE FROM %s WHERE pk = 1 AND ck = 1");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ // The columns metric should account for all regular columns.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForRangeDelete() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v0 int static, v1 int, v2 int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "DELETE FROM %s WHERE pk = 1 AND ck > 1");
+
+ // The range delete is intended to delete at least one row, but that is only a lower bound.
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ // The columns metric should account for all regular columns.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForPartitionDelete() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v0 int static, v1 int, v2 int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "DELETE FROM %s WHERE pk = 1");
+
+ // A partition deletion intends to delete at least one row.
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ // If we delete one row, we intended to delete all its regular and static columns.
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForIntraRowBatch() throws Exception
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+
+ String first = String.format("INSERT INTO %s.%s (pk, ck, v1, v2) VALUES (1, 2, 3, 4)", KEYSPACE, currentTable());
+ String second = String.format("DELETE FROM %s.%s WHERE pk = 1 AND ck > 1", KEYSPACE, currentTable());
+
+ List<List<ByteBuffer>> values = ImmutableList.of(Collections.emptyList(), Collections.emptyList());
+ BatchMessage batch = new BatchMessage(BatchStatement.Type.LOGGED, ImmutableList.of(first, second), values, QueryOptions.DEFAULT);
+ client.execute(batch);
+
+ // Both operations affect the same row, but writes and deletes are distinct.
+ assertEquals(2, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(4, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+ }
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForIfNotExistsV1() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v1);
+ shouldRecordWriteMetricsForIfNotExists();
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForIfNotExistsV2() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v2);
+ shouldRecordWriteMetricsForIfNotExists();
+ }
+
+ public void shouldRecordWriteMetricsForIfNotExists() throws Exception
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int, v3 int)");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+ client.execute(new QueryMessage(String.format("INSERT INTO %s.%s (pk, v1, v2, v3) VALUES (1, 2, 3, 4) IF NOT EXISTS", KEYSPACE, currentTable()), QueryOptions.DEFAULT));
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+
+ // We read internally, but don't reflect that in our read metrics.
+ assertEquals(0, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ assertEquals(0, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForCASV1() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v1);
+ shouldRecordWriteMetricsForCAS();
+ }
+
+ @Test
+ public void shouldRecordWriteMetricsForCASV2() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v2);
+ shouldRecordWriteMetricsForCAS();
+ }
+
+ public void shouldRecordWriteMetricsForCAS() throws Exception
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int)");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+ client.execute(new QueryMessage(String.format("INSERT INTO %s.%s (pk, v1, v2) VALUES (1, 2, 3)", KEYSPACE, currentTable()), QueryOptions.DEFAULT));
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+
+ client.execute(new QueryMessage(String.format("UPDATE %s.%s SET v2 = 4 WHERE pk = 1 IF v1 = 2", KEYSPACE, currentTable()), QueryOptions.DEFAULT));
+
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+
+ // We read internally, but don't reflect that in our read metrics.
+ assertEquals(0, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ assertEquals(0, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+ }
+
+ @Test
+ public void shouldNotRecordWriteMetricsForFailedCASV1() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v1);
+ shouldNotRecordWriteMetricsForFailedCAS();
+ }
+
+ @Test
+ public void shouldNotRecordWriteMetricsForFailedCASV2() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v2);
+ shouldNotRecordWriteMetricsForFailedCAS();
+ }
+
+ public void shouldNotRecordWriteMetricsForFailedCAS() throws Exception
+ {
+ createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int)");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+ client.execute(new QueryMessage(String.format("INSERT INTO %s.%s (pk, v1, v2) VALUES (1, 2, 3)", KEYSPACE, currentTable()), QueryOptions.DEFAULT));
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsWritten.getCount());
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+
+ client.execute(new QueryMessage(String.format("UPDATE %s.%s SET v2 = 4 WHERE pk = 1 IF v1 = 4", KEYSPACE, currentTable()), QueryOptions.DEFAULT));
+
+ // We didn't actually write anything, so don't reflect a write against the metrics.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsWritten.getCount());
+
+ // Don't reflect in our read metrics the result returned to the client by the failed CAS write.
+ assertEquals(0, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ assertEquals(0, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+ }
+
+ @Test
+ public void shouldRecordReadMetricsOnSerialReadV1() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v1);
+ shouldRecordReadMetricsOnSerialRead();
+ }
+
+ @Test
+ public void shouldRecordReadMetricsOnSerialReadV2() throws Exception
+ {
+ Paxos.setPaxosVariant(Config.PaxosVariant.v2);
+ shouldRecordReadMetricsOnSerialRead();
+ }
+
+ public void shouldRecordReadMetricsOnSerialRead() throws Exception
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ try (SimpleClient client = new SimpleClient(nativeAddr.getHostAddress(), nativePort, CURRENT))
+ {
+ client.connect(false);
+ client.execute(new QueryMessage(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (1, 1, 1)", KEYSPACE, currentTable()), QueryOptions.DEFAULT));
+
+ QueryMessage query = new QueryMessage(String.format("SELECT * FROM %s.%s WHERE pk = 1 AND ck = 1", KEYSPACE, currentTable()),
+ QueryOptions.forInternalCalls(ConsistencyLevel.SERIAL, Collections.emptyList()));
+ client.execute(query);
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+
+ // Both the partition key and clustering key are provided by the client in the request.
+ assertEquals(1, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForGlobalIndexQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ createIndex("CREATE INDEX on %s (v)");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (2, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE v = 1");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The index search term is provided by the client in the request, so we don't consider that column read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForPartitionRestrictedIndexQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ createIndex("CREATE INDEX on %s (v)");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE pk = 1 AND v = 1");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The partition key and index search term are provided by the client, so we don't consider those columns read.
+ assertEquals(1, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForClusteringKeyIndexQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+ createIndex("CREATE INDEX on %s (ck)");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE ck = 2");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The index search term is provided by the client in the request, so we don't consider that column read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForFilteringQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (2, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE v = 1 ALLOW FILTERING");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The filtering term is provided by the client in the request, so we don't consider that column read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForRangeFilteringQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (2, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE v > 1 ALLOW FILTERING");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The value column is restricted over a range, not bound to a particular value.
+ assertEquals(3, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForINFilteringQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (2, 2, 2)");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE v IN (1) ALLOW FILTERING");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The filtering term is provided by the client in the request, so we don't consider that column read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+
+ @Test
+ public void shouldRecordReadMetricsForContainsQuery() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, ck int, v set<int>, PRIMARY KEY (pk, ck))");
+
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (1, 1, {1, 2, 3} )");
+ executeNet(CURRENT, "INSERT INTO %s (pk, ck, v) VALUES (2, 2, {4, 5, 6})");
+ executeNet(CURRENT, "SELECT * FROM %s WHERE v CONTAINS 1 ALLOW FILTERING");
+
+ assertEquals(1, ClientRequestSizeMetrics.totalRowsRead.getCount());
+ // The filtering term is provided by the client in the request, so we don't consider that column read.
+ assertEquals(2, ClientRequestSizeMetrics.totalColumnsRead.getCount());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/metrics/SamplerTest.java b/test/unit/org/apache/cassandra/metrics/SamplerTest.java
index dba19a3..862abbb 100644
--- a/test/unit/org/apache/cassandra/metrics/SamplerTest.java
+++ b/test/unit/org/apache/cassandra/metrics/SamplerTest.java
@@ -79,7 +79,12 @@
return true;
}
- public void beginSampling(int capacity, int durationMillis)
+ public boolean isActive()
+ {
+ return true;
+ }
+
+ public void beginSampling(int capacity, long durationMillis)
{
}
diff --git a/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java b/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
index 4c9de77..bbcced4 100644
--- a/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/TableMetricsTest.java
@@ -19,10 +19,12 @@
package org.apache.cassandra.metrics;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,6 +38,7 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.service.StorageService;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -47,6 +50,7 @@
private static final String KEYSPACE = "junit";
private static final String TABLE = "tablemetricstest";
private static final String COUNTER_TABLE = "tablemetricscountertest";
+ private static final String TWCS_TABLE = "tablemetricstesttwcs";
private static EmbeddedCassandraService cassandra;
private static Cluster cluster;
@@ -68,6 +72,15 @@
return recreateTable(TABLE);
}
+ private ColumnFamilyStore recreateTWCSTable()
+ {
+ session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, TWCS_TABLE));
+ session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int, val1 text, val2 text, PRIMARY KEY(id, val1)) " +
+ " WITH compaction = {'class': 'TimeWindowCompactionStrategy', 'compaction_window_unit': 'MINUTES', 'compaction_window_size': 1};",
+ KEYSPACE, TWCS_TABLE));
+ return ColumnFamilyStore.getIfExists(KEYSPACE, TWCS_TABLE);
+ }
+
private ColumnFamilyStore recreateTable(String table)
{
session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, table));
@@ -130,6 +143,37 @@
}
@Test
+ public void testMaxSSTableSize() throws Exception
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ assertEquals(0, cfs.metric.maxSSTableSize.getValue().longValue());
+
+ for (int i = 0; i < 1000; i++)
+ {
+ session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
+ }
+
+ StorageService.instance.forceKeyspaceFlush(KEYSPACE);
+
+ assertGreaterThan(cfs.metric.maxSSTableSize.getValue().doubleValue(), 0);
+ }
+
+ @Test
+ public void testMaxSSTableDuration() throws Exception
+ {
+ ColumnFamilyStore cfs = recreateTWCSTable();
+ assertEquals(0, cfs.metric.maxSSTableDuration.getValue().longValue());
+
+ session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TWCS_TABLE, 1, "val1", "val1"));
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
+ session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TWCS_TABLE, 2, "val2", "val2"));
+
+ StorageService.instance.forceKeyspaceFlush(KEYSPACE);
+
+ assertGreaterThan(cfs.metric.maxSSTableDuration.getValue().doubleValue(), 0);
+ }
+
+ @Test
public void testPreparedStatementsExecuted()
{
ColumnFamilyStore cfs = recreateTable();
diff --git a/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java b/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java
new file mode 100644
index 0000000..62d8f7c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/metrics/TrieMemtableMetricsTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.OverrideConfigurationLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.service.StorageService;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_SHARD_COUNT;
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(BMUnitRunner.class)
+public class TrieMemtableMetricsTest extends SchemaLoader
+{
+ private static final int NUM_SHARDS = 13;
+
+ private static final Logger logger = LoggerFactory.getLogger(TrieMemtableMetricsTest.class);
+ private static Session session;
+
+ private static final String KEYSPACE = "triememtable";
+ private static final String TABLE = "metricstest";
+
+ @BeforeClass
+ public static void loadSchema() throws ConfigurationException
+ {
+ // shadow superclass method; we'll call it directly
+ // after tinkering with the Config
+ }
+
+ @BeforeClass
+ public static void setup() throws ConfigurationException, IOException
+ {
+ OverrideConfigurationLoader.override((config) -> {
+ config.partitioner = "Murmur3Partitioner";
+ });
+ MEMTABLE_SHARD_COUNT.setInt(NUM_SHARDS);
+
+ SchemaLoader.loadSchema();
+
+ EmbeddedCassandraService cassandra = new EmbeddedCassandraService();
+ cassandra.start();
+
+ Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ session = cluster.connect();
+
+ session.execute(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };", KEYSPACE));
+ }
+
+ private ColumnFamilyStore recreateTable()
+ {
+ return recreateTable(TABLE);
+ }
+
+ private ColumnFamilyStore recreateTable(String table)
+ {
+ session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, table));
+ session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int, val1 text, val2 text, PRIMARY KEY(id, val1)) WITH MEMTABLE = 'test_memtable_metrics';", KEYSPACE, table));
+ return ColumnFamilyStore.getIfExists(KEYSPACE, table);
+ }
+
+ @Test
+ public void testRegularStatementsAreCounted()
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ TrieMemtableMetricsView metrics = getMemtableMetrics(cfs);
+ assertEquals(0, metrics.contendedPuts.getCount());
+ assertEquals(0, metrics.uncontendedPuts.getCount());
+
+ for (int i = 0; i < 10; i++)
+ {
+ session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
+ }
+
+ long allPuts = metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount();
+ assertEquals(10, allPuts);
+ }
+
+ @Test
+ public void testFlushRelatedMetrics() throws IOException, ExecutionException, InterruptedException
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ TrieMemtableMetricsView metrics = getMemtableMetrics(cfs);
+
+ StorageService.instance.forceKeyspaceFlush(KEYSPACE, TABLE);
+ assertEquals(0, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
+
+ writeAndFlush(10);
+ assertEquals(10, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
+
+ // verify that metrics survive flush / memtable switching
+ writeAndFlush(10);
+ assertEquals(20, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
+ assertEquals(metrics.lastFlushShardDataSizes.toString(), NUM_SHARDS, metrics.lastFlushShardDataSizes.numSamplesGauge.getValue().intValue());
+ }
+
+ @Test
+ @BMRules(rules = { @BMRule(name = "Delay memtable update",
+ targetClass = "InMemoryTrie",
+ targetMethod = "putSingleton",
+ action = "java.lang.Thread.sleep(10)")})
+ public void testContentionMetrics() throws IOException, ExecutionException, InterruptedException
+ {
+ ColumnFamilyStore cfs = recreateTable();
+ TrieMemtableMetricsView metrics = getMemtableMetrics(cfs);
+ assertEquals(0, (int) metrics.lastFlushShardDataSizes.numSamplesGauge.getValue());
+
+ StorageService.instance.forceKeyspaceFlush(KEYSPACE, TABLE);
+
+ writeAndFlush(100);
+
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ metrics.contentionTime.latency.getSnapshot().dump(stream);
+
+ assertEquals(100, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
+ assertThat(metrics.contendedPuts.getCount(), greaterThan(0L));
+ assertThat(metrics.contentionTime.totalLatency.getCount(), greaterThan(0L));
+ }
+
+ @Test
+ public void testMetricsCleanupOnDrop()
+ {
+ String tableName = TABLE + "_metrics_cleanup";
+ CassandraMetricsRegistry registry = CassandraMetricsRegistry.Metrics;
+ Supplier<Stream<String>> metrics = () -> registry.getNames().stream().filter(m -> m.contains(tableName));
+
+ // no metrics before creating
+ assertEquals(0, metrics.get().count());
+
+ recreateTable(tableName);
+ // some metrics
+ assertTrue(metrics.get().count() > 0);
+
+ session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, tableName));
+ // no metrics after drop
+ assertEquals(metrics.get().collect(Collectors.joining(",")), 0, metrics.get().count());
+ }
+
+ private TrieMemtableMetricsView getMemtableMetrics(ColumnFamilyStore cfs)
+ {
+ return new TrieMemtableMetricsView(cfs.keyspace.getName(), cfs.name);
+ }
+
+ private void writeAndFlush(int rows) throws IOException, ExecutionException, InterruptedException
+ {
+ logger.info("writing {} rows", rows);
+ Future[] futures = new Future[rows];
+ for (int i = 0; i < rows; i++)
+ {
+ logger.info("writing {} row", i);
+ futures[i] = session.executeAsync(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
+ }
+ for (int i = 0; i < rows; i++)
+ {
+ futures[i].get();
+ logger.info("writing {} row completed", i);
+ }
+ logger.info("forcing flush");
+ StorageService.instance.forceKeyspaceFlush(KEYSPACE, TABLE);
+ logger.info("table flushed");
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ session.close();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/net/ConnectionTest.java b/test/unit/org/apache/cassandra/net/ConnectionTest.java
index ec447aa..53bc7c5 100644
--- a/test/unit/org/apache/cassandra/net/ConnectionTest.java
+++ b/test/unit/org/apache/cassandra/net/ConnectionTest.java
@@ -55,10 +55,10 @@
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelOutboundHandlerAdapter;
import io.netty.channel.ChannelPromise;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
@@ -71,6 +71,7 @@
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SSL_STORAGE_PORT;
import static org.apache.cassandra.net.MessagingService.VERSION_30;
import static org.apache.cassandra.net.MessagingService.VERSION_3014;
import static org.apache.cassandra.net.MessagingService.VERSION_40;
@@ -573,9 +574,7 @@
@Test
public void testPendingOutboundConnectionUpdatesMessageVersionOnReconnectAttempt() throws Throwable
{
- final String storagePortProperty = Config.PROPERTY_PREFIX + "ssl_storage_port";
- final String originalStoragePort = System.getProperty(storagePortProperty);
- try
+ try (WithProperties properties = new WithProperties().set(SSL_STORAGE_PORT, 7011))
{
// Set up an inbound connection listening *only* on the SSL storage port to
// replicate a 3.x node. Force the messaging version to be incorrectly set to 4.0
@@ -586,7 +585,6 @@
MessagingService.instance().versions.set(FBUtilities.getBroadcastAddressAndPort(),
MessagingService.VERSION_40);
- System.setProperty(storagePortProperty, "7011");
final InetAddressAndPort legacySSLAddrsAndPort = endpoint.withPort(DatabaseDescriptor.getSSLStoragePort());
InboundConnectionSettings inboundSettings = settings.inbound.apply(new InboundConnectionSettings().withEncryption(encryptionOptions))
.withBindAddress(legacySSLAddrsAndPort)
@@ -653,10 +651,6 @@
{
MessagingService.instance().versions.set(FBUtilities.getBroadcastAddressAndPort(),
current_version);
- if (originalStoragePort != null)
- System.setProperty(storagePortProperty, originalStoragePort);
- else
- System.clearProperty(storagePortProperty);
}
}
diff --git a/test/unit/org/apache/cassandra/net/FramingTest.java b/test/unit/org/apache/cassandra/net/FramingTest.java
index 81f95f3..18e3179 100644
--- a/test/unit/org/apache/cassandra/net/FramingTest.java
+++ b/test/unit/org/apache/cassandra/net/FramingTest.java
@@ -71,13 +71,13 @@
public void serialize(byte[] t, DataOutputPlus out, int version) throws IOException
{
- out.writeUnsignedVInt(t.length);
+ out.writeUnsignedVInt32(t.length);
out.write(t);
}
public byte[] deserialize(DataInputPlus in, int version) throws IOException
{
- byte[] r = new byte[(int) in.readUnsignedVInt()];
+ byte[] r = new byte[in.readUnsignedVInt32()];
in.readFully(r);
return r;
}
diff --git a/test/unit/org/apache/cassandra/net/HandshakeTest.java b/test/unit/org/apache/cassandra/net/HandshakeTest.java
index 75ae103..36439f5 100644
--- a/test/unit/org/apache/cassandra/net/HandshakeTest.java
+++ b/test/unit/org/apache/cassandra/net/HandshakeTest.java
@@ -19,10 +19,20 @@
package org.apache.cassandra.net;
import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import com.google.common.net.InetAddresses;
+
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.gms.GossipDigestSyn;
+import org.apache.cassandra.security.DefaultSslContextFactory;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -42,11 +52,15 @@
import static org.apache.cassandra.net.MessagingService.minimum_version;
import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
import static org.apache.cassandra.net.OutboundConnectionInitiator.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
// TODO: test failure due to exception, timeout, etc
public class HandshakeTest
{
private static final SocketFactory factory = new SocketFactory();
+ static final InetAddressAndPort TO_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.2"), 7012);
+ static final InetAddressAndPort FROM_ADDR = InetAddressAndPort.getByAddressOverrideDefaults(InetAddresses.forString("127.0.0.1"), 7012);
@BeforeClass
public static void startup()
@@ -80,6 +94,7 @@
Future<Result<MessagingSuccess>> future =
initiateMessaging(eventLoop,
SMALL_MESSAGES,
+ SslFallbackConnectionType.SERVER_CONFIG,
new OutboundConnectionSettings(endpoint)
.withAcceptVersions(acceptOutbound)
.withDefaults(ConnectionCategory.MESSAGING),
@@ -92,6 +107,7 @@
}
}
+
@Test
public void testBothCurrentVersion() throws InterruptedException, ExecutionException
{
@@ -172,7 +188,7 @@
}
catch (ExecutionException e)
{
- Assert.assertTrue(e.getCause() instanceof ClosedChannelException);
+ assertTrue(e.getCause() instanceof ClosedChannelException);
}
}
@@ -186,7 +202,7 @@
}
catch (ExecutionException e)
{
- Assert.assertTrue(e.getCause() instanceof ClosedChannelException);
+ assertTrue(e.getCause() instanceof ClosedChannelException);
}
}
@@ -207,7 +223,7 @@
}
catch (ExecutionException e)
{
- Assert.assertTrue(e.getCause() instanceof ClosedChannelException);
+ assertTrue(e.getCause() instanceof ClosedChannelException);
}
}
@@ -218,4 +234,165 @@
Assert.assertEquals(Result.Outcome.SUCCESS, result.outcome);
Assert.assertEquals(VERSION_30, result.success().messagingVersion);
}
+
+ @Test
+ public void testOutboundConnectionfFallbackDuringUpgrades() throws ClosedChannelException, InterruptedException
+ {
+ // Upgrade from Non-SSL -> Optional SSL
+ // Outbound connection from Optional SSL(new node) -> Non-SSL (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.SSL, true, SslFallbackConnectionType.NO_SSL, false);
+
+ // Upgrade from Optional SSL -> Strict SSL
+ // Outbound connection from Strict SSL(new node) -> Optional SSL (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.SSL, false, SslFallbackConnectionType.SSL, true);
+
+ // Upgrade from Optional SSL -> Strict MTLS
+ // Outbound connection from Strict MTLS(new node) -> Optional SSL (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.MTLS, false, SslFallbackConnectionType.SSL, true);
+
+ // Upgrade from Strict SSL -> Optional MTLS
+ // Outbound connection from Optional MTLS(new node) -> Strict SSL (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.MTLS, true, SslFallbackConnectionType.SSL, false);
+
+ // Upgrade from Strict Optional MTLS -> Strict MTLS
+ // Outbound connection from Strict TLS(new node) -> Optional TLS (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.MTLS, false, SslFallbackConnectionType.MTLS, true);
+ }
+
+ @Test
+ public void testOutboundConnectionfFallbackDuringDowngrades() throws ClosedChannelException, InterruptedException
+ {
+ // From Strict MTLS -> Optional MTLS
+ // Outbound connection from Optional TLS(new node) -> Strict MTLS (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.MTLS, true, SslFallbackConnectionType.MTLS, false);
+
+ // From Optional MTLS -> Strict SSL
+ // Outbound connection from Strict SSL(new node) -> Optional MTLS (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.SSL, false, SslFallbackConnectionType.MTLS, true);
+
+ // From Strict MTLS -> Optional SSL
+ // Outbound connection from Optional SSL(new node) -> Strict MTLS (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.SSL, true, SslFallbackConnectionType.MTLS, false);
+
+ // From Strict SSL -> Optional SSL
+ // Outbound connection from Optional SSL(new node) -> Strict SSL (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.SSL, true, SslFallbackConnectionType.SSL, false);
+
+ // From Optional SSL -> Non-SSL
+ // Outbound connection from Non-SSL(new node) -> Optional SSL (old node)
+ testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType.NO_SSL, false, SslFallbackConnectionType.SSL, true);
+ }
+
+ @Test
+ public void testOutboundConnectionDoesntFallbackWhenErrorIsNotSSLRelated() throws ClosedChannelException, InterruptedException
+ {
+ // Configuring nodes in Optional SSL mode
+ // when optional mode is enabled, if the connection error is SSL related, fallback to another SSL strategy should happen,
+ // otherwise it should use same SSL strategy and retry
+ ServerEncryptionOptions serverEncryptionOptions = getServerEncryptionOptions(SslFallbackConnectionType.SSL, true);
+ InboundSockets inbound = getInboundSocket(serverEncryptionOptions);
+ try
+ {
+ InetAddressAndPort endpoint = inbound.sockets().stream().map(s -> s.settings.bindAddress).findFirst().get();
+
+ // Open outbound connections before server starts listening
+ // The connection should be accepted after opening inbound connections, with the same SSL context without fallback
+ OutboundConnection outboundConnection = initiateOutbound(endpoint, SslFallbackConnectionType.SSL, true);
+
+ // Let the outbound connection be tried for 4 times atleast
+ while (outboundConnection.connectionAttempts() < SslFallbackConnectionType.values().length)
+ {
+ Thread.sleep(1000);
+ }
+ assertFalse(outboundConnection.isConnected());
+ inbound.open();
+ // As soon as the node accepts inbound connections, the connection must be established with right SSL context
+ waitForConnection(outboundConnection);
+ assertTrue(outboundConnection.isConnected());
+ }
+ finally
+ {
+ inbound.close().await(10L, TimeUnit.SECONDS);
+ }
+ }
+
+ private ServerEncryptionOptions getServerEncryptionOptions(SslFallbackConnectionType sslConnectionType, boolean optional)
+ {
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions().withOptional(optional)
+ .withKeyStore("test/conf/cassandra_ssl_test.keystore")
+ .withKeyStorePassword("cassandra")
+ .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore")
+ .withOutboundKeystorePassword("cassandra")
+ .withTrustStore("test/conf/cassandra_ssl_test.truststore")
+ .withTrustStorePassword("cassandra")
+ .withSslContextFactory((new ParameterizedClass(DefaultSslContextFactory.class.getName(),
+ new HashMap<>())));
+ if (sslConnectionType == SslFallbackConnectionType.MTLS)
+ {
+ serverEncryptionOptions = serverEncryptionOptions.withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all)
+ .withRequireClientAuth(true);
+ }
+ else if (sslConnectionType == SslFallbackConnectionType.SSL)
+ {
+ serverEncryptionOptions = serverEncryptionOptions.withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.all)
+ .withRequireClientAuth(false);
+ }
+ return serverEncryptionOptions;
+ }
+
+ private InboundSockets getInboundSocket(ServerEncryptionOptions serverEncryptionOptions)
+ {
+ InboundConnectionSettings settings = new InboundConnectionSettings().withAcceptMessaging(new AcceptVersions(minimum_version, current_version))
+ .withEncryption(serverEncryptionOptions)
+ .withBindAddress(TO_ADDR);
+ List<InboundConnectionSettings> settingsList = new ArrayList<>();
+ settingsList.add(settings);
+ return new InboundSockets(settingsList);
+ }
+
+ private OutboundConnection initiateOutbound(InetAddressAndPort endpoint, SslFallbackConnectionType connectionType, boolean optional) throws ClosedChannelException
+ {
+ final OutboundConnectionSettings settings = new OutboundConnectionSettings(endpoint)
+ .withAcceptVersions(new AcceptVersions(minimum_version, current_version))
+ .withDefaults(ConnectionCategory.MESSAGING)
+ .withEncryption(getServerEncryptionOptions(connectionType, optional))
+ .withFrom(FROM_ADDR);
+ OutboundConnections outboundConnections = OutboundConnections.tryRegister(new ConcurrentHashMap<>(), TO_ADDR, settings);
+ GossipDigestSyn syn = new GossipDigestSyn("cluster", "partitioner", new ArrayList<>(0));
+ Message<GossipDigestSyn> message = Message.out(Verb.GOSSIP_DIGEST_SYN, syn);
+ OutboundConnection outboundConnection = outboundConnections.connectionFor(message);
+ outboundConnection.enqueue(message);
+ outboundConnection.initiate();
+ return outboundConnection;
+ }
+
+ private void testOutboundFallbackOnSSLHandshakeFailure(SslFallbackConnectionType fromConnectionType, boolean fromOptional,
+ SslFallbackConnectionType toConnectionType, boolean toOptional) throws ClosedChannelException, InterruptedException
+ {
+ // Configures inbound connections to be optional mTLS
+ InboundSockets inbound = getInboundSocket(getServerEncryptionOptions(toConnectionType, toOptional));
+ try
+ {
+ InetAddressAndPort endpoint = inbound.sockets().stream().map(s -> s.settings.bindAddress).findFirst().get();
+ inbound.open();
+
+ // Open outbound connections, and wait until connection is established
+ OutboundConnection outboundConnection = initiateOutbound(endpoint, fromConnectionType, fromOptional);
+ waitForConnection(outboundConnection);
+ assertTrue(outboundConnection.isConnected());
+ }
+ finally
+ {
+ inbound.close().await(10L, TimeUnit.SECONDS);
+ }
+ }
+
+ private void waitForConnection(OutboundConnection outboundConnection) throws InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (!outboundConnection.isConnected() && System.currentTimeMillis() - startTime < 60000)
+ {
+ Thread.sleep(1000);
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java b/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java
index 5a73cbd..4391d24 100644
--- a/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java
+++ b/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java
@@ -40,6 +40,9 @@
import org.assertj.core.api.Assertions;
import org.mockito.Mockito;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_APPROX;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_PRECISE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
import static org.apache.cassandra.net.Message.serializer;
import static org.apache.cassandra.utils.CassandraGenerators.MESSAGE_GEN;
import static org.apache.cassandra.utils.FailingConsumer.orFail;
@@ -50,10 +53,10 @@
@BeforeClass
public static void beforeClass()
{
- System.setProperty("org.apache.cassandra.disable_mbean_registration", "true");
+ ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true);
// message serialization uses the MonotonicClock class for precise and approx timestamps, so mock it out
- System.setProperty("cassandra.monotonic_clock.precise", FixedMonotonicClock.class.getName());
- System.setProperty("cassandra.monotonic_clock.approx", FixedMonotonicClock.class.getName());
+ CLOCK_MONOTONIC_PRECISE.setString(FixedMonotonicClock.class.getName());
+ CLOCK_MONOTONIC_APPROX.setString(FixedMonotonicClock.class.getName());
DatabaseDescriptor.daemonInitialization();
}
diff --git a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
index 349d865..32d5050 100644
--- a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
+++ b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java
@@ -25,6 +25,7 @@
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.channels.AsynchronousSocketChannel;
+import java.security.cert.Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -35,31 +36,35 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.*;
import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import com.google.common.net.InetAddresses;
-
-import com.codahale.metrics.Timer;
-
-import org.apache.cassandra.auth.IInternodeAuthenticator;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.metrics.MessagingMetrics;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.utils.FBUtilities;
-import org.awaitility.Awaitility;
-import org.caffinitas.ohc.histo.EstimatedHistogram;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.auth.IInternodeAuthenticator;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.MessagingMetrics;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.awaitility.Awaitility;
+import org.caffinitas.ohc.histo.EstimatedHistogram;
+
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
public class MessagingServiceTest
{
@@ -67,7 +72,8 @@
public static AtomicInteger rejectedConnections = new AtomicInteger();
public static final IInternodeAuthenticator ALLOW_NOTHING_AUTHENTICATOR = new IInternodeAuthenticator()
{
- public boolean authenticate(InetAddress remoteAddress, int remotePort)
+ public boolean authenticate(InetAddress remoteAddress, int remotePort,
+ Certificate[] certificates, InternodeConnectionDirection connectionType)
{
rejectedConnections.incrementAndGet();
return false;
@@ -78,6 +84,25 @@
}
};
+
+ public static final IInternodeAuthenticator REJECT_OUTBOUND_AUTHENTICATOR = new IInternodeAuthenticator()
+ {
+ public boolean authenticate(InetAddress remoteAddress, int remotePort,
+ Certificate[] certificates, InternodeConnectionDirection connectionType)
+ {
+ if (connectionType == InternodeConnectionDirection.OUTBOUND)
+ {
+ rejectedConnections.incrementAndGet();
+ return false;
+ }
+ return true;
+ }
+
+ public void validateConfiguration() throws ConfigurationException
+ {
+
+ }
+ };
private static IInternodeAuthenticator originalAuthenticator;
private static ServerEncryptionOptions originalServerEncryptionOptions;
private static InetAddressAndPort originalListenAddress;
@@ -228,19 +253,38 @@
@Test
public void testFailedOutboundInternodeAuth() throws Exception
{
- MessagingService ms = MessagingService.instance();
- DatabaseDescriptor.setInternodeAuthenticator(ALLOW_NOTHING_AUTHENTICATOR);
- InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.250");
+ // Listen on serverside for connections
+ ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions()
+ .withInternodeEncryption(ServerEncryptionOptions.InternodeEncryption.none);
- //Should return null
- int rejectedBefore = rejectedConnections.get();
- Message<?> messageOut = Message.out(Verb.ECHO_REQ, NoPayload.noPayload);
- ms.send(messageOut, address);
- Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> rejectedConnections.get() > rejectedBefore);
+ DatabaseDescriptor.setInternodeAuthenticator(REJECT_OUTBOUND_AUTHENTICATOR);
+ InetAddress listenAddress = FBUtilities.getJustLocalAddress();
- //Should tolerate null
- ms.closeOutbound(address);
- ms.send(messageOut, address);
+ InboundConnectionSettings settings = new InboundConnectionSettings().withEncryption(serverEncryptionOptions);
+ InboundSockets connections = new InboundSockets(settings);
+
+ try
+ {
+ connections.open().await();
+ Assert.assertTrue(connections.isListening());
+
+ MessagingService ms = MessagingService.instance();
+ //Should return null
+ int rejectedBefore = rejectedConnections.get();
+ Message<?> messageOut = Message.out(Verb.ECHO_REQ, NoPayload.noPayload);
+ InetAddressAndPort address = InetAddressAndPort.getByAddress(listenAddress);
+ ms.send(messageOut, address);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> rejectedConnections.get() > rejectedBefore);
+
+ //Should tolerate null
+ ms.closeOutbound(address);
+ ms.send(messageOut, address);
+ }
+ finally
+ {
+ connections.close().await();
+ Assert.assertFalse(connections.isListening());
+ }
}
@Test
@@ -262,6 +306,11 @@
int rejectedBefore = rejectedConnections.get();
Future<Void> connectFuture = testChannel.connect(new InetSocketAddress(listenAddress, DatabaseDescriptor.getStoragePort()));
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(connectFuture::isDone);
+
+ // Since authentication doesn't happen during connect, try writing a dummy string which triggers
+ // authentication handler.
+ testChannel.write(ByteBufferUtil.bytes("dummy string"));
Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> rejectedConnections.get() > rejectedBefore);
connectFuture.cancel(true);
diff --git a/test/unit/org/apache/cassandra/net/SocketUtils.java b/test/unit/org/apache/cassandra/net/SocketUtils.java
index 78a49bd..23d1cc5 100644
--- a/test/unit/org/apache/cassandra/net/SocketUtils.java
+++ b/test/unit/org/apache/cassandra/net/SocketUtils.java
@@ -21,8 +21,6 @@
import java.io.IOException;
import java.net.ServerSocket;
-import com.google.common.base.Throwables;
-
public class SocketUtils
{
public static synchronized int findAvailablePort() throws RuntimeException
@@ -37,7 +35,7 @@
}
catch (IOException e)
{
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
finally
{
@@ -49,7 +47,7 @@
}
catch (IOException e)
{
- Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
}
diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
index 47d50a2..4409b3c 100644
--- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
+++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
@@ -98,7 +98,7 @@
NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
task.run();
- assertEquals(0, task.get().numberOfDifferences);
+ assertTrue(task.stat.differences.isEmpty());
}
@Test
@@ -145,7 +145,7 @@
}
// ensure that the changed range was recorded
- assertEquals("Wrong differing ranges", interesting.size(), task.stat.numberOfDifferences);
+ assertEquals("Wrong differing ranges", interesting.size(), task.stat.differences.size());
}
@Test
diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
index c7c68c4..03ac2ed 100644
--- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
@@ -305,8 +305,8 @@
assertThat(results)
.hasSize(2)
- .extracting(s -> s.numberOfDifferences)
- .containsOnly(1L);
+ .extracting(s -> s.differences.size())
+ .containsOnly(1);
assertThat(messages)
.hasSize(2)
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
index a6ca084..0483fcf 100644
--- a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java
@@ -23,7 +23,6 @@
import java.util.Map;
import java.util.Set;
-import org.junit.Assert;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -32,6 +31,7 @@
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.repair.RepairParallelism;
+import org.apache.cassandra.streaming.PreviewKind;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -149,17 +149,76 @@
// default value
option = RepairOption.parse(options, Murmur3Partitioner.instance);
- Assert.assertFalse(option.isForcedRepair());
+ assertFalse(option.isForcedRepair());
// explicit true
options.put(RepairOption.FORCE_REPAIR_KEY, "true");
option = RepairOption.parse(options, Murmur3Partitioner.instance);
- Assert.assertTrue(option.isForcedRepair());
+ assertTrue(option.isForcedRepair());
// explicit false
options.put(RepairOption.FORCE_REPAIR_KEY, "false");
option = RepairOption.parse(options, Murmur3Partitioner.instance);
- Assert.assertFalse(option.isForcedRepair());
+ assertFalse(option.isForcedRepair());
+ }
+
+ @Test
+ public void testOptimiseStreams()
+ {
+ boolean optFull = DatabaseDescriptor.autoOptimiseFullRepairStreams();
+ boolean optInc = DatabaseDescriptor.autoOptimiseIncRepairStreams();
+ boolean optPreview = DatabaseDescriptor.autoOptimisePreviewRepairStreams();
+ try
+ {
+ for (PreviewKind previewKind : PreviewKind.values())
+ for (boolean inc : new boolean[] {true, false})
+ assertOptimise(previewKind, inc);
+ }
+ finally
+ {
+ setOptimise(optFull, optInc, optPreview);
+ }
+ }
+
+ private void assertHelper(Map<String, String> options, boolean full, boolean inc, boolean preview, boolean expected)
+ {
+ setOptimise(full, inc, preview);
+ assertEquals(expected, RepairOption.parse(options, Murmur3Partitioner.instance).optimiseStreams());
+ }
+
+ private void setOptimise(boolean full, boolean inc, boolean preview)
+ {
+ DatabaseDescriptor.setAutoOptimiseFullRepairStreams(full);
+ DatabaseDescriptor.setAutoOptimiseIncRepairStreams(inc);
+ DatabaseDescriptor.setAutoOptimisePreviewRepairStreams(preview);
+ }
+
+ private void assertOptimise(PreviewKind previewKind, boolean incremental)
+ {
+ Map<String, String> options = new HashMap<>();
+ options.put(RepairOption.PREVIEW, previewKind.toString());
+ options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(incremental));
+ for (boolean a : new boolean[]{ true, false })
+ {
+ for (boolean b : new boolean[]{ true, false })
+ {
+ if (previewKind.isPreview())
+ {
+ assertHelper(options, a, b, true, true);
+ assertHelper(options, a, b, false, false);
+ }
+ else if (incremental)
+ {
+ assertHelper(options, a, true, b, true);
+ assertHelper(options, a, false, b, false);
+ }
+ else
+ {
+ assertHelper(options, true, a, b, true);
+ assertHelper(options, false, a, b, false);
+ }
+ }
+ }
}
private void assertParseThrowsIllegalArgumentExceptionWithMessage(Map<String, String> optionsToParse, String expectedErrorMessage)
diff --git a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
index e1556b7..16ef782 100644
--- a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
+++ b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
@@ -24,7 +24,6 @@
import java.util.function.Supplier;
import com.google.common.collect.ImmutableMap;
-import org.apache.cassandra.io.util.File;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -43,8 +42,9 @@
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.utils.FBUtilities;
@@ -473,7 +473,7 @@
// check
assertTrue(cfs.indexManager.listIndexes().isEmpty());
LifecycleTransaction.waitForDeletions();
- assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
+ assertFalse(desc.fileFor(Components.DATA).exists());
}
@Test
diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java
index f5c9986..2306120 100644
--- a/test/unit/org/apache/cassandra/schema/MockSchema.java
+++ b/test/unit/org/apache/cassandra/schema/MockSchema.java
@@ -33,27 +33,41 @@
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.memtable.SkipListMemtable;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
+import org.apache.cassandra.io.sstable.format.bti.BtiTableReader;
+import org.apache.cassandra.io.sstable.format.bti.PartitionIndex;
+import org.apache.cassandra.io.sstable.indexsummary.IndexSummary;
+import org.apache.cassandra.io.sstable.keycache.KeyCache;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.Memory;
-import org.apache.cassandra.utils.AlwaysPresentFilter;
+import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.Throwables;
import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
@@ -74,19 +88,28 @@
new Object[]{ Util.newUUIDGen() });
}
+ private static final File tempFile = temp("mocksegmentedfile");
+
static
{
Memory offsets = Memory.allocate(4);
offsets.setInt(0, 0);
indexSummary = new IndexSummary(Murmur3Partitioner.instance, offsets, 0, Memory.allocate(4), 0, 0, 0, 1);
+
+ try (DataOutputStreamPlus out = tempFile.newOutputStream(File.WriteMode.OVERWRITE))
+ {
+ out.write(new byte[10]);
+ }
+ catch (IOException ex)
+ {
+ throw Throwables.throwAsUncheckedException(ex);
+ }
}
private static final AtomicInteger id = new AtomicInteger();
public static final Keyspace ks = Keyspace.mockKS(KeyspaceMetadata.create("mockks", KeyspaceParams.simpleTransient(1)));
public static final IndexSummary indexSummary;
- private static final File tempFile = temp("mocksegmentedfile");
-
public static Memtable memtable(ColumnFamilyStore cfs)
{
return SkipListMemtable.FACTORY.create(null, cfs.metadata, cfs);
@@ -102,6 +125,11 @@
return sstable(generation, 0, false, first, last, cfs);
}
+ public static SSTableReader sstable(int generation, long first, long last, int minLocalDeletionTime, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, 0, false, first, last, 0, cfs, minLocalDeletionTime);
+ }
+
public static SSTableReader sstable(int generation, boolean keepRef, ColumnFamilyStore cfs)
{
return sstable(generation, 0, keepRef, cfs);
@@ -111,6 +139,7 @@
{
return sstable(generation, size, false, cfs);
}
+
public static SSTableReader sstable(int generation, int size, boolean keepRef, ColumnFamilyStore cfs)
{
return sstable(generation, size, keepRef, generation, generation, cfs);
@@ -128,7 +157,12 @@
public static SSTableReader sstableWithTimestamp(int generation, long timestamp, ColumnFamilyStore cfs)
{
- return sstable(generation, 0, false, 0, 1000, 0, Integer.MAX_VALUE, timestamp, cfs);
+ return sstable(generation, 0, false, 0, 1000, 0, cfs, Integer.MAX_VALUE, timestamp);
+ }
+
+ public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs)
+ {
+ return sstable(generation, size, keepRef, firstToken, lastToken, level, cfs, Integer.MAX_VALUE);
}
public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, ColumnFamilyStore cfs)
@@ -136,55 +170,117 @@
return sstable(generation, size, keepRef, firstToken, lastToken, 0, cfs);
}
- public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs)
+ public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs, int minLocalDeletionTime)
{
- return sstable(generation, size, keepRef, firstToken, lastToken, level, Integer.MAX_VALUE, System.currentTimeMillis() * 1000, cfs);
+ return sstable(generation, size, keepRef, firstToken, lastToken, level, cfs, minLocalDeletionTime, System.currentTimeMillis() * 1000);
}
- public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, int minLocalDeletionTime, long timestamp, ColumnFamilyStore cfs)
+ public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs, int minLocalDeletionTime, long timestamp)
{
+ SSTableFormat<?, ?> format = DatabaseDescriptor.getSelectedSSTableFormat();
Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(),
cfs.keyspace.getName(),
cfs.getTableName(),
- sstableId(generation), SSTableFormat.Type.BIG);
- Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
- for (Component component : components)
- {
- File file = new File(descriptor.filenameFor(component));
- file.createFileIfNotExists();
- }
- // .complete() with size to make sstable.onDiskLength work
- try (FileHandle.Builder builder = new FileHandle.Builder(new ChannelProxy(tempFile)).bufferSize(size);
- FileHandle fileHandle = builder.complete(size))
- {
- if (size > 0)
- {
- try
- {
- File file = new File(descriptor.filenameFor(Component.DATA));
- Util.setFileLength(file, size);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
- SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
- MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator);
- collector.update(new DeletionTime(timestamp, minLocalDeletionTime));
- StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level)
- .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header)
- .get(MetadataType.STATS);
- SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata,
- fileHandle.sharedCopy(), fileHandle.sharedCopy(), indexSummary.sharedCopy(),
- new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header);
- reader.first = readerBounds(firstToken);
- reader.last = readerBounds(lastToken);
- if (!keepRef)
- reader.selfRef().release();
- return reader;
- }
+ sstableId(generation),
+ format);
+ if (BigFormat.is(format))
+ {
+ Set<Component> components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC);
+ for (Component component : components)
+ {
+ File file = descriptor.fileFor(component);
+ file.createFileIfNotExists();
+ }
+ // .complete() with size to make sstable.onDiskLength work
+ try (FileHandle fileHandle = new FileHandle.Builder(tempFile).bufferSize(size).withLengthOverride(size).complete())
+ {
+ maybeSetDataLength(descriptor, size);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
+ MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator);
+ collector.update(new DeletionTime(timestamp, minLocalDeletionTime));
+ BufferDecoratedKey first = readerBounds(firstToken);
+ BufferDecoratedKey last = readerBounds(lastToken);
+ StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, first.retainable().getKey().slice(), last.retainable().getKey().slice())
+ .get(MetadataType.STATS);
+ BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components)
+ .setTableMetadataRef(cfs.metadata)
+ .setDataFile(fileHandle.sharedCopy())
+ .setIndexFile(fileHandle.sharedCopy())
+ .setIndexSummary(indexSummary.sharedCopy())
+ .setFilter(FilterFactory.AlwaysPresent)
+ .setMaxDataAge(1L)
+ .setStatsMetadata(metadata)
+ .setOpenReason(SSTableReader.OpenReason.NORMAL)
+ .setSerializationHeader(header)
+ .setFirst(first)
+ .setLast(last)
+ .setKeyCache(cfs.metadata().params.caching.cacheKeys ? new KeyCache(CacheService.instance.keyCache) : KeyCache.NO_CACHE)
+ .build(cfs, false, false);
+ if (!keepRef)
+ reader.selfRef().release();
+ return reader;
+ }
+ }
+ else if (BtiFormat.is(format))
+ {
+ Set<Component> components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC);
+ for (Component component : components)
+ {
+ File file = descriptor.fileFor(component);
+ file.createFileIfNotExists();
+ }
+ // .complete() with size to make sstable.onDiskLength work
+ try (FileHandle fileHandle = new FileHandle.Builder(tempFile).bufferSize(size).withLengthOverride(size).complete())
+ {
+ maybeSetDataLength(descriptor, size);
+ SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
+ MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator);
+ collector.update(new DeletionTime(timestamp, minLocalDeletionTime));
+ BufferDecoratedKey first = readerBounds(firstToken);
+ BufferDecoratedKey last = readerBounds(lastToken);
+ StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, first.retainable().getKey(), last.retainable().getKey())
+ .get(MetadataType.STATS);
+ BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
+ .setTableMetadataRef(cfs.metadata)
+ .setDataFile(fileHandle.sharedCopy())
+ .setPartitionIndex(new PartitionIndex(fileHandle.sharedCopy(), 0, 0, readerBounds(firstToken), readerBounds(lastToken)))
+ .setRowIndexFile(fileHandle.sharedCopy())
+ .setFilter(FilterFactory.AlwaysPresent)
+ .setMaxDataAge(1L)
+ .setStatsMetadata(metadata)
+ .setOpenReason(SSTableReader.OpenReason.NORMAL)
+ .setSerializationHeader(header)
+ .setFirst(readerBounds(firstToken))
+ .setLast(readerBounds(lastToken))
+ .build(cfs, false, false);
+ if (!keepRef)
+ reader.selfRef().release();
+ return reader;
+ }
+ }
+ else
+ {
+ throw Util.testMustBeImplementedForSSTableFormat();
+ }
+ }
+
+ private static void maybeSetDataLength(Descriptor descriptor, long size)
+ {
+ if (size > 0)
+ {
+ try
+ {
+ File file = descriptor.fileFor(Components.DATA);
+ Util.setFileLength(file, size);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
}
public static ColumnFamilyStore newCFS()
diff --git a/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java b/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java
index 9c3271b..683e52f 100644
--- a/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java
+++ b/test/unit/org/apache/cassandra/schema/RemoveWithoutDroppingTest.java
@@ -28,12 +28,14 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
import org.mockito.Mockito;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SCHEMA_UPDATE_HANDLER_FACTORY_CLASS;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.assertj.core.api.Assertions.assertThat;
@@ -46,7 +48,9 @@
@BeforeClass
public static void beforeClass()
{
- System.setProperty(SchemaUpdateHandlerFactoryProvider.SUH_FACTORY_CLASS_PROPERTY, TestSchemaUpdateHandlerFactory.class.getName());
+ ServerTestUtils.daemonInitialization();
+
+ SCHEMA_UPDATE_HANDLER_FACTORY_CLASS.setString(TestSchemaUpdateHandlerFactory.class.getName());
CQLTester.prepareServer();
Schema.instance.registerListener(listener);
}
diff --git a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
index e807730..089077f 100644
--- a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
@@ -33,11 +33,13 @@
import com.google.common.collect.ImmutableMap;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
@@ -61,6 +63,7 @@
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
@RunWith(BMUnitRunner.class)
public class SchemaKeyspaceTest
@@ -181,6 +184,42 @@
}
+ @Test
+ public void testAutoSnapshotEnabledOnTable()
+ {
+ Assume.assumeTrue(DatabaseDescriptor.isAutoSnapshot());
+ String keyspaceName = "AutoSnapshot";
+ String tableName = "table1";
+
+ createTable(keyspaceName, "CREATE TABLE " + tableName + " (a text primary key, b int) WITH allow_auto_snapshot = true");
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName);
+
+ assertTrue(cfs.isAutoSnapshotEnabled());
+
+ SchemaTestUtil.announceTableDrop(keyspaceName, tableName);
+
+ assertFalse(cfs.listSnapshots().isEmpty());
+ }
+
+ @Test
+ public void testAutoSnapshotDisabledOnTable()
+ {
+ Assume.assumeTrue(DatabaseDescriptor.isAutoSnapshot());
+ String keyspaceName = "AutoSnapshot";
+ String tableName = "table2";
+
+ createTable(keyspaceName, "CREATE TABLE " + tableName + " (a text primary key, b int) WITH allow_auto_snapshot = false");
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName);
+
+ assertFalse(cfs.isAutoSnapshotEnabled());
+
+ SchemaTestUtil.announceTableDrop(keyspaceName, tableName);
+
+ assertTrue(cfs.listSnapshots().isEmpty());
+ }
+
private static void updateTable(String keyspace, TableMetadata oldTable, TableMetadata newTable)
{
KeyspaceMetadata ksm = Schema.instance.getKeyspaceInstance(keyspace).getMetadata();
@@ -215,7 +254,7 @@
UnfilteredRowIterators.filter(serializedCD.unfilteredIterator(), FBUtilities.nowInSeconds()));
Set<ColumnMetadata> columns = new HashSet<>();
for (UntypedResultSet.Row row : columnsRows)
- columns.add(SchemaKeyspace.createColumnFromRow(row, Types.none()));
+ columns.add(SchemaKeyspace.createColumnFromRow(row, Types.none(), UserFunctions.none()));
assertEquals(metadata.params, params);
assertEquals(new HashSet<>(metadata.columns()), columns);
diff --git a/test/unit/org/apache/cassandra/security/CustomSslContextFactoryConfigTest.java b/test/unit/org/apache/cassandra/security/CustomSslContextFactoryConfigTest.java
index c1ab4a4..daa0f16 100644
--- a/test/unit/org/apache/cassandra/security/CustomSslContextFactoryConfigTest.java
+++ b/test/unit/org/apache/cassandra/security/CustomSslContextFactoryConfigTest.java
@@ -25,19 +25,24 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
+
public class CustomSslContextFactoryConfigTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-sslcontextfactory.yaml");
+ properties = new WithProperties().set(CASSANDRA_CONFIG, "cassandra-sslcontextfactory.yaml");
}
@AfterClass
public static void tearDownDatabaseDescriptor() {
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test
diff --git a/test/unit/org/apache/cassandra/security/CustomSslContextFactoryInvalidConfigTest.java b/test/unit/org/apache/cassandra/security/CustomSslContextFactoryInvalidConfigTest.java
index 79e7d52..dfadea7 100644
--- a/test/unit/org/apache/cassandra/security/CustomSslContextFactoryInvalidConfigTest.java
+++ b/test/unit/org/apache/cassandra/security/CustomSslContextFactoryInvalidConfigTest.java
@@ -24,18 +24,22 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
public class CustomSslContextFactoryInvalidConfigTest
{
+ static WithProperties properties;
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-sslcontextfactory-invalidconfiguration.yaml");
+ properties = new WithProperties().set(CASSANDRA_CONFIG, "cassandra-sslcontextfactory-invalidconfiguration.yaml");
}
@AfterClass
public static void tearDownDatabaseDescriptor() {
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test(expected = IllegalArgumentException.class)
diff --git a/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java b/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java
index 0657eb6..bcd13ef 100644
--- a/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java
@@ -33,6 +33,9 @@
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslProvider;
import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.distributed.shared.WithProperties;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
public class DefaultSslContextFactoryTest
{
@@ -53,15 +56,23 @@
config.put("keystore_password", "cassandra");
}
+ private void addOutboundKeystoreOptions(Map<String, Object> config)
+ {
+ config.put("outbound_keystore", "test/conf/cassandra_ssl_test_outbound.keystore");
+ config.put("outbound_keystore_password", "cassandra");
+ }
+
@Test
public void getSslContextOpenSSL() throws IOException
{
- EncryptionOptions options = new EncryptionOptions().withTrustStore("test/conf/cassandra_ssl_test.truststore")
- .withTrustStorePassword("cassandra")
- .withKeyStore("test/conf/cassandra_ssl_test.keystore")
- .withKeyStorePassword("cassandra")
- .withRequireClientAuth(false)
- .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA");
+ EncryptionOptions.ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions().withTrustStore("test/conf/cassandra_ssl_test.truststore")
+ .withTrustStorePassword("cassandra")
+ .withKeyStore("test/conf/cassandra_ssl_test.keystore")
+ .withKeyStorePassword("cassandra")
+ .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore")
+ .withOutboundKeystorePassword("cassandra")
+ .withRequireClientAuth(false)
+ .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA");
SslContext sslContext = SSLFactory.getOrCreateSslContext(options, true, ISslContextFactory.SocketType.CLIENT);
Assert.assertNotNull(sslContext);
if (OpenSsl.isAvailable())
@@ -78,7 +89,7 @@
config.put("truststore", "/this/is/probably/not/a/file/on/your/test/machine");
DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
- defaultSslContextFactoryImpl.checkedExpiry = false;
+ defaultSslContextFactoryImpl.keystoreContext.checkedExpiry = false;
defaultSslContextFactoryImpl.buildTrustManagerFactory();
}
@@ -90,7 +101,7 @@
config.put("truststore_password", "HomeOfBadPasswords");
DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
- defaultSslContextFactoryImpl.checkedExpiry = false;
+ defaultSslContextFactoryImpl.keystoreContext.checkedExpiry = false;
defaultSslContextFactoryImpl.buildTrustManagerFactory();
}
@@ -101,7 +112,7 @@
config.putAll(commonConfig);
DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
- defaultSslContextFactoryImpl.checkedExpiry = false;
+ defaultSslContextFactoryImpl.keystoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = defaultSslContextFactoryImpl.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
}
@@ -112,10 +123,10 @@
Map<String,Object> config = new HashMap<>();
config.putAll(commonConfig);
config.put("keystore", "/this/is/probably/not/a/file/on/your/test/machine");
- config.put("keystore_password","ThisWontMatter");
+ config.put("keystore_password", "ThisWontMatter");
DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
- defaultSslContextFactoryImpl.checkedExpiry = false;
+ defaultSslContextFactoryImpl.keystoreContext.checkedExpiry = false;
defaultSslContextFactoryImpl.buildKeyManagerFactory();
}
@@ -139,32 +150,84 @@
DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
// Make sure the exiry check didn't happen so far for the private key
- Assert.assertFalse(defaultSslContextFactoryImpl.checkedExpiry);
+ Assert.assertFalse(defaultSslContextFactoryImpl.keystoreContext.checkedExpiry);
addKeystoreOptions(config);
DefaultSslContextFactory defaultSslContextFactoryImpl2 = new DefaultSslContextFactory(config);
// Trigger the private key loading. That will also check for expired private key
defaultSslContextFactoryImpl2.buildKeyManagerFactory();
// Now we should have checked the private key's expiry
- Assert.assertTrue(defaultSslContextFactoryImpl2.checkedExpiry);
+ Assert.assertTrue(defaultSslContextFactoryImpl2.keystoreContext.checkedExpiry);
// Make sure that new factory object preforms the fresh private key expiry check
DefaultSslContextFactory defaultSslContextFactoryImpl3 = new DefaultSslContextFactory(config);
- Assert.assertFalse(defaultSslContextFactoryImpl3.checkedExpiry);
+ Assert.assertFalse(defaultSslContextFactoryImpl3.keystoreContext.checkedExpiry);
defaultSslContextFactoryImpl3.buildKeyManagerFactory();
- Assert.assertTrue(defaultSslContextFactoryImpl3.checkedExpiry);
+ Assert.assertTrue(defaultSslContextFactoryImpl3.keystoreContext.checkedExpiry);
+ }
+
+ @Test(expected = IOException.class)
+ public void buildOutboundKeyManagerFactoryWithInvalidKeystoreFile() throws IOException
+ {
+ Map<String, Object> config = new HashMap<>();
+ config.putAll(commonConfig);
+ config.put("outbound_keystore", "/this/is/probably/not/a/file/on/your/test/machine");
+ config.put("outbound_keystore_password", "ThisWontMatter");
+
+ DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
+ defaultSslContextFactoryImpl.outboundKeystoreContext.checkedExpiry = false;
+ defaultSslContextFactoryImpl.buildOutboundKeyManagerFactory();
+ }
+
+ @Test(expected = IOException.class)
+ public void buildOutboundKeyManagerFactoryWithBadPassword() throws IOException
+ {
+ Map<String, Object> config = new HashMap<>();
+ config.putAll(commonConfig);
+ addOutboundKeystoreOptions(config);
+ config.put("outbound_keystore_password", "HomeOfBadPasswords");
+
+ DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
+ defaultSslContextFactoryImpl.buildOutboundKeyManagerFactory();
+ }
+
+ @Test
+ public void buildOutboundKeyManagerFactoryHappyPath() throws IOException
+ {
+ Map<String, Object> config = new HashMap<>();
+ config.putAll(commonConfig);
+
+ DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
+ // Make sure the exiry check didn't happen so far for the private key
+ Assert.assertFalse(defaultSslContextFactoryImpl.outboundKeystoreContext.checkedExpiry);
+
+ addOutboundKeystoreOptions(config);
+ DefaultSslContextFactory defaultSslContextFactoryImpl2 = new DefaultSslContextFactory(config);
+ // Trigger the private key loading. That will also check for expired private key
+ defaultSslContextFactoryImpl2.buildOutboundKeyManagerFactory();
+ // Now we should have checked the private key's expiry
+ Assert.assertTrue(defaultSslContextFactoryImpl2.outboundKeystoreContext.checkedExpiry);
+ Assert.assertFalse(defaultSslContextFactoryImpl2.keystoreContext.checkedExpiry);
+
+ // Make sure that new factory object preforms the fresh private key expiry check
+ DefaultSslContextFactory defaultSslContextFactoryImpl3 = new DefaultSslContextFactory(config);
+ Assert.assertFalse(defaultSslContextFactoryImpl3.outboundKeystoreContext.checkedExpiry);
+ defaultSslContextFactoryImpl3.buildOutboundKeyManagerFactory();
+ Assert.assertTrue(defaultSslContextFactoryImpl3.outboundKeystoreContext.checkedExpiry);
+ Assert.assertFalse(defaultSslContextFactoryImpl2.keystoreContext.checkedExpiry);
}
@Test
public void testDisableOpenSslForInJvmDtests() {
// The configuration name below is hard-coded intentionally to make sure we don't break the contract without
// changing the documentation appropriately
- System.setProperty("cassandra.disable_tcactive_openssl","true");
- Map<String,Object> config = new HashMap<>();
- config.putAll(commonConfig);
+ try (WithProperties properties = new WithProperties().set(DISABLE_TCACTIVE_OPENSSL, true))
+ {
+ Map<String,Object> config = new HashMap<>();
+ config.putAll(commonConfig);
- DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
- Assert.assertEquals(SslProvider.JDK, defaultSslContextFactoryImpl.getSslProvider());
- System.clearProperty("cassandra.disable_tcactive_openssl");
+ DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
+ Assert.assertEquals(SslProvider.JDK, defaultSslContextFactoryImpl.getSslProvider());
+ }
}
}
diff --git a/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java b/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java
index be49d16..fc84b3a 100644
--- a/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java
@@ -32,6 +32,9 @@
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.distributed.shared.WithProperties;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
public class FileBasedSslContextFactoryTest
{
@@ -39,16 +42,19 @@
private EncryptionOptions.ServerEncryptionOptions encryptionOptions;
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra.yaml");
+ CASSANDRA_CONFIG.reset();
+ properties = new WithProperties();
}
@AfterClass
public static void tearDownDatabaseDescriptor()
{
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Before
@@ -62,7 +68,9 @@
.withRequireClientAuth(false)
.withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA")
.withKeyStore("test/conf/cassandra_ssl_test.keystore")
- .withKeyStorePassword("cassandra");
+ .withKeyStorePassword("cassandra")
+ .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore")
+ .withOutboundKeystorePassword("cassandra");
}
@Test
@@ -73,6 +81,7 @@
Assert.assertEquals("org.apache.cassandra.security.FileBasedSslContextFactoryTest$TestFileBasedSSLContextFactory",
localEncryptionOptions.ssl_context_factory.class_name);
Assert.assertNotNull("keystore_password must not be null", localEncryptionOptions.keystore_password);
+ Assert.assertNotNull("outbound_keystore_password must not be null", localEncryptionOptions.outbound_keystore_password);
TestFileBasedSSLContextFactory sslContextFactory =
(TestFileBasedSSLContextFactory) localEncryptionOptions.sslContextFactoryInstance;
@@ -86,11 +95,12 @@
@Test(expected = IllegalArgumentException.class)
public void testEmptyKeystorePasswords() throws SSLException
{
- EncryptionOptions.ServerEncryptionOptions localEncryptionOptions = encryptionOptions.withKeyStorePassword(null);
+ EncryptionOptions.ServerEncryptionOptions localEncryptionOptions = encryptionOptions.withKeyStorePassword(null).withOutboundKeystorePassword(null);
Assert.assertEquals("org.apache.cassandra.security.FileBasedSslContextFactoryTest$TestFileBasedSSLContextFactory",
localEncryptionOptions.ssl_context_factory.class_name);
Assert.assertNull("keystore_password must be null", localEncryptionOptions.keystore_password);
+ Assert.assertNull("outbound_keystore_password must be null", localEncryptionOptions.outbound_keystore_password);
TestFileBasedSSLContextFactory sslContextFactory =
(TestFileBasedSSLContextFactory) localEncryptionOptions.sslContextFactoryInstance;
@@ -117,6 +127,7 @@
Assert.assertEquals("org.apache.cassandra.security.FileBasedSslContextFactoryTest$TestFileBasedSSLContextFactory",
localEncryptionOptions.ssl_context_factory.class_name);
Assert.assertNull("keystore_password must be null", localEncryptionOptions.keystore_password);
+ Assert.assertNotNull("outbound_keystore_password must not be null", localEncryptionOptions.outbound_keystore_password);
TestFileBasedSSLContextFactory sslContextFactory =
(TestFileBasedSSLContextFactory) localEncryptionOptions.sslContextFactoryInstance;
@@ -132,6 +143,26 @@
}
}
+ /**
+ * Tests for the empty password for the {@code outbound_keystore}. Since the {@code outbound_keystore_password} defaults
+ * to the {@code keystore_password}, this test should pass without exceptions.
+ */
+ @Test
+ public void testOnlyEmptyOutboundKeystorePassword() throws SSLException
+ {
+ EncryptionOptions.ServerEncryptionOptions localEncryptionOptions = encryptionOptions.withOutboundKeystorePassword(null);
+
+ Assert.assertEquals("org.apache.cassandra.security.FileBasedSslContextFactoryTest$TestFileBasedSSLContextFactory",
+ localEncryptionOptions.ssl_context_factory.class_name);
+ Assert.assertNotNull("keystore_password must not be null", localEncryptionOptions.keystore_password);
+ Assert.assertNull("outbound_keystore_password must be null", localEncryptionOptions.outbound_keystore_password);
+
+ TestFileBasedSSLContextFactory sslContextFactory =
+ (TestFileBasedSSLContextFactory) localEncryptionOptions.sslContextFactoryInstance;
+ sslContextFactory.buildKeyManagerFactory();
+ sslContextFactory.buildTrustManagerFactory();
+ }
+
@Test
public void testEmptyTruststorePassword() throws SSLException
{
@@ -139,6 +170,7 @@
Assert.assertEquals("org.apache.cassandra.security.FileBasedSslContextFactoryTest$TestFileBasedSSLContextFactory",
localEncryptionOptions.ssl_context_factory.class_name);
Assert.assertNotNull("keystore_password must not be null", localEncryptionOptions.keystore_password);
+ Assert.assertNotNull("outbound_keystore_password must not be null", localEncryptionOptions.outbound_keystore_password);
Assert.assertNull("truststore_password must be null", localEncryptionOptions.truststore_password);
TestFileBasedSSLContextFactory sslContextFactory =
diff --git a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigTest.java b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigTest.java
index ab6b00a..e27be4e 100644
--- a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigTest.java
+++ b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigTest.java
@@ -27,19 +27,24 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
public class PEMBasedSslContextFactoryConfigTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-pem-sslcontextfactory.yaml");
+ properties = new WithProperties().set(CASSANDRA_CONFIG, "cassandra-pem-sslcontextfactory.yaml");
}
@AfterClass
public static void tearDownDatabaseDescriptor()
{
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test
diff --git a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithMismatchingPasswordsTest.java b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithMismatchingPasswordsTest.java
index c1e2dbe..698e7a7 100644
--- a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithMismatchingPasswordsTest.java
+++ b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithMismatchingPasswordsTest.java
@@ -27,20 +27,25 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
+
public class PEMBasedSslContextFactoryConfigWithMismatchingPasswordsTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-pem-sslcontextfactory-mismatching-passwords.yaml");
+ properties = new WithProperties().set(CASSANDRA_CONFIG, "cassandra-pem-sslcontextfactory-mismatching-passwords.yaml");
}
@AfterClass
public static void tearDownDatabaseDescriptor()
{
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test(expected = ConfigurationException.class)
diff --git a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithUnencryptedKeysTest.java b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithUnencryptedKeysTest.java
index 066de8f..2b63d05 100644
--- a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithUnencryptedKeysTest.java
+++ b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithUnencryptedKeysTest.java
@@ -27,20 +27,27 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
public class PEMBasedSslContextFactoryConfigWithUnencryptedKeysTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-pem-sslcontextfactory-unencryptedkeys.yaml");
- System.setProperty("cassandra.disable_tcactive_openssl", "true");
+ properties = new WithProperties()
+ .set(CASSANDRA_CONFIG, "cassandra-pem-sslcontextfactory-unencryptedkeys.yaml")
+ .set(DISABLE_TCACTIVE_OPENSSL, true);
}
@AfterClass
public static void tearDownDatabaseDescriptor()
{
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test
diff --git a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryInvalidConfigTest.java b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryInvalidConfigTest.java
index 2281137..bb157bb 100644
--- a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryInvalidConfigTest.java
+++ b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryInvalidConfigTest.java
@@ -27,20 +27,25 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
+
public class PEMBasedSslContextFactoryInvalidConfigTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-pem-sslcontextfactory-invalidconfiguration.yaml");
+ properties = new WithProperties().set(CASSANDRA_CONFIG, "cassandra-pem-sslcontextfactory-invalidconfiguration.yaml");
}
@AfterClass
public static void tearDownDatabaseDescriptor()
{
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test(expected = ConfigurationException.class)
diff --git a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java
index 243d300..ee9c610 100644
--- a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java
@@ -35,7 +35,9 @@
import io.netty.handler.ssl.SslProvider;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL;
import static org.apache.cassandra.security.PEMBasedSslContextFactory.ConfigKey.ENCODED_CERTIFICATES;
import static org.apache.cassandra.security.PEMBasedSslContextFactory.ConfigKey.ENCODED_KEY;
import static org.apache.cassandra.security.PEMBasedSslContextFactory.ConfigKey.KEY_PASSWORD;
@@ -216,6 +218,27 @@
.withRequireClientAuth(false)
.withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA")
.withSslContextFactory(sslContextFactory);
+ SslContext sslContext = SSLFactory.getOrCreateSslContext(options, true, ISslContextFactory.SocketType.SERVER);
+ Assert.assertNotNull(sslContext);
+ if (OpenSsl.isAvailable())
+ Assert.assertTrue(sslContext instanceof OpenSslContext);
+ else
+ Assert.assertTrue(sslContext instanceof SslContext);
+ }
+
+ @Test
+ public void getSslContextOpenSSLOutboundKeystore() throws IOException
+ {
+ ParameterizedClass sslContextFactory = new ParameterizedClass(PEMBasedSslContextFactory.class.getSimpleName()
+ , new HashMap<>());
+ EncryptionOptions.ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions().withTrustStore("test/conf/cassandra_ssl_test.truststore.pem")
+ .withKeyStore("test/conf/cassandra_ssl_test.keystore.pem")
+ .withKeyStorePassword("cassandra")
+ .withOutboundKeystore("test/conf/cassandra_ssl_test.keystore.pem")
+ .withOutboundKeystorePassword("cassandra")
+ .withRequireClientAuth(false)
+ .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA")
+ .withSslContextFactory(sslContextFactory);
SslContext sslContext = SSLFactory.getOrCreateSslContext(options, true, ISslContextFactory.SocketType.CLIENT);
Assert.assertNotNull(sslContext);
if (OpenSsl.isAvailable())
@@ -233,7 +256,7 @@
config.put("truststore", "/this/is/probably/not/a/file/on/your/test/machine");
DefaultSslContextFactory defaultSslContextFactoryImpl = new DefaultSslContextFactory(config);
- defaultSslContextFactoryImpl.checkedExpiry = false;
+ defaultSslContextFactoryImpl.keystoreContext.checkedExpiry = false;
defaultSslContextFactoryImpl.buildTrustManagerFactory();
}
@@ -244,7 +267,7 @@
config.putAll(commonConfig);
PEMBasedSslContextFactory sslContextFactory = new PEMBasedSslContextFactory(config);
- sslContextFactory.checkedExpiry = false;
+ sslContextFactory.keystoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = sslContextFactory.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
}
@@ -258,7 +281,7 @@
addFileBaseTrustStoreOptions(config);
PEMBasedSslContextFactory sslContextFactory = new PEMBasedSslContextFactory(config);
- sslContextFactory.checkedExpiry = false;
+ sslContextFactory.keystoreContext.checkedExpiry = false;
TrustManagerFactory trustManagerFactory = sslContextFactory.buildTrustManagerFactory();
Assert.assertNotNull(trustManagerFactory);
}
@@ -271,7 +294,7 @@
config.put("keystore", "/this/is/probably/not/a/file/on/your/test/machine");
PEMBasedSslContextFactory sslContextFactory = new PEMBasedSslContextFactory(config);
- sslContextFactory.checkedExpiry = false;
+ sslContextFactory.keystoreContext.checkedExpiry = false;
sslContextFactory.buildKeyManagerFactory();
}
@@ -295,20 +318,20 @@
PEMBasedSslContextFactory sslContextFactory1 = new PEMBasedSslContextFactory(config);
// Make sure the exiry check didn't happen so far for the private key
- Assert.assertFalse(sslContextFactory1.checkedExpiry);
+ Assert.assertFalse(sslContextFactory1.keystoreContext.checkedExpiry);
addKeyStoreOptions(config);
PEMBasedSslContextFactory sslContextFactory2 = new PEMBasedSslContextFactory(config);
// Trigger the private key loading. That will also check for expired private key
sslContextFactory2.buildKeyManagerFactory();
// Now we should have checked the private key's expiry
- Assert.assertTrue(sslContextFactory2.checkedExpiry);
+ Assert.assertTrue(sslContextFactory2.keystoreContext.checkedExpiry);
// Make sure that new factory object preforms the fresh private key expiry check
PEMBasedSslContextFactory sslContextFactory3 = new PEMBasedSslContextFactory(config);
- Assert.assertFalse(sslContextFactory3.checkedExpiry);
+ Assert.assertFalse(sslContextFactory3.keystoreContext.checkedExpiry);
sslContextFactory3.buildKeyManagerFactory();
- Assert.assertTrue(sslContextFactory3.checkedExpiry);
+ Assert.assertTrue(sslContextFactory3.keystoreContext.checkedExpiry);
}
@Test(expected = IllegalArgumentException.class)
@@ -343,20 +366,20 @@
PEMBasedSslContextFactory sslContextFactory1 = new PEMBasedSslContextFactory(config);
// Make sure the expiry check didn't happen so far for the private key
- Assert.assertFalse(sslContextFactory1.checkedExpiry);
+ Assert.assertFalse(sslContextFactory1.keystoreContext.checkedExpiry);
addFileBaseKeyStoreOptions(config);
PEMBasedSslContextFactory sslContextFactory2 = new PEMBasedSslContextFactory(config);
// Trigger the private key loading. That will also check for expired private key
sslContextFactory2.buildKeyManagerFactory();
// Now we should have checked the private key's expiry
- Assert.assertTrue(sslContextFactory2.checkedExpiry);
+ Assert.assertTrue(sslContextFactory2.keystoreContext.checkedExpiry);
// Make sure that new factory object preforms the fresh private key expiry check
PEMBasedSslContextFactory sslContextFactory3 = new PEMBasedSslContextFactory(config);
- Assert.assertFalse(sslContextFactory3.checkedExpiry);
+ Assert.assertFalse(sslContextFactory3.keystoreContext.checkedExpiry);
sslContextFactory3.buildKeyManagerFactory();
- Assert.assertTrue(sslContextFactory3.checkedExpiry);
+ Assert.assertTrue(sslContextFactory3.keystoreContext.checkedExpiry);
}
@Test
@@ -392,13 +415,14 @@
{
// The configuration name below is hard-coded intentionally to make sure we don't break the contract without
// changing the documentation appropriately
- System.setProperty("cassandra.disable_tcactive_openssl", "true");
- Map<String, Object> config = new HashMap<>();
- config.putAll(commonConfig);
+ try (WithProperties properties = new WithProperties().set(DISABLE_TCACTIVE_OPENSSL, true))
+ {
+ Map<String, Object> config = new HashMap<>();
+ config.putAll(commonConfig);
- PEMBasedSslContextFactory sslContextFactory = new PEMBasedSslContextFactory(config);
- Assert.assertEquals(SslProvider.JDK, sslContextFactory.getSslProvider());
- System.clearProperty("cassandra.disable_tcactive_openssl");
+ PEMBasedSslContextFactory sslContextFactory = new PEMBasedSslContextFactory(config);
+ Assert.assertEquals(SslProvider.JDK, sslContextFactory.getSslProvider());
+ }
}
@Test(expected = IllegalArgumentException.class)
diff --git a/test/unit/org/apache/cassandra/security/PEMJKSSslContextFactoryConfigTest.java b/test/unit/org/apache/cassandra/security/PEMJKSSslContextFactoryConfigTest.java
index 8efd3e4..f187e2c 100644
--- a/test/unit/org/apache/cassandra/security/PEMJKSSslContextFactoryConfigTest.java
+++ b/test/unit/org/apache/cassandra/security/PEMJKSSslContextFactoryConfigTest.java
@@ -27,18 +27,23 @@
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
public class PEMJKSSslContextFactoryConfigTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void setupDatabaseDescriptor()
{
- System.setProperty("cassandra.config", "cassandra-pem-jks-sslcontextfactory.yaml");
+ properties = new WithProperties().set(CASSANDRA_CONFIG, "cassandra-pem-jks-sslcontextfactory.yaml");
}
@AfterClass
public static void tearDownDatabaseDescriptor() {
- System.clearProperty("cassandra.config");
+ properties.close();
}
@Test
diff --git a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
index e5aa4b1..ff3bab9 100644
--- a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
@@ -18,11 +18,19 @@
*/
package org.apache.cassandra.security;
-import org.apache.cassandra.io.util.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.util.HashMap;
import java.util.Map;
+import javax.net.ssl.X509KeyManager;
import org.apache.commons.io.FileUtils;
import org.junit.Assert;
@@ -31,12 +39,19 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import io.netty.handler.ssl.OpenSslClientContext;
+import io.netty.handler.ssl.OpenSslServerContext;
+import io.netty.handler.ssl.OpenSslSessionContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.io.util.File;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class SSLFactoryTest
{
@@ -65,13 +80,17 @@
.withTrustStore("test/conf/cassandra_ssl_test.truststore")
.withTrustStorePassword("cassandra")
.withRequireClientAuth(false)
- .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA");
+ .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA")
+ .withSslContextFactory(new ParameterizedClass(TestFileBasedSSLContextFactory.class.getName(),
+ new HashMap<>()));
}
private ServerEncryptionOptions addKeystoreOptions(ServerEncryptionOptions options)
{
return options.withKeyStore("test/conf/cassandra_ssl_test.keystore")
- .withKeyStorePassword("cassandra");
+ .withKeyStorePassword("cassandra")
+ .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore")
+ .withOutboundKeystorePassword("cassandra");
}
private ServerEncryptionOptions addPEMKeystoreOptions(ServerEncryptionOptions options)
@@ -81,6 +100,8 @@
return options.withSslContextFactory(sslContextFactoryClass)
.withKeyStore("test/conf/cassandra_ssl_test.keystore.pem")
.withKeyStorePassword("cassandra")
+ .withOutboundKeystore("test/conf/cassandra_ssl_test.keystore.pem")
+ .withOutboundKeystorePassword("cassandra")
.withTrustStore("test/conf/cassandra_ssl_test.truststore.pem");
}
@@ -117,7 +138,41 @@
}
@Test
- public void testPEMSslContextReload_HappyPath() throws IOException, InterruptedException
+ public void testServerSocketShouldUseKeystore() throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException
+ {
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withOutboundKeystore("dummyKeystore")
+ .withOutboundKeystorePassword("dummyPassword");
+
+ // Server socket type should create a keystore with keystore & keystore password
+ final OpenSslServerContext context = (OpenSslServerContext) SSLFactory.createNettySslContext(options, true, ISslContextFactory.SocketType.SERVER);
+ assertNotNull(context);
+
+ // Verify if right certificate is loaded into SslContext
+ final Certificate loadedCertificate = getCertificateLoadedInSslContext(context.sessionContext());
+ final Certificate certificate = getCertificates("test/conf/cassandra_ssl_test.keystore", "cassandra");
+ assertEquals(loadedCertificate, certificate);
+ }
+
+ @Test
+ public void testClientSocketShouldUseOutboundKeystore() throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException, NoSuchFieldException, ClassNotFoundException, InvocationTargetException, IllegalAccessException, NoSuchMethodException
+ {
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions)
+ .withKeyStore("dummyKeystore")
+ .withKeyStorePassword("dummyPassword");
+
+ // Client socket type should create a keystore with outbound Keystore & outbound password
+ final OpenSslClientContext context = (OpenSslClientContext) SSLFactory.createNettySslContext(options, true, ISslContextFactory.SocketType.CLIENT);
+ assertNotNull(context);
+
+ // Verify if right certificate is loaded into SslContext
+ final Certificate loadedCertificate = getCertificateLoadedInSslContext(context.sessionContext());
+ final Certificate certificate = getCertificates("test/conf/cassandra_ssl_test_outbound.keystore", "cassandra");
+ assertEquals(loadedCertificate, certificate);
+ }
+
+ @Test
+ public void testPEMSslContextReload_HappyPath() throws IOException
{
try
{
@@ -223,8 +278,7 @@
@Test
public void getSslContext_ParamChanges() throws IOException
{
- EncryptionOptions options = addKeystoreOptions(encryptionOptions)
- .withEnabled(true)
+ ServerEncryptionOptions options = addKeystoreOptions(encryptionOptions)
.withCipherSuites("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256");
SslContext ctx1 = SSLFactory.getOrCreateSslContext(options, true,
@@ -301,4 +355,36 @@
Assert.assertNotEquals(cacheKey1, cacheKey2);
}
+
+ public static class TestFileBasedSSLContextFactory extends FileBasedSslContextFactory {
+ public TestFileBasedSSLContextFactory(Map<String, Object> parameters)
+ {
+ super(parameters);
+ }
+ }
+
+ private static Certificate getCertificates(final String filename, final String password) throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException
+ {
+ FileInputStream is = new FileInputStream(filename);
+ KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType());
+ char[] passwd = password.toCharArray();
+ keystore.load(is, passwd);
+ return keystore.getCertificate("cassandra_ssl_test");
+ }
+
+ private static Certificate getCertificateLoadedInSslContext(final OpenSslSessionContext session)
+ throws ClassNotFoundException, InvocationTargetException, IllegalAccessException, NoSuchMethodException, NoSuchFieldException
+ {
+ Field providerField = OpenSslSessionContext.class.getDeclaredField("provider");
+ providerField.setAccessible(true);
+
+ Class<?> keyMaterialProvider = Class.forName("io.netty.handler.ssl.OpenSslKeyMaterialProvider");
+ Object provider = keyMaterialProvider.cast(providerField.get(session));
+
+ Method keyManager = provider.getClass().getDeclaredMethod("keyManager");
+ keyManager.setAccessible(true);
+ X509KeyManager keyManager1 = (X509KeyManager) keyManager.invoke(provider);
+ final Certificate[] certificates = keyManager1.getCertificateChain("cassandra_ssl_test");
+ return certificates[0];
+ }
}
diff --git a/test/unit/org/apache/cassandra/serializers/MapSerializerTest.java b/test/unit/org/apache/cassandra/serializers/MapSerializerTest.java
new file mode 100644
index 0000000..b012123
--- /dev/null
+++ b/test/unit/org/apache/cassandra/serializers/MapSerializerTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.Range;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.MapType;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER;
+import static org.junit.Assert.assertEquals;
+
+public class MapSerializerTest
+{
+ @Test
+ public void testGetIndexFromSerialized()
+ {
+ testGetIndexFromSerialized(true);
+ testGetIndexFromSerialized(false);
+ }
+
+ private static void testGetIndexFromSerialized(boolean isMultiCell)
+ {
+ MapType<Integer, Integer> type = MapType.getInstance(Int32Type.instance, Int32Type.instance, isMultiCell);
+ AbstractType<Integer> nameType = type.nameComparator();
+ MapSerializer<Integer, Integer> serializer = type.getSerializer();
+
+ Map<Integer, Integer> map = new HashMap<>(4);
+ map.put(1, 10);
+ map.put(3, 30);
+ map.put(4, 40);
+ map.put(6, 60);
+ ByteBuffer bb = type.decompose(map);
+
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(0), nameType));
+ assertEquals(0, serializer.getIndexFromSerialized(bb, nameType.decompose(1), nameType));
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(2), nameType));
+ assertEquals(1, serializer.getIndexFromSerialized(bb, nameType.decompose(3), nameType));
+ assertEquals(2, serializer.getIndexFromSerialized(bb, nameType.decompose(4), nameType));
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(5), nameType));
+ assertEquals(3, serializer.getIndexFromSerialized(bb, nameType.decompose(6), nameType));
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(7), nameType));
+
+ assertEquals(Range.closed(0, Integer.MAX_VALUE), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, UNSET_BYTE_BUFFER, nameType));
+
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(3), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(2, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(4), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(3, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(5), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(3, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(6), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(7), UNSET_BYTE_BUFFER, nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(0, 2), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(0, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(0, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(1, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(1, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(1, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(1, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(3), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(2, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(4), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(3, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(5), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(3, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(6), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(7), nameType.decompose(7), nameType));
+
+ // interval with lower bound greater than upper bound
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(7), nameType.decompose(0), nameType));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/serializers/SetSerializerTest.java b/test/unit/org/apache/cassandra/serializers/SetSerializerTest.java
new file mode 100644
index 0000000..07522e2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/serializers/SetSerializerTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.serializers;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.Range;
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.SetType;
+import static org.junit.Assert.assertEquals;
+import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER;
+
+public class SetSerializerTest
+{
+ @Test
+ public void testGetIndexFromSerialized()
+ {
+ testGetIndexFromSerialized(true);
+ testGetIndexFromSerialized(false);
+ }
+
+ private static void testGetIndexFromSerialized(boolean isMultiCell)
+ {
+ SetType<Integer> type = SetType.getInstance(Int32Type.instance, isMultiCell);
+ AbstractType<Integer> nameType = type.nameComparator();
+ SetSerializer<Integer> serializer = type.getSerializer();
+
+ Set<Integer> set = new HashSet<>(Arrays.asList(1, 3, 4, 6));
+ ByteBuffer bb = type.decompose(set);
+
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(0), nameType));
+ assertEquals(0, serializer.getIndexFromSerialized(bb, nameType.decompose(1), nameType));
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(2), nameType));
+ assertEquals(1, serializer.getIndexFromSerialized(bb, nameType.decompose(3), nameType));
+ assertEquals(2, serializer.getIndexFromSerialized(bb, nameType.decompose(4), nameType));
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(5), nameType));
+ assertEquals(3, serializer.getIndexFromSerialized(bb, nameType.decompose(6), nameType));
+ assertEquals(-1, serializer.getIndexFromSerialized(bb, nameType.decompose(7), nameType));
+
+ assertEquals(Range.closed(0, Integer.MAX_VALUE), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, UNSET_BYTE_BUFFER, nameType));
+
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(3), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(2, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(4), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(3, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(5), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(3, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(6), UNSET_BYTE_BUFFER, nameType));
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(7), UNSET_BYTE_BUFFER, nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(0, 2), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, UNSET_BYTE_BUFFER, nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(0, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(0, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(0, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(1, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(1, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(1, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(1, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(7), nameType));
+
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(0), nameType.decompose(0), nameType));
+ assertEquals(Range.closedOpen(0, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(1), nameType.decompose(1), nameType));
+ assertEquals(Range.closedOpen(1, 1), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(2), nameType.decompose(2), nameType));
+ assertEquals(Range.closedOpen(1, 2), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(3), nameType.decompose(3), nameType));
+ assertEquals(Range.closedOpen(2, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(4), nameType.decompose(4), nameType));
+ assertEquals(Range.closedOpen(3, 3), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(5), nameType.decompose(5), nameType));
+ assertEquals(Range.closedOpen(3, 4), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(6), nameType.decompose(6), nameType));
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(7), nameType.decompose(7), nameType));
+
+ // interval with lower bound greater than upper bound
+ assertEquals(Range.closedOpen(0, 0), serializer.getIndexesRangeFromSerialized(bb, nameType.decompose(7), nameType.decompose(0), nameType));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java b/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java
index 0fc8559..cae7de0 100644
--- a/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java
+++ b/test/unit/org/apache/cassandra/service/AbstractFilesystemOwnershipCheckTest.java
@@ -36,6 +36,7 @@
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.StartupChecksOptions;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.util.File;
@@ -66,11 +67,14 @@
protected StartupChecksOptions options = new StartupChecksOptions();
+ static WithProperties properties;
+
protected void setup()
{
cleanTempDir();
tempDir = new File(com.google.common.io.Files.createTempDir());
token = makeRandomString(10);
+ properties = new WithProperties();
System.clearProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.getKey());
System.clearProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey());
System.clearProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_ENABLE.getKey());
@@ -188,6 +192,7 @@
public void teardown() throws IOException
{
cleanTempDir();
+ properties.close();
}
// tests for enabling/disabling/configuring the check
@@ -205,14 +210,14 @@
{
// no exceptions thrown from the supplier because the check is skipped
options.enable(check_filesystem_ownership);
- System.setProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_ENABLE.getKey(), "false");
+ CassandraRelevantProperties.FILE_SYSTEM_CHECK_ENABLE.setBoolean(false);
AbstractFilesystemOwnershipCheckTest.checker(() -> { throw new RuntimeException("FAIL"); }).execute(options);
}
@Test
public void checkEnabledButClusterPropertyIsEmpty()
{
- System.setProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey(), "");
+ CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.setString("");
AbstractFilesystemOwnershipCheckTest.executeAndFail(AbstractFilesystemOwnershipCheckTest.checker(tempDir), options, MISSING_PROPERTY, CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey());
}
@@ -220,7 +225,7 @@
public void checkEnabledButClusterPropertyIsUnset()
{
Assume.assumeFalse(options.getConfig(check_filesystem_ownership).containsKey("ownership_token"));
- System.clearProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey());
+ CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.clearValue(); // checkstyle: suppress nearby 'clearValueSystemPropertyUsage'
AbstractFilesystemOwnershipCheckTest.executeAndFail(AbstractFilesystemOwnershipCheckTest.checker(tempDir), options, MISSING_PROPERTY, CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey());
}
@@ -356,7 +361,7 @@
File leafDir = AbstractFilesystemOwnershipCheckTest.mkdirs(tempDir, "cassandra/data");
writeFile(leafDir.parent(), "other_file", AbstractFilesystemOwnershipCheckTest.makeProperties(1, 1, token));
AbstractFilesystemOwnershipCheckTest.executeAndFail(AbstractFilesystemOwnershipCheckTest.checker(leafDir), options, NO_OWNERSHIP_FILE, quote(leafDir.absolutePath()));
- System.setProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.getKey(), "other_file");
+ CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.setString("other_file");
AbstractFilesystemOwnershipCheckTest.checker(leafDir).execute(options);
}
diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
index 9c1660a..93fd9f2 100644
--- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
+++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
@@ -79,6 +79,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
public class ActiveRepairServiceTest
{
@@ -469,6 +470,39 @@
}
}
+ @Test
+ public void testRepairSessionSpaceInMiB()
+ {
+ ActiveRepairService activeRepairService = ActiveRepairService.instance;
+ int previousSize = activeRepairService.getRepairSessionSpaceInMiB();
+ try
+ {
+ Assert.assertEquals((Runtime.getRuntime().maxMemory() / (1024 * 1024) / 16),
+ activeRepairService.getRepairSessionSpaceInMiB());
+
+ int targetSize = (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024) / 4) + 1;
+
+ activeRepairService.setRepairSessionSpaceInMiB(targetSize);
+ Assert.assertEquals(targetSize, activeRepairService.getRepairSessionSpaceInMiB());
+
+ activeRepairService.setRepairSessionSpaceInMiB(10);
+ Assert.assertEquals(10, activeRepairService.getRepairSessionSpaceInMiB());
+
+ try
+ {
+ activeRepairService.setRepairSessionSpaceInMiB(0);
+ fail("Should have received an IllegalArgumentException for depth of 0");
+ }
+ catch (IllegalArgumentException ignored) { }
+
+ Assert.assertEquals(10, activeRepairService.getRepairSessionSpaceInMiB());
+ }
+ finally
+ {
+ activeRepairService.setRepairSessionSpaceInMiB(previousSize);
+ }
+ }
+
private static class Task implements Runnable
{
private final Condition blocked;
diff --git a/test/unit/org/apache/cassandra/service/ClientStateTest.java b/test/unit/org/apache/cassandra/service/ClientStateTest.java
index 56d0893..04bddb4 100644
--- a/test/unit/org/apache/cassandra/service/ClientStateTest.java
+++ b/test/unit/org/apache/cassandra/service/ClientStateTest.java
@@ -36,19 +36,23 @@
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.auth.Roles;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class ClientStateTest
{
+ static WithProperties properties;
+
@BeforeClass
public static void beforeClass()
{
- System.setProperty("org.apache.cassandra.disable_mbean_registration", "true");
+ properties = new WithProperties().set(ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION, true);
SchemaLoader.prepareServer();
DatabaseDescriptor.setAuthFromRoot(true);
// create the system_auth keyspace so the IRoleManager can function as normal
@@ -62,7 +66,7 @@
@AfterClass
public static void afterClass()
{
- System.clearProperty("org.apache.cassandra.disable_mbean_registration");
+ properties.close();
}
@Test
diff --git a/test/unit/org/apache/cassandra/service/PartitionDenylistTest.java b/test/unit/org/apache/cassandra/service/PartitionDenylistTest.java
index ed51518..4a93c67 100644
--- a/test/unit/org/apache/cassandra/service/PartitionDenylistTest.java
+++ b/test/unit/org/apache/cassandra/service/PartitionDenylistTest.java
@@ -23,6 +23,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -44,6 +45,8 @@
@BeforeClass
public static void init()
{
+ ServerTestUtils.daemonInitialization();
+
CQLTester.prepareServer();
KeyspaceMetadata schema = KeyspaceMetadata.create(ks_cql,
diff --git a/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java b/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java
index 5545d93..6bd5fd2 100644
--- a/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java
+++ b/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java
@@ -25,12 +25,15 @@
import java.util.Set;
import java.util.stream.Collectors;
+import org.junit.BeforeClass;
import org.junit.Test;
+import com.googlecode.concurrenttrees.common.Iterables;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.VersionAndType;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.File;
import org.assertj.core.util.Files;
import org.quicktheories.core.Gen;
@@ -47,6 +50,12 @@
private static final int MAX_VERSION_LIST_SIZE = 10;
private static final int MAX_UPDATES_PER_GEN = 100;
+ @BeforeClass
+ public static void beforeClass()
+ {
+ DatabaseDescriptor.clientInitialization();
+ }
+
/**
* Ensures that the tracker properly maintains the set of versions in use.
*
@@ -59,15 +68,15 @@
{
qt().forAll(lists().of(updates()).ofSizeBetween(0, MAX_UPDATES_PER_GEN),
sstableFormatTypes())
- .checkAssert((updates, formatType) -> {
- SSTablesGlobalTracker tracker = new SSTablesGlobalTracker(formatType);
+ .checkAssert((updates, format) -> {
+ SSTablesGlobalTracker tracker = new SSTablesGlobalTracker(format);
Set<Descriptor> all = new HashSet<>();
- Set<VersionAndType> previous = Collections.emptySet();
+ Set<Version> previous = Collections.emptySet();
for (Update update : updates)
{
update.applyTo(all);
boolean triggerUpdate = tracker.handleSSTablesChange(update.removed, update.added);
- Set<VersionAndType> expectedInUse = versionAndTypes(all);
+ Set<Version> expectedInUse = versionAndTypes(all);
assertEquals(expectedInUse, tracker.versionsInUse());
assertEquals(!expectedInUse.equals(previous), triggerUpdate);
previous = expectedInUse;
@@ -75,9 +84,9 @@
});
}
- private Set<VersionAndType> versionAndTypes(Set<Descriptor> descriptors)
+ private Set<Version> versionAndTypes(Set<Descriptor> descriptors)
{
- return descriptors.stream().map(SSTablesGlobalTracker::version).collect(Collectors.toSet());
+ return descriptors.stream().map(d -> d.version).collect(Collectors.toSet());
}
private Gen<String> keyspaces()
@@ -109,16 +118,16 @@
return lists().of(descriptors()).ofSizeBetween(minSize, MAX_VERSION_LIST_SIZE);
}
- private Gen<SSTableFormat.Type> sstableFormatTypes()
+ private Gen<SSTableFormat<?, ?>> sstableFormatTypes()
{
- return Generate.enumValues(SSTableFormat.Type.class);
+ return Generate.pick(Iterables.toList(DatabaseDescriptor.getSSTableFormats().values()));
}
private Gen<String> sstableVersionString()
{
// We want to somewhat favor the current version, as that is technically more realistic so we generate it 50%
// of the time, and generate something random 50% of the time.
- return Generate.constant(SSTableFormat.Type.current().info.getLatestVersion().getVersion())
+ return Generate.constant(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion().version)
.mix(strings().betweenCodePoints('a', 'z').ofLength(2));
}
@@ -157,4 +166,4 @@
'}';
}
}
-}
\ No newline at end of file
+}
diff --git a/test/unit/org/apache/cassandra/service/StartupChecksTest.java b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
index 72f8804..6502e7d 100644
--- a/test/unit/org/apache/cassandra/service/StartupChecksTest.java
+++ b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
@@ -37,6 +37,7 @@
import org.apache.cassandra.utils.Clock;
import static java.util.Collections.singletonList;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_INVALID_LEGACY_SSTABLE_ROOT;
import static org.apache.cassandra.io.util.FileUtils.createTempFile;
import static org.apache.cassandra.service.DataResurrectionCheck.HEARTBEAT_FILE_CONFIG_PROPERTY;
import static org.apache.cassandra.service.StartupChecks.StartupCheckType.check_data_resurrection;
@@ -46,7 +47,6 @@
public class StartupChecksTest
{
- public static final String INVALID_LEGACY_SSTABLE_ROOT_PROP = "invalid-legacy-sstable-root";
StartupChecks startupChecks;
Path sstableDir;
static File heartbeatFile;
@@ -112,6 +112,14 @@
Files.createDirectories(backupDir);
copyInvalidLegacySSTables(backupDir);
startupChecks.verify(options);
+
+ // and in the system directory as of CASSANDRA-17777
+ new File(backupDir).deleteRecursive();
+ File dataDir = new File(DatabaseDescriptor.getAllDataFileLocations()[0]);
+ Path systemDir = Paths.get(dataDir.absolutePath(), "system", "InvalidSystemDirectory");
+ Files.createDirectories(systemDir);
+ copyInvalidLegacySSTables(systemDir);
+ startupChecks.verify(options);
}
@Test
@@ -160,9 +168,9 @@
private void copyLegacyNonSSTableFiles(Path targetDir) throws IOException
{
- Path legacySSTableRoot = Paths.get(System.getProperty(INVALID_LEGACY_SSTABLE_ROOT_PROP),
- "Keyspace1",
- "Standard1");
+ Path legacySSTableRoot = Paths.get(TEST_INVALID_LEGACY_SSTABLE_ROOT.getString(),
+ "Keyspace1",
+ "Standard1");
for (String filename : new String[]{"Keyspace1-Standard1-ic-0-TOC.txt",
"Keyspace1-Standard1-ic-0-Digest.sha1",
"legacyleveled.json"})
@@ -198,9 +206,9 @@
private void copyInvalidLegacySSTables(Path targetDir) throws IOException
{
- File legacySSTableRoot = new File(Paths.get(System.getProperty(INVALID_LEGACY_SSTABLE_ROOT_PROP),
- "Keyspace1",
- "Standard1"));
+ File legacySSTableRoot = new File(Paths.get(TEST_INVALID_LEGACY_SSTABLE_ROOT.getString(),
+ "Keyspace1",
+ "Standard1"));
for (File f : legacySSTableRoot.tryList())
Files.copy(f.toPath(), targetDir.resolve(f.name()));
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceDrainTest.java b/test/unit/org/apache/cassandra/service/StorageServiceDrainTest.java
new file mode 100644
index 0000000..f8da096
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/StorageServiceDrainTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service;
+
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.concurrent.Executors;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertTrue;
+
+public class StorageServiceDrainTest
+{
+ private static final String KEYSPACE = "keyspace";
+ private static final String TABLE = "table";
+ private static final String COLUMN = "column";
+ private static final int ROWS = 1000;
+
+ @Before
+ public void before() throws UnknownHostException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
+
+ CommitLog.instance.start();
+
+ CompactionManager.instance.disableAutoCompaction();
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+
+ StorageService.instance
+ .getTokenMetadata()
+ .updateNormalToken(new BytesToken((new byte[]{50})), InetAddressAndPort.getByName("127.0.0.1"));
+
+ final ColumnFamilyStore table = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
+ for (int row = 0; row < ROWS; row++)
+ {
+ final ByteBuffer value = ByteBufferUtil.bytes(String.valueOf(row));
+ new RowUpdateBuilder(table.metadata(), System.currentTimeMillis(), value)
+ .clustering(ByteBufferUtil.bytes(COLUMN))
+ .add("val", value)
+ .build()
+ .applyUnsafe();
+ }
+ Util.flush(table);
+ }
+
+ @Test
+ public void testSSTablesImportAbort()
+ {
+ final ColumnFamilyStore table = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
+
+ assertTrue(table
+ .importNewSSTables(Collections.emptySet(), false, false, false, false, false, false, false)
+ .isEmpty());
+
+ Executors.newSingleThreadExecutor().execute(() -> {
+ try
+ {
+ StorageService.instance.drain();
+ }
+ catch (final Exception exception)
+ {
+ throw new RuntimeException(exception);
+ }});
+
+ while (!StorageService.instance.isDraining())
+ Thread.yield();
+
+ assertThatThrownBy(() -> table
+ .importNewSSTables(Collections.emptySet(), false, false, false, false, false, false, false))
+ .isInstanceOf(RuntimeException.class)
+ .hasCauseInstanceOf(InterruptedException.class);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java
index bb78c83..8c4ab9b 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceServerM3PTest.java
@@ -26,13 +26,13 @@
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.PropertyFileSnitch;
import static org.apache.cassandra.ServerTestUtils.cleanup;
import static org.apache.cassandra.ServerTestUtils.mkdirs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
import static org.junit.Assert.assertTrue;
public class StorageServiceServerM3PTest
@@ -40,7 +40,7 @@
@BeforeClass
public static void setUp() throws ConfigurationException
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
IEndpointSnitch snitch = new PropertyFileSnitch();
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
index 119b810..d4cf450 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
@@ -41,6 +41,7 @@
import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
@@ -55,6 +56,8 @@
import static org.apache.cassandra.ServerTestUtils.cleanup;
import static org.apache.cassandra.ServerTestUtils.mkdirs;
+import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -63,7 +66,7 @@
@BeforeClass
public static void setUp() throws ConfigurationException
{
- System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
+ GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true);
DatabaseDescriptor.daemonInitialization();
CommitLog.instance.start();
IEndpointSnitch snitch = new PropertyFileSnitch();
@@ -633,7 +636,7 @@
@Test
public void isReplacingSameHostAddressAndHostIdTest() throws UnknownHostException
{
- try
+ try (WithProperties properties = new WithProperties())
{
UUID differentHostId = UUID.randomUUID();
Assert.assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId));
@@ -643,20 +646,16 @@
Gossiper.instance.initializeNodeUnsafe(FBUtilities.getBroadcastAddressAndPort(), localHostId, 1);
// Check detects replacing the same host address with the same hostid
- System.setProperty("cassandra.replace_address", hostAddress);
+ REPLACE_ADDRESS.setString(hostAddress);
Assert.assertTrue(StorageService.instance.isReplacingSameHostAddressAndHostId(localHostId));
// Check detects replacing the same host address with a different host id
- System.setProperty("cassandra.replace_address", hostAddress);
+ REPLACE_ADDRESS.setString(hostAddress);
Assert.assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId));
// Check tolerates the DNS entry going away for the replace_address
- System.setProperty("cassandra.replace_address", "unresolvable.host.local.");
+ REPLACE_ADDRESS.setString("unresolvable.host.local.");
Assert.assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId));
}
- finally
- {
- System.clearProperty("cassandra.replace_address");
- }
}
}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceTest.java b/test/unit/org/apache/cassandra/service/StorageServiceTest.java
index e4589d6..e070095 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceTest.java
@@ -204,13 +204,135 @@
}
@Test
+ public void testRepairSessionMaximumTreeDepth()
+ {
+ StorageService storageService = StorageService.instance;
+ int previousDepth = storageService.getRepairSessionMaximumTreeDepth();
+ try
+ {
+ Assert.assertEquals(20, storageService.getRepairSessionMaximumTreeDepth());
+ storageService.setRepairSessionMaximumTreeDepth(10);
+ Assert.assertEquals(10, storageService.getRepairSessionMaximumTreeDepth());
+
+ try
+ {
+ storageService.setRepairSessionMaximumTreeDepth(9);
+ fail("Should have received a IllegalArgumentException for depth of 9");
+ }
+ catch (IllegalArgumentException ignored) { }
+ Assert.assertEquals(10, storageService.getRepairSessionMaximumTreeDepth());
+
+ try
+ {
+ storageService.setRepairSessionMaximumTreeDepth(-20);
+ fail("Should have received a IllegalArgumentException for depth of -20");
+ }
+ catch (IllegalArgumentException ignored) { }
+ Assert.assertEquals(10, storageService.getRepairSessionMaximumTreeDepth());
+
+ storageService.setRepairSessionMaximumTreeDepth(22);
+ Assert.assertEquals(22, storageService.getRepairSessionMaximumTreeDepth());
+ }
+ finally
+ {
+ storageService.setRepairSessionMaximumTreeDepth(previousDepth);
+ }
+ }
+
+ @Test
+ public void testColumnIndexSizeInKiB()
+ {
+ StorageService storageService = StorageService.instance;
+ int previousColumnIndexSize = storageService.getColumnIndexSizeInKiB();
+ try
+ {
+ storageService.setColumnIndexSizeInKiB(1024);
+ Assert.assertEquals(1024, storageService.getColumnIndexSizeInKiB());
+
+ try
+ {
+ storageService.setColumnIndexSizeInKiB(2 * 1024 * 1024);
+ fail("Should have received an IllegalArgumentException column_index_size = 2GiB");
+ }
+ catch (IllegalArgumentException ignored) { }
+ Assert.assertEquals(1024, storageService.getColumnIndexSizeInKiB());
+ }
+ finally
+ {
+ storageService.setColumnIndexSizeInKiB(previousColumnIndexSize);
+ }
+ }
+
+ @Test
+ public void testColumnIndexCacheSizeInKiB()
+ {
+ StorageService storageService = StorageService.instance;
+ int previousColumnIndexCacheSize = storageService.getColumnIndexCacheSizeInKiB();
+ try
+ {
+ storageService.setColumnIndexCacheSizeInKiB(1024);
+ Assert.assertEquals(1024, storageService.getColumnIndexCacheSizeInKiB());
+
+ try
+ {
+ storageService.setColumnIndexCacheSizeInKiB(2 * 1024 * 1024);
+ fail("Should have received an IllegalArgumentException column_index_cache_size= 2GiB");
+ }
+ catch (IllegalArgumentException ignored) { }
+ Assert.assertEquals(1024, storageService.getColumnIndexCacheSizeInKiB());
+ }
+ finally
+ {
+ storageService.setColumnIndexCacheSizeInKiB(previousColumnIndexCacheSize);
+ }
+ }
+
+ @Test
+ public void testBatchSizeWarnThresholdInKiB()
+ {
+ StorageService storageService = StorageService.instance;
+ int previousBatchSizeWarnThreshold = storageService.getBatchSizeWarnThresholdInKiB();
+ try
+ {
+ storageService.setBatchSizeWarnThresholdInKiB(1024);
+ Assert.assertEquals(1024, storageService.getBatchSizeWarnThresholdInKiB());
+
+ try
+ {
+ storageService.setBatchSizeWarnThresholdInKiB(2 * 1024 * 1024);
+ fail("Should have received an IllegalArgumentException batch_size_warn_threshold = 2GiB");
+ }
+ catch (IllegalArgumentException ignored) { }
+ Assert.assertEquals(1024, storageService.getBatchSizeWarnThresholdInKiB());
+ }
+ finally
+ {
+ storageService.setBatchSizeWarnThresholdInKiB(previousBatchSizeWarnThreshold);
+ }
+ }
+
+ @Test
+ public void testLocalDatacenterNodesExcludedDuringRebuild()
+ {
+ try
+ {
+ getStorageService().rebuild(DatabaseDescriptor.getLocalDataCenter(), "StorageServiceTest", null, null, true);
+ fail();
+ }
+ catch (IllegalArgumentException e)
+ {
+ Assert.assertEquals("Cannot set source data center to be local data center, when excludeLocalDataCenter flag is set", e.getMessage());
+ }
+ }
+
+ @Test
public void testRebuildFailOnNonExistingDatacenter()
{
String nonExistentDC = "NON_EXISTENT_DC";
try
{
- getStorageService().rebuild(nonExistentDC, "StorageServiceTest", null, null);
+ getStorageService().rebuild(nonExistentDC, "StorageServiceTest", null, null, true);
fail();
}
catch (IllegalArgumentException ex)
diff --git a/test/unit/org/apache/cassandra/service/SystemPropertiesBasedFileSystemOwnershipCheckTest.java b/test/unit/org/apache/cassandra/service/SystemPropertiesBasedFileSystemOwnershipCheckTest.java
index cb54f81..557b26d 100644
--- a/test/unit/org/apache/cassandra/service/SystemPropertiesBasedFileSystemOwnershipCheckTest.java
+++ b/test/unit/org/apache/cassandra/service/SystemPropertiesBasedFileSystemOwnershipCheckTest.java
@@ -21,6 +21,7 @@
import org.junit.Before;
import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.shared.WithProperties;
public class SystemPropertiesBasedFileSystemOwnershipCheckTest extends AbstractFilesystemOwnershipCheckTest
{
@@ -28,7 +29,7 @@
public void setup()
{
super.setup();
- System.setProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey(), token);
- System.setProperty(CassandraRelevantProperties.FILE_SYSTEM_CHECK_ENABLE.getKey(), "true");
+ properties = new WithProperties().set(CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN, token)
+ .set(CassandraRelevantProperties.FILE_SYSTEM_CHECK_ENABLE, true);
}
}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java
index 387ec65..bcd9714 100644
--- a/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java
+++ b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java
@@ -48,6 +48,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.cassandra.config.CassandraRelevantProperties.PARTITIONER;
import static org.apache.cassandra.dht.Range.deoverlap;
import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE;
import static org.apache.cassandra.service.paxos.Ballot.none;
@@ -62,7 +63,7 @@
private static final AtomicInteger tableNum = new AtomicInteger();
static
{
- System.setProperty("cassandra.partitioner", Murmur3Partitioner.class.getName());
+ PARTITIONER.setString(Murmur3Partitioner.class.getName());
DatabaseDescriptor.daemonInitialization();
assert DatabaseDescriptor.getPartitioner() instanceof Murmur3Partitioner;
}
diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java
index ee0878c..2e8668d 100644
--- a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java
+++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java
@@ -42,6 +42,7 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.ColumnMetadata;
@@ -52,6 +53,7 @@
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.utils.ByteBufferUtil;
+import static org.apache.cassandra.config.CassandraRelevantProperties.FORCE_PAXOS_STATE_REBUILD;
import static org.apache.cassandra.service.paxos.uncommitted.PaxosStateTracker.stateDirectory;
import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.*;
import static org.apache.cassandra.service.paxos.uncommitted.UncommittedTableDataTest.assertIteratorContents;
@@ -94,38 +96,6 @@
directories = new File[]{directory1, directory2};
}
- private static class SystemProp implements AutoCloseable
- {
- private final String prop;
- private final String prev;
-
- public SystemProp(String prop, String prev)
- {
- this.prop = prop;
- this.prev = prev;
- }
-
- public void close()
- {
- if (prev == null)
- System.clearProperty(prop);
- else
- System.setProperty(prop, prev);
- }
-
- public static SystemProp set(String prop, String val)
- {
- String prev = System.getProperty(prop);
- System.setProperty(prop, val);
- return new SystemProp(prop, prev);
- }
-
- public static SystemProp set(String prop, boolean val)
- {
- return set(prop, Boolean.toString(val));
- }
- }
-
private static PartitionUpdate update(TableMetadata cfm, int k, Ballot ballot)
{
ColumnMetadata col = cfm.getColumn(new ColumnIdentifier("v", false));
@@ -206,7 +176,7 @@
SystemKeyspace.savePaxosWritePromise(dk(0), cfm1, ballots[2]);
SystemKeyspace.savePaxosProposal(commit(cfm1, 2, ballots[3]));
- try (SystemProp forceRebuild = SystemProp.set(PaxosStateTracker.FORCE_REBUILD_PROP, true))
+ try (WithProperties with = new WithProperties().set(FORCE_PAXOS_STATE_REBUILD, true))
{
PaxosStateTracker tracker = PaxosStateTracker.create(directories);
Assert.assertTrue(tracker.isRebuildNeeded());
diff --git a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java
index 259a65f..32ebeff 100644
--- a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java
+++ b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java
@@ -34,10 +34,12 @@
import org.apache.cassandra.db.PartitionRangeReadCommand;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.partitions.CachedPartition;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.StubIndex;
import org.apache.cassandra.schema.IndexMetadata;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MAX_CONCURRENT_RANGE_REQUESTS;
import static org.apache.cassandra.db.ConsistencyLevel.ONE;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.junit.Assert.assertEquals;
@@ -47,18 +49,20 @@
*/
public class RangeCommandsTest extends CQLTester
{
+
+ static WithProperties properties;
private static final int MAX_CONCURRENCY_FACTOR = 4;
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
- System.setProperty("cassandra.max_concurrent_range_requests", String.valueOf(MAX_CONCURRENCY_FACTOR));
+ properties = new WithProperties().set(MAX_CONCURRENT_RANGE_REQUESTS, MAX_CONCURRENCY_FACTOR);
}
@AfterClass
public static void cleanup()
{
- System.clearProperty("cassandra.max_concurrent_range_requests");
+ properties.close();
}
@Test
diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
index d03a2dd..9c2303f 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
@@ -27,6 +27,7 @@
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
+import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -123,9 +124,57 @@
Paths.get(baseDir.toString(), DATA_DIR_3)));
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(3);
- assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files));
- assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files));
- assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files, false));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files, false));
+
+ // Verify snapshot loading for a specific keyspace
+ loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+ Paths.get(baseDir.toString(), DATA_DIR_2),
+ Paths.get(baseDir.toString(), DATA_DIR_3)));
+
+ snapshots = loader.loadSnapshots(KEYSPACE_1);
+ assertThat(snapshots).hasSize(2);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files, false));
+
+ loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+ Paths.get(baseDir.toString(), DATA_DIR_2),
+ Paths.get(baseDir.toString(), DATA_DIR_3)));
+ snapshots = loader.loadSnapshots(KEYSPACE_2);
+ assertThat(snapshots).hasSize(1);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files, false));
+ }
+
+ @Test
+ public void testEphemeralSnapshotWithoutManifest() throws IOException
+ {
+ Set<File> tag1Files = new HashSet<>();
+
+ // Create one snapshot per table - without manifests:
+ // - ks1.t1 : tag1
+ File baseDir = new File(tmpDir.newFolder());
+ boolean ephemeralFileCreated = false;
+ for (String dataDir : DATA_DIRS)
+ {
+ File dir = createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1);
+ tag1Files.add(dir);
+ if (!ephemeralFileCreated)
+ {
+ createEphemeralMarkerFile(dir);
+ ephemeralFileCreated = true;
+ }
+ }
+
+ // Verify snapshot is found correctly from data directories
+ SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+ Paths.get(baseDir.toString(), DATA_DIR_2),
+ Paths.get(baseDir.toString(), DATA_DIR_3)));
+
+ Set<TableSnapshot> snapshots = loader.loadSnapshots();
+ assertThat(snapshots).hasSize(1);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, true));
+ Assert.assertTrue(snapshots.stream().findFirst().get().isEphemeral());
}
@Test
@@ -169,9 +218,26 @@
Paths.get(baseDir.toString(), DATA_DIR_3)));
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(3);
- assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files));
- assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files));
- assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files, false));
+
+ // Verify snapshot loading for a specific keyspace
+ loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+ Paths.get(baseDir.toString(), DATA_DIR_2),
+ Paths.get(baseDir.toString(), DATA_DIR_3)));
+
+ snapshots = loader.loadSnapshots(KEYSPACE_1);
+ assertThat(snapshots).hasSize(2);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
+
+ loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+ Paths.get(baseDir.toString(), DATA_DIR_2),
+ Paths.get(baseDir.toString(), DATA_DIR_3)));
+ snapshots = loader.loadSnapshots(KEYSPACE_2);
+ assertThat(snapshots).hasSize(1);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files, false));
}
@Test
@@ -208,7 +274,7 @@
private void writeManifest(File snapshotDir, Instant creationTime, DurationSpec.IntSecondsBound ttl) throws IOException
{
- SnapshotManifest manifest = new SnapshotManifest(Lists.newArrayList("f1", "f2", "f3"), ttl, creationTime);
+ SnapshotManifest manifest = new SnapshotManifest(Lists.newArrayList("f1", "f2", "f3"), ttl, creationTime, false);
manifest.serializeToJsonFile(getManifestFile(snapshotDir));
}
@@ -219,6 +285,11 @@
return file;
}
+ private static void createEphemeralMarkerFile(File dir)
+ {
+ Assert.assertTrue(new File(dir, "ephemeral.snapshot").createFileIfNotExists());
+ }
+
static String tableDirName(String tableName, UUID tableId)
{
return String.format("%s-%s", tableName, removeDashes(tableId));
diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
index edbfff2..f9233bb 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
@@ -57,7 +57,7 @@
@ClassRule
public static TemporaryFolder temporaryFolder = new TemporaryFolder();
- private TableSnapshot generateSnapshotDetails(String tag, Instant expiration)
+ private TableSnapshot generateSnapshotDetails(String tag, Instant expiration, boolean ephemeral)
{
try
{
@@ -67,7 +67,8 @@
tag,
Instant.EPOCH,
expiration,
- createFolders(temporaryFolder));
+ createFolders(temporaryFolder),
+ ephemeral);
}
catch (Exception ex)
{
@@ -75,13 +76,11 @@
}
}
-
@Test
- public void testLoadSnapshots() throws Exception
- {
- TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH);
- TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusSeconds(ONE_DAY_SECS));
- TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null);
+ public void testLoadSnapshots() throws Exception {
+ TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false);
+ TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusSeconds(ONE_DAY_SECS), false);
+ TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null, false);
List<TableSnapshot> snapshots = Arrays.asList(expired, nonExpired, nonExpiring);
// Create SnapshotManager with 3 snapshots: expired, non-expired and non-expiring
@@ -95,14 +94,13 @@
}
@Test
- public void testClearExpiredSnapshots()
- {
+ public void testClearExpiredSnapshots() throws Exception {
SnapshotManager manager = new SnapshotManager(3, 3);
// Add 3 snapshots: expired, non-expired and non-expiring
- TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH);
- TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS));
- TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null);
+ TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false);
+ TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false);
+ TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null, false);
manager.addSnapshot(expired);
manager.addSnapshot(nonExpired);
manager.addSnapshot(nonExpiring);
@@ -125,8 +123,7 @@
}
@Test
- public void testScheduledCleanup() throws Exception
- {
+ public void testScheduledCleanup() throws Exception {
SnapshotManager manager = new SnapshotManager(0, 1);
try
{
@@ -134,8 +131,9 @@
manager.start();
// Add 2 expiring snapshots: one to expire in 2 seconds, another in 1 day
- TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(2));
- TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS));
+ int TTL_SECS = 2;
+ TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS), false);
+ TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false);
manager.addSnapshot(toExpire);
manager.addSnapshot(nonExpired);
@@ -146,10 +144,11 @@
assertThat(manager.getExpiringSnapshots()).contains(toExpire);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
- await().pollInterval(2, SECONDS)
- .timeout(10, SECONDS)
- .until(() -> manager.getExpiringSnapshots().size() == 1);
+ // Sleep 4 seconds
+ Thread.sleep((TTL_SECS + 2) * 1000L);
+ // Snapshot with ttl=2s should be gone, while other should remain
+ assertThat(manager.getExpiringSnapshots()).hasSize(1);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
assertThat(toExpire.exists()).isFalse();
assertThat(nonExpired.exists()).isTrue();
@@ -160,6 +159,24 @@
}
}
+ @Test
+ public void testClearSnapshot() throws Exception
+ {
+ // Given
+ SnapshotManager manager = new SnapshotManager(1, 3);
+ TableSnapshot expiringSnapshot = generateSnapshotDetails("snapshot", now().plusMillis(50000), false);
+ manager.addSnapshot(expiringSnapshot);
+ assertThat(manager.getExpiringSnapshots()).contains(expiringSnapshot);
+ assertThat(expiringSnapshot.exists()).isTrue();
+
+ // When
+ manager.clearSnapshot(expiringSnapshot);
+
+ // Then
+ assertThat(manager.getExpiringSnapshots()).doesNotContain(expiringSnapshot);
+ assertThat(expiringSnapshot.exists()).isFalse();
+ }
+
@Test // see CASSANDRA-18211
public void testConcurrentClearingOfSnapshots() throws Exception
{
@@ -185,12 +202,12 @@
}
};
- TableSnapshot expiringSnapshot = generateSnapshotDetails("mysnapshot", Instant.now().plusSeconds(15));
+ TableSnapshot expiringSnapshot = generateSnapshotDetails("mysnapshot", Instant.now().plusSeconds(15), false);
manager.addSnapshot(expiringSnapshot);
manager.resumeSnapshotCleanup();
- Thread nonExpiringSnapshotCleanupThred = new Thread(() -> manager.clearSnapshot(generateSnapshotDetails("mysnapshot2", null)));
+ Thread nonExpiringSnapshotCleanupThred = new Thread(() -> manager.clearSnapshot(generateSnapshotDetails("mysnapshot2", null, false)));
// wait until the first snapshot expires
await().pollInterval(1, SECONDS)
diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManifestTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManifestTest.java
index d3b11c0..5c12359 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManifestTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManifestTest.java
@@ -37,6 +37,7 @@
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.utils.JsonUtils;
public class SnapshotManifestTest
{
@@ -70,7 +71,7 @@
map.put("expires_at", expiresAt);
map.put("files", Arrays.asList("db1", "db2", "db3"));
- ObjectMapper mapper = new ObjectMapper();
+ ObjectMapper mapper = JsonUtils.JSON_OBJECT_MAPPER;
File manifestFile = new File(tempFolder.newFile("manifest.json"));
mapper.writeValue((OutputStream) new FileOutputStreamPlus(manifestFile), map);
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(manifestFile);
@@ -84,7 +85,7 @@
public void testOptionalFields() throws IOException {
Map<String, Object> map = new HashMap<>();
map.put("files", Arrays.asList("db1", "db2", "db3"));
- ObjectMapper mapper = new ObjectMapper();
+ ObjectMapper mapper = JsonUtils.JSON_OBJECT_MAPPER;
File manifestFile = new File(tempFolder.newFile("manifest.json"));
mapper.writeValue((OutputStream) new FileOutputStreamPlus(manifestFile), map);
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(manifestFile);
@@ -99,7 +100,7 @@
Map<String, Object> map = new HashMap<>();
map.put("files", Arrays.asList("db1", "db2", "db3"));
map.put("dummy", "dummy");
- ObjectMapper mapper = new ObjectMapper();
+ ObjectMapper mapper = JsonUtils.JSON_OBJECT_MAPPER;
File manifestFile = new File(tempFolder.newFile("manifest.json"));
mapper.writeValue((OutputStream) new FileOutputStreamPlus(manifestFile), map);
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(manifestFile);
@@ -108,7 +109,7 @@
@Test
public void testSerializeAndDeserialize() throws Exception {
- SnapshotManifest manifest = new SnapshotManifest(Arrays.asList("db1", "db2", "db3"), new DurationSpec.IntSecondsBound("2m"), Instant.ofEpochMilli(currentTimeMillis()));
+ SnapshotManifest manifest = new SnapshotManifest(Arrays.asList("db1", "db2", "db3"), new DurationSpec.IntSecondsBound("2m"), Instant.ofEpochMilli(currentTimeMillis()), false);
File manifestFile = new File(tempFolder.newFile("manifest.json"));
manifest.serializeToJsonFile(manifestFile);
diff --git a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
index 4bb1756..d592dd9 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
@@ -21,8 +21,10 @@
import java.io.IOException;
import java.nio.file.Paths;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -35,9 +37,11 @@
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertTrue;
public class TableSnapshotTest
{
@@ -50,15 +54,18 @@
@ClassRule
public static TemporaryFolder tempFolder = new TemporaryFolder();
- public static Set<File> createFolders(TemporaryFolder temp) throws IOException {
+ public static Set<File> createFolders(TemporaryFolder temp) throws IOException
+ {
File folder = new File(temp.newFolder());
Set<File> folders = new HashSet<>();
- for (String folderName : Arrays.asList("foo", "bar", "buzz")) {
+ for (String folderName : Arrays.asList("foo", "bar", "buzz"))
+ {
File subfolder = new File(folder, folderName);
subfolder.tryCreateDirectories();
assertThat(subfolder.exists());
folders.add(subfolder);
- };
+ }
+ ;
return folders;
}
@@ -74,7 +81,9 @@
"some",
null,
null,
- folders);
+ folders,
+ false
+ );
assertThat(snapshot.exists()).isTrue();
@@ -95,7 +104,9 @@
"some",
null,
null,
- folders);
+ folders,
+ false
+ );
assertThat(snapshot.isExpiring()).isFalse();
assertThat(snapshot.isExpired(now())).isFalse();
@@ -107,7 +118,9 @@
"some",
now(),
null,
- folders);
+ folders,
+ false
+ );
assertThat(snapshot.isExpiring()).isFalse();
assertThat(snapshot.isExpired(now())).isFalse();
@@ -119,7 +132,9 @@
"some",
now(),
now().plusSeconds(1000),
- folders);
+ folders,
+ false
+ );
assertThat(snapshot.isExpiring()).isTrue();
assertThat(snapshot.isExpired(now())).isFalse();
@@ -131,7 +146,8 @@
"some",
now(),
now().minusSeconds(1000),
- folders);
+ folders,
+ false);
assertThat(snapshot.isExpiring()).isTrue();
assertThat(snapshot.isExpired(now())).isTrue();
@@ -159,7 +175,8 @@
"some",
null,
null,
- folders);
+ folders,
+ false);
Long res = 0L;
@@ -185,7 +202,9 @@
"some",
null,
null,
- folders);
+ folders,
+ false
+ );
Long res = 0L;
@@ -214,7 +233,10 @@
"some1",
createdAt,
null,
- folders);
+ folders,
+ false
+ );
+
assertThat(withCreatedAt.getCreatedAt()).isEqualTo(createdAt);
// When createdAt is null, it should return the snapshot folder minimum update time
@@ -225,11 +247,82 @@
"some1",
null,
null,
- folders);
+ folders,
+ false
+ );
+
assertThat(withoutCreatedAt.getCreatedAt()).isEqualTo(Instant.ofEpochMilli(folders.stream().mapToLong(f -> f.lastModified()).min().getAsLong()));
}
@Test
+ public void testShouldClearSnapshot() throws Exception
+ {
+ // TableSnapshot variables -> ephemeral / true / false, createdAt -> null / notnull
+
+ Instant now = Instant.now();
+
+ String keyspace = "ks";
+ String table = "tbl";
+ UUID id = UUID.randomUUID();
+ String tag = "someTag";
+ Instant snapshotCreation = now.minusSeconds(60);
+ Set<File> folders = createFolders(tempFolder);
+
+ List<TableSnapshot> snapshots = new ArrayList<>();
+
+ for (boolean ephemeral : new boolean[]{ true, false })
+ for (Instant createdAt : new Instant[]{ snapshotCreation, null })
+ snapshots.add(new TableSnapshot(keyspace,
+ table,
+ id,
+ tag,
+ createdAt, // variable
+ null,
+ folders,
+ ephemeral)); // variable
+
+ List<Pair<String, Long>> testingMethodInputs = new ArrayList<>();
+
+ for (String testingTag : new String[] {null, "", tag, "someothertag"})
+ // 0 to deactive byTimestamp logic, now.toEpochMilli as true, snapshot minus 60s as false
+ for (long olderThanTimestamp : new long[] {0, now.toEpochMilli(), snapshotCreation.minusSeconds(60).toEpochMilli()})
+ testingMethodInputs.add(Pair.create(testingTag, olderThanTimestamp));
+
+ for (Pair<String, Long> methodInput : testingMethodInputs)
+ {
+ String testingTag = methodInput.left();
+ Long olderThanTimestamp = methodInput.right;
+ for (TableSnapshot snapshot : snapshots)
+ {
+ // if shouldClear method returns true, it is only in case
+ // 1. snapshot to clear is not ephemeral
+ // 2. tag to clear is null, empty, or it is equal to snapshot tag
+ // 3. byTimestamp is true
+ if (TableSnapshot.shouldClearSnapshot(testingTag, olderThanTimestamp).test(snapshot))
+ {
+ // shouldClearTag = true
+ boolean shouldClearTag = (testingTag == null || testingTag.isEmpty()) || snapshot.getTag().equals(testingTag);
+ // notEphemeral
+ boolean notEphemeral = !snapshot.isEphemeral();
+ // byTimestamp
+ boolean byTimestamp = true;
+
+ if (olderThanTimestamp > 0L)
+ {
+ Instant createdAt = snapshot.getCreatedAt();
+ if (createdAt != null)
+ byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp));
+ }
+
+ assertTrue(notEphemeral);
+ assertTrue(shouldClearTag);
+ assertTrue(byTimestamp);
+ }
+ }
+ }
+ }
+
+ @Test
public void testGetLiveFileFromSnapshotFile()
{
testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db",
diff --git a/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java b/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java
index 34ce09c..5a8f4a5 100644
--- a/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java
+++ b/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java
@@ -212,7 +212,8 @@
peer,
Collections.emptyList(),
Collections.emptyList(),
- StreamSession.State.INITIALIZED));
+ StreamSession.State.INITIALIZED,
+ null));
StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer);
session.init(future);
diff --git a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
index 45172fe..778b6b2 100644
--- a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
+++ b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java
@@ -46,7 +46,7 @@
}
StreamSummary sending = new StreamSummary(tableId, 10, 100);
- SessionInfo info = new SessionInfo(local, 0, local, summaries, Collections.singleton(sending), StreamSession.State.PREPARING);
+ SessionInfo info = new SessionInfo(local, 0, local, summaries, Collections.singleton(sending), StreamSession.State.PREPARING, null);
assert info.getTotalFilesToReceive() == 45;
assert info.getTotalFilesToSend() == 10;
diff --git a/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java b/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java
new file mode 100644
index 0000000..7ef3ff9
--- /dev/null
+++ b/test/unit/org/apache/cassandra/streaming/StreamSessionTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.attribute.FileAttributeView;
+import java.nio.file.attribute.FileStoreAttributeView;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import org.apache.cassandra.io.util.File;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.agent.ByteBuddyAgent;
+import net.bytebuddy.dynamic.loading.ClassReloadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.Util;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class StreamSessionTest extends CQLTester
+{
+ private static Directories dirs;
+ private static Directories dirs2;
+
+ private static List<File> files;
+ private static List<Directories.DataDirectory> datadirs;
+
+ private static ColumnFamilyStore cfs;
+ private static ColumnFamilyStore cfs2;
+ private static List<FakeFileStore> filestores = Lists.newArrayList(new FakeFileStore(), new FakeFileStore(), new FakeFileStore());
+
+ @BeforeClass
+ public static void before()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ ByteBuddyAgent.install();
+ new ByteBuddy().redefine(ColumnFamilyStore.class)
+ .method(named("getIfExists").and(takesArguments(1)))
+ .intercept(MethodDelegation.to(BBKeyspaceHelper.class))
+ .make()
+ .load(ColumnFamilyStore.class.getClassLoader(), ClassReloadingStrategy.fromInstalledAgent());
+ files = Lists.newArrayList(new File("/tmp/1"),
+ new File("/tmp/2"),
+ new File("/tmp/3"));
+ datadirs = files.stream().map(Directories.DataDirectory::new).collect(Collectors.toList());
+ DatabaseDescriptor.setMinFreeSpacePerDriveInMebibytes(0);
+ DatabaseDescriptor.setMaxSpaceForCompactionsPerDrive(1.0);
+ }
+
+ @Test
+ public void basicDiskSpaceTest() throws InterruptedException
+ {
+ createTable("create table %s (k int primary key, i int)");
+ dirs = new Directories(getCurrentColumnFamilyStore().metadata(), datadirs);
+ cfs = new MockCFS(getCurrentColumnFamilyStore(), dirs);
+
+ Map<TableId, Long> perTableIdIncomingBytes = new HashMap<>();
+ perTableIdIncomingBytes.put(cfs.metadata.id, 999L);
+
+ filestores.get(0).usableSpace = 334;
+ filestores.get(1).usableSpace = 334;
+ filestores.get(2).usableSpace = 334;
+
+ Keyspace.all().forEach(ks -> ks.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+ do
+ {
+ Thread.sleep(100);
+ } while (!CompactionManager.instance.active.getCompactions().isEmpty());
+
+ assertTrue(StreamSession.checkDiskSpace(perTableIdIncomingBytes, nextTimeUUID(), filestoreMapper));
+
+ filestores.get(0).usableSpace = 332;
+ assertFalse(StreamSession.checkDiskSpace(perTableIdIncomingBytes, nextTimeUUID(), filestoreMapper));
+
+ }
+
+ @Test
+ public void multiTableDiskSpaceTest() throws InterruptedException
+ {
+ createTable("create table %s (k int primary key, i int)");
+ dirs = new Directories(getCurrentColumnFamilyStore().metadata(), datadirs.subList(0,2));
+ cfs = new MockCFS(getCurrentColumnFamilyStore(), dirs);
+ createTable("create table %s (k int primary key, i int)");
+ dirs2 = new Directories(getCurrentColumnFamilyStore().metadata(), datadirs.subList(1,3));
+ cfs2 = new MockCFS(getCurrentColumnFamilyStore(), dirs2);
+
+ Map<TableId, Long> perTableIdIncomingBytes = new HashMap<>();
+ // cfs has datadirs 0, 1
+ // cfs2 has datadirs 1, 2
+ // this means that the datadir 1 will get 1000 bytes streamed, and the other ddirs 500bytes:
+ perTableIdIncomingBytes.put(cfs.metadata.id, 1000L);
+ perTableIdIncomingBytes.put(cfs2.metadata.id, 1000L);
+
+ filestores.get(0).usableSpace = 501;
+ filestores.get(1).usableSpace = 1001;
+ filestores.get(2).usableSpace = 501;
+
+ Keyspace.all().forEach(ks -> ks.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+ do
+ {
+ Thread.sleep(100);
+ } while (!CompactionManager.instance.active.getCompactions().isEmpty());
+
+ assertTrue(StreamSession.checkDiskSpace(perTableIdIncomingBytes, nextTimeUUID(), filestoreMapper));
+
+ filestores.get(1).usableSpace = 999;
+ assertFalse(StreamSession.checkDiskSpace(perTableIdIncomingBytes, nextTimeUUID(), filestoreMapper));
+
+ }
+
+ static Function<File, FileStore> filestoreMapper = (f) -> {
+ for (int i = 0; i < files.size(); i++)
+ {
+ if (f.toPath().startsWith(files.get(i).toPath()))
+ return filestores.get(i);
+ }
+ throw new RuntimeException("Bad file: "+f);
+ };
+
+ private static class MockCFS extends ColumnFamilyStore
+ {
+ MockCFS(ColumnFamilyStore cfs, Directories dirs)
+ {
+ super(cfs.keyspace, cfs.getTableName(), Util.newSeqGen(), cfs.metadata, dirs, false, false, true);
+ }
+ }
+
+ // return our mocked tables:
+ public static class BBKeyspaceHelper
+ {
+ public static ColumnFamilyStore getIfExists(TableId id)
+ {
+ if (id == cfs.metadata.id)
+ return cfs;
+ return cfs2;
+ }
+ }
+
+ public static class FakeFileStore extends FileStore
+ {
+ public long usableSpace;
+
+ @Override
+ public long getUsableSpace()
+ {
+ return usableSpace;
+ }
+
+ @Override
+ public String name() {return null;}
+ @Override
+ public String type() {return null;}
+ @Override
+ public boolean isReadOnly() {return false;}
+ @Override
+ public long getTotalSpace() {return 0;}
+ @Override
+ public long getUnallocatedSpace() {return 0;}
+ @Override
+ public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {return false;}
+ @Override
+ public boolean supportsFileAttributeView(String name) {return false;}
+ @Override
+ public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {return null;}
+ @Override
+ public Object getAttribute(String attribute) throws IOException {return null;}
+ }
+}
diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
index cffd3b2..0527648 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
@@ -26,6 +26,7 @@
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.cassandra.io.sstable.format.big.BigFormatPartitionWriter;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -341,13 +342,13 @@
// add columns of size slightly less than column_index_size to force insert column index
updates.clustering(1)
- .add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize() - 64]))
+ .add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY) - 64]))
.build()
.apply();
updates = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key);
updates.clustering(6)
- .add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize()]))
+ .add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY)]))
.build()
.apply();
diff --git a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
index 619e0d9..391d589 100644
--- a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
+++ b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
@@ -17,14 +17,14 @@
*/
package org.apache.cassandra.streaming.compression;
-import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
-import org.apache.cassandra.io.util.File;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -33,18 +33,21 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.streaming.CompressedInputStream;
+import org.apache.cassandra.db.streaming.CompressionInfo;
import org.apache.cassandra.io.compress.CompressedSequentialWriter;
import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.sstable.format.CompressionInfoComponent;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.io.util.SequentialWriterOption;
import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.db.streaming.CompressedInputStream;
-import org.apache.cassandra.db.streaming.CompressionInfo;
import org.apache.cassandra.utils.ChecksumType;
import static org.junit.Assert.assertEquals;
@@ -121,12 +124,12 @@
// write compressed data file of longs
File parentDir = new File(tempFolder.newFolder());
Descriptor desc = new Descriptor(parentDir, "ks", "cf", new SequenceBasedSSTableId(1));
- File tmp = new File(desc.filenameFor(Component.DATA));
+ File tmp = desc.fileFor(Components.DATA);
MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
CompressionParams param = CompressionParams.snappy(32, minCompressRatio);
Map<Long, Long> index = new HashMap<Long, Long>();
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp,
- desc.filenameFor(Component.COMPRESSION_INFO),
+ desc.fileFor(Components.COMPRESSION_INFO),
null,
SequentialWriterOption.DEFAULT,
param, collector))
@@ -139,7 +142,7 @@
writer.finish();
}
- CompressionMetadata comp = CompressionMetadata.create(tmp.absolutePath());
+ CompressionMetadata comp = CompressionInfoComponent.load(desc);
List<SSTableReader.PartitionPositionBounds> sections = new ArrayList<>();
for (long l : valuesToCheck)
{
@@ -184,7 +187,7 @@
testException(sections, info);
return;
}
- CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(toRead)), info, ChecksumType.CRC32, () -> 1.0);
+ CompressedInputStream input = new CompressedInputStream(new DataInputBuffer(toRead), info, ChecksumType.CRC32, () -> 1.0);
try (DataInputStream in = new DataInputStream(input))
{
@@ -199,7 +202,7 @@
private static void testException(List<SSTableReader.PartitionPositionBounds> sections, CompressionInfo info) throws IOException
{
- CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(new byte[0])), info, ChecksumType.CRC32, () -> 1.0);
+ CompressedInputStream input = new CompressedInputStream(new DataInputBuffer(new byte[0]), info, ChecksumType.CRC32, () -> 1.0);
try (DataInputStream in = new DataInputStream(input))
{
@@ -218,4 +221,3 @@
}
}
}
-
diff --git a/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java b/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
index 507aa53..fb08971 100644
--- a/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
+++ b/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
@@ -39,7 +39,8 @@
assertNoUnexpectedThreadsStarted(new String[] { "ObjectCleanerThread",
"Shutdown-checker",
- "cluster[0-9]-connection-reaper-[0-9]" });
+ "cluster[0-9]-connection-reaper-[0-9]" },
+ false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -65,7 +66,11 @@
"globalEventExecutor-[1-9]-[1-9]",
"globalEventExecutor-[1-9]-[1-9]",
"Shutdown-checker",
- "cluster[0-9]-connection-reaper-[0-9]" });
+ "cluster[0-9]-connection-reaper-[0-9]",
+ "Attach Listener",
+ "process reaper",
+ "JNA Cleaner"},
+ false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -93,11 +98,15 @@
"globalEventExecutor-[1-9]-[1-9]",
"globalEventExecutor-[1-9]-[1-9]",
"Shutdown-checker",
- "cluster[0-9]-connection-reaper-[0-9]" });
- assertSchemaNotLoaded();
- assertCLSMNotLoaded();
- assertSystemKSNotLoaded();
- assertKeyspaceNotLoaded();
+ "cluster[0-9]-connection-reaper-[0-9]",
+ "Attach Listener",
+ "process reaper",
+ "JNA Cleaner"},
+ false);
+ assertSchemaNotLoaded();
+ assertCLSMNotLoaded();
+ assertSystemKSNotLoaded();
+ assertKeyspaceNotLoaded();
assertServerNotLoaded();
}
@@ -121,10 +130,14 @@
"globalEventExecutor-[1-9]-[1-9]",
"globalEventExecutor-[1-9]-[1-9]",
"Shutdown-checker",
- "cluster[0-9]-connection-reaper-[0-9]" });
- assertSchemaNotLoaded();
- assertCLSMNotLoaded();
- assertSystemKSNotLoaded();
+ "cluster[0-9]-connection-reaper-[0-9]",
+ "Attach Listener",
+ "process reaper",
+ "JNA Cleaner"},
+ false);
+ assertSchemaNotLoaded();
+ assertCLSMNotLoaded();
+ assertSystemKSNotLoaded();
assertKeyspaceNotLoaded();
assertServerNotLoaded();
}
diff --git a/test/unit/org/apache/cassandra/tools/GetVersionTest.java b/test/unit/org/apache/cassandra/tools/GetVersionTest.java
index 9eaf57f..b3ddb58 100644
--- a/test/unit/org/apache/cassandra/tools/GetVersionTest.java
+++ b/test/unit/org/apache/cassandra/tools/GetVersionTest.java
@@ -29,7 +29,7 @@
{
ToolResult tool = ToolRunner.invokeClass(GetVersion.class);
tool.assertOnCleanExit();
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java b/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java
index ede3d6c..59485de 100644
--- a/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java
+++ b/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java
@@ -18,33 +18,35 @@
package org.apache.cassandra.tools;
-import java.util.Arrays;
import java.util.List;
-import com.google.common.collect.Lists;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import com.datastax.driver.core.SimpleStatement;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.io.sstable.format.bti.BtiFormat;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.GCInspector;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.apache.cassandra.transport.ProtocolVersion;
import org.assertj.core.api.Assertions;
+import static com.google.common.collect.Lists.newArrayList;
+
/**
* This class is to monitor the JMX compatability cross different versions, and relies on a gold set of metrics which
* were generated following the instructions below. These tests only check for breaking changes, so will ignore any
* new metrics added in a release. If the latest release is not finalized yet then the latest version might fail
* if a unrelesed metric gets renamed, if this happens then the gold set should be updated for the latest version.
- *
+ * <p>
* If a test fails for a previous version, then this means we have a JMX compatability regression, if the metric has
* gone through proper deprecation then the metric can be excluded using the patterns used in other tests, if the metric
* has not gone through proper deprecation then the change should be looked at more carfuly to avoid breaking users.
- *
+ * <p>
* In order to generate the dump for another version, launch a cluster then run the following
* {@code
* create keyspace cql_test_keyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'};
@@ -64,6 +66,9 @@
@BeforeClass
public static void setup() throws Exception
{
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setColumnIndexSizeInKiB(0); // make sure the column index is created
+
startJMXServer();
}
@@ -90,72 +95,89 @@
@Test
public void diff30() throws Throwable
{
- List<String> excludeObjects = Arrays.asList("org.apache.cassandra.metrics:type=ThreadPools.*",
- "org.apache.cassandra.internal:.*",
- "org.apache.cassandra.metrics:type=DroppedMessage.*",
- "org.apache.cassandra.metrics:type=ClientRequest,scope=CASRead,name=ConditionNotMet",
- "org.apache.cassandra.metrics:type=Client,name=connectedThriftClients", // removed in CASSANDRA-11115
- "org.apache.cassandra.request:type=ReadRepairStage", // removed in CASSANDRA-13910
- "org.apache.cassandra.db:type=HintedHandoffManager", // removed in CASSANDRA-15939
+ List<String> excludeObjects = newArrayList("org.apache.cassandra.metrics:type=ThreadPools.*",
+ "org.apache.cassandra.internal:.*",
+ "org.apache.cassandra.metrics:type=DroppedMessage.*",
+ "org.apache.cassandra.metrics:type=ClientRequest,scope=CASRead,name=ConditionNotMet",
+ "org.apache.cassandra.metrics:type=Client,name=connectedThriftClients", // removed in CASSANDRA-11115
+ "org.apache.cassandra.request:type=ReadRepairStage", // removed in CASSANDRA-13910
+ "org.apache.cassandra.db:type=HintedHandoffManager", // removed in CASSANDRA-15939
- // dropped tables
- "org.apache.cassandra.metrics:type=Table,keyspace=system,scope=(schema_aggregates|schema_columnfamilies|schema_columns|schema_functions|schema_keyspaces|schema_triggers|schema_usertypes),name=.*",
- ".*keyspace=system,(scope|table|columnfamily)=views_builds_in_progress.*",
- ".*keyspace=system,(scope|table|columnfamily)=range_xfers.*",
- ".*keyspace=system,(scope|table|columnfamily)=hints.*",
- ".*keyspace=system,(scope|table|columnfamily)=batchlog.*");
- List<String> excludeAttributes = Arrays.asList("RPCServerRunning", // removed in CASSANDRA-11115
- "MaxNativeProtocolVersion");
- List<String> excludeOperations = Arrays.asList("startRPCServer", "stopRPCServer", // removed in CASSANDRA-11115
- // nodetool apis that were changed,
- "decommission", // -> decommission(boolean)
- "forceRepairAsync", // -> repairAsync
- "forceRepairRangeAsync", // -> repairAsync
- "beginLocalSampling", // -> beginLocalSampling(p1: java.lang.String, p2: int, p3: int): void
- "finishLocalSampling" // -> finishLocalSampling(p1: java.lang.String, p2: int): java.util.List
+ // dropped tables
+ "org.apache.cassandra.metrics:type=Table,keyspace=system,scope=(schema_aggregates|schema_columnfamilies|schema_columns|schema_functions|schema_keyspaces|schema_triggers|schema_usertypes),name=.*",
+ ".*keyspace=system,(scope|table|columnfamily)=views_builds_in_progress.*",
+ ".*keyspace=system,(scope|table|columnfamily)=range_xfers.*",
+ ".*keyspace=system,(scope|table|columnfamily)=hints.*",
+ ".*keyspace=system,(scope|table|columnfamily)=batchlog.*");
+ List<String> excludeAttributes = newArrayList("RPCServerRunning", // removed in CASSANDRA-11115
+ "MaxNativeProtocolVersion");
+ List<String> excludeOperations = newArrayList("startRPCServer", "stopRPCServer", // removed in CASSANDRA-11115
+ // nodetool apis that were changed,
+ "decommission", // -> decommission(boolean)
+ "forceRepairAsync", // -> repairAsync
+ "forceRepairRangeAsync", // -> repairAsync
+ "beginLocalSampling", // -> beginLocalSampling(p1: java.lang.String, p2: int, p3: int): void
+ "finishLocalSampling" // -> finishLocalSampling(p1: java.lang.String, p2: int): java.util.List
);
+ if (BtiFormat.isSelected())
+ {
+ excludeObjects.add("org.apache.cassandra.metrics:type=(ColumnFamily|Keyspace|Table).*,name=IndexSummary.*"); // -> when BTI format is used, index summary is not used (CASSANDRA-17056)
+ }
+
diff(excludeObjects, excludeAttributes, excludeOperations, "test/data/jmxdump/cassandra-3.0-jmx.yaml");
}
@Test
public void diff311() throws Throwable
{
- List<String> excludeObjects = Arrays.asList("org.apache.cassandra.metrics:type=ThreadPools.*", //lazy initialization in 4.0
- "org.apache.cassandra.internal:.*",
- "org.apache.cassandra.metrics:type=DroppedMessage,scope=PAGED_RANGE.*", //it was deprecated in the previous major version
- "org.apache.cassandra.metrics:type=Client,name=connectedThriftClients", // removed in CASSANDRA-11115
- "org.apache.cassandra.request:type=ReadRepairStage", // removed in CASSANDRA-13910
- "org.apache.cassandra.db:type=HintedHandoffManager", // removed in CASSANDRA-15939
+ List<String> excludeObjects = newArrayList("org.apache.cassandra.metrics:type=ThreadPools.*", //lazy initialization in 4.0
+ "org.apache.cassandra.internal:.*",
+ "org.apache.cassandra.metrics:type=DroppedMessage,scope=PAGED_RANGE.*", //it was deprecated in the previous major version
+ "org.apache.cassandra.metrics:type=Client,name=connectedThriftClients", // removed in CASSANDRA-11115
+ "org.apache.cassandra.request:type=ReadRepairStage", // removed in CASSANDRA-13910
+ "org.apache.cassandra.db:type=HintedHandoffManager", // removed in CASSANDRA-15939
- // dropped tables
- "org.apache.cassandra.metrics:type=Table,keyspace=system,scope=(schema_aggregates|schema_columnfamilies|schema_columns|schema_functions|schema_keyspaces|schema_triggers|schema_usertypes),name=.*",
- ".*keyspace=system,(scope|table|columnfamily)=views_builds_in_progress.*",
- ".*keyspace=system,(scope|table|columnfamily)=range_xfers.*",
- ".*keyspace=system,(scope|table|columnfamily)=hints.*",
- ".*keyspace=system,(scope|table|columnfamily)=batchlog.*"
+ // dropped tables
+ "org.apache.cassandra.metrics:type=Table,keyspace=system,scope=(schema_aggregates|schema_columnfamilies|schema_columns|schema_functions|schema_keyspaces|schema_triggers|schema_usertypes),name=.*",
+ ".*keyspace=system,(scope|table|columnfamily)=views_builds_in_progress.*",
+ ".*keyspace=system,(scope|table|columnfamily)=range_xfers.*",
+ ".*keyspace=system,(scope|table|columnfamily)=hints.*",
+ ".*keyspace=system,(scope|table|columnfamily)=batchlog.*"
);
- List<String> excludeAttributes = Arrays.asList("RPCServerRunning", // removed in CASSANDRA-11115
- "MaxNativeProtocolVersion",
- "StreamingSocketTimeout");
- List<String> excludeOperations = Arrays.asList("startRPCServer", "stopRPCServer", // removed in CASSANDRA-11115
- // nodetool apis that were changed,
- "decommission", // -> decommission(boolean)
- "forceRepairAsync", // -> repairAsync
- "forceRepairRangeAsync", // -> repairAsync
- "beginLocalSampling", // -> beginLocalSampling(p1: java.lang.String, p2: int, p3: int): void
- "finishLocalSampling" // -> finishLocalSampling(p1: java.lang.String, p2: int): java.util.List
+ List<String> excludeAttributes = newArrayList("RPCServerRunning", // removed in CASSANDRA-11115
+ "MaxNativeProtocolVersion",
+ "StreamingSocketTimeout");
+ List<String> excludeOperations = newArrayList("startRPCServer", "stopRPCServer", // removed in CASSANDRA-11115
+ // nodetool apis that were changed,
+ "decommission", // -> decommission(boolean)
+ "forceRepairAsync", // -> repairAsync
+ "forceRepairRangeAsync", // -> repairAsync
+ "beginLocalSampling", // -> beginLocalSampling(p1: java.lang.String, p2: int, p3: int): void
+ "finishLocalSampling" // -> finishLocalSampling(p1: java.lang.String, p2: int): java.util.List
);
+ if (BtiFormat.isSelected())
+ {
+ excludeObjects.add("org.apache.cassandra.metrics:type=(ColumnFamily|Keyspace|Table).*,name=IndexSummary.*"); // -> when BTI format is used, index summary is not used (CASSANDRA-17056)
+ excludeObjects.add("org.apache.cassandra.metrics:type=Index,scope=RowIndexEntry.*");
+ }
+
diff(excludeObjects, excludeAttributes, excludeOperations, "test/data/jmxdump/cassandra-3.11-jmx.yaml");
}
@Test
public void diff40() throws Throwable
{
- List<String> excludeObjects = Arrays.asList();
- List<String> excludeAttributes = Arrays.asList();
- List<String> excludeOperations = Arrays.asList();
+ List<String> excludeObjects = newArrayList();
+ List<String> excludeAttributes = newArrayList();
+ List<String> excludeOperations = newArrayList();
+
+ if (BtiFormat.isSelected())
+ {
+ excludeObjects.add("org.apache.cassandra.metrics:type=(ColumnFamily|Keyspace|Table).*,name=IndexSummary.*"); // -> when BTI format is used, index summary is not used (CASSANDRA-17056)
+ excludeObjects.add("org.apache.cassandra.metrics:type=Index,scope=RowIndexEntry.*");
+ }
diff(excludeObjects, excludeAttributes, excludeOperations, "test/data/jmxdump/cassandra-4.0-jmx.yaml");
}
@@ -163,9 +185,15 @@
@Test
public void diff41() throws Throwable
{
- List<String> excludeObjects = Arrays.asList();
- List<String> excludeAttributes = Arrays.asList();
- List<String> excludeOperations = Arrays.asList();
+ List<String> excludeObjects = newArrayList();
+ List<String> excludeAttributes = newArrayList();
+ List<String> excludeOperations = newArrayList();
+
+ if (BtiFormat.isSelected())
+ {
+ excludeObjects.add("org.apache.cassandra.metrics:type=(ColumnFamily|Keyspace|Table).*,name=IndexSummary.*"); // -> when BTI format is used, index summary is not used (CASSANDRA-17056)
+ excludeObjects.add("org.apache.cassandra.metrics:type=Index,scope=RowIndexEntry.*");
+ }
diff(excludeObjects, excludeAttributes, excludeOperations, "test/data/jmxdump/cassandra-4.1-jmx.yaml");
}
@@ -174,10 +202,10 @@
{
setupStandardTables();
- List<String> args = Lists.newArrayList("tools/bin/jmxtool", "diff",
- "-f", "yaml",
- "--ignore-missing-on-left",
- original, TMP.getRoot().getAbsolutePath() + "/out.yaml");
+ List<String> args = newArrayList("tools/bin/jmxtool", "diff",
+ "-f", "yaml",
+ "--ignore-missing-on-left",
+ original, TMP.getRoot().getAbsolutePath() + "/out.yaml");
excludeObjects.forEach(a -> {
args.add("--exclude-object");
args.add(a);
diff --git a/test/unit/org/apache/cassandra/tools/NodeProbeTest.java b/test/unit/org/apache/cassandra/tools/NodeProbeTest.java
index 22554e6..2947a26 100644
--- a/test/unit/org/apache/cassandra/tools/NodeProbeTest.java
+++ b/test/unit/org/apache/cassandra/tools/NodeProbeTest.java
@@ -37,6 +37,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
probe = new NodeProbe(jmxHost, jmxPort);
}
diff --git a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
index 3bb2825..9a5ae8e 100644
--- a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
+++ b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
@@ -30,7 +30,12 @@
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.File;
import org.apache.commons.io.FileUtils;
import org.junit.BeforeClass;
@@ -72,9 +77,15 @@
"process reaper", // spawned by the jvm when executing external processes
// and may still be active when we check
"Attach Listener", // spawned in intellij IDEA
+ "JNA Cleaner", // spawned by JNA
};
- public void assertNoUnexpectedThreadsStarted(String[] optionalThreadNames)
+ static final String[] NON_DEFAULT_MEMTABLE_THREADS =
+ {
+ "((Native|Slab|Heap)Pool|Logged)Cleaner"
+ };
+
+ public void assertNoUnexpectedThreadsStarted(String[] optionalThreadNames, boolean allowNonDefaultMemtableThreads)
{
ThreadMXBean threads = ManagementFactory.getThreadMXBean();
@@ -87,10 +98,15 @@
.filter(Objects::nonNull)
.map(ThreadInfo::getThreadName)
.collect(Collectors.toSet());
+ Iterable<String> optionalNames = optionalThreadNames != null
+ ? Arrays.asList(optionalThreadNames)
+ : Collections.emptyList();
+ if (allowNonDefaultMemtableThreads && DatabaseDescriptor.getMemtableConfigurations().containsKey("default"))
+ optionalNames = Iterables.concat(optionalNames, Arrays.asList(NON_DEFAULT_MEMTABLE_THREADS));
- List<Pattern> optional = optionalThreadNames != null
- ? Arrays.stream(optionalThreadNames).map(Pattern::compile).collect(Collectors.toList())
- : Collections.emptyList();
+ List<Pattern> optional = StreamSupport.stream(optionalNames.spliterator(), false)
+ .map(Pattern::compile)
+ .collect(Collectors.toList());
current.removeAll(initial);
@@ -180,7 +196,7 @@
@BeforeClass
public static void setupTester()
{
- System.setProperty("cassandra.partitioner", "org.apache.cassandra.dht.Murmur3Partitioner");
+ CassandraRelevantProperties.PARTITIONER.setString("org.apache.cassandra.dht.Murmur3Partitioner");
// may start an async appender
LoggerFactory.getLogger(OfflineToolUtils.class);
@@ -219,7 +235,7 @@
protected void assertCorrectEnvPostTest()
{
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, true);
assertSchemaLoaded();
assertServerNotLoaded();
}
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExpiredBlockersTest.java b/test/unit/org/apache/cassandra/tools/SSTableExpiredBlockersTest.java
index e76769e..10b3f5d 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExpiredBlockersTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExpiredBlockersTest.java
@@ -37,7 +37,7 @@
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java
index 1530a2e..b713144 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportSchemaLoadingTest.java
@@ -28,6 +28,7 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.MismatchedInputException;
+import org.apache.cassandra.utils.JsonUtils;
import org.assertj.core.api.Assertions;
import static org.hamcrest.CoreMatchers.startsWith;
@@ -44,7 +45,7 @@
*/
public class SSTableExportSchemaLoadingTest extends OfflineToolUtils
{
- private static final ObjectMapper mapper = new ObjectMapper();
+ private static final ObjectMapper mapper = JsonUtils.JSON_OBJECT_MAPPER;
private static final TypeReference<List<Map<String, Object>>> jacksonListOfMapsType = new TypeReference<List<Map<String, Object>>>() {};
private static String sstable;
@@ -182,7 +183,7 @@
*/
private void assertPostTestEnv()
{
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, true);
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
assertKeyspaceNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index a5b70a2..3a4ead4 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@ -108,7 +108,7 @@
*/
private void assertPostTestEnv()
{
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTableLevelResetterTest.java b/test/unit/org/apache/cassandra/tools/SSTableLevelResetterTest.java
index e6d7adc..0d609d3 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableLevelResetterTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableLevelResetterTest.java
@@ -36,7 +36,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTableMetadataViewerTest.java b/test/unit/org/apache/cassandra/tools/SSTableMetadataViewerTest.java
index 4998b6e..6728fdf 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableMetadataViewerTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableMetadataViewerTest.java
@@ -55,7 +55,7 @@
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Options:"));
assertEquals(1, tool.getExitCode());
}
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -108,7 +108,7 @@
assertTrue(tool.getStdout(), CharMatcher.ascii().matchesAllOf(tool.getStdout()));
Assertions.assertThat(tool.getStdout()).doesNotContain("Widest Partitions");
Assertions.assertThat(tool.getStdout()).contains(sstable.replaceAll("-Data\\.db$", ""));
- assertTrue(tool.getStderr(), tool.getStderr().isEmpty());
+ assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
assertGoodEnvPostTest();
}
@@ -123,7 +123,7 @@
ToolResult tool = ToolRunner.invokeClass(SSTableMetadataViewer.class, arg, sstable);
Assertions.assertThat(tool.getStdout()).contains(Util.BLUE);
Assertions.assertThat(tool.getStdout()).contains(sstable.replaceAll("-Data\\.db$", ""));
- assertTrue("Arg: [" + arg + "]\n" + tool.getStderr(), tool.getStderr().isEmpty());
+ assertTrue("Arg: [" + arg + "]\n" + tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
assertGoodEnvPostTest();
});
@@ -139,7 +139,7 @@
ToolResult tool = ToolRunner.invokeClass(SSTableMetadataViewer.class, arg, sstable);
assertTrue(tool.getStdout(), !CharMatcher.ascii().matchesAllOf(tool.getStdout()));
Assertions.assertThat(tool.getStdout()).contains(sstable.replaceAll("-Data\\.db$", ""));
- assertTrue("Arg: [" + arg + "]\n" + tool.getStderr(), tool.getStderr().isEmpty());
+ assertTrue("Arg: [" + arg + "]\n" + tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
assertGoodEnvPostTest();
});
@@ -211,14 +211,14 @@
ToolResult tool = ToolRunner.invokeClass(SSTableMetadataViewer.class, arg, sstable);
Assertions.assertThat(tool.getStdout()).contains("Widest Partitions");
Assertions.assertThat(tool.getStdout()).contains(sstable.replaceAll("-Data\\.db$", ""));
- assertTrue("Arg: [" + arg + "]\n" + tool.getStderr(), tool.getStderr().isEmpty());
+ assertTrue("Arg: [" + arg + "]\n" + tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
});
}
private void assertGoodEnvPostTest()
{
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTableOfflineRelevelTest.java b/test/unit/org/apache/cassandra/tools/SSTableOfflineRelevelTest.java
index 3f4314c..15f6297 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableOfflineRelevelTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableOfflineRelevelTest.java
@@ -36,7 +36,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java b/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java
new file mode 100644
index 0000000..247fa8a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.tools.ToolRunner.ToolResult;
+import org.apache.cassandra.utils.FBUtilities;
+import org.assertj.core.api.AbstractStringAssert;
+import org.assertj.core.api.Assertions;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for {@link SSTablePartitions}.
+ */
+public class SSTablePartitionsTest extends OfflineToolUtils
+{
+ private static final String SSTABLE_1 = sstable("legacy_ma_simple");
+ private static final String SSTABLE_2 = sstable("legacy_ma_clust");
+ private static final String HEADER_1 = "\nProcessing #1 (big-ma) (0.169 KiB uncompressed, 0.086 KiB on disk)\n";
+ private static final String HEADER_2 = "\nProcessing #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n";
+ private static final String BACKUPS_HEADER_1 = "\nProcessing Backup:backups #1 (big-ma) (0.169 KiB uncompressed, 0.086 KiB on disk)\n";
+ private static final String BACKUPS_HEADER_2 = "\nProcessing Backup:backups #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n";
+ private static final String SNAPSHOTS_HEADER_1 = "\nProcessing Snapshot:snapshot-1 #1 (big-ma) (0.169 KiB uncompressed, 0.086 KiB on disk)\n";
+ private static final String SNAPSHOTS_HEADER_2 = "\nProcessing Snapshot:snapshot-1 #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n";
+ private static final String SUMMARY_1 = " Partition size Row count Cell count Tombstone count\n" +
+ " ~p50 0.034 KiB 1 1 0\n" +
+ " ~p75 0.034 KiB 1 1 0\n" +
+ " ~p90 0.034 KiB 1 1 0\n" +
+ " ~p95 0.034 KiB 1 1 0\n" +
+ " ~p99 0.034 KiB 1 1 0\n" +
+ " ~p999 0.034 KiB 1 1 0\n" +
+ " min 0.032 KiB 1 1 0\n" +
+ " max 0.034 KiB 1 1 0\n" +
+ " count 5\n";
+ private static final String SUMMARY_2 = " Partition size Row count Cell count Tombstone count\n" +
+ " ~p50 71.735 KiB 50 50 0\n" +
+ " ~p75 71.735 KiB 50 50 0\n" +
+ " ~p90 71.735 KiB 50 50 0\n" +
+ " ~p95 71.735 KiB 50 50 0\n" +
+ " ~p99 71.735 KiB 50 50 0\n" +
+ " ~p999 71.735 KiB 50 50 0\n" +
+ " min 65.625 KiB 50 50 0\n" +
+ " max 65.630 KiB 50 50 0\n" +
+ " count 5\n";
+
+ @BeforeClass
+ public static void prepareDirectories()
+ {
+ createBackupsAndSnapshots(SSTABLE_1);
+ createBackupsAndSnapshots(SSTABLE_2);
+ }
+
+ private static void createBackupsAndSnapshots(String sstable)
+ {
+ File parentDir = new File(sstable).parent();
+
+ File backupsDir = new File(parentDir, Directories.BACKUPS_SUBDIR);
+ backupsDir.tryCreateDirectory();
+
+ File snapshotsDir = new File(parentDir, Directories.SNAPSHOT_SUBDIR);
+ snapshotsDir.tryCreateDirectory();
+
+ File snapshotDir = new File(snapshotsDir, "snapshot-1");
+ snapshotDir.tryCreateDirectory();
+
+ for (File f : parentDir.tryList(File::isFile))
+ {
+ FileUtils.copyWithOutConfirm(f, new File(backupsDir, f.name()));
+ FileUtils.copyWithOutConfirm(f, new File(snapshotDir, f.name()));
+ }
+ }
+
+ /**
+ * Runs post-test assertions about loaded classed and started threads.
+ */
+ @After
+ public void assertPostTestEnv()
+ {
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
+ assertCLSMNotLoaded();
+ assertSystemKSNotLoaded();
+ assertKeyspaceNotLoaded();
+ assertServerNotLoaded();
+ }
+
+ /**
+ * Verify that the tool prints help when no arguments are provided.
+ */
+ @Test
+ public void testNoArgsPrintsHelp()
+ {
+ ToolResult tool = ToolRunner.invokeClass(SSTablePartitions.class);
+ Assertions.assertThat(tool.getExitCode()).isOne();
+ Assertions.assertThat(tool.getCleanedStderr()).contains("You must supply at least one sstable or directory");
+ Assertions.assertThat(tool.getStdout()).contains("usage");
+ }
+
+ /**
+ * Verify that the tool prints the right help contents.
+ * If you added, modified options or help, please update docs if necessary.
+ */
+ @Test
+ public void testMaybeChangeDocs()
+ {
+ ToolResult tool = ToolRunner.invokeClass(SSTablePartitions.class);
+ Assertions.assertThat(tool.getStdout())
+ .isEqualTo("usage: sstablepartitions <options> <sstable files or directories>\n" +
+ " \n" +
+ "Print partition statistics of one or more sstables.\n" +
+ " -b,--backups include backups present in data\n" +
+ " directories (recursive scans)\n" +
+ " -c,--min-cells <arg> partition cell count threshold\n" +
+ " -k,--key <arg> Partition keys to include\n" +
+ " -m,--csv CSV output (machine readable)\n" +
+ " -o,--min-tombstones <arg> partition tombstone count threshold\n" +
+ " -r,--recursive scan for sstables recursively\n" +
+ " -s,--snapshots include snapshots present in data\n" +
+ " directories (recursive scans)\n" +
+ " -t,--min-size <arg> partition size threshold, expressed as\n" +
+ " either the number of bytes or a size with unit of the form 10KiB, 20MiB,\n" +
+ " 30GiB, etc.\n" +
+ " -u,--current-timestamp <arg> timestamp (seconds since epoch, unit time)\n" +
+ " for TTL expired calculation\n" +
+ " -w,--min-rows <arg> partition row count threshold\n" +
+ " -x,--exclude-key <arg> Excluded partition key(s) from partition\n" +
+ " detailed row/cell/tombstone information (irrelevant, if --partitions-only\n" +
+ " is given)\n" +
+ " -y,--partitions-only Do not process per-partition detailed\n" +
+ " row/cell/tombstone information, only brief information\n");
+ }
+
+ /**
+ * Verify that the tool can select single sstable file.
+ */
+ @Test
+ public void testSingleSSTable()
+ {
+ assertThatToolSucceds(SSTABLE_1).isEqualTo(HEADER_1 + SUMMARY_1);
+ assertThatToolSucceds(SSTABLE_2).isEqualTo(HEADER_2 + SUMMARY_2);
+ }
+
+ /**
+ * Verify that the tool can select multiple sstable files.
+ */
+ @Test
+ public void testMultipleSSTables()
+ {
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2)
+ .isEqualTo(HEADER_2 + SUMMARY_2 + HEADER_1 + SUMMARY_1);
+ }
+
+ /**
+ * Verify that the tool can select all the sstable files in a directory.
+ */
+ @Test
+ public void testDirectory()
+ {
+ assertThatToolSucceds(new File(SSTABLE_1).parentPath())
+ .isEqualTo(HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds("-r", new File(SSTABLE_1).parent().parentPath())
+ .contains(HEADER_1 + SUMMARY_1)
+ .contains(HEADER_2 + SUMMARY_2);
+
+ assertThatToolSucceds("--recursive", new File(SSTABLE_2).parent().parentPath())
+ .contains(HEADER_1 + SUMMARY_1)
+ .contains(HEADER_2 + SUMMARY_2);
+ }
+
+ /**
+ * Test the flag for collecting and printing sstable partition sizes only.
+ */
+ @Test
+ public void testPartitionsOnly()
+ {
+ testPartitionsOnly("-y");
+ testPartitionsOnly("--partitions-only");
+ }
+
+ private static void testPartitionsOnly(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, option)
+ .isEqualTo(HEADER_1 +
+ " Partition size\n" +
+ " ~p50 0.034 KiB\n" +
+ " ~p75 0.034 KiB\n" +
+ " ~p90 0.034 KiB\n" +
+ " ~p95 0.034 KiB\n" +
+ " ~p99 0.034 KiB\n" +
+ " ~p999 0.034 KiB\n" +
+ " min 0.032 KiB\n" +
+ " max 0.034 KiB\n" +
+ " count 5\n");
+
+ assertThatToolSucceds(SSTABLE_2, "--partitions-only")
+ .isEqualTo(HEADER_2 +
+ " Partition size\n" +
+ " ~p50 71.735 KiB\n" +
+ " ~p75 71.735 KiB\n" +
+ " ~p90 71.735 KiB\n" +
+ " ~p95 71.735 KiB\n" +
+ " ~p99 71.735 KiB\n" +
+ " ~p999 71.735 KiB\n" +
+ " min 65.625 KiB\n" +
+ " max 65.630 KiB\n" +
+ " count 5\n");
+ }
+
+ /**
+ * Test the flag for detecting partitions over a certain size threshold.
+ */
+ @Test
+ public void testMinSize()
+ {
+ testMinSize("-t");
+ testMinSize("--min-size");
+ }
+
+ private static void testMinSize(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "35")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 +
+ HEADER_1 +
+ " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 4 partitions match\n" +
+ " Keys: 1 2 3 4\n" +
+ SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "36")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 + HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "67201")
+ .isEqualTo(HEADER_2 +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 4 partitions match\n" +
+ " Keys: 1 2 3 4\n" +
+ SUMMARY_2 + HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "67201B")
+ .isEqualTo(HEADER_2 +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 4 partitions match\n" +
+ " Keys: 1 2 3 4\n" +
+ SUMMARY_2 + HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "65KiB")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 + HEADER_1 + SUMMARY_1);
+ }
+
+ /**
+ * Test the flag for detecting partitions with more cells than a certain threshold.
+ */
+ @Test
+ public void testMinCells()
+ {
+ testMinCells("-c");
+ testMinCells("--min-cells");
+ }
+
+ private static void testMinCells(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 +
+ HEADER_1 +
+ " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "2")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 + HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "51")
+ .isEqualTo(HEADER_2 + SUMMARY_2 +
+ HEADER_1 + SUMMARY_1);
+ }
+
+ /**
+ * Test the flag for detecting partitions with more rows than a certain threshold.
+ */
+ @Test
+ public void testMinRows()
+ {
+ testMinRows("-w");
+ testMinRows("--min-rows");
+ }
+
+ private static void testMinRows(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 +
+ HEADER_1 +
+ " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "50")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 + HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "51")
+ .isEqualTo(HEADER_2 + SUMMARY_2 +
+ HEADER_1 + SUMMARY_1);
+ }
+
+ /**
+ * Test the flag for detecting partitions with more tombstones than a certain threshold.
+ */
+ @Test
+ public void testMinTombstones()
+ {
+ testMinTombstones("-o");
+ testMinTombstones("--min-tombstones");
+ }
+
+ private static void testMinTombstones(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0")
+ .isEqualTo(HEADER_2 +
+ " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_2 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_2 +
+ HEADER_1 +
+ " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 5 partitions match\n" +
+ " Keys: 0 1 2 3 4\n" +
+ SUMMARY_1);
+
+ assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-tombstones", "1")
+ .isEqualTo(HEADER_2 + SUMMARY_2 + HEADER_1 + SUMMARY_1);
+ }
+
+ /**
+ * Test the flag for providing a current time.
+ */
+ @Test
+ public void testCurrentTimestamp()
+ {
+ testCurrentTimestamp("-u");
+ testCurrentTimestamp("--current-timestamp");
+ }
+
+ private static void testCurrentTimestamp(String option)
+ {
+ String now = String.valueOf(FBUtilities.nowInSeconds());
+ assertThatToolSucceds(SSTABLE_1, option, now).isEqualTo(HEADER_1 + SUMMARY_1);
+ }
+
+ /**
+ * Test the flag for including backup sstables.
+ */
+ @Test
+ public void testBackups()
+ {
+ testBackups("-b");
+ testBackups("--backups");
+ }
+
+ private static void testBackups(String option)
+ {
+ assertThatToolSucceds(new File(SSTABLE_1).parentPath(), "-r", option)
+ .isEqualTo(HEADER_1 + SUMMARY_1 + BACKUPS_HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(new File(SSTABLE_2).parentPath(), "-r", option)
+ .isEqualTo(HEADER_2 + SUMMARY_2 + BACKUPS_HEADER_2 + SUMMARY_2);
+ }
+
+ /**
+ * Test the flag for including snapshot sstables.
+ */
+ @Test
+ public void testSnapshots()
+ {
+ testSnapshots("-s");
+ testSnapshots("--snapshots");
+ }
+
+ private static void testSnapshots(String option)
+ {
+ assertThatToolSucceds(new File(SSTABLE_1).parentPath(), "-r", option)
+ .isEqualTo(HEADER_1 + SUMMARY_1 + SNAPSHOTS_HEADER_1 + SUMMARY_1);
+
+ assertThatToolSucceds(new File(SSTABLE_2).parentPath(), "-r", option)
+ .isEqualTo(HEADER_2 + SUMMARY_2 + SNAPSHOTS_HEADER_2 + SUMMARY_2);
+ }
+
+ /**
+ * Test the flag for specifying partiton keys to be considered.
+ */
+ @Test
+ public void testIncludedKeys()
+ {
+ testIncludedKeys("-k");
+ testIncludedKeys("--key");
+ }
+
+ private static void testIncludedKeys(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "1", option, "3")
+ .contains(HEADER_1 +
+ " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 2 partitions match\n" +
+ " Keys: 1 3\n")
+ .contains("count 2\n");
+
+ assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "0", option, "2", option, "4")
+ .contains(HEADER_1 +
+ " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 3 partitions match\n" +
+ " Keys: 0 2 4\n")
+ .contains("count 3\n");
+
+ assertThatToolSucceds(SSTABLE_1, "-y","--min-size", "0", option, "0", option, "2", option, "4")
+ .contains(HEADER_1 +
+ " Partition: '0' (30) live, size: 0.032 KiB\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 3 partitions match\n" +
+ " Keys: 0 2 4\n")
+ .contains("count 3\n");
+ }
+
+ /**
+ * Test the flag for specifying partiton keys to be excluded.
+ */
+ @Test
+ public void testExcludedKeys()
+ {
+ testExcludedKeys("-x");
+ testExcludedKeys("--exclude-key");
+ }
+
+ private static void testExcludedKeys(String option)
+ {
+ assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "1", option, "3")
+ .contains(HEADER_1 +
+ " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 3 partitions match\n" +
+ " Keys: 0 2 4\n")
+ .contains("count 3\n");
+
+ assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "0", option, "2", option, "4")
+ .contains(HEADER_1 +
+ " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" +
+ "Summary of #1 (big-ma):\n" +
+ " File: " + SSTABLE_1 + "\n" +
+ " 2 partitions match\n" +
+ " Keys: 1 3\n")
+ .contains("count 2\n");
+ }
+
+ /**
+ * Test the flag for producing machine-readable CSV output.
+ */
+ @Test
+ public void testCSV()
+ {
+ testCSV("-m");
+ testCSV("--csv");
+ }
+
+ private static void testCSV(String option)
+ {
+ assertThatToolSucceds(option, "--min-size", "35", SSTABLE_1, SSTABLE_2)
+ .isEqualTo(format("key,keyBinary,live,offset,size,rowCount,cellCount,tombstoneCount," +
+ "rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount,cellTombstoneCount," +
+ "rowTtlExpired,cellTtlExpired,directory,keyspace,table,index,snapshot,backup," +
+ "generation,format,version\n" +
+ "\"0\",30,true,0,67200,50,50,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" +
+ "\"1\",31,true,67200,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+ "\"2\",32,true,134405,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+ "\"3\",33,true,201610,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+ "\"4\",34,true,268815,67205,50,50,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+ "\"1\",31,true,33,35,1,1,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" +
+ "\"2\",32,true,68,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+ "\"3\",33,true,103,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n" +
+ "\"4\",34,true,138,35,1,1,0,0,0,0,0,0,0,%<s,,,,,,1,big,ma\n",
+ SSTABLE_2, SSTABLE_1));
+ }
+
+ private static AbstractStringAssert<?> assertThatToolSucceds(String... args)
+ {
+ ToolResult tool = invokeTool(args);
+ Assertions.assertThat(tool.getExitCode()).isZero();
+ tool.assertOnCleanExit();
+ return Assertions.assertThat(tool.getStdout());
+ }
+
+ private static String sstable(String table)
+ {
+ try
+ {
+ return findOneSSTable("legacy_sstables", table);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static ToolResult invokeTool(String... args)
+ {
+ return ToolRunner.invokeClass(SSTablePartitions.class, args);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/SSTableRepairedAtSetterTest.java b/test/unit/org/apache/cassandra/tools/SSTableRepairedAtSetterTest.java
index 3531075..9d3a609 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableRepairedAtSetterTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableRepairedAtSetterTest.java
@@ -41,7 +41,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -79,7 +79,7 @@
"--is-repaired",
findOneSSTable("legacy_sstables", "legacy_ma_simple"));
tool.assertOnCleanExit();
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -95,7 +95,7 @@
"--is-unrepaired",
findOneSSTable("legacy_sstables", "legacy_ma_simple"));
tool.assertOnCleanExit();
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -113,7 +113,7 @@
String file = tmpFile.absolutePath();
ToolResult tool = ToolRunner.invokeClass(SSTableRepairedAtSetter.class, "--really-set", "--is-repaired", "-f", file);
tool.assertOnCleanExit();
- assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
+ assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneSplitterTest.java b/test/unit/org/apache/cassandra/tools/StandaloneSplitterTest.java
index d28558f..05e7b34 100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneSplitterTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneSplitterTest.java
@@ -25,6 +25,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
import org.hamcrest.CoreMatchers;
@@ -42,7 +43,7 @@
// the legacy tables use a different partitioner :(
// (Don't use ByteOrderedPartitioner.class.getName() as that would initialize the class and work
// against the goal of this test to check classes and threads initialized by the tool.)
- System.setProperty("cassandra.partitioner", "org.apache.cassandra.dht.ByteOrderedPartitioner");
+ CassandraRelevantProperties.PARTITIONER.setString("org.apache.cassandra.dht.ByteOrderedPartitioner");
}
@Test
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java b/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java
index 944b8de..40f075a 100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java
@@ -35,6 +35,7 @@
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -124,6 +125,6 @@
assertTrue("Generated sstable must be at least 1MiB", (new File(sstableFileName)).length() > 1024*1024);
sstablesDir = new File(sstableFileName).parent();
origSstables = Arrays.asList(sstablesDir.tryList());
- System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
+ TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.setBoolean(true);
}
}
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
index e9d2070..13705f9 100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
@@ -18,51 +18,52 @@
package org.apache.cassandra.tools;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.apache.cassandra.io.util.File;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.sstable.LegacySSTableTest;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
-import org.assertj.core.util.Lists;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.junit.Assert.assertEquals;
/*
* SStableUpdater should be run with the server shutdown, but we need to set up a certain env to be able to
* load/swap/drop sstable files under the test's feet. Hence why we need a separate file vs StandaloneUpgraderTest.
- *
+ *
* Caution: heavy hacking ahead.
*/
public class StandaloneUpgraderOnSStablesTest
{
+ static WithProperties properties;
+
String legacyId = LegacySSTableTest.legacyVersions[LegacySSTableTest.legacyVersions.length - 1];
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
LegacySSTableTest.defineSchema();
- System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
+ properties = new WithProperties().set(TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST, true);
}
@AfterClass
public static void clearClassEnv()
{
- System.clearProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST);
+ properties.close();
}
@Test
@@ -76,9 +77,7 @@
"-k",
"legacy_tables",
"legacy_" + legacyId + "_simple");
- Assertions.assertThat(tool.getStdout()).contains("Found 1 sstables that need upgrading.");
- Assertions.assertThat(tool.getStdout()).contains("legacy_tables/legacy_" + legacyId + "_simple");
- Assertions.assertThat(tool.getStdout()).contains("-Data.db");
+ checkUpgradeToolOutput(tool, origFiles);
tool.assertOnCleanExit();
List<String> newFiles = getSStableFiles("legacy_tables", "legacy_" + legacyId + "_simple");
@@ -105,13 +104,19 @@
"wrongsnapshot");
Assertions.assertThat(tool.getStdout()).contains("Found 0 sstables that need upgrading.");
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists("legacy_tables", "legacy_" + legacyId + "_simple");
+ List<String> names = cfs.getDirectories()
+ .sstableLister(Directories.OnTxnErr.IGNORE)
+ .snapshots("testsnapshot").list().keySet().stream()
+ .map(descriptor -> descriptor.baseFile().toString())
+ .collect(Collectors.toList());
+
tool = ToolRunner.invokeClass(StandaloneUpgrader.class,
"legacy_tables",
"legacy_" + legacyId + "_simple",
"testsnapshot");
- Assertions.assertThat(tool.getStdout()).contains("Found 1 sstables that need upgrading.");
- Assertions.assertThat(tool.getStdout()).contains("legacy_tables/legacy_" + legacyId + "_simple");
- Assertions.assertThat(tool.getStdout()).contains("-Data.db");
+ checkUpgradeToolOutput(tool, names);
+
tool.assertOnCleanExit();
}
@@ -125,9 +130,8 @@
ToolResult tool = ToolRunner.invokeClass(StandaloneUpgrader.class,
"legacy_tables",
"legacy_" + legacyId + "_simple");
- Assertions.assertThat(tool.getStdout()).contains("Found 1 sstables that need upgrading.");
- Assertions.assertThat(tool.getStdout()).contains("legacy_tables/legacy_" + legacyId + "_simple");
- Assertions.assertThat(tool.getStdout()).contains("-Data.db");
+
+ checkUpgradeToolOutput(tool, origFiles);
tool.assertOnCleanExit();
List<String> newFiles = getSStableFiles("legacy_tables", "legacy_" + legacyId + "_simple");
@@ -138,22 +142,25 @@
Keyspace.open("legacy_tables").getColumnFamilyStore("legacy_" + legacyId + "_simple").loadNewSSTables();
}
+ private static void checkUpgradeToolOutput(ToolResult tool, List<String> names)
+ {
+ Assertions.assertThat(tool.getStdout()).contains("Found " + names.size() + " sstables that need upgrading.");
+ for (String name : names)
+ {
+ Assertions.assertThat(tool.getStdout()).matches("(?s).*Upgrading.*" + Pattern.quote(name) + ".*");
+ Assertions.assertThat(tool.getStdout()).matches("(?s).*Upgrade of.*" + Pattern.quote(name) + ".*complete.*");
+ }
+ }
+
private List<String> getSStableFiles(String ks, String table) throws StartupException
{
ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(table);
org.apache.cassandra.Util.flush(cfs);
ColumnFamilyStore.scrubDataDirectories(cfs.metadata());
- Set<SSTableReader> sstables = cfs.getLiveSSTables();
- if (sstables.isEmpty())
- return Lists.emptyList();
-
- String sstableFileName = sstables.iterator().next().getFilename();
- File sstablesDir = new File(sstableFileName).parent();
- return Arrays.asList(sstablesDir.tryList())
- .stream()
- .filter(f -> f.isFile())
- .map(file -> file.toString())
- .collect(Collectors.toList());
+ return cfs.getDirectories()
+ .sstableLister(Directories.OnTxnErr.IGNORE).list().keySet().stream()
+ .map(descriptor -> descriptor.baseFile().toString())
+ .collect(Collectors.toList());
}
}
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
index 65dd125..b1eb139 100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
@@ -29,12 +29,16 @@
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.distributed.shared.WithProperties;
+import org.apache.cassandra.io.sstable.VerifyTest;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigTableVerifier;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
@@ -42,28 +46,31 @@
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.SchemaLoader.standardCFMD;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.junit.Assert.assertEquals;
/**
* Class that tests tables for {@link StandaloneVerifier} by updating using {@link SchemaLoader}
* Similar in vein to other {@link SchemaLoader} type tests, as well as {@link StandaloneUpgraderOnSStablesTest}.
- * Since the tool mainly exercises the {@link org.apache.cassandra.db.compaction.Verifier}, we elect to
- * not run every conceivable option as many tests are already covered by {@link org.apache.cassandra.db.VerifyTest}.
+ * Since the tool mainly exercises the {@link BigTableVerifier}, we elect to
+ * not run every conceivable option as many tests are already covered by {@link VerifyTest}.
*
* Note: the complete coverage is composed of:
* - {@link StandaloneVerifierOnSSTablesTest}
* - {@link StandaloneVerifierTest}
- * - {@link org.apache.cassandra.db.VerifyTest}
+ * - {@link VerifyTest}
*/
public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils
{
+ static WithProperties properties;
+
@BeforeClass
public static void setup()
{
// since legacy tables test data uses ByteOrderedPartitioner that's what we need
// for the check version to work
- System.setProperty("cassandra.partitioner", "org.apache.cassandra.dht.ByteOrderedPartitioner");
- System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing`
+ CassandraRelevantProperties.PARTITIONER.setString("org.apache.cassandra.dht.ByteOrderedPartitioner");
+ properties = new WithProperties().set(TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST, true);
SchemaLoader.loadSchema();
StorageService.instance.initServer();
}
@@ -72,7 +79,7 @@
public static void teardown() throws Exception
{
SchemaLoader.cleanupAndLeaveDirs();
- System.clearProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST);
+ properties.close();
}
@Test
@@ -129,7 +136,7 @@
String corruptStatsTable = "corruptStatsTable";
createAndPopulateTable(keyspaceName, corruptStatsTable, cfs -> {
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.STATS), "rw"))
+ try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.fileFor(Components.STATS).toJavaIOFile(), "rw"))
{
file.seek(0);
file.writeBytes(StringUtils.repeat('z', 2));
@@ -150,8 +157,8 @@
createAndPopulateTable(keyspaceName, corruptDataTable, cfs -> {
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
- long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
+ long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ);
+ long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ);
long startPosition = Math.min(row0Start, row1Start);
try (RandomAccessFile file = new RandomAccessFile(sstable.getFilename(), "rw"))
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java b/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java
index 9d8f797..f94a5d6 100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java
@@ -22,6 +22,7 @@
import org.junit.Test;
+import org.apache.cassandra.io.sstable.VerifyTest;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
import org.hamcrest.CoreMatchers;
@@ -33,7 +34,7 @@
* Note: the complete coverage is composed of:
* - {@link StandaloneVerifierOnSSTablesTest}
* - {@link StandaloneVerifierTest}
- * - {@link org.apache.cassandra.db.VerifyTest}
+ * - {@link VerifyTest}
*/
public class StandaloneVerifierTest extends OfflineToolUtils
{
@@ -219,4 +220,4 @@
assertEquals(1, tool.getExitCode());
});
}
-}
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/tools/ToolRunner.java b/test/unit/org/apache/cassandra/tools/ToolRunner.java
index e3b9595..d3b9f10 100644
--- a/test/unit/org/apache/cassandra/tools/ToolRunner.java
+++ b/test/unit/org/apache/cassandra/tools/ToolRunner.java
@@ -402,7 +402,7 @@
{
String raw = getStderr();
String cleaned = getCleanedStderr();
- assertTrue("Failed to clean stderr completely.\nRaw (length=" + raw.length() + "):\n" + raw +
+ assertTrue("Failed to clean stderr completely.\nRaw (length=" + raw.length() + "):\n" + raw +
"\nCleaned (length=" + cleaned.length() + "):\n" + cleaned,
cleaned.trim().isEmpty());
}
diff --git a/test/unit/org/apache/cassandra/tools/ToolsSchemaLoadingTest.java b/test/unit/org/apache/cassandra/tools/ToolsSchemaLoadingTest.java
index 1a99643..d238327 100644
--- a/test/unit/org/apache/cassandra/tools/ToolsSchemaLoadingTest.java
+++ b/test/unit/org/apache/cassandra/tools/ToolsSchemaLoadingTest.java
@@ -34,7 +34,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -49,7 +49,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -64,7 +64,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("No sstables to split"));
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -79,7 +79,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@@ -94,7 +94,7 @@
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
assertEquals(1, tool.getExitCode());
- assertNoUnexpectedThreadsStarted(null);
+ assertNoUnexpectedThreadsStarted(null, false);
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
diff --git a/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java b/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
index 975ad15..d9cfdee 100644
--- a/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
+++ b/test/unit/org/apache/cassandra/tools/TopPartitionsTest.java
@@ -18,16 +18,20 @@
package org.apache.cassandra.tools;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.management.openmbean.CompositeData;
import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,18 +40,30 @@
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.metrics.Sampler;
+import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import static java.lang.String.format;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+/**
+ * Includes test cases for both the 'toppartitions' command and its successor 'profileload'
+ */
public class TopPartitionsTest
{
+ public static String KEYSPACE = TopPartitionsTest.class.getSimpleName().toLowerCase();
+ public static String TABLE = "test";
+
@BeforeClass
public static void loadSchema() throws ConfigurationException
{
SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1));
+ executeInternal(format("CREATE TABLE %s.%s (k text, c text, v text, PRIMARY KEY (k, c))", KEYSPACE, TABLE));
}
@Test
@@ -59,7 +75,7 @@
{
try
{
- q.put(StorageService.instance.samplePartitions(1000, 100, 10, Lists.newArrayList("READS", "WRITES")));
+ q.put(StorageService.instance.samplePartitions(null, 1000, 100, 10, Lists.newArrayList("READS", "WRITES")));
}
catch (Exception e)
{
@@ -82,4 +98,132 @@
List<CompositeData> result = ColumnFamilyStore.getIfExists("system", "local").finishLocalSampling("READS", 5);
assertEquals("If this failed you probably have to raise the beginLocalSampling duration", 1, result.size());
}
+
+ @Test
+ public void testTopPartitionsRowTombstoneAndSSTableCount() throws Exception
+ {
+ int count = 10;
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ cfs.disableAutoCompaction();
+
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'a', 'a')", KEYSPACE, TABLE));
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'b', 'a')", KEYSPACE, TABLE));
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('a', 'c', 'a')", KEYSPACE, TABLE));
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('b', 'b', 'b')", KEYSPACE, TABLE));
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'c', 'c')", KEYSPACE, TABLE));
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'd', 'a')", KEYSPACE, TABLE));
+ executeInternal(format("INSERT INTO %s.%s(k,c,v) VALUES ('c', 'e', 'a')", KEYSPACE, TABLE));
+ executeInternal(format("DELETE FROM %s.%s WHERE k='a' AND c='a'", KEYSPACE, TABLE));
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+
+ // test multi-partition read
+ cfs.beginLocalSampling("READ_ROW_COUNT", count, 240000);
+ cfs.beginLocalSampling("READ_TOMBSTONE_COUNT", count, 240000);
+ cfs.beginLocalSampling("READ_SSTABLE_COUNT", count, 240000);
+
+ executeInternal(format("SELECT * FROM %s.%s", KEYSPACE, TABLE));
+ Thread.sleep(2000); // simulate waiting before finishing sampling
+
+ List<CompositeData> rowCounts = cfs.finishLocalSampling("READ_ROW_COUNT", count);
+ List<CompositeData> tsCounts = cfs.finishLocalSampling("READ_TOMBSTONE_COUNT", count);
+ List<CompositeData> sstCounts = cfs.finishLocalSampling("READ_SSTABLE_COUNT", count);
+
+ assertEquals(0, sstCounts.size()); // not tracked on range reads
+ assertEquals(3, rowCounts.size()); // 3 partitions read (a, b, c)
+ assertEquals(1, tsCounts.size()); // 1 partition w tombstones (a)
+
+ for (CompositeData data : rowCounts)
+ {
+ String partitionKey = (String) data.get("value");
+ long numRows = (long) data.get("count");
+ if (partitionKey.equalsIgnoreCase("a"))
+ {
+ assertEquals(2, numRows);
+ }
+ else if (partitionKey.equalsIgnoreCase("b"))
+ assertEquals(1, numRows);
+ else if (partitionKey.equalsIgnoreCase("c"))
+ assertEquals(3, numRows);
+ }
+
+ assertEquals("a", tsCounts.get(0).get("value"));
+ assertEquals(1, (long) tsCounts.get(0).get("count"));
+
+ // test single partition read
+ cfs.beginLocalSampling("READ_ROW_COUNT", count, 240000);
+ cfs.beginLocalSampling("READ_TOMBSTONE_COUNT", count, 240000);
+ cfs.beginLocalSampling("READ_SSTABLE_COUNT", count, 240000);
+
+ executeInternal(format("SELECT * FROM %s.%s WHERE k='a'", KEYSPACE, TABLE));
+ executeInternal(format("SELECT * FROM %s.%s WHERE k='b'", KEYSPACE, TABLE));
+ executeInternal(format("SELECT * FROM %s.%s WHERE k='c'", KEYSPACE, TABLE));
+ Thread.sleep(2000); // simulate waiting before finishing sampling
+
+ rowCounts = cfs.finishLocalSampling("READ_ROW_COUNT", count);
+ tsCounts = cfs.finishLocalSampling("READ_TOMBSTONE_COUNT", count);
+ sstCounts = cfs.finishLocalSampling("READ_SSTABLE_COUNT", count);
+
+ assertEquals(3, sstCounts.size()); // 3 partitions read
+ assertEquals(3, rowCounts.size()); // 3 partitions read
+ assertEquals(1, tsCounts.size()); // 3 partitions read only one containing tombstones
+
+ for (CompositeData data : sstCounts)
+ {
+ String partitionKey = (String) data.get("value");
+ long numRows = (long) data.get("count");
+ if (partitionKey.equalsIgnoreCase("a"))
+ {
+ assertEquals(2, numRows);
+ }
+ else if (partitionKey.equalsIgnoreCase("b"))
+ assertEquals(1, numRows);
+ else if (partitionKey.equalsIgnoreCase("c"))
+ assertEquals(1, numRows);
+ }
+
+ for (CompositeData data : rowCounts)
+ {
+ String partitionKey = (String) data.get("value");
+ long numRows = (long) data.get("count");
+ if (partitionKey.equalsIgnoreCase("a"))
+ {
+ assertEquals(2, numRows);
+ }
+ else if (partitionKey.equalsIgnoreCase("b"))
+ assertEquals(1, numRows);
+ else if (partitionKey.equalsIgnoreCase("c"))
+ assertEquals(3, numRows);
+ }
+
+ assertEquals("a", tsCounts.get(0).get("value"));
+ assertEquals(1, (long) tsCounts.get(0).get("count"));
+ }
+
+ @Test
+ public void testStartAndStopScheduledSampling()
+ {
+ List<String> allSamplers = Arrays.stream(Sampler.SamplerType.values()).map(Enum::toString).collect(Collectors.toList());
+ StorageService ss = StorageService.instance;
+
+ assertTrue("Scheduling new sampled tasks should be allowed",
+ ss.startSamplingPartitions(null, null, 10, 10, 100, 10, allSamplers));
+
+ assertEquals(Collections.singletonList("*.*"), ss.getSampleTasks());
+
+ assertFalse("Sampling with duplicate keys should be disallowed",
+ ss.startSamplingPartitions(null, null, 20, 20, 100, 10, allSamplers));
+
+ assertTrue("Existing scheduled sampling tasks should be cancellable", ss.stopSamplingPartitions(null, null));
+
+ int timeout = 10;
+ while (timeout-- > 0 && ss.getSampleTasks().size() > 0)
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+
+ assertEquals("Scheduled sampled tasks should be removed", Collections.emptyList(), ss.getSampleTasks());
+
+ assertTrue("When nothing is scheduled, you should be able to stop all scheduled sampling tasks",
+ ss.stopSamplingPartitions(null, null));
+ }
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
index 0e172f3..379d02a 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
@@ -19,7 +19,14 @@
package org.apache.cassandra.tools.nodetool;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
import javax.management.openmbean.TabularData;
import org.junit.AfterClass;
@@ -27,19 +34,34 @@
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.tools.ToolRunner.ToolResult;
+import static java.lang.String.format;
+import static java.time.temporal.ChronoUnit.HOURS;
+import static java.time.temporal.ChronoUnit.SECONDS;
+import static java.util.Collections.emptyMap;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAllDataFileLocations;
+import static org.apache.cassandra.tools.ToolRunner.invokeNodetool;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertTrue;
public class ClearSnapshotTest extends CQLTester
{
+ private static final Pattern DASH_PATTERN = Pattern.compile("-");
private static NodeProbe probe;
@BeforeClass
public static void setup() throws Exception
{
startJMXServer();
+ requireNetwork();
probe = new NodeProbe(jmxHost, jmxPort);
}
@@ -50,62 +72,254 @@
}
@Test
- public void testClearSnapshot_NoArgs()
- {
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clearsnapshot");
- assertThat(tool.getExitCode()).isEqualTo(2);
- assertThat(tool.getCleanedStderr()).contains("Specify snapshot name or --all");
-
- tool = ToolRunner.invokeNodetool("clearsnapshot", "--all");
- tool.assertOnCleanExit();
- }
-
- @Test
- public void testClearSnapshot_AllAndName()
- {
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clearsnapshot", "-t", "some-name", "--all");
- assertThat(tool.getExitCode()).isEqualTo(2);
- assertThat(tool.getCleanedStderr()).contains("Specify only one of snapshot name or --all");
- }
-
- @Test
public void testClearSnapshot_RemoveByName()
{
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("snapshot","-t","some-name");
+ ToolResult tool = invokeNodetool("snapshot", "-t", "some-name");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).isNotEmpty();
-
- Map<String, TabularData> snapshots_before = probe.getSnapshotDetails();
+
+ Map<String, TabularData> snapshots_before = probe.getSnapshotDetails(emptyMap());
assertThat(snapshots_before).containsKey("some-name");
- tool = ToolRunner.invokeNodetool("clearsnapshot","-t","some-name");
+ tool = invokeNodetool("clearsnapshot", "-t", "some-name");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).isNotEmpty();
-
- Map<String, TabularData> snapshots_after = probe.getSnapshotDetails();
+
+ Map<String, TabularData> snapshots_after = probe.getSnapshotDetails(emptyMap());
assertThat(snapshots_after).doesNotContainKey("some-name");
}
@Test
public void testClearSnapshot_RemoveMultiple()
{
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("snapshot","-t","some-name");
+ ToolResult tool = invokeNodetool("snapshot", "-t", "some-name");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).isNotEmpty();
- tool = ToolRunner.invokeNodetool("snapshot","-t","some-other-name");
+ tool = invokeNodetool("snapshot", "-t", "some-other-name");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).isNotEmpty();
- Map<String, TabularData> snapshots_before = probe.getSnapshotDetails();
+ Map<String, TabularData> snapshots_before = probe.getSnapshotDetails(emptyMap());
assertThat(snapshots_before).hasSize(2);
- tool = ToolRunner.invokeNodetool("clearsnapshot","--all");
+ tool = invokeNodetool("clearsnapshot", "--all");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).isNotEmpty();
-
- Map<String, TabularData> snapshots_after = probe.getSnapshotDetails();
+
+ Map<String, TabularData> snapshots_after = probe.getSnapshotDetails(emptyMap());
assertThat(snapshots_after).isEmpty();
}
-
+
+ @Test
+ public void testClearSnapshotWithOlderThanFlag() throws Throwable
+ {
+ Instant start = Instant.ofEpochMilli(currentTimeMillis());
+ prepareData(start);
+
+ // wait 10 seconds for the sake of the test
+ await().timeout(15, TimeUnit.SECONDS).until(() -> Instant.now().isAfter(start.plusSeconds(10)));
+
+ // clear all snapshots for specific keyspace older than 3 hours for a specific keyspace
+ invokeNodetool("clearsnapshot", "--older-than", "3h", "--all", "--", KEYSPACE).assertOnCleanExit();
+
+ await().until(() -> {
+ String output = invokeNodetool("listsnapshots").getStdout();
+ return !output.contains("snapshot-to-clear-ks1-tb1") &&
+ output.contains("some-other-snapshot-ks1-tb1") &&
+ output.contains("last-snapshot-ks1-tb1") &&
+ output.contains("snapshot-to-clear-ks2-tb2") &&
+ output.contains("some-other-snapshot-ks2-tb2") &&
+ output.contains("last-snapshot-ks2-tb2");
+ });
+
+ // clear all snapshots older than 2 hours for all keyspaces
+ invokeNodetool("clearsnapshot", "--older-than", "2h", "--all").assertOnCleanExit();
+
+ await().until(() -> {
+ String output = invokeNodetool("listsnapshots").getStdout();
+
+ return !output.contains("some-other-snapshot-ks1-tb1") &&
+ output.contains("last-snapshot-ks1-tb1") &&
+ !output.contains("snapshot-to-clear-ks2-tb2") &&
+ !output.contains("some-other-snapshot-ks2-tb2") &&
+ output.contains("last-snapshot-ks2-tb2");
+ });
+
+ // clear all snapshosts older than 1 second
+ invokeNodetool("clearsnapshot", "--older-than", "1s", "--all", "--", currentKeyspace()).assertOnCleanExit();
+
+ await().until(() -> {
+ String output = invokeNodetool("listsnapshots").getStdout();
+ return output.contains("last-snapshot-ks1-tb1") &&
+ !output.contains("last-snapshot-ks2-tb2");
+ });
+
+ invokeNodetool("clearsnapshot", "--older-than", "1s", "--all").assertOnCleanExit();
+ await().until(() -> !invokeNodetool("listsnapshots").getStdout().contains("last-snapshot-ks1-tb1"));
+ }
+
+
+ @Test
+ public void testClearSnapshotWithOlderThanTimestampFlag() throws Throwable
+ {
+ Instant start = Instant.ofEpochMilli(currentTimeMillis());
+ prepareData(start);
+
+ // wait 10 seconds for the sake of the test
+ await().timeout(15, TimeUnit.SECONDS).until(() -> Instant.now().isAfter(start.plusSeconds(10)));
+
+ // clear all snapshots for specific keyspace older than 3 hours for a specific keyspace
+ invokeNodetool("clearsnapshot", "--older-than-timestamp",
+ Instant.now().minus(3, HOURS).toString(),
+ "--all", "--", KEYSPACE).assertOnCleanExit();
+
+ await().until(() -> {
+ String output = invokeNodetool("listsnapshots").getStdout();
+ return !output.contains("snapshot-to-clear-ks1-tb1") &&
+ output.contains("some-other-snapshot-ks1-tb1") &&
+ output.contains("last-snapshot-ks1-tb1") &&
+ output.contains("snapshot-to-clear-ks2-tb2") &&
+ output.contains("some-other-snapshot-ks2-tb2") &&
+ output.contains("last-snapshot-ks2-tb2");
+ });
+
+ // clear all snapshots older than 2 hours for all keyspaces
+ invokeNodetool("clearsnapshot", "--older-than-timestamp",
+ Instant.now().minus(2, HOURS).toString(),
+ "--all").assertOnCleanExit();
+
+ await().until(() -> {
+ String output = invokeNodetool("listsnapshots").getStdout();
+
+ return !output.contains("some-other-snapshot-ks1-tb1") &&
+ output.contains("last-snapshot-ks1-tb1") &&
+ !output.contains("snapshot-to-clear-ks2-tb2") &&
+ !output.contains("some-other-snapshot-ks2-tb2") &&
+ output.contains("last-snapshot-ks2-tb2");
+ });
+
+ // clear all snapshots older than now for all keyspaces
+ invokeNodetool("clearsnapshot", "--older-than-timestamp",
+ Instant.now().toString(),
+ "--all").assertOnCleanExit();
+
+ await().until(() -> {
+ String output = invokeNodetool("listsnapshots").getStdout();
+ return !output.contains("last-snapshot-ks1-tb1") &&
+ !output.contains("last-snapshot-ks2-tb2");
+ });
+ }
+
+ @Test
+ public void testIncompatibleFlags()
+ {
+ ToolResult invalidCommand1 = invokeNodetool("clearsnapshot",
+ "--older-than-timestamp", Instant.now().toString(),
+ "--older-than", "3h",
+ "--all");
+ invalidCommand1.asserts().failure();
+ assertTrue(invalidCommand1.getStdout().contains("Specify only one of --older-than or --older-than-timestamp"));
+
+ ToolResult invalidCommand2 = invokeNodetool("clearsnapshot", "-t", "some-snapshot-tag", "--all");
+ invalidCommand2.asserts().failure();
+ assertTrue(invalidCommand2.getStdout().contains("Specify only one of snapshot name or --all"));
+
+ ToolResult invalidCommand3 = invokeNodetool("clearsnapshot", "--", "keyspace");
+ invalidCommand3.asserts().failure();
+ assertTrue(invalidCommand3.getStdout().contains("Specify snapshot name or --all"));
+
+ ToolResult invalidCommand4 = invokeNodetool("clearsnapshot",
+ "--older-than-timestamp", Instant.now().toString(),
+ "-t", "some-snapshot-tag");
+ invalidCommand4.asserts().failure();
+ assertTrue(invalidCommand4.getStdout().contains("Specifying snapshot name together with --older-than-timestamp flag is not allowed"));
+
+ ToolResult invalidCommand5 = invokeNodetool("clearsnapshot",
+ "--older-than", "3h",
+ "-t", "some-snapshot-tag");
+ invalidCommand5.asserts().failure();
+ assertTrue(invalidCommand5.getStdout().contains("Specifying snapshot name together with --older-than flag is not allowed"));
+
+ ToolResult invalidCommand6 = invokeNodetool("clearsnapshot",
+ "--older-than-timestamp", "123",
+ "--all", "--", "somekeyspace");
+ invalidCommand6.asserts().failure();
+ assertTrue(invalidCommand6.getStdout().contains("Parameter --older-than-timestamp has to be a valid instant in ISO format."));
+
+ ToolResult invalidCommand7 = invokeNodetool("clearsnapshot",
+ "--older-than", "3k",
+ "--all", "--", "somekeyspace");
+ invalidCommand7.asserts().failure();
+ assertTrue(invalidCommand7.getStdout().contains("Invalid duration: 3k"));
+ }
+
+ private void rewriteManifest(String tableId,
+ String[] dataDirs,
+ String keyspace,
+ String tableName,
+ String snapshotName,
+ Instant createdAt) throws Exception
+ {
+ Path manifestPath = findManifest(dataDirs, keyspace, tableId, tableName, snapshotName);
+ SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(new File(manifestPath));
+ SnapshotManifest manifestWithEphemeralFlag = new SnapshotManifest(manifest.files, null, createdAt, false);
+ manifestWithEphemeralFlag.serializeToJsonFile(new File(manifestPath));
+ }
+
+ private Path findManifest(String[] dataDirs, String keyspace, String tableId, String tableName, String snapshotName)
+ {
+ for (String dataDir : dataDirs)
+ {
+ Path manifest = Paths.get(dataDir)
+ .resolve(keyspace)
+ .resolve(format("%s-%s", tableName, tableId))
+ .resolve("snapshots")
+ .resolve(snapshotName)
+ .resolve("manifest.json");
+
+ if (Files.exists(manifest))
+ {
+ return manifest;
+ }
+ }
+
+ throw new IllegalStateException("Unable to find manifest!");
+ }
+
+ private void prepareData(Instant start) throws Throwable
+ {
+ String tableName = createTable(KEYSPACE, "CREATE TABLE %s (id int primary key)");
+ execute("INSERT INTO %s (id) VALUES (?)", 1);
+ flush(KEYSPACE);
+
+ String keyspace2 = createKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+ String tableName2 = createTable(keyspace2, "CREATE TABLE %s (id int primary key)");
+ execute(formatQuery(keyspace2, "INSERT INTO %s (id) VALUES (?)"), 1);
+ flush(keyspace2);
+
+ invokeNodetool("snapshot", "-t", "snapshot-to-clear-ks1-tb1", "-cf", tableName, "--", KEYSPACE).assertOnCleanExit();
+ invokeNodetool("snapshot", "-t", "some-other-snapshot-ks1-tb1", "-cf", tableName, "--", KEYSPACE).assertOnCleanExit();
+ invokeNodetool("snapshot", "-t", "last-snapshot-ks1-tb1", "-cf", tableName, "--", KEYSPACE).assertOnCleanExit();
+
+ invokeNodetool("snapshot", "-t", "snapshot-to-clear-ks2-tb2", "-cf", tableName2, "--", keyspace2).assertOnCleanExit();
+ invokeNodetool("snapshot", "-t", "some-other-snapshot-ks2-tb2", "-cf", tableName2, "--", keyspace2).assertOnCleanExit();
+ invokeNodetool("snapshot", "-t", "last-snapshot-ks2-tb2", "-cf", tableName2, "--", keyspace2).assertOnCleanExit();
+
+ Optional<TableMetadata> tableMetadata = Keyspace.open(KEYSPACE).getMetadata().tables.get(tableName);
+ Optional<TableMetadata> tableMetadata2 = Keyspace.open(keyspace2).getMetadata().tables.get(tableName2);
+
+ String tableId = DASH_PATTERN.matcher(tableMetadata.orElseThrow(() -> new IllegalStateException(format("no metadata found for %s.%s", KEYSPACE, tableName)))
+ .id.asUUID().toString()).replaceAll("");
+
+ String tableId2 = DASH_PATTERN.matcher(tableMetadata2.orElseThrow(() -> new IllegalStateException(format("no metadata found for %s.%s", keyspace2, tableName2)))
+ .id.asUUID().toString()).replaceAll("");
+
+ rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "snapshot-to-clear-ks1-tb1", start.minus(5, HOURS));
+ rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "some-other-snapshot-ks1-tb1", start.minus(2, HOURS));
+ rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "last-snapshot-ks1-tb1", start.minus(1, SECONDS));
+ rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "snapshot-to-clear-ks2-tb2", start.minus(5, HOURS));
+ rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "some-other-snapshot-ks2-tb2", start.minus(2, HOURS));
+ rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "last-snapshot-ks2-tb2", start.minus(1, SECONDS));
+ }
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java
index 5975f66..d39fc63 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java
@@ -42,6 +42,7 @@
import org.apache.cassandra.tools.ToolRunner;
import org.assertj.core.groups.Tuple;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
import static org.assertj.core.api.Assertions.assertThat;
public class ClientStatsTest
@@ -56,7 +57,7 @@
{
// Since we run EmbeddedCassandraServer, we need to manually associate JMX address; otherwise it won't start
int jmxPort = CQLTester.getAutomaticallyAllocatedPort(InetAddress.getLoopbackAddress());
- System.setProperty("cassandra.jmx.local.port", String.valueOf(jmxPort));
+ CASSANDRA_JMX_LOCAL_PORT.setInt(jmxPort);
cassandra = ServerTestUtils.startEmbeddedCassandraService();
cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/CompactTest.java b/test/unit/org/apache/cassandra/tools/nodetool/CompactTest.java
index 34059aa..928f885 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/CompactTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/CompactTest.java
@@ -35,6 +35,7 @@
@BeforeClass
public static void setup() throws Throwable
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/CompactionHistoryTest.java b/test/unit/org/apache/cassandra/tools/nodetool/CompactionHistoryTest.java
new file mode 100644
index 0000000..804b75a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/CompactionHistoryTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.tools.ToolRunner.ToolResult;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
+import static org.apache.cassandra.tools.ToolRunner.invokeNodetool;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class CompactionHistoryTest extends CQLTester
+{
+ @Parameter
+ public List<String> cmd;
+
+ @Parameter(1)
+ public String compactionType;
+
+ @Parameter(2)
+ public int systemTableRecord;
+
+ @Parameters(name = "{index}: cmd={0} compactionType={1} systemTableRecord={2}")
+ public static Collection<Object[]> data()
+ {
+ List<Object[]> result = new ArrayList<>();
+ result.add(new Object[]{ Lists.newArrayList("compact"), OperationType.MAJOR_COMPACTION.type, 1 });
+ result.add(new Object[]{ Lists.newArrayList("garbagecollect"), OperationType.GARBAGE_COLLECT.type, 10 });
+ result.add(new Object[]{ Lists.newArrayList("upgradesstables", "-a"), OperationType.UPGRADE_SSTABLES.type, 10 });
+ return result;
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ requireNetwork();
+ startJMXServer();
+ }
+
+ @Test
+ public void testCompactionProperties() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id text, value text, PRIMARY KEY ((id)))");
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+ cfs.disableAutoCompaction();
+ // write SSTables for the specific key
+ for (int i = 0; i < 10; i++)
+ {
+ execute("INSERT INTO %s (id, value) VALUES (?, ?)", "key" + i, "value" + i);
+ flush(keyspace());
+ }
+
+ int expectedSSTablesCount = 10;
+ assertThat(cfs.getTracker().getView().liveSSTables()).hasSize(expectedSSTablesCount);
+
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ List<String> cmds = builder.addAll(cmd).add(keyspace()).add(currentTable()).build();
+ compactionHistoryResultVerify(keyspace(), currentTable(), ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType), cmds);
+
+ String cql = "select keyspace_name,columnfamily_name,compaction_properties from system." + SystemKeyspace.COMPACTION_HISTORY +
+ " where keyspace_name = '" + keyspace() + "' AND columnfamily_name = '" + currentTable() + "' ALLOW FILTERING";
+ Object[][] objects = new Object[systemTableRecord][];
+ for (int i = 0; i != systemTableRecord; ++i)
+ {
+ objects[i] = row(keyspace(), currentTable(), ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType));
+ }
+ assertRows(execute(cql), objects);
+ }
+
+ private void compactionHistoryResultVerify(String keyspace, String table, Map<String, String> properties, List<String> cmds)
+ {
+ ToolResult toolCompact = invokeNodetool(cmds);
+ toolCompact.assertOnCleanExit();
+
+ ToolResult toolHistory = invokeNodetool("compactionhistory");
+ toolHistory.assertOnCleanExit();
+ assertCompactionHistoryOutPut(toolHistory, keyspace, table, properties);
+ }
+
+ public static void assertCompactionHistoryOutPut(ToolResult toolHistory, String keyspace, String table, Map<String, String> properties)
+ {
+ String stdout = toolHistory.getStdout();
+ String[] resultArray = stdout.split(System.lineSeparator());
+ assertTrue(Arrays.stream(resultArray)
+ .anyMatch(result -> result.contains('{' + FBUtilities.toString(properties) + '}')
+ && result.contains(keyspace)
+ && result.contains(table)));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
index b42a166..a626dae 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.tools.nodetool;
import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -35,6 +36,7 @@
import org.apache.cassandra.tools.ToolRunner;
import org.apache.cassandra.utils.TimeUUID;
import org.assertj.core.api.Assertions;
+import org.awaitility.Awaitility;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.assertj.core.api.Assertions.assertThat;
@@ -105,8 +107,8 @@
long bytesTotal = 123456;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
- .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
- .collect(Collectors.toList());
+ .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
+ .collect(Collectors.toList());
CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
{
public CompactionInfo getCompactionInfo()
@@ -121,21 +123,15 @@
};
CompactionManager.instance.active.beginCompaction(compactionHolder);
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("compactionstats");
- tool.assertOnCleanExit();
- String stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 1");
+ String stdout = waitForNumberOfPendingTasks(1, "compactionstats");
Assertions.assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress");
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%",
- compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), bytesCompacted, bytesTotal,
- CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
+ compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), bytesCompacted, bytesTotal,
+ CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
Assertions.assertThat(stdout).containsPattern(expectedStatsPattern);
CompactionManager.instance.active.finishCompaction(compactionHolder);
- tool = ToolRunner.invokeNodetool("compactionstats");
- tool.assertOnCleanExit();
- stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 0");
+ waitForNumberOfPendingTasks(0, "compactionstats");
}
@Test
@@ -148,13 +144,27 @@
long bytesTotal = 123456;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
- .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
- .collect(Collectors.toList());
+ .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
+ .collect(Collectors.toList());
+ String targetDirectory = "/some/dir/" + cfs.metadata.keyspace + '/' + cfs.metadata.name + '-' + cfs.metadata.id.asUUID();
CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
{
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
+ return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+
+ CompactionInfo.Holder nonCompactionHolder = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
}
public boolean isGlobal()
@@ -164,21 +174,23 @@
};
CompactionManager.instance.active.beginCompaction(compactionHolder);
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("compactionstats", "-V");
- tool.assertOnCleanExit();
- String stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 1");
- Assertions.assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit");
- String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
- CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
- OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES);
+ CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
+ String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V");
+ Assertions.assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
+ String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+ CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+ OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES,
+ targetDirectory);
Assertions.assertThat(stdout).containsPattern(expectedStatsPattern);
+ String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+ CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+ OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES);
+ Assertions.assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
+
CompactionManager.instance.active.finishCompaction(compactionHolder);
- tool = ToolRunner.invokeNodetool("compactionstats", "-V");
- tool.assertOnCleanExit();
- stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 0");
+ CompactionManager.instance.active.finishCompaction(nonCompactionHolder);
+ waitForNumberOfPendingTasks(0, "compactionstats", "-V");
}
@Test
@@ -191,8 +203,8 @@
long bytesTotal = 123456;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
- .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
- .collect(Collectors.toList());
+ .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
+ .collect(Collectors.toList());
CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
{
public CompactionInfo getCompactionInfo()
@@ -207,21 +219,15 @@
};
CompactionManager.instance.active.beginCompaction(compactionHolder);
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("compactionstats", "--human-readable");
- tool.assertOnCleanExit();
- String stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 1");
+ String stdout = waitForNumberOfPendingTasks(1, "compactionstats", "--human-readable");
Assertions.assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress");
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%",
- compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), "123 bytes", "120.56 KiB",
- CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
+ compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), "123 bytes", "120.56 KiB",
+ CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100);
Assertions.assertThat(stdout).containsPattern(expectedStatsPattern);
CompactionManager.instance.active.finishCompaction(compactionHolder);
- tool = ToolRunner.invokeNodetool("compactionstats", "--human-readable");
- tool.assertOnCleanExit();
- stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 0");
+ waitForNumberOfPendingTasks(0, "compactionstats", "--human-readable");
}
@Test
@@ -234,13 +240,27 @@
long bytesTotal = 123456;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
- .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
- .collect(Collectors.toList());
+ .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
+ .collect(Collectors.toList());
+ String targetDirectory = "/some/dir/" + cfs.metadata.keyspace + '/' + cfs.metadata.name + '-' + cfs.metadata.id.asUUID();
CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder()
{
public CompactionInfo getCompactionInfo()
{
- return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
+ return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
+ }
+
+ public boolean isGlobal()
+ {
+ return false;
+ }
+ };
+
+ CompactionInfo.Holder nonCompactionHolder = new CompactionInfo.Holder()
+ {
+ public CompactionInfo getCompactionInfo()
+ {
+ return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
}
public boolean isGlobal()
@@ -250,20 +270,35 @@
};
CompactionManager.instance.active.beginCompaction(compactionHolder);
- ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("compactionstats", "--vtable", "--human-readable");
- tool.assertOnCleanExit();
- String stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 1");
- Assertions.assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit");
- String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
- CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
- OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES);
+ CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
+ String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable");
+ Assertions.assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
+ String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+ CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+ OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES,
+ targetDirectory);
Assertions.assertThat(stdout).containsPattern(expectedStatsPattern);
+ String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
+ CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
+ OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES);
+ Assertions.assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
CompactionManager.instance.active.finishCompaction(compactionHolder);
- tool = ToolRunner.invokeNodetool("compactionstats", "--vtable", "--human-readable");
- tool.assertOnCleanExit();
- stdout = tool.getStdout();
- assertThat(stdout).contains("pending tasks: 0");
+ CompactionManager.instance.active.finishCompaction(nonCompactionHolder);
+ waitForNumberOfPendingTasks(0, "compactionstats", "--vtable", "--human-readable");
}
-}
+
+ private String waitForNumberOfPendingTasks(int pendingTasksToWaitFor, String... args)
+ {
+ AtomicReference<String> stdout = new AtomicReference<>();
+ Awaitility.await().until(() -> {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(args);
+ tool.assertOnCleanExit();
+ String output = tool.getStdout();
+ stdout.set(output);
+ return output.contains("pending tasks: " + pendingTasksToWaitFor);
+ });
+
+ return stdout.get();
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ForceCompactionTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ForceCompactionTest.java
new file mode 100644
index 0000000..04d369e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/ForceCompactionTest.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class ForceCompactionTest extends CQLTester
+{
+ private final static int NUM_PARTITIONS = 10;
+ private final static int NUM_ROWS = 100;
+
+ @Before
+ public void setup() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key text, c1 text, c2 text, c3 text, PRIMARY KEY (key, c1))");
+
+ for (int partitionCount = 0; partitionCount < NUM_PARTITIONS; partitionCount++)
+ {
+ for (int rowCount = 0; rowCount < NUM_ROWS; rowCount++)
+ {
+ execute("INSERT INTO %s (key, c1, c2, c3) VALUES (?, ?, ?, ?)",
+ "k" + partitionCount, "c1_" + rowCount, "c2_" + rowCount, "c3_" + rowCount);
+ }
+ }
+
+ // Disable auto compaction
+ // NOTE: We can only disable the auto compaction once the table is created because the setting is on
+ // the table level. And we don't need to re-enable it back because the table will be dropped after the test.
+ disableCompaction();
+ }
+
+ @Test
+ public void forceCompactPartitionTombstoneTest() throws Throwable
+ {
+ String keyToPurge = "k0";
+
+ testHelper("DELETE FROM %s WHERE key = ?", keyToPurge);
+ }
+
+ @Test
+ public void forceCompactMultiplePartitionsTombstoneTest() throws Throwable
+ {
+ List<String> keysToPurge = new ArrayList<>();
+ Random rand = new Random();
+
+ int numPartitionsToPurge = 1 + rand.nextInt(NUM_PARTITIONS);
+ for (int count = 0; count < numPartitionsToPurge; count++)
+ {
+ String key = "k" + rand.nextInt(NUM_PARTITIONS);
+
+ execute("DELETE FROM %s WHERE key = ?", key);
+ keysToPurge.add(key);
+ }
+
+ flush();
+
+ String[] keys = new String[keysToPurge.size()];
+ keys = keysToPurge.toArray(keys);
+ forceCompact(keys);
+
+ verifyNotContainsTombstones();
+ }
+
+ @Test
+ public void forceCompactRowTombstoneTest() throws Throwable
+ {
+ String keyToPurge = "k0";
+
+ testHelper("DELETE FROM %s WHERE key = ? AND c1 = 'c1_0'", keyToPurge);
+ }
+
+ @Test
+ public void forceCompactMultipleRowsTombstoneTest() throws Throwable
+ {
+ List<String> keysToPurge = new ArrayList<>();
+
+ Random randPartition = new Random();
+ Random randRow = new Random();
+
+ for (int count = 0; count < 10; count++)
+ {
+ String partitionKey = "k" + randPartition.nextInt(NUM_PARTITIONS);
+ String clusteringKey = "c1_" + randRow.nextInt(NUM_ROWS);
+
+ execute("DELETE FROM %s WHERE key = ? AND c1 = ?", partitionKey, clusteringKey);
+ keysToPurge.add(partitionKey);
+ }
+
+ flush();
+
+ String[] keys = new String[keysToPurge.size()];
+ keys = keysToPurge.toArray(keys);
+ forceCompact(keys);
+
+ verifyNotContainsTombstones();
+ }
+
+ @Test
+ public void forceCompactCellTombstoneTest() throws Throwable
+ {
+ String keyToPurge = "k0";
+ testHelper("DELETE c2 FROM %s WHERE key = ? AND c1 = 'c1_0'", keyToPurge);
+ }
+
+ @Test
+ public void forceCompactMultipleCellsTombstoneTest() throws Throwable
+ {
+ List<String> keysToPurge = new ArrayList<>();
+
+ Random randPartition = new Random();
+ Random randRow = new Random();
+
+ for (int count = 0; count < 10; count++)
+ {
+ String partitionKey = "k" + randPartition.nextInt(NUM_PARTITIONS);
+ String clusteringKey = "c1_" + randRow.nextInt(NUM_ROWS);
+
+ execute("DELETE c2, c3 FROM %s WHERE key = ? AND c1 = ?", partitionKey, clusteringKey);
+ keysToPurge.add(partitionKey);
+ }
+
+ flush();
+
+ String[] keys = new String[keysToPurge.size()];
+ keys = keysToPurge.toArray(keys);
+ forceCompact(keys);
+
+ verifyNotContainsTombstones();
+ }
+
+ @Test
+ public void forceCompactUpdateCellTombstoneTest() throws Throwable
+ {
+ String keyToPurge = "k0";
+ testHelper("UPDATE %s SET c2 = null WHERE key = ? AND c1 = 'c1_0'", keyToPurge);
+ }
+
+ @Test
+ public void forceCompactTTLExpiryTest() throws Throwable
+ {
+ int ttlSec = 2;
+ String keyToPurge = "k0";
+
+ execute("UPDATE %s USING TTL ? SET c2 = 'bbb' WHERE key = ? AND c1 = 'c1_0'", ttlSec, keyToPurge);
+
+ flush();
+
+ // Wait until the TTL has been expired
+ // NOTE: we double the wait time of the ttl to be on the safer side and avoid the flakiness of the test
+ Thread.sleep(ttlSec * 1000 * 2);
+
+ String[] keysToPurge = new String[]{keyToPurge};
+ forceCompact(keysToPurge);
+
+ verifyNotContainsTombstones();
+ }
+
+ @Test
+ public void forceCompactCompositePartitionKeysTest() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key1 text, key2 text, c1 text, c2 text, PRIMARY KEY ((key1, key2), c1))");
+
+ for (int partitionCount = 0; partitionCount < NUM_PARTITIONS; partitionCount++)
+ {
+ for (int rowCount = 0; rowCount < NUM_ROWS; rowCount++)
+ {
+ execute("INSERT INTO %s (key1, key2, c1, c2) VALUES (?, ?, ?, ?)",
+ "k1_" + partitionCount, "k2_" + partitionCount, "c1_" + rowCount, "c2_" + rowCount);
+ }
+ }
+
+ // Disable auto compaction
+ // NOTE: We can only disable the auto compaction once the table is created because the setting is on
+ // the table level. And we don't need to re-enable it back because the table will be dropped after the test.
+ disableCompaction();
+
+ String keyToPurge = "k1_0:k2_0";
+
+ execute("DELETE FROM %s WHERE key1 = 'k1_0' and key2 = 'k2_0'");
+
+ flush();
+
+ String[] keysToPurge = new String[]{keyToPurge};
+ forceCompact(keysToPurge);
+
+ verifyNotContainsTombstones();
+ }
+
+ private void testHelper(String cqlStatement, String keyToPurge) throws Throwable
+ {
+ execute(cqlStatement, keyToPurge);
+
+ flush();
+
+ String[] keysToPurge = new String[]{keyToPurge};
+ forceCompact(keysToPurge);
+
+ verifyNotContainsTombstones();
+ }
+
+ private void forceCompact(String[] partitionKeysIgnoreGcGrace)
+ {
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ if (cfs != null)
+ {
+ cfs.forceCompactionKeysIgnoringGcGrace(partitionKeysIgnoreGcGrace);
+ }
+ }
+
+ private void verifyNotContainsTombstones()
+ {
+ // Get sstables
+ ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+ Collection<SSTableReader> sstables = cfs.getLiveSSTables();
+
+ // always run a major compaction before calling this
+ assertTrue(sstables.size() == 1);
+
+ SSTableReader sstable = sstables.iterator().next();
+ int actualPurgedTombstoneCount = 0;
+ try (ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext())
+ {
+ try (UnfilteredRowIterator iter = scanner.next())
+ {
+ // Partition should be all alive
+ assertTrue(iter.partitionLevelDeletion().isLive());
+
+ while (iter.hasNext())
+ {
+ Unfiltered atom = iter.next();
+ if (atom.isRow())
+ {
+ Row r = (Row)atom;
+
+ // Row should be all alive
+ assertTrue(r.deletion().isLive());
+
+ // Cell should be alive as well
+ for (Cell c : r.cells())
+ {
+ assertFalse(c.isTombstone());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/GetAuditLogTest.java b/test/unit/org/apache/cassandra/tools/nodetool/GetAuditLogTest.java
index 069710b..b96187f 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/GetAuditLogTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/GetAuditLogTest.java
@@ -32,6 +32,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/GetAuthCacheConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/GetAuthCacheConfigTest.java
index 6afc179..90484e3 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/GetAuthCacheConfigTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/GetAuthCacheConfigTest.java
@@ -42,6 +42,7 @@
{
CQLTester.setUpClass();
CQLTester.requireAuthentication();
+ requireNetwork();
startJMXServer();
}
@@ -95,17 +96,17 @@
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("getauthcacheconfig");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("Required option '--cache-name' is missing"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("getauthcacheconfig", "--cache-name");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("Required values for option 'cache-name' not provided"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("getauthcacheconfig", "--cache-name", "wrong");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("Unknown cache name: wrong"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
}
@Test
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/GetFullQueryLogTest.java b/test/unit/org/apache/cassandra/tools/nodetool/GetFullQueryLogTest.java
index 61d34ec..dfc1099 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/GetFullQueryLogTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/GetFullQueryLogTest.java
@@ -38,6 +38,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateCredentialsCacheTest.java b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateCredentialsCacheTest.java
index 40d7ffb..d83c846 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateCredentialsCacheTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateCredentialsCacheTest.java
@@ -62,6 +62,7 @@
.newAuthenticator((EndPoint) null, null)
.initialResponse());
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateJmxPermissionsCacheTest.java b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateJmxPermissionsCacheTest.java
index f44a274..54f2de7 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateJmxPermissionsCacheTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateJmxPermissionsCacheTest.java
@@ -65,6 +65,7 @@
AuthCacheService.initializeAndRegisterCaches();
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateNetworkPermissionsCacheTest.java b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateNetworkPermissionsCacheTest.java
index c54e526..4122066 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateNetworkPermissionsCacheTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateNetworkPermissionsCacheTest.java
@@ -47,6 +47,7 @@
roleManager.createRole(AuthenticatedUser.SYSTEM_USER, ROLE_B, AuthTestUtils.getLoginRoleOptions());
AuthCacheService.initializeAndRegisterCaches();
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/InvalidatePermissionsCacheTest.java b/test/unit/org/apache/cassandra/tools/nodetool/InvalidatePermissionsCacheTest.java
index e6d2ba1..c8affb2 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/InvalidatePermissionsCacheTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/InvalidatePermissionsCacheTest.java
@@ -85,6 +85,7 @@
authorizer.grant(AuthenticatedUser.SYSTEM_USER, permissions, resource, ROLE_B);
}
+ requireNetwork();
startJMXServer();
}
@@ -185,44 +186,44 @@
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("No resource options allowed without a <role> being specified"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("invalidatepermissionscache", "role1");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("No resource options specified"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("invalidatepermissionscache", "role1", "--invalid-option");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("A single <role> is only supported / you have a typo in the resource options spelling"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("invalidatepermissionscache", "role1", "--all-tables");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("--all-tables option should be passed along with --keyspace option"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("invalidatepermissionscache", "role1", "--table", "t1");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("--table option should be passed along with --keyspace option"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("invalidatepermissionscache", "role1", "--function", "f[Int32Type]");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("--function option should be passed along with --functions-in-keyspace option"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("invalidatepermissionscache", "role1", "--functions-in-keyspace",
KEYSPACE, "--function", "f[x]");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout())
.isEqualTo(wrapByDefaultNodetoolMessage("An error was encountered when looking up function definition: Unable to find abstract-type class 'org.apache.cassandra.db.marshal.x'"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
}
@Test
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateRolesCacheTest.java b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateRolesCacheTest.java
index bb8fb92..08ba676 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/InvalidateRolesCacheTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/InvalidateRolesCacheTest.java
@@ -47,6 +47,7 @@
roleManager.createRole(AuthenticatedUser.SYSTEM_USER, ROLE_B, AuthTestUtils.getLoginRoleOptions());
AuthCacheService.initializeAndRegisterCaches();
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java
index f8bd530..7bddc9b 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java
@@ -117,7 +117,8 @@
InetAddressAndPort.getLocalHost(),
streamSummaries,
streamSummaries,
- State.COMPLETE);
+ State.COMPLETE,
+ null);
try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream out = new PrintStream(baos))
{
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java b/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java
index 00c8bba..28fffd9 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -155,7 +156,7 @@
{
// Bad KS
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("ring", "mockks");
- tool.assertOnCleanExit();
+ Assert.assertEquals(1, tool.getExitCode());
assertThat(tool.getStdout()).contains("The keyspace mockks, does not exist");
// Good KS
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java
new file mode 100644
index 0000000..a007ce7
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ScrubTest;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.tools.StandaloneScrubber;
+import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Throwables;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.SchemaLoader.counterCFMD;
+import static org.apache.cassandra.SchemaLoader.createKeyspace;
+import static org.apache.cassandra.SchemaLoader.getCompressionParameters;
+import static org.apache.cassandra.SchemaLoader.loadSchema;
+import static org.apache.cassandra.SchemaLoader.standardCFMD;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
+import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX1;
+import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX1_BYTEORDERED;
+import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX2;
+import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX2_BYTEORDERED;
+import static org.apache.cassandra.io.sstable.ScrubTest.CF_UUID;
+import static org.apache.cassandra.io.sstable.ScrubTest.COMPRESSION_CHUNK_LENGTH;
+import static org.apache.cassandra.io.sstable.ScrubTest.COUNTER_CF;
+import static org.apache.cassandra.io.sstable.ScrubTest.assertOrderedAll;
+import static org.apache.cassandra.io.sstable.ScrubTest.fillCF;
+import static org.apache.cassandra.io.sstable.ScrubTest.fillCounterCF;
+import static org.apache.cassandra.io.sstable.ScrubTest.overrideWithGarbage;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class ScrubToolTest
+{
+ private static final String CF = "scrub_tool_test";
+ private static final AtomicInteger seq = new AtomicInteger();
+
+ String ksName;
+ Keyspace keyspace;
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ loadSchema();
+ }
+
+ @Before
+ public void setup()
+ {
+ ksName = "scrub_test_" + seq.incrementAndGet();
+ createKeyspace(ksName,
+ KeyspaceParams.simple(1),
+ standardCFMD(ksName, CF),
+ counterCFMD(ksName, COUNTER_CF).compression(getCompressionParameters(COMPRESSION_CHUNK_LENGTH)),
+ standardCFMD(ksName, CF_UUID, 0, UUIDType.instance),
+ SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1, true),
+ SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2, true),
+ SchemaLoader.keysIndexCFMD(ksName, CF_INDEX1_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance),
+ SchemaLoader.compositeIndexCFMD(ksName, CF_INDEX2_BYTEORDERED, true).partitioner(ByteOrderedPartitioner.instance));
+ keyspace = Keyspace.open(ksName);
+
+ CompactionManager.instance.disableAutoCompaction();
+ TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.setBoolean(true);
+ }
+
+ @Test
+ public void testScrubOnePartitionWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
+ Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
+ tool.assertOnCleanExit();
+
+ // check data is still there
+ assertOrderedAll(cfs, 1);
+ }
+
+ @Test
+ public void testSkipScrubCorruptedCounterPartitionWithTool() throws IOException, WriteTimeoutException
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
+ int numPartitions = 1000;
+
+ fillCounterCF(cfs, numPartitions);
+ assertOrderedAll(cfs, numPartitions);
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ ScrubTest.overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x7A);
+
+ // with skipCorrupted == true, the corrupt rows will be skipped
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-s", ksName, COUNTER_CF);
+ Assertions.assertThat(tool.getStdout()).contains("0 empty");
+ Assertions.assertThat(tool.getStdout()).contains("partitions that were skipped");
+ tool.assertOnCleanExit();
+
+ assertEquals(1, cfs.getLiveSSTables().size());
+ }
+
+ @Test
+ public void testNoSkipScrubCorruptedCounterPartitionWithTool() throws IOException, WriteTimeoutException
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
+ int numPartitions = 1000;
+
+ fillCounterCF(cfs, numPartitions);
+ assertOrderedAll(cfs, numPartitions);
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+ //use 0x00 instead of the usual 0x7A because if by any chance it's able to iterate over the corrupt
+ //section, then we get many out-of-order errors, which we don't want
+ overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x0);
+
+ // with skipCorrupted == false, the scrub is expected to fail
+ try
+ {
+ ToolRunner.invokeClass(StandaloneScrubber.class, ksName, COUNTER_CF);
+ fail("Expected a CorruptSSTableException to be thrown");
+ }
+ catch (IOError err)
+ {
+ Throwables.assertAnyCause(err, CorruptSSTableException.class);
+ }
+ }
+
+ @Test
+ public void testNoCheckScrubMultiPartitionWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 10);
+ assertOrderedAll(cfs, 10);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-n", ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
+ Assertions.assertThat(tool.getStdout()).contains("10 partitions in new sstable and 0 empty");
+ tool.assertOnCleanExit();
+
+ // check data is still there
+ assertOrderedAll(cfs, 10);
+ }
+
+ @Test
+ public void testHeaderFixValidateOnlyWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate_only", ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Not continuing with scrub, since '--header-fix validate-only' was specified.");
+ tool.assertOnCleanExit();
+ assertOrderedAll(cfs, 1);
+ }
+
+ @Test
+ public void testHeaderFixValidateWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate", ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
+ Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
+ tool.assertOnCleanExit();
+ assertOrderedAll(cfs, 1);
+ }
+
+ @Test
+ public void testHeaderFixFixOnlyWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix-only", ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Not continuing with scrub, since '--header-fix fix-only' was specified.");
+ tool.assertOnCleanExit();
+ assertOrderedAll(cfs, 1);
+ }
+
+ @Test
+ public void testHeaderFixWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix", ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
+ Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
+ tool.assertOnCleanExit();
+ assertOrderedAll(cfs, 1);
+ }
+
+ @Test
+ public void testHeaderFixNoChecksWithTool()
+ {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ fillCF(cfs, 1);
+ assertOrderedAll(cfs, 1);
+
+ ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "off", ksName, CF);
+ Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into");
+ Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty");
+ tool.assertOnCleanExit();
+ assertOrderedAll(cfs, 1);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAuthCacheConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAuthCacheConfigTest.java
index 2c90486..79329c6 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetAuthCacheConfigTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAuthCacheConfigTest.java
@@ -42,6 +42,7 @@
{
CQLTester.setUpClass();
CQLTester.requireAuthentication();
+ requireNetwork();
startJMXServer();
}
@@ -113,17 +114,17 @@
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("setauthcacheconfig");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("Required option '--cache-name' is missing"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("setauthcacheconfig", "--cache-name");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("Required values for option 'cache-name' not provided"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
tool = ToolRunner.invokeNodetool("setauthcacheconfig", "--cache-name", "wrong", "--validity-period", "1");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("Unknown cache name: wrong"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
}
@Test
@@ -132,7 +133,7 @@
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("setauthcacheconfig", "--cache-name", "PermissionCache");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("At least one optional parameter need to be passed"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
}
@Test
@@ -144,7 +145,7 @@
"--disable-active-update");
assertThat(tool.getExitCode()).isEqualTo(1);
assertThat(tool.getStdout()).isEqualTo(wrapByDefaultNodetoolMessage("enable-active-update and disable-active-update cannot be used together"));
- assertThat(tool.getStderr()).isEmpty();
+ assertThat(tool.getCleanedStderr()).isEmpty();
}
@Test
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetColumnIndexSizeTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetColumnIndexSizeTest.java
index 40f82b0..1a19f09 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetColumnIndexSizeTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetColumnIndexSizeTest.java
@@ -36,6 +36,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java
index d15cc0c..24ee9e5 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetCompactionThroughputTest.java
@@ -38,6 +38,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableInterDCStreamThroughputTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableInterDCStreamThroughputTest.java
index 4aea013..d6d3253 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableInterDCStreamThroughputTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableInterDCStreamThroughputTest.java
@@ -39,6 +39,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableStreamThroughputTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableStreamThroughputTest.java
index 028f32e..f250699 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableStreamThroughputTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetEntireSSTableStreamThroughputTest.java
@@ -39,6 +39,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetInterDCStreamThroughputTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetInterDCStreamThroughputTest.java
index 0ef6b2c..6cbb948 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetInterDCStreamThroughputTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetInterDCStreamThroughputTest.java
@@ -46,6 +46,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetGetStreamThroughputTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetGetStreamThroughputTest.java
index 911e0dc..cf688d6 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SetGetStreamThroughputTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SetGetStreamThroughputTest.java
@@ -47,6 +47,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SnapshotTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SnapshotTest.java
index 2fbaf95..9742972 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/SnapshotTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/SnapshotTest.java
@@ -36,6 +36,7 @@
@BeforeClass
public static void setup() throws Exception
{
+ requireNetwork();
startJMXServer();
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/TableStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/TableStatsTest.java
index 578bd88..6ba2151 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/TableStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/TableStatsTest.java
@@ -28,9 +28,9 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.utils.JsonUtils;
import org.yaml.snakeyaml.Yaml;
import static org.assertj.core.api.Assertions.assertThat;
@@ -111,7 +111,8 @@
" off_heap_memory_used_total, pending_flushes, percent_repaired,\n" +
" read_latency, reads, space_used_by_snapshots_total, space_used_live,\n" +
" space_used_total, sstable_compression_ratio, sstable_count,\n" +
- " table_name, write_latency, writes)\n" +
+ " table_name, write_latency, writes, max_sstable_size,\n" +
+ " local_read_write_ratio, twcs_max_duration)\n" +
"\n" +
" -t <top>, --top <top>\n" +
" Show only the top K tables for the sort key (specify the number K of\n" +
@@ -137,12 +138,12 @@
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("tablestats");
tool.assertOnCleanExit();
- assertThat(tool.getStdout()).contains("Keyspace : system_schema");
+ assertThat(tool.getStdout()).contains("Keyspace: system_schema");
assertThat(StringUtils.countMatches(tool.getStdout(), "Table:")).isGreaterThan(1);
tool = ToolRunner.invokeNodetool("tablestats", "system_distributed");
tool.assertOnCleanExit();
- assertThat(tool.getStdout()).contains("Keyspace : system_distributed");
+ assertThat(tool.getStdout()).contains("Keyspace: system_distributed");
assertThat(tool.getStdout()).doesNotContain("Keyspace : system_schema");
assertThat(StringUtils.countMatches(tool.getStdout(), "Table:")).isGreaterThan(1);
}
@@ -152,7 +153,7 @@
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("tablestats", "-i", "system_schema.aggregates");
tool.assertOnCleanExit();
- assertThat(tool.getStdout()).contains("Keyspace : system_schema");
+ assertThat(tool.getStdout()).contains("Keyspace: system_schema");
assertThat(tool.getStdout()).doesNotContain("Table: system_schema.aggregates");
assertThat(StringUtils.countMatches(tool.getStdout(), "Table:")).isGreaterThan(1);
}
@@ -234,7 +235,7 @@
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("tablestats", arg, "json");
tool.assertOnCleanExit();
String json = tool.getStdout();
- assertThatCode(() -> new ObjectMapper().readTree(json)).doesNotThrowAnyException();
+ assertThatCode(() -> JsonUtils.JSON_OBJECT_MAPPER.readTree(json)).doesNotThrowAnyException();
assertThat(json).containsPattern("\"sstable_count\"\\s*:\\s*[0-9]+")
.containsPattern("\"old_sstable_count\"\\s*:\\s*[0-9]+");
});
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java
index 124f297..e9548fb 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/TpStatsTest.java
@@ -29,13 +29,13 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.NoPayload;
import org.apache.cassandra.tools.ToolRunner;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JsonUtils;
import org.yaml.snakeyaml.Yaml;
import static org.apache.cassandra.net.Verb.ECHO_REQ;
@@ -163,8 +163,7 @@
{
try
{
- ObjectMapper mapper = new ObjectMapper();
- mapper.readTree(str);
+ JsonUtils.JSON_OBJECT_MAPPER.readTree(str);
return true;
}
catch(IOException e)
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/UninitializedServerTest.java b/test/unit/org/apache/cassandra/tools/nodetool/UninitializedServerTest.java
new file mode 100644
index 0000000..2ddd270
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/UninitializedServerTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.tools.ToolRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class UninitializedServerTest extends CQLTester
+{
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ startJMXServer();
+ }
+
+ @Test
+ public void testUnintializedServer()
+ {
+ // CASSANDRA-11537
+ // fails, not finished initializing node because test never calls requireNetwork()
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("status");
+ assertThat(tool.getException() instanceof IllegalArgumentException);
+ assertThat(tool.getStderr().contains("Server is not initialized yet, cannot run nodetool."));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java b/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java
index ea49899..1710395 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java
@@ -212,6 +212,11 @@
"table1 > table3 > table5 > table2 > table4 > table6",
humanReadable,
ascending);
+ runCompareTest(testTables,
+ "twcs_max_duration",
+ "table2 > table4 > table1 > table3 > table6 > table5",
+ humanReadable,
+ ascending);
}
@Test
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java
index 81687ba..0ddef9e 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinterTest.java
@@ -22,13 +22,16 @@
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
import org.junit.Test;
+import org.apache.cassandra.utils.JsonUtils;
import org.assertj.core.api.Assertions;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
public class TableStatsPrinterTest extends TableStatsTestBase
{
@@ -36,10 +39,11 @@
"\tTable: %s\n" +
"\tSSTable count: 60000\n" +
"\tOld SSTable count: 0\n" +
+ "\tMax SSTable size: 0.000KiB\n" +
"\tSpace used (live): 0\n" +
"\tSpace used (total): 9001\n" +
"\tSpace used by snapshots (total): 1111\n" +
- "\tSSTable Compression Ratio: 0.68\n" +
+ "\tSSTable Compression Ratio: 0.68000\n" +
"\tNumber of partitions (estimate): 111111\n" +
"\tMemtable cell count: 111\n" +
"\tMemtable data size: 0\n" +
@@ -48,6 +52,7 @@
"\tLocal read latency: 2.000 ms\n" +
"\tLocal write count: 5\n" +
"\tLocal write latency: 0.050 ms\n" +
+ "\tLocal read/write ratio: 0.00000\n" +
"\tPending flushes: 11111\n" +
"\tPercent repaired: 100.0\n" +
"\tBytes repaired: 0.000KiB\n" +
@@ -71,11 +76,12 @@
"\tTable: %s\n" +
"\tSSTable count: 3000\n" +
"\tOld SSTable count: 0\n" +
+ "\tMax SSTable size: 0.000KiB\n" +
"\tSpace used (live): 22\n" +
"\tSpace used (total): 1024\n" +
"\tSpace used by snapshots (total): 222\n" +
"\tOff heap memory used (total): 314159367\n" +
- "\tSSTable Compression Ratio: 0.68\n" +
+ "\tSSTable Compression Ratio: 0.68000\n" +
"\tNumber of partitions (estimate): 22222\n" +
"\tMemtable cell count: 22\n" +
"\tMemtable data size: 900\n" +
@@ -85,6 +91,7 @@
"\tLocal read latency: 3.000 ms\n" +
"\tLocal write count: 4\n" +
"\tLocal write latency: 0.000 ms\n" +
+ "\tLocal read/write ratio: 0.00000\n" +
"\tPending flushes: 222222\n" +
"\tPercent repaired: 99.9\n" +
"\tBytes repaired: 0.000KiB\n" +
@@ -111,10 +118,11 @@
"\tTable: %s\n" +
"\tSSTable count: 50000\n" +
"\tOld SSTable count: 0\n" +
+ "\tMax SSTable size: 0.000KiB\n" +
"\tSpace used (live): 0\n" +
"\tSpace used (total): 512\n" +
"\tSpace used by snapshots (total): 0\n" +
- "\tSSTable Compression Ratio: 0.32\n" +
+ "\tSSTable Compression Ratio: 0.32000\n" +
"\tNumber of partitions (estimate): 3333\n" +
"\tMemtable cell count: 333333\n" +
"\tMemtable data size: 1999\n" +
@@ -123,6 +131,7 @@
"\tLocal read latency: 4.000 ms\n" +
"\tLocal write count: 3\n" +
"\tLocal write latency: NaN ms\n" +
+ "\tLocal read/write ratio: 0.00000\n" +
"\tPending flushes: 333\n" +
"\tPercent repaired: 99.8\n" +
"\tBytes repaired: 0.000KiB\n" +
@@ -146,11 +155,12 @@
"\tTable: %s\n" +
"\tSSTable count: 2000\n" +
"\tOld SSTable count: 0\n" +
+ "\tMax SSTable size: 0.000KiB\n" +
"\tSpace used (live): 4444\n" +
"\tSpace used (total): 256\n" +
"\tSpace used by snapshots (total): 44\n" +
"\tOff heap memory used (total): 441213818\n" +
- "\tSSTable Compression Ratio: 0.95\n" +
+ "\tSSTable Compression Ratio: 0.95000\n" +
"\tNumber of partitions (estimate): 444\n" +
"\tMemtable cell count: 4\n" +
"\tMemtable data size: 3000\n" +
@@ -160,6 +170,7 @@
"\tLocal read latency: NaN ms\n" +
"\tLocal write count: 2\n" +
"\tLocal write latency: 2.000 ms\n" +
+ "\tLocal read/write ratio: 0.00000\n" +
"\tPending flushes: 4444\n" +
"\tPercent repaired: 50.0\n" +
"\tBytes repaired: 0.000KiB\n" +
@@ -186,10 +197,11 @@
"\tTable: %s\n" +
"\tSSTable count: 40000\n" +
"\tOld SSTable count: 0\n" +
+ "\tMax SSTable size: 0.000KiB\n" +
"\tSpace used (live): 55555\n" +
"\tSpace used (total): 64\n" +
"\tSpace used by snapshots (total): 55555\n" +
- "\tSSTable Compression Ratio: 0.99\n" +
+ "\tSSTable Compression Ratio: 0.99000\n" +
"\tNumber of partitions (estimate): 55\n" +
"\tMemtable cell count: 55555\n" +
"\tMemtable data size: 20000\n" +
@@ -198,6 +210,7 @@
"\tLocal read latency: 0.000 ms\n" +
"\tLocal write count: 1\n" +
"\tLocal write latency: 1.000 ms\n" +
+ "\tLocal read/write ratio: 0.00000\n" +
"\tPending flushes: 5\n" +
"\tPercent repaired: 93.0\n" +
"\tBytes repaired: 0.000KiB\n" +
@@ -221,11 +234,12 @@
"\tTable: %s\n" +
"\tSSTable count: 1000\n" +
"\tOld SSTable count: 0\n" +
+ "\tMax SSTable size: 0.000KiB\n" +
"\tSpace used (live): 666666\n" +
"\tSpace used (total): 0\n" +
"\tSpace used by snapshots (total): 0\n" +
"\tOff heap memory used (total): 162470810\n" +
- "\tSSTable Compression Ratio: 0.68\n" +
+ "\tSSTable Compression Ratio: 0.68000\n" +
"\tNumber of partitions (estimate): 6\n" +
"\tMemtable cell count: 6666\n" +
"\tMemtable data size: 1000000\n" +
@@ -235,6 +249,7 @@
"\tLocal read latency: 1.000 ms\n" +
"\tLocal write count: 0\n" +
"\tLocal write latency: 0.500 ms\n" +
+ "\tLocal read/write ratio: 0.00000\n" +
"\tPending flushes: 66\n" +
"\tPercent repaired: 0.0\n" +
"\tBytes repaired: 0.000KiB\n" +
@@ -263,9 +278,9 @@
* without leaking test implementation into the TableStatsHolder implementation.
*/
public static final String expectedDefaultPrinterOutput =
- "Total number of tables: 0\n" +
+ "Total number of tables: 6\n" +
"----------------\n" +
- "Keyspace : keyspace1\n" +
+ "Keyspace: keyspace1\n" +
"\tRead Count: 3\n" +
"\tRead Latency: 0.0 ms\n" +
"\tWrite Count: 12\n" +
@@ -275,7 +290,7 @@
String.format(duplicateTabs(expectedDefaultTable2Output), "table2") +
String.format(duplicateTabs(expectedDefaultTable3Output), "table3") +
"----------------\n" +
- "Keyspace : keyspace2\n" +
+ "Keyspace: keyspace2\n" +
"\tRead Count: 7\n" +
"\tRead Latency: 0.0 ms\n" +
"\tWrite Count: 3\n" +
@@ -284,7 +299,7 @@
String.format(duplicateTabs(expectedDefaultTable4Output), "table4") +
String.format(duplicateTabs(expectedDefaultTable5Output), "table5") +
"----------------\n" +
- "Keyspace : keyspace3\n" +
+ "Keyspace: keyspace3\n" +
"\tRead Count: 5\n" +
"\tRead Latency: 0.0 ms\n" +
"\tWrite Count: 0\n" +
@@ -341,7 +356,8 @@
@Test
public void testDefaultPrinter() throws Exception
{
- StatsHolder holder = new TestTableStatsHolder(testKeyspaces, "", 0);
+ TestTableStatsHolder holder = new TestTableStatsHolder(testKeyspaces, "", 0);
+ holder.numberOfTables = testKeyspaces.stream().map(ks -> ks.tables.size()).mapToInt(Integer::intValue).sum();
StatsPrinter<StatsHolder> printer = TableStatsPrinter.from("", false);
try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream())
{
@@ -393,12 +409,13 @@
Assertions.assertThat(byteStream.toString())
.isEqualTo("{\n" +
" \"keyspace3\" : {\n" +
- " \"write_latency_ms\" : \"NaN\",\n" +
+ " \"write_latency_ms\" : null,\n" +
" \"tables\" : {\n" +
" \"table6\" : {\n" +
" \"average_tombstones_per_slice_last_five_minutes\" : 6.0,\n" +
" \"top_tombstone_partitions\" : null,\n" +
" \"bloom_filter_off_heap_memory_used\" : \"667408\",\n" +
+ " \"twcs\" : null,\n" +
" \"bytes_pending_repair\" : 0,\n" +
" \"memtable_switch_count\" : 6,\n" +
" \"maximum_tombstones_per_slice_last_five_minutes\" : 6,\n" +
@@ -412,6 +429,7 @@
" \"compacted_partition_minimum_bytes\" : 6,\n" +
" \"local_read_count\" : 5,\n" +
" \"sstable_compression_ratio\" : 0.68,\n" +
+ " \"max_sstable_size\" : 0,\n" +
" \"dropped_mutations\" : \"666666\",\n" +
" \"top_size_partitions\" : null,\n" +
" \"bloom_filter_false_positives\" : 400,\n" +
@@ -426,6 +444,7 @@
" \"local_write_count\" : 0,\n" +
" \"droppable_tombstone_ratio\" : \"0.66667\",\n" +
" \"compression_metadata_off_heap_memory_used\" : \"1\",\n" +
+ " \"local_read_write_ratio\" : \"0.00000\",\n" +
" \"number_of_partitions_estimate\" : 6,\n" +
" \"bytes_repaired\" : 0,\n" +
" \"maximum_live_cells_per_slice_last_five_minutes\" : 2,\n" +
@@ -465,6 +484,7 @@
" average_tombstones_per_slice_last_five_minutes: 6.0\n" +
" top_tombstone_partitions: null\n" +
" bloom_filter_off_heap_memory_used: '667408'\n" +
+ " twcs: null\n" +
" bytes_pending_repair: 0\n" +
" memtable_switch_count: 6\n" +
" maximum_tombstones_per_slice_last_five_minutes: 6\n" +
@@ -478,6 +498,7 @@
" compacted_partition_minimum_bytes: 6\n" +
" local_read_count: 5\n" +
" sstable_compression_ratio: 0.68\n" +
+ " max_sstable_size: 0\n" +
" dropped_mutations: '666666'\n" +
" top_size_partitions: null\n" +
" bloom_filter_false_positives: 400\n" +
@@ -492,6 +513,7 @@
" local_write_count: 0\n" +
" droppable_tombstone_ratio: '0.66667'\n" +
" compression_metadata_off_heap_memory_used: '1'\n" +
+ " local_read_write_ratio: '0.00000'\n" +
" number_of_partitions_estimate: 6\n" +
" bytes_repaired: 0\n" +
" maximum_live_cells_per_slice_last_five_minutes: 2\n" +
@@ -512,6 +534,61 @@
}
}
+ @Test
+ public void testJsonPrinter2() throws Exception
+ {
+ final StatsPrinter<StatsHolder> printer = TableStatsPrinter.from("json", false);
+ StatsHolder holder = new TestTableStatsHolder(testKeyspaces, "reads", 0);
+ try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream())
+ {
+ printer.print(holder, new PrintStream(byteStream));
+ byte[] json = byteStream.toByteArray();
+ Map<String, Object> result = JsonUtils.fromJsonMap(json);
+ assertEquals(0, result.remove("total_number_of_tables"));
+ assertEquals(6, result.size());
+
+ // One relatively easy way to verify serialization is to check converted-to-Map
+ // intermediate form and round-trip (serialize-deserialize) for equivalence.
+ // But there are couple of minor gotchas to consider
+
+ Map<String, Object> expectedData = holder.convert2Map();
+ expectedData.remove("total_number_of_tables");
+ assertEquals(6, expectedData.size());
+
+ for (Map.Entry<String, Object> entry : result.entrySet())
+ {
+ Map<String, Object> expTable = (Map<String, Object>) expectedData.get(entry.getKey());
+ Map<String, Object> actualTable = (Map<String, Object>) entry.getValue();
+
+ assertEquals(expTable.size(), actualTable.size());
+
+ for (Map.Entry<String, Object> tableEntry : actualTable.entrySet())
+ {
+ Object expValue = expTable.get(tableEntry.getKey());
+ Object actualValue = tableEntry.getValue();
+
+ // Some differences to expect: Long that fits in Integer may get deserialized as latter:
+ if (expValue instanceof Long && actualValue instanceof Integer)
+ {
+ actualValue = ((Number) actualValue).longValue();
+ }
+
+ // And then a bit more exotic case: Not-a-Numbers should be coerced into nulls
+ // (existing behavior as of 4.0.0)
+ if (expValue instanceof Double && !Double.isFinite((Double) expValue))
+ {
+ assertNull("Entry '" + tableEntry.getKey() + "' of table '" + entry.getKey() + "' should be coerced from NaN to null:",
+ actualValue);
+ continue;
+ }
+ assertEquals("Entry '" + tableEntry.getKey() + "' of table '" + entry.getKey() + "' does not match",
+ expValue, actualValue);
+ }
+ }
+ }
+ }
+
+
/**
* A test version of TableStatsHolder to hold a test vector instead of gathering stats from a live cluster.
*/
@@ -531,5 +608,4 @@
return true;
}
}
-
}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java
index 8ef0ea3..ab49c6f 100644
--- a/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java
+++ b/test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java
@@ -71,6 +71,7 @@
template.oldSSTableCount = 0L;
template.spaceUsedLive = "0";
template.spaceUsedTotal = "0";
+ template.maxSSTableSize = 0L;
template.spaceUsedBySnapshotsTotal = "0";
template.percentRepaired = 1.0D;
template.bytesRepaired = 0L;
@@ -99,6 +100,8 @@
template.averageTombstonesPerSliceLastFiveMinutes = Double.NaN;
template.maximumTombstonesPerSliceLastFiveMinutes = 0L;
template.droppedMutations = "0";
+ template.twcs = null;
+ template.twcsDurationInMillis = 0L;
return template;
}
@@ -337,6 +340,10 @@
table2.memtableOffHeapMemoryUsed = "314159265";
table4.memtableOffHeapMemoryUsed = "141421356";
table6.memtableOffHeapMemoryUsed = "161803398";
+ // twcs max duration: 2 > 4 > 1 = 3 = 6 = 5
+ table2.twcsDurationInMillis = 2000L;
+ table4.twcsDurationInMillis = 1000L;
+ table5.twcsDurationInMillis = null;
// create test keyspaces from templates
testKeyspaces = new ArrayList<>();
StatsKeyspace keyspace1 = createStatsKeyspaceTemplate("keyspace1");
diff --git a/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java b/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
index 120de2a..8bc9f28 100644
--- a/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
+++ b/test/unit/org/apache/cassandra/transport/CQLUserAuditTest.java
@@ -49,6 +49,7 @@
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.EmbeddedCassandraService;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS;
import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQueue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -70,7 +71,7 @@
config.audit_logging_options.logger = new ParameterizedClass("DiagnosticEventAuditLogger", null);
});
- System.setProperty("cassandra.superuser_setup_delay_ms", "0");
+ SUPERUSER_SETUP_DELAY_MS.setLong(0);
embedded = ServerTestUtils.startEmbeddedCassandraService();
diff --git a/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java b/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java
new file mode 100644
index 0000000..0c70315
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.channel.Channel;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.metrics.ClientMetrics;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.messages.AuthResponse;
+
+public class MessageDispatcherTest
+{
+ static final Message.Request AUTH_RESPONSE_REQUEST = new AuthResponse(new byte[0])
+ {
+ public Response execute(QueryState queryState, long queryStartNanoTime, boolean traceRequest)
+ {
+ return null;
+ }
+ };
+
+ private static AuthTestDispatcher dispatch;
+ private static int maxAuthThreadsBeforeTests;
+
+ @BeforeClass
+ public static void init() throws Exception
+ {
+ DatabaseDescriptor.daemonInitialization();
+ ClientMetrics.instance.init(Collections.emptyList());
+ maxAuthThreadsBeforeTests = DatabaseDescriptor.getNativeTransportMaxAuthThreads();
+ dispatch = new AuthTestDispatcher();
+ }
+
+ @AfterClass
+ public static void restoreAuthSize()
+ {
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(maxAuthThreadsBeforeTests);
+ }
+
+ @Test
+ public void testAuthRateLimiter() throws Exception
+ {
+ long startRequests = completedRequests();
+
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(1);
+ long auths = tryAuth(this::completedAuth);
+ Assert.assertEquals(auths, 1);
+
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(100);
+ auths = tryAuth(this::completedAuth);
+ Assert.assertEquals(auths, 1);
+
+ // Make sure no tasks executed on the regular pool
+ Assert.assertEquals(startRequests, completedRequests());
+ }
+
+ @Test
+ public void testAuthRateLimiterNotUsed() throws Exception
+ {
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(1);
+ for (Message.Type type : Message.Type.values())
+ {
+ if (type == Message.Type.AUTH_RESPONSE || type == Message.Type.CREDENTIALS || type.direction != Message.Direction.REQUEST)
+ continue;
+
+ long auths = completedAuth();
+ long requests = tryAuth(this::completedRequests, new Message.Request(type)
+ {
+ public Response execute(QueryState queryState, long queryStartNanoTime, boolean traceRequest)
+ {
+ return null;
+ }
+ });
+ Assert.assertEquals(requests, 1);
+ Assert.assertEquals(completedAuth() - auths, 0);
+ }
+ }
+
+ @Test
+ public void testAuthRateLimiterDisabled() throws Exception
+ {
+ long startAuthRequests = completedAuth();
+
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(0);
+ long requests = tryAuth(this::completedRequests);
+ Assert.assertEquals(requests, 1);
+
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(-1);
+ requests = tryAuth(this::completedRequests);
+ Assert.assertEquals(requests, 1);
+
+ DatabaseDescriptor.setNativeTransportMaxAuthThreads(-1000);
+ requests = tryAuth(this::completedRequests);
+ Assert.assertEquals(requests, 1);
+
+ // Make sure no tasks executed on the auth pool
+ Assert.assertEquals(startAuthRequests, completedAuth());
+ }
+
+ private long completedRequests()
+ {
+ return Dispatcher.requestExecutor.getCompletedTaskCount();
+ }
+
+ private long completedAuth()
+ {
+ return Dispatcher.authExecutor.getCompletedTaskCount();
+ }
+
+ public long tryAuth(Callable<Long> check) throws Exception
+ {
+ return tryAuth(check, AUTH_RESPONSE_REQUEST);
+ }
+
+ @SuppressWarnings("UnstableApiUsage")
+ public long tryAuth(Callable<Long> check, Message.Request request) throws Exception
+ {
+ long start = check.call();
+ dispatch.dispatch(null, request, (channel,req,response) -> null, ClientResourceLimits.Overload.NONE);
+
+ // While this is timeout based, we should be *well below* a full second on any of this processing in any sane environment.
+ long timeout = System.currentTimeMillis();
+ while(start == check.call() && System.currentTimeMillis() - timeout < 1000)
+ {
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+ }
+ return check.call() - start;
+ }
+
+ public static class AuthTestDispatcher extends Dispatcher
+ {
+ public AuthTestDispatcher()
+ {
+ super(false);
+ }
+
+ @Override
+ void processRequest(Channel channel,
+ Message.Request request,
+ FlushItemConverter forFlusher,
+ ClientResourceLimits.Overload backpressure,
+ long approxStartTimeNanos)
+ {
+ // noop
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
index f0b45c3..1bd1a8f 100644
--- a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
+++ b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java
@@ -47,6 +47,7 @@
import org.apache.cassandra.transport.messages.QueryMessage;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.MD5Digest;
+import org.apache.cassandra.utils.ReflectionUtils;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@ -66,7 +67,7 @@
cqlQueryHandlerField = ClientState.class.getDeclaredField("cqlQueryHandler");
cqlQueryHandlerField.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
+ Field modifiersField = ReflectionUtils.getModifiersField();
modifiersAccessible = modifiersField.isAccessible();
modifiersField.setAccessible(true);
modifiersField.setInt(cqlQueryHandlerField, cqlQueryHandlerField.getModifiers() & ~Modifier.FINAL);
@@ -84,7 +85,7 @@
return;
try
{
- Field modifiersField = Field.class.getDeclaredField("modifiers");
+ Field modifiersField = ReflectionUtils.getModifiersField();
modifiersField.setAccessible(true);
modifiersField.setInt(cqlQueryHandlerField, cqlQueryHandlerField.getModifiers() | Modifier.FINAL);
diff --git a/test/unit/org/apache/cassandra/transport/SerDeserTest.java b/test/unit/org/apache/cassandra/transport/SerDeserTest.java
index 6033203..11b7489 100644
--- a/test/unit/org/apache/cassandra/transport/SerDeserTest.java
+++ b/test/unit/org/apache/cassandra/transport/SerDeserTest.java
@@ -18,21 +18,46 @@
package org.apache.cassandra.transport;
import java.nio.ByteBuffer;
-import java.util.*;
-
-import org.apache.commons.lang3.RandomStringUtils;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import io.netty.buffer.Unpooled;
import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.RandomStringUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.Constants;
+import org.apache.cassandra.cql3.FieldIdentifier;
+import org.apache.cassandra.cql3.Lists;
+import org.apache.cassandra.cql3.Maps;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.ResultSet;
+import org.apache.cassandra.cql3.Sets;
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.cql3.UserTypes;
import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.ClientState;
@@ -44,16 +69,15 @@
import org.apache.cassandra.utils.Pair;
import static org.junit.Assert.assertEquals;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
/**
* Serialization/deserialization tests for protocol objects and messages.
*/
public class SerDeserTest
{
-
@BeforeClass
public static void setupDD()
{
@@ -64,12 +88,6 @@
@Test
public void collectionSerDeserTest() throws Exception
{
- for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
- collectionSerDeserTest(version);
- }
-
- public void collectionSerDeserTest(ProtocolVersion version) throws Exception
- {
// Lists
ListType<?> lt = ListType.getInstance(Int32Type.instance, true);
List<Integer> l = Arrays.asList(2, 6, 1, 9);
@@ -78,18 +96,17 @@
for (Integer i : l)
lb.add(Int32Type.instance.decompose(i));
- assertEquals(l, lt.getSerializer().deserializeForNativeProtocol(CollectionSerializer.pack(lb, lb.size(), version), version));
+ assertEquals(l, lt.getSerializer().deserialize(CollectionSerializer.pack(lb, lb.size())));
// Sets
SetType<?> st = SetType.getInstance(UTF8Type.instance, true);
- Set<String> s = new LinkedHashSet<>();
- s.addAll(Arrays.asList("bar", "foo", "zee"));
+ Set<String> s = new LinkedHashSet<>(Arrays.asList("bar", "foo", "zee"));
List<ByteBuffer> sb = new ArrayList<>(s.size());
for (String t : s)
sb.add(UTF8Type.instance.decompose(t));
- assertEquals(s, st.getSerializer().deserializeForNativeProtocol(CollectionSerializer.pack(sb, sb.size(), version), version));
+ assertEquals(s, st.getSerializer().deserialize(CollectionSerializer.pack(sb, sb.size())));
// Maps
MapType<?, ?> mt = MapType.getInstance(UTF8Type.instance, LongType.instance, true);
@@ -105,52 +122,49 @@
mb.add(LongType.instance.decompose(entry.getValue()));
}
- assertEquals(m, mt.getSerializer().deserializeForNativeProtocol(CollectionSerializer.pack(mb, m.size(), version), version));
+ assertEquals(m, mt.getSerializer().deserialize(CollectionSerializer.pack(mb, m.size())));
}
@Test(expected = MarshalException.class)
public void setsMayNotContainNullsTest()
{
- ProtocolVersion version = ProtocolVersion.MIN_SUPPORTED_VERSION;
SetType<?> st = SetType.getInstance(UTF8Type.instance, true);
List<ByteBuffer> sb = new ArrayList<>(1);
sb.add(null);
- st.getSerializer().deserializeForNativeProtocol(CollectionSerializer.pack(sb, sb.size(), version), version);
+ st.getSerializer().deserialize(CollectionSerializer.pack(sb, sb.size()));
}
@Test(expected = MarshalException.class)
public void mapKeysMayNotContainNullsTest()
{
- ProtocolVersion version = ProtocolVersion.MIN_SUPPORTED_VERSION;
MapType<?, ?> mt = MapType.getInstance(UTF8Type.instance, LongType.instance, true);
List<ByteBuffer> mb = new ArrayList<>(2);
mb.add(null);
mb.add(LongType.instance.decompose(999L));
- mt.getSerializer().deserializeForNativeProtocol(CollectionSerializer.pack(mb, mb.size(), version), version);
+ mt.getSerializer().deserialize(CollectionSerializer.pack(mb, mb.size()));
}
@Test(expected = MarshalException.class)
public void mapValueMayNotContainNullsTest()
{
- ProtocolVersion version = ProtocolVersion.MIN_SUPPORTED_VERSION;
MapType<?, ?> mt = MapType.getInstance(UTF8Type.instance, LongType.instance, true);
List<ByteBuffer> mb = new ArrayList<>(2);
mb.add(UTF8Type.instance.decompose("danger"));
mb.add(null);
- mt.getSerializer().deserializeForNativeProtocol(CollectionSerializer.pack(mb, mb.size(), version), version);
+ mt.getSerializer().deserialize(CollectionSerializer.pack(mb, mb.size()));
}
@Test
- public void eventSerDeserTest() throws Exception
+ public void eventSerDeserTest()
{
for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
eventSerDeserTest(version);
}
- public void eventSerDeserTest(ProtocolVersion version) throws Exception
+ public void eventSerDeserTest(ProtocolVersion version)
{
List<Event> events = new ArrayList<>();
@@ -228,14 +242,14 @@
}
@Test
- public void udtSerDeserTest() throws Exception
+ public void udtSerDeserTest()
{
for (ProtocolVersion version : ProtocolVersion.SUPPORTED)
udtSerDeserTest(version);
}
- public void udtSerDeserTest(ProtocolVersion version) throws Exception
+ public void udtSerDeserTest(ProtocolVersion version)
{
ListType<?> lt = ListType.getInstance(Int32Type.instance, true);
SetType<?> st = SetType.getInstance(UTF8Type.instance, true);
@@ -274,7 +288,7 @@
ByteBuffer serialized = t.bindAndGet(options);
- ByteBuffer[] fields = udt.split(serialized);
+ ByteBuffer[] fields = udt.split(ByteBufferAccessor.instance, serialized);
assertEquals(4, fields.length);
@@ -284,16 +298,15 @@
// a UDT should alway be serialized with version 3 of the protocol. Which is why we don't use 'version'
// on purpose below.
- assertEquals(Arrays.asList(3, 1), lt.getSerializer().deserializeForNativeProtocol(fields[1], ProtocolVersion.V3));
+ assertEquals(Arrays.asList(3, 1), lt.getSerializer().deserialize(fields[1]));
- LinkedHashSet<String> s = new LinkedHashSet<>();
- s.addAll(Arrays.asList("bar", "foo"));
- assertEquals(s, st.getSerializer().deserializeForNativeProtocol(fields[2], ProtocolVersion.V3));
+ LinkedHashSet<String> s = new LinkedHashSet<>(Arrays.asList("bar", "foo"));
+ assertEquals(s, st.getSerializer().deserialize(fields[2]));
LinkedHashMap<String, Long> m = new LinkedHashMap<>();
m.put("bar", 12L);
m.put("foo", 24L);
- assertEquals(m, mt.getSerializer().deserializeForNativeProtocol(fields[3], ProtocolVersion.V3));
+ assertEquals(m, mt.getSerializer().deserialize(fields[3]));
}
@Test
diff --git a/test/unit/org/apache/cassandra/transport/WriteBytesTest.java b/test/unit/org/apache/cassandra/transport/WriteBytesTest.java
new file mode 100644
index 0000000..1dd1c79
--- /dev/null
+++ b/test/unit/org/apache/cassandra/transport/WriteBytesTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Generators;
+import org.assertj.core.api.Assertions;
+
+import static org.quicktheories.QuickTheory.qt;
+
+
+public class WriteBytesTest
+{
+ @Test
+ public void test()
+ {
+ int maxBytes = 10_000;
+ ByteBuf buf = Unpooled.buffer(maxBytes);
+ qt().forAll(Generators.bytesAnyType(0, maxBytes)).checkAssert(bb -> {
+ buf.clear();
+
+ int size = bb.remaining();
+ int pos = bb.position();
+
+ CBUtil.addBytes(bb, buf);
+
+ // test for consumption
+ Assertions.assertThat(bb.remaining()).isEqualTo(size);
+ Assertions.assertThat(bb.position()).isEqualTo(pos);
+
+ Assertions.assertThat(buf.writerIndex()).isEqualTo(size);
+ for (int i = 0; i < size; i++)
+ Assertions.assertThat(buf.getByte(buf.readerIndex() + i)).describedAs("byte mismatch at index %d", i).isEqualTo(bb.get(bb.position() + i));
+ FileUtils.clean(bb);
+ });
+ }
+
+}
diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
index 96464c9..7f08d6c 100644
--- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
+++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
@@ -18,10 +18,7 @@
*/
package org.apache.cassandra.utils;
-import org.apache.cassandra.io.util.*;
-
import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashSet;
@@ -29,10 +26,20 @@
import java.util.Random;
import java.util.Set;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.apache.cassandra.Util;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.IFilter.FilterKey;
import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator;
import org.apache.cassandra.utils.obs.IBitSet;
@@ -55,11 +62,11 @@
}
else
{
- BloomFilterSerializer.serialize((BloomFilter) f, out);
+ BloomFilterSerializer.forVersion(false).serialize((BloomFilter) f, out);
}
ByteArrayInputStream in = new ByteArrayInputStream(out.getData(), 0, out.getLength());
- IFilter f2 = BloomFilterSerializer.deserialize(new DataInputStream(in), oldBfFormat);
+ IFilter f2 = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(Util.DataInputStreamPlusImpl.wrap(in));
assert f2.isPresent(FilterTestHelper.bytes("a"));
assert !f2.isPresent(FilterTestHelper.bytes("b"));
@@ -210,13 +217,14 @@
File file = FileUtils.createDeletableTempFile("bloomFilterTest-", ".dat");
BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d);
filter.add(FilterTestHelper.wrap(test));
- DataOutputStreamPlus out = new FileOutputStreamPlus(file);
- BloomFilterSerializer.serialize(filter, out);
+ FileOutputStreamPlus out = file.newOutputStream(File.WriteMode.OVERWRITE);
+ BloomFilterSerializer serializer = BloomFilterSerializer.forVersion(false);
+ serializer.serialize(filter, out);
out.close();
filter.close();
- DataInputStream in = new DataInputStream(new FileInputStreamPlus(file));
- BloomFilter filter2 = BloomFilterSerializer.deserialize(in, false);
+ FileInputStreamPlus in = file.newInputStream();
+ BloomFilter filter2 = BloomFilterSerializer.forVersion(false).deserialize(in);
Assert.assertTrue(filter2.isPresent(FilterTestHelper.wrap(test)));
FileUtils.closeQuietly(in);
filter2.close();
diff --git a/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java b/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java
index 4ae8626..2b831b9 100644
--- a/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java
+++ b/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java
@@ -356,4 +356,37 @@
}
}
}
+
+ @Test
+ public void testEqualsWithShortLength() throws IOException
+ {
+ ByteBuffer bb = ByteBufferUtil.bytes(s);
+ checkEquals(bb);
+
+ bb = fromStringWithPosition(s, 10, false);
+ checkEquals(bb);
+
+ bb = fromStringWithPosition(s, 10, true);
+ checkEquals(bb);
+ }
+
+ private void checkEquals(ByteBuffer bb) throws IOException
+ {
+ DataOutputBuffer out = new DataOutputBuffer();
+ ByteBufferUtil.writeWithShortLength(bb, out);
+
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assert ByteBufferUtil.equalsWithShortLength(in, bb);
+
+ int index = ThreadLocalRandom.current().nextInt(bb.remaining());
+
+ in = new DataInputStream(new ByteArrayInputStream(out.toByteArray()));
+ bb.put(bb.position() + index, (byte) (bb.get(index) ^ 0x55));
+ assert !ByteBufferUtil.equalsWithShortLength(in, bb);
+ bb.put(bb.position() + index, (byte) (bb.get(index) ^ 0x55)); // revert change
+
+ in = new DataInputStream(new ByteArrayInputStream(out.toByteArray()));
+ bb.limit(bb.position() + index);
+ assert !ByteBufferUtil.equalsWithShortLength(in, bb);
+ }
}
diff --git a/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java b/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java
index 7693b45..f400461 100644
--- a/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java
+++ b/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java
@@ -18,10 +18,6 @@
*/
package org.apache.cassandra.utils;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
@@ -29,11 +25,18 @@
import org.junit.Test;
+import org.apache.cassandra.Util;
import org.apache.cassandra.io.util.BytesReadTracker;
+import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.io.util.TrackedDataInputPlus;
import org.apache.cassandra.io.util.TrackedInputStream;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class BytesReadTrackerTest
{
@@ -99,9 +102,9 @@
out.close();
}
- DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
- BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
- DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker;
+ DataInputStreamPlus in = new DataInputBuffer(testData);
+ BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
+ DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker;
try
{
@@ -172,9 +175,9 @@
out.close();
}
- DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
- BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
- DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker;
+ DataInputStreamPlus in = new DataInputBuffer(testData);
+ BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
+ DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker;
try
{
@@ -200,9 +203,9 @@
String testStr = "1234567890";
byte[] testData = testStr.getBytes();
- DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
- BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
- DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker;
+ DataInputStreamPlus in = new DataInputBuffer(testData);
+ BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
+ DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker;
try
{
@@ -233,8 +236,8 @@
public void internalTestReadLine(boolean inputStream) throws Exception
{
DataInputStream in = new DataInputStream(new ByteArrayInputStream("1".getBytes()));
- BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
- DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker;
+ BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
+ DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker;
try
{
diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
index ad9cb6b..34547dc 100644
--- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
+++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
@@ -18,20 +18,26 @@
package org.apache.cassandra.utils;
import java.lang.reflect.Modifier;
+import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.builder.MultilineRecursiveToStringStyle;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.FieldIdentifier;
import org.apache.cassandra.db.ReadCommand;
@@ -50,6 +56,10 @@
import org.apache.cassandra.dht.OrderPreservingPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.HeartBeatState;
+import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.ConnectionType;
import org.apache.cassandra.net.Message;
@@ -202,7 +212,7 @@
}
ColumnIdentifier name = new ColumnIdentifier(str, true);
int position = !kind.isPrimaryKeyKind() ? -1 : (int) rnd.next(Constraint.between(0, 30));
- return new ColumnMetadata(ks, table, name, typeGen.generate(rnd), position, kind);
+ return new ColumnMetadata(ks, table, name, typeGen.generate(rnd), position, kind, null);
}
public static Gen<ByteBuffer> partitionKeyDataGen(TableMetadata metadata)
@@ -316,4 +326,166 @@
}
}, true);
}
+
+ public static Gen<Token> murmurToken()
+ {
+ Constraint token = Constraint.between(Long.MIN_VALUE, Long.MAX_VALUE);
+ return rs -> new Murmur3Partitioner.LongToken(rs.next(token));
+ }
+
+ public static Gen<Token> byteOrderToken()
+ {
+ Constraint size = Constraint.between(0, 10);
+ Constraint byteRange = Constraint.between(Byte.MIN_VALUE, Byte.MAX_VALUE);
+ return rs -> {
+ byte[] token = new byte[Math.toIntExact(rs.next(size))];
+ for (int i = 0; i < token.length; i++)
+ token[i] = (byte) rs.next(byteRange);
+ return new ByteOrderedPartitioner.BytesToken(token);
+ };
+ }
+
+ public static Gen<Token> randomPartitionerToken()
+ {
+ Constraint domain = Constraint.none();
+ return rs -> new RandomPartitioner.BigIntegerToken(BigInteger.valueOf(rs.next(domain)));
+ }
+
+ public static Gen<Token> localPartitionerToken(LocalPartitioner partitioner)
+ {
+ Gen<ByteBuffer> bytes = AbstractTypeGenerators.getTypeSupport(partitioner.getTokenValidator()).bytesGen();
+ return rs -> partitioner.getToken(bytes.generate(rs));
+ }
+
+ public static Gen<Token> orderPreservingToken()
+ {
+ Gen<String> string = Generators.utf8(0, 10);
+ return rs -> new OrderPreservingPartitioner.StringToken(string.generate(rs));
+ }
+
+ public static Gen<Token> token(IPartitioner partitioner)
+ {
+ if (partitioner instanceof Murmur3Partitioner) return murmurToken();
+ if (partitioner instanceof ByteOrderedPartitioner) return byteOrderToken();
+ if (partitioner instanceof RandomPartitioner) return randomPartitionerToken();
+ if (partitioner instanceof LocalPartitioner) return localPartitionerToken((LocalPartitioner) partitioner);
+ if (partitioner instanceof OrderPreservingPartitioner) return orderPreservingToken();
+ throw new UnsupportedOperationException("Unsupported partitioner: " + partitioner.getClass());
+ }
+
+ public static Gen<? extends Collection<Token>> tokens(IPartitioner partitioner)
+ {
+ Gen<Token> tokenGen = token(partitioner);
+ return SourceDSL.lists().of(tokenGen).ofSizeBetween(1, 16);
+ }
+
+ public static Gen<HeartBeatState> heartBeatStates()
+ {
+ Constraint generationDomain = Constraint.between(0, Integer.MAX_VALUE);
+ Constraint versionDomain = Constraint.between(-1, Integer.MAX_VALUE);
+ return rs -> new HeartBeatState(Math.toIntExact(rs.next(generationDomain)), Math.toIntExact(rs.next(versionDomain)));
+ }
+
+ private static Gen<Map<ApplicationState, VersionedValue>> gossipApplicationStates()
+ {
+ //TODO support all application states...
+ // atm only used by a single test, which only looks at status
+ Gen<Boolean> statusWithPort = SourceDSL.booleans().all();
+ Gen<VersionedValue> statusGen = gossipStatusValue();
+
+ return rs -> {
+ ApplicationState statusState = statusWithPort.generate(rs) ? ApplicationState.STATUS_WITH_PORT : ApplicationState.STATUS;
+ VersionedValue vv = statusGen.generate(rs);
+ if (vv == null) return ImmutableMap.of();
+ return ImmutableMap.of(statusState, vv);
+ };
+ }
+
+ private static Gen<String> gossipStatus()
+ {
+ return SourceDSL.arbitrary()
+ .pick(VersionedValue.STATUS_NORMAL,
+ VersionedValue.STATUS_BOOTSTRAPPING_REPLACE,
+ VersionedValue.STATUS_BOOTSTRAPPING,
+ VersionedValue.STATUS_MOVING,
+ VersionedValue.STATUS_LEAVING,
+ VersionedValue.STATUS_LEFT,
+
+ //TODO would be good to prefix with STATUS_ like others
+ VersionedValue.REMOVING_TOKEN,
+ VersionedValue.REMOVED_TOKEN,
+ VersionedValue.HIBERNATE + VersionedValue.DELIMITER + true,
+ VersionedValue.HIBERNATE + VersionedValue.DELIMITER + false,
+ VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + true,
+ VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + false,
+ ""
+ );
+ }
+
+ private static Gen<VersionedValue> gossipStatusValue()
+ {
+ IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+ Gen<String> statusGen = gossipStatus();
+ Gen<Token> tokenGen = token(partitioner);
+ Gen<? extends Collection<Token>> tokensGen = tokens(partitioner);
+ Gen<InetAddress> addressGen = Generators.INET_ADDRESS_GEN;
+ Gen<InetAddressAndPort> addressAndGenGen = INET_ADDRESS_AND_PORT_GEN;
+ Gen<Boolean> bool = SourceDSL.booleans().all();
+ Constraint millis = Constraint.between(0, Long.MAX_VALUE);
+ Constraint version = Constraint.between(0, Integer.MAX_VALUE);
+ Gen<UUID> hostId = Generators.UUID_RANDOM_GEN;
+ VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(partitioner);
+ return rs -> {
+ String status = statusGen.generate(rs);
+ switch (status)
+ {
+ case "":
+ return null;
+ case VersionedValue.STATUS_NORMAL:
+ return factory.normal(tokensGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_BOOTSTRAPPING:
+ return factory.bootstrapping(tokensGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_BOOTSTRAPPING_REPLACE:
+ if (bool.generate(rs)) return factory.bootReplacingWithPort(addressAndGenGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ else return factory.bootReplacing(addressGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_MOVING:
+ return factory.moving(tokenGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_LEAVING:
+ return factory.leaving(tokensGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_LEFT:
+ return factory.left(tokensGen.generate(rs), rs.next(millis)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.REMOVING_TOKEN:
+ return factory.removingNonlocal(hostId.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.REMOVED_TOKEN:
+ return factory.removedNonlocal(hostId.generate(rs), rs.next(millis)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.HIBERNATE + VersionedValue.DELIMITER + true:
+ return factory.hibernate(true).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.HIBERNATE + VersionedValue.DELIMITER + false:
+ return factory.hibernate(false).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + true:
+ return factory.shutdown(true).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + false:
+ return factory.shutdown(false).withVersion(Math.toIntExact(rs.next(version)));
+ default:
+ throw new AssertionError("Unexpected status: " + status);
+ }
+ };
+ }
+
+ public static Gen<EndpointState> endpointStates()
+ {
+ Gen<HeartBeatState> hbGen = heartBeatStates();
+ Gen<Map<ApplicationState, VersionedValue>> appStates = gossipApplicationStates();
+ Gen<Boolean> alive = SourceDSL.booleans().all();
+ Constraint updateTimestamp = Constraint.between(0, Long.MAX_VALUE);
+ return rs -> {
+ EndpointState state = new EndpointState(hbGen.generate(rs));
+ Map<ApplicationState, VersionedValue> map = appStates.generate(rs);
+ if (!map.isEmpty()) state.addApplicationStates(map);
+ if (alive.generate(rs)) state.markAlive();
+ else state.markDead();
+ state.unsafeSetUpdateTimestamp(rs.next(updateTimestamp));
+ return state;
+ };
+ }
}
diff --git a/test/unit/org/apache/cassandra/utils/Generators.java b/test/unit/org/apache/cassandra/utils/Generators.java
index 179c0f4..1baa16e 100644
--- a/test/unit/org/apache/cassandra/utils/Generators.java
+++ b/test/unit/org/apache/cassandra/utils/Generators.java
@@ -39,6 +39,8 @@
import org.quicktheories.generators.SourceDSL;
import org.quicktheories.impl.Constraint;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_BLOB_SHARED_SEED;
+
public final class Generators
{
private static final Logger logger = LoggerFactory.getLogger(Generators.class);
@@ -300,6 +302,16 @@
public static Gen<ByteBuffer> bytes(int min, int max)
{
+ return bytes(min, max, SourceDSL.arbitrary().constant(BBCases.HEAP));
+ }
+
+ public static Gen<ByteBuffer> bytesAnyType(int min, int max)
+ {
+ return bytes(min, max, SourceDSL.arbitrary().enumValues(BBCases.class));
+ }
+
+ private static Gen<ByteBuffer> bytes(int min, int max, Gen<BBCases> cases)
+ {
if (min < 0)
throw new IllegalArgumentException("Asked for negative bytes; given " + min);
if (max > MAX_BLOB_LENGTH)
@@ -314,11 +326,31 @@
// to add more randomness, also shift offset in the array so the same size doesn't yield the same bytes
int offset = (int) rnd.next(Constraint.between(0, MAX_BLOB_LENGTH - size));
- return ByteBuffer.wrap(LazySharedBlob.SHARED_BYTES, offset, size);
+ return handleCases(cases, rnd, offset, size);
};
+ };
+
+ private enum BBCases { HEAP, READ_ONLY_HEAP, DIRECT, READ_ONLY_DIRECT }
+
+ private static ByteBuffer handleCases(Gen<BBCases> cases, RandomnessSource rnd, int offset, int size) {
+ switch (cases.generate(rnd))
+ {
+ case HEAP: return ByteBuffer.wrap(LazySharedBlob.SHARED_BYTES, offset, size);
+ case READ_ONLY_HEAP: return ByteBuffer.wrap(LazySharedBlob.SHARED_BYTES, offset, size).asReadOnlyBuffer();
+ case DIRECT: return directBufferFromSharedBlob(offset, size);
+ case READ_ONLY_DIRECT: return directBufferFromSharedBlob(offset, size).asReadOnlyBuffer();
+ default: throw new AssertionError("can't wait for jdk 17!");
+ }
}
- /**
+ private static ByteBuffer directBufferFromSharedBlob(int offset, int size) {
+ ByteBuffer bb = ByteBuffer.allocateDirect(size);
+ bb.put(LazySharedBlob.SHARED_BYTES, offset, size);
+ bb.flip();
+ return bb;
+ }
+
+ /**
* Implements a valid utf-8 generator.
*
* Implementation note, currently relies on getBytes to strip out non-valid utf-8 chars, so is slow
@@ -349,7 +381,7 @@
static
{
- long blobSeed = Long.parseLong(System.getProperty("cassandra.test.blob.shared.seed", Long.toString(System.currentTimeMillis())));
+ long blobSeed = TEST_BLOB_SHARED_SEED.getLong(System.currentTimeMillis());
logger.info("Shared blob Gen used seed {}", blobSeed);
Random random = new Random(blobSeed);
diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java
index be235e6..e23bb38 100644
--- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java
@@ -18,10 +18,8 @@
*/
package org.apache.cassandra.utils;
-import java.io.DataInputStream;
import java.io.IOException;
-import org.apache.cassandra.io.util.FileInputStreamPlus;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -31,12 +29,12 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.utils.obs.OffHeapBitSet;
-
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.utils.obs.OffHeapBitSet;
public class SerializationsTest extends AbstractSerializationsTester
{
@@ -65,7 +63,7 @@
if (oldBfFormat)
serializeOldBfFormat((BloomFilter) bf, out);
else
- BloomFilterSerializer.serialize((BloomFilter) bf, out);
+ BloomFilterSerializer.forVersion(false).serialize((BloomFilter) bf, out);
}
}
}
@@ -80,7 +78,7 @@
}
try (FileInputStreamPlus in = getInput("4.0", "utils.BloomFilter1000.bin");
- IFilter filter = BloomFilterSerializer.deserialize(in, false))
+ IFilter filter = BloomFilterSerializer.forVersion(false).deserialize(in))
{
boolean present;
for (int i = 0 ; i < 1000 ; i++)
@@ -96,7 +94,7 @@
}
try (FileInputStreamPlus in = getInput("3.0", "utils.BloomFilter1000.bin");
- IFilter filter = BloomFilterSerializer.deserialize(in, true))
+ IFilter filter = BloomFilterSerializer.forVersion(true).deserialize(in))
{
boolean present;
for (int i = 0 ; i < 1000 ; i++)
@@ -118,12 +116,12 @@
testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db", true);
}
- private static void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception
+ private void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception
{
Murmur3Partitioner partitioner = new Murmur3Partitioner();
- try (DataInputStream in = new DataInputStream(new FileInputStreamPlus(new File(file)));
- IFilter filter = BloomFilterSerializer.deserialize(in, oldBfFormat))
+ try (FileInputStreamPlus in = new File(file).newInputStream();
+ IFilter filter = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(in))
{
for (int i = 1; i <= 10; i++)
{
diff --git a/test/unit/org/apache/cassandra/utils/SimpleGraph.java b/test/unit/org/apache/cassandra/utils/SimpleGraph.java
new file mode 100644
index 0000000..71b1fb2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/SimpleGraph.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Ordering;
+
+/**
+ * A directed graph. Main usage is the {@link #findPaths(Object, Object)} method which is used to find all paths between
+ * 2 vertices.
+ */
+public class SimpleGraph<V>
+{
+ private final ImmutableMap<V, ImmutableSet<V>> edges;
+
+ private SimpleGraph(ImmutableMap<V, ImmutableSet<V>> edges)
+ {
+ if (edges == null || edges.isEmpty())
+ throw new AssertionError("Edges empty");
+ this.edges = edges;
+ }
+
+ public static <T extends Comparable<T>> NavigableSet<T> sortedVertices(SimpleGraph<T> graph)
+ {
+ return new TreeSet<>(graph.vertices());
+ }
+
+ public static <T extends Comparable<T>> T min(SimpleGraph<T> graph)
+ {
+ return Ordering.natural().min(graph.vertices());
+ }
+
+ public static <T extends Comparable<T>> T max(SimpleGraph<T> graph)
+ {
+ return Ordering.natural().max(graph.vertices());
+ }
+
+ public boolean hasEdge(V a, V b)
+ {
+ ImmutableSet<V> matches = edges.get(a);
+ return matches != null && matches.contains(b);
+ }
+
+ public ImmutableSet<V> vertices()
+ {
+ ImmutableSet.Builder<V> b = ImmutableSet.builder();
+ b.addAll(edges.keySet());
+ edges.values().forEach(b::addAll);
+ return b.build();
+ }
+
+ public List<List<V>> findPaths(V from, V to)
+ {
+ List<List<V>> matches = new ArrayList<>();
+ findPaths0(Collections.singletonList(from), from, to, matches::add);
+ return matches;
+ }
+
+ private void findPaths0(List<V> accum, V from, V to, Consumer<List<V>> onMatch)
+ {
+ ImmutableSet<V> check = edges.get(from);
+ if (check == null)
+ return; // no matches
+ for (V next : check)
+ {
+ if (accum.contains(next))
+ return; // ignore walking recursive
+ List<V> nextAccum = new ArrayList<>(accum);
+ nextAccum.add(next);
+ if (next.equals(to))
+ {
+ onMatch.accept(nextAccum);
+ }
+ else
+ {
+ findPaths0(nextAccum, next, to, onMatch);
+ }
+ }
+ }
+
+ public static class Builder<V>
+ {
+ private final Map<V, Set<V>> edges = new HashMap<>();
+
+ public Builder<V> addEdge(V from, V to)
+ {
+ edges.computeIfAbsent(from, ignore -> new HashSet<>()).add(to);
+ return this;
+ }
+
+ public SimpleGraph<V> build()
+ {
+ ImmutableMap.Builder<V, ImmutableSet<V>> builder = ImmutableMap.builder();
+ for (Map.Entry<V, Set<V>> e : edges.entrySet())
+ builder.put(e.getKey(), ImmutableSet.copyOf(e.getValue()));
+ return new SimpleGraph(builder.build());
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/SimpleGraphTest.java b/test/unit/org/apache/cassandra/utils/SimpleGraphTest.java
new file mode 100644
index 0000000..6adee36
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/SimpleGraphTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.assertj.core.api.Assertions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SimpleGraphTest
+{
+ @Test
+ public void empty()
+ {
+ Assertions.assertThatThrownBy(() -> new SimpleGraph.Builder<String>().build())
+ .isInstanceOf(AssertionError.class)
+ .hasMessage("Edges empty");
+ }
+
+ /**
+ * If vertices have edges that form a circle this should not cause {@link SimpleGraph#findPaths(Object, Object)} to
+ * hang.
+ */
+ @Test
+ public void recursive()
+ {
+ SimpleGraph<String> graph = of("A", "B",
+ "B", "C",
+ "C", "A");
+ // no paths to identity
+ assertThat(graph.findPaths("A", "A")).isEmpty();
+ assertThat(graph.findPaths("B", "B")).isEmpty();
+ assertThat(graph.findPaths("C", "C")).isEmpty();
+
+ assertThat(graph.findPaths("C", "B")).isEqualTo(Collections.singletonList(Arrays.asList("C", "A", "B")));
+
+ // all options return and don't have duplicate keys
+ for (String i : graph.vertices())
+ {
+ for (String j : graph.vertices())
+ {
+ List<List<String>> paths = graph.findPaths(i, j);
+ for (List<String> path : paths)
+ {
+ Map<String, Integer> distinct = countDistinct(path);
+ for (Map.Entry<String, Integer> e : distinct.entrySet())
+ assertThat(e.getValue()).describedAs("Duplicate vertex %s found; %s", e.getKey(), path).isEqualTo(1);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void simple()
+ {
+ SimpleGraph<String> graph = of("A", "B",
+ "B", "C",
+ "C", "D");
+
+ assertThat(graph.findPaths("A", "B")).isEqualTo(Collections.singletonList(Arrays.asList("A", "B")));
+ assertThat(graph.findPaths("A", "C")).isEqualTo(Collections.singletonList(Arrays.asList("A", "B", "C")));
+ assertThat(graph.findPaths("B", "D")).isEqualTo(Collections.singletonList(Arrays.asList("B", "C", "D")));
+
+ assertThat(graph.hasEdge("A", "B")).isTrue();
+ assertThat(graph.hasEdge("C", "D")).isTrue();
+ assertThat(graph.hasEdge("B", "A")).isFalse();
+ assertThat(graph.hasEdge("C", "B")).isFalse();
+ }
+
+ private static <T> Map<T, Integer> countDistinct(List<T> list)
+ {
+ Map<T, Integer> map = new HashMap<>();
+ for (T t : list)
+ map.compute(t, (ignore, accum) -> accum == null ? 1 : accum + 1);
+ return map;
+ }
+
+ static <T> SimpleGraph<T> of(T... values)
+ {
+ assert values.length % 2 == 0: "graph requires even number of values, but given " + values.length;
+ SimpleGraph.Builder<T> builder = new SimpleGraph.Builder<>();
+ for (int i = 0; i < values.length; i = i + 2)
+ builder.addEdge(values[i], values[i + 1]);
+ return builder.build();
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/TeeDataInputPlusTest.java b/test/unit/org/apache/cassandra/utils/TeeDataInputPlusTest.java
new file mode 100644
index 0000000..baac9ca
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/TeeDataInputPlusTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.TeeDataInputPlus;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TeeDataInputPlusTest
+{
+ @Test
+ public void testTeeBuffer() throws Exception
+ {
+ DataOutputBuffer out = new DataOutputBuffer();
+ byte[] testData;
+
+ // boolean
+ out.writeBoolean(true);
+ // byte
+ out.writeByte(0x1);
+ // char
+ out.writeChar('a');
+ // short
+ out.writeShort(1);
+ // int
+ out.writeInt(1);
+ // long
+ out.writeLong(1L);
+ // float
+ out.writeFloat(1.0f);
+ // double
+ out.writeDouble(1.0d);
+ // vint
+ out.writeVInt(-1337L);
+ //unsigned vint
+ out.writeUnsignedVInt(1337L);
+ // String
+ out.writeUTF("abc");
+ //Another string to test skip
+ out.writeUTF("garbagetoskipattheend");
+ testData = out.toByteArray();
+
+ int LIMITED_SIZE = 40;
+ DataInputBuffer reader = new DataInputBuffer(testData);
+ DataInputBuffer reader2 = new DataInputBuffer(testData);
+ DataOutputBuffer teeOut = new DataOutputBuffer();
+ DataOutputBuffer limitedTeeOut = new DataOutputBuffer();
+ TeeDataInputPlus tee = new TeeDataInputPlus(reader, teeOut);
+ TeeDataInputPlus limitedTee = new TeeDataInputPlus(reader2, limitedTeeOut, LIMITED_SIZE);
+
+ // boolean = 1byte
+ boolean bool = tee.readBoolean();
+ assertTrue(bool);
+ bool = limitedTee.readBoolean();
+ assertTrue(bool);
+ // byte = 1byte
+ byte b = tee.readByte();
+ assertEquals(b, 0x1);
+ b = limitedTee.readByte();
+ assertEquals(b, 0x1);
+ // char = 2byte
+ char c = tee.readChar();
+ assertEquals('a', c);
+ c = limitedTee.readChar();
+ assertEquals('a', c);
+ // short = 2bytes
+ short s = tee.readShort();
+ assertEquals(1, s);
+ s = limitedTee.readShort();
+ assertEquals(1, s);
+ // int = 4bytes
+ int i = tee.readInt();
+ assertEquals(1, i);
+ i = limitedTee.readInt();
+ assertEquals(1, i);
+ // long = 8bytes
+ long l = tee.readLong();
+ assertEquals(1L, l);
+ l = limitedTee.readLong();
+ assertEquals(1L, l);
+ // float = 4bytes
+ float f = tee.readFloat();
+ assertEquals(1.0f, f, 0);
+ f = limitedTee.readFloat();
+ assertEquals(1.0f, f, 0);
+ // double = 8bytes
+ double d = tee.readDouble();
+ assertEquals(1.0d, d, 0);
+ d = limitedTee.readDouble();
+ assertEquals(1.0d, d, 0);
+ long vint = tee.readVInt();
+ assertEquals(-1337L, vint);
+ vint = limitedTee.readVInt();
+ assertEquals(-1337L, vint);
+ long uvint = tee.readUnsignedVInt();
+ assertEquals(1337L, uvint);
+ uvint = limitedTee.readUnsignedVInt();
+ assertEquals(1337L, uvint);
+ // String("abc") = 2(string size) + 3 = 5 bytes
+ String str = tee.readUTF();
+ assertEquals("abc", str);
+ str = limitedTee.readUTF();
+ assertEquals("abc", str);
+ int skipped = tee.skipBytes(100);
+ assertEquals(23, skipped);
+ skipped = limitedTee.skipBytes(100);
+ assertEquals(23, skipped);
+
+ byte[] teeData = teeOut.toByteArray();
+ assertFalse(tee.isLimitReached());
+ assertTrue(Arrays.equals(testData, teeData));
+
+ byte[] limitedTeeData = limitedTeeOut.toByteArray();
+ assertTrue(limitedTee.isLimitReached());
+ assertTrue(Arrays.equals(Arrays.copyOf(testData, LIMITED_SIZE - 1 ), limitedTeeData));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java
index 76e7098..e764b3f 100644
--- a/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java
+++ b/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.utils.btree;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BTREE_BRANCH_SHIFT;
import static org.apache.cassandra.utils.btree.BTreeRemoval.remove;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -37,7 +38,7 @@
{
static
{
- System.setProperty("cassandra.btree.branchshift", "3");
+ BTREE_BRANCH_SHIFT.setInt(3);
}
private static final Comparator<Integer> CMP = new Comparator<Integer>()
diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
index 73f0ff7..004dfe0 100644
--- a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
@@ -25,6 +25,7 @@
import org.junit.Assert;
+import static org.apache.cassandra.config.CassandraRelevantProperties.BTREE_BRANCH_SHIFT;
import static org.junit.Assert.*;
public class BTreeTest
@@ -32,9 +33,9 @@
static Integer[] ints = new Integer[20];
static
{
- System.setProperty("cassandra.btree.branchshift", "2");
+ BTREE_BRANCH_SHIFT.setInt(2);
for (int i = 0 ; i < ints.length ; i++)
- ints[i] = new Integer(i);
+ ints[i] = Integer.valueOf(i);
}
static final UpdateFunction<Integer, Integer> updateF = new UpdateFunction<Integer, Integer>()
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java
new file mode 100644
index 0000000..a9b187d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java
@@ -0,0 +1,1015 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.cql3.Term;
+import org.apache.cassandra.db.marshal.*;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.cql3.Duration;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LengthPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.serializers.SimpleDateSerializer;
+import org.apache.cassandra.serializers.TypeSerializer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.UUIDGen;
+
+@RunWith(Parameterized.class)
+public class AbstractTypeByteSourceTest
+{
+ private static final String ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()";
+
+ @Parameterized.Parameters(name = "version={0}")
+ public static Iterable<ByteComparable.Version> versions()
+ {
+ return ImmutableList.of(ByteComparable.Version.OSS50);
+ }
+
+ private final ByteComparable.Version version;
+
+ public AbstractTypeByteSourceTest(ByteComparable.Version version)
+ {
+ this.version = version;
+ }
+
+ private <T> void testValuesForType(AbstractType<T> type, T... values)
+ {
+ testValuesForType(type, Arrays.asList(values));
+ }
+
+ private <T> void testValuesForType(AbstractType<T> type, List<T> values)
+ {
+ for (T initial : values)
+ decodeAndAssertEquals(type, initial);
+ if (IntegerType.instance.equals(type))
+ // IntegerType tests go through A LOT of values, so short of randomly picking up to, let's say 1000
+ // values to combine with, we'd rather skip the comparison tests for them.
+ return;
+ for (int i = 0; i < values.size(); ++i)
+ {
+ for (int j = i + 1; j < values.size(); ++j)
+ {
+ ByteBuffer left = type.decompose(values.get(i));
+ ByteBuffer right = type.decompose(values.get(j));
+ int compareBuffers = Integer.signum(type.compare(left, right));
+ ByteSource leftSource = type.asComparableBytes(left.duplicate(), version);
+ ByteSource rightSource = type.asComparableBytes(right.duplicate(), version);
+ int compareBytes = Integer.signum(ByteComparable.compare(v -> leftSource, v -> rightSource, version));
+ Assert.assertEquals(compareBuffers, compareBytes);
+ }
+ }
+ }
+
+ private <T> void testValuesForType(AbstractType<T> type, Stream<T> values)
+ {
+ values.forEach(initial -> decodeAndAssertEquals(type, initial));
+ }
+
+ private <T> void decodeAndAssertEquals(AbstractType<T> type, T initial)
+ {
+ ByteBuffer initialBuffer = type.decompose(initial);
+ // Assert that fromComparableBytes decodes correctly.
+ ByteSource.Peekable peekableBytes = ByteSource.peekable(type.asComparableBytes(initialBuffer, version));
+ ByteBuffer decodedBuffer = type.fromComparableBytes(peekableBytes, version);
+ Assert.assertEquals("For " + ByteSourceComparisonTest.safeStr(initial),
+ ByteBufferUtil.bytesToHex(initialBuffer),
+ ByteBufferUtil.bytesToHex(decodedBuffer));
+ // Assert that the value composed from fromComparableBytes is the correct one.
+ peekableBytes = ByteSource.peekable(type.asComparableBytes(initialBuffer, version));
+ T decoded = type.compose(type.fromComparableBytes(peekableBytes, version));
+ Assert.assertEquals(initial, decoded);
+ }
+
+ private static String newRandomAlphanumeric(Random prng, int length)
+ {
+ StringBuilder random = new StringBuilder(length);
+ for (int i = 0; i < length; ++i)
+ random.append(ALPHABET.charAt(prng.nextInt(ALPHABET.length())));
+ return random.toString();
+ }
+
+ @Test
+ public void testAsciiType()
+ {
+ String[] asciiStrings = new String[]
+ {
+ "",
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890",
+ "!@#$%^&*()",
+ };
+ testValuesForType(AsciiType.instance, asciiStrings);
+
+ Random prng = new Random();
+ Stream<String> asciiStream = Stream.generate(() -> newRandomAlphanumeric(prng, 10)).limit(1000);
+ testValuesForType(AsciiType.instance, asciiStream);
+ }
+
+ @Test
+ public void testBooleanType()
+ {
+ testValuesForType(BooleanType.instance, Boolean.TRUE, Boolean.FALSE, null);
+ }
+
+ @Test
+ public void testBytesType()
+ {
+ List<ByteBuffer> byteBuffers = new ArrayList<>();
+ Random prng = new Random();
+ byte[] byteArray;
+ int[] arrayLengths = new int[] {1, 10, 100, 1000};
+ for (int length : arrayLengths)
+ {
+ byteArray = new byte[length];
+ for (int i = 0; i < 1000; ++i)
+ {
+ prng.nextBytes(byteArray);
+ byteBuffers.add(ByteBuffer.wrap(byteArray));
+ }
+ }
+ testValuesForType(BytesType.instance, byteBuffers.toArray(new ByteBuffer[0]));
+ }
+
+ @Test
+ public void testByteType()
+ {
+ testValuesForType(ByteType.instance, new Byte[] { null });
+
+ Stream<Byte> allBytes = IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE + 1)
+ .mapToObj(value -> (byte) value);
+ testValuesForType(ByteType.instance, allBytes);
+ }
+
+ @Test
+ public void testCompositeType()
+ {
+ CompositeType compType = CompositeType.getInstance(UTF8Type.instance, TimeUUIDType.instance, IntegerType.instance);
+ List<ByteBuffer> byteBuffers = new ArrayList<>();
+ Random prng = new Random();
+ // Test with complete CompositeType rows
+ for (int i = 0; i < 1000; ++i)
+ {
+ String randomString = newRandomAlphanumeric(prng, 10);
+ TimeUUID randomUuid = TimeUUID.Generator.nextTimeUUID();
+ BigInteger randomVarint = BigInteger.probablePrime(80, prng);
+ byteBuffers.add(compType.decompose(randomString, randomUuid, randomVarint));
+ }
+ // Test with incomplete CompositeType rows, where only the first element is present
+ ByteBuffer[] incompleteComposite = new ByteBuffer[1];
+ incompleteComposite[0] = UTF8Type.instance.decompose(newRandomAlphanumeric(prng, 10));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, true, incompleteComposite));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, false, incompleteComposite));
+ // ...and the last end-of-component byte is not 0.
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, true, incompleteComposite, (byte) 1));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, false, incompleteComposite, (byte) 1));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, true, incompleteComposite, (byte) -1));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, false, incompleteComposite, (byte) -1));
+ // Test with incomplete CompositeType rows, where only the last element is not present
+ incompleteComposite = new ByteBuffer[2];
+ incompleteComposite[0] = UTF8Type.instance.decompose(newRandomAlphanumeric(prng, 10));
+ incompleteComposite[1] = TimeUUIDType.instance.decompose(TimeUUID.Generator.nextTimeUUID());
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, true, incompleteComposite));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, false, incompleteComposite));
+ // ...and the last end-of-component byte is not 0.
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, true, incompleteComposite, (byte) 1));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, false, incompleteComposite, (byte) 1));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, true, incompleteComposite, (byte) -1));
+ byteBuffers.add(CompositeType.build(ByteBufferAccessor.instance, false, incompleteComposite, (byte) -1));
+
+ testValuesForType(compType, byteBuffers.toArray(new ByteBuffer[0]));
+ }
+
+ @Test
+ public void testDateType()
+ {
+ Stream<Date> dates = Stream.of(null,
+ new Date(Long.MIN_VALUE),
+ new Date(Long.MAX_VALUE),
+ new Date());
+ testValuesForType(DateType.instance, dates);
+
+ dates = new Random().longs(1000).mapToObj(Date::new);
+ testValuesForType(DateType.instance, dates);
+ }
+
+ @Test
+ public void testDecimalType()
+ {
+ // We won't be using testValuesForType for DecimalType (i.e. we won't also be comparing the initial and decoded
+ // ByteBuffer values). That's because the same BigDecimal value can be represented with a couple of different,
+ // even if equivalent pairs of <mantissa, scale> (e.g. 0.1 is 1 * e-1, as well as 10 * e-2, as well as...).
+ // And in practice it's easier to just convert to BigDecimals and then compare, instead of trying to manually
+ // decode and convert to canonical representations, which then to compare. For example of generating canonical
+ // decimals in the first place, see testReversedType().
+ Consumer<BigDecimal> bigDecimalConsumer = initial ->
+ {
+ ByteSource byteSource = DecimalType.instance.asComparableBytes(DecimalType.instance.decompose(initial), version);
+ BigDecimal decoded = DecimalType.instance.compose(DecimalType.instance.fromComparableBytes(ByteSource.peekable(byteSource), version));
+ if (initial == null)
+ Assert.assertNull(decoded);
+ else
+ Assert.assertEquals(0, initial.compareTo(decoded));
+ };
+ // Test some interesting predefined BigDecimal values.
+ Stream.of(null,
+ BigDecimal.ZERO,
+ BigDecimal.ONE,
+ BigDecimal.ONE.add(BigDecimal.ONE),
+ BigDecimal.TEN,
+ BigDecimal.valueOf(0.0000000000000000000000000000000001),
+ BigDecimal.valueOf(-0.0000000000000000000000000000000001),
+ BigDecimal.valueOf(0.0000000000000001234567891011121314),
+ BigDecimal.valueOf(-0.0000000000000001234567891011121314),
+ BigDecimal.valueOf(12345678910111213.141516171819202122),
+ BigDecimal.valueOf(-12345678910111213.141516171819202122),
+ new BigDecimal(BigInteger.TEN, Integer.MIN_VALUE),
+ new BigDecimal(BigInteger.TEN.negate(), Integer.MIN_VALUE),
+ new BigDecimal(BigInteger.TEN, Integer.MAX_VALUE),
+ new BigDecimal(BigInteger.TEN.negate(), Integer.MAX_VALUE),
+ new BigDecimal(BigInteger.TEN.pow(1000), Integer.MIN_VALUE),
+ new BigDecimal(BigInteger.TEN.pow(1000).negate(), Integer.MIN_VALUE),
+ new BigDecimal(BigInteger.TEN.pow(1000), Integer.MAX_VALUE),
+ new BigDecimal(BigInteger.TEN.pow(1000).negate(), Integer.MAX_VALUE))
+ .forEach(bigDecimalConsumer);
+ // Test BigDecimals created from random double values with predefined range modifiers.
+ double[] bounds = {
+ Double.MIN_VALUE,
+ -1_000_000_000.0,
+ -100_000.0,
+ -1.0,
+ 1.0,
+ 100_000.0,
+ 1_000_000_000.0,
+ Double.MAX_VALUE};
+ for (double bound : bounds)
+ {
+ new Random().doubles(1000)
+ .mapToObj(initial -> BigDecimal.valueOf(initial * bound))
+ .forEach(bigDecimalConsumer);
+ }
+ }
+
+ @Test
+ public void testDoubleType()
+ {
+ Stream<Double> doubles = Stream.of(null,
+ Double.NaN,
+ Double.POSITIVE_INFINITY,
+ Double.NEGATIVE_INFINITY,
+ Double.MAX_VALUE,
+ Double.MIN_VALUE,
+ +0.0,
+ -0.0,
+ +1.0,
+ -1.0,
+ +12345678910.111213141516,
+ -12345678910.111213141516);
+ testValuesForType(DoubleType.instance, doubles);
+
+ doubles = new Random().doubles(1000).boxed();
+ testValuesForType(DoubleType.instance, doubles);
+ }
+
+ @Test
+ public void testDurationType()
+ {
+ Random prng = new Random();
+ Stream<Duration> posDurations = Stream.generate(() ->
+ {
+ int months = prng.nextInt(12) + 1;
+ int days = prng.nextInt(28) + 1;
+ long nanos = (Math.abs(prng.nextLong() % 86_400_000_000_000L)) + 1;
+ return Duration.newInstance(months, days, nanos);
+ })
+ .limit(1000);
+ testValuesForType(DurationType.instance, posDurations);
+ Stream<Duration> negDurations = Stream.generate(() ->
+ {
+ int months = prng.nextInt(12) + 1;
+ int days = prng.nextInt(28) + 1;
+ long nanos = (Math.abs(prng.nextLong() % 86_400_000_000_000L)) + 1;
+ return Duration.newInstance(-months, -days, -nanos);
+ })
+ .limit(1000);
+ testValuesForType(DurationType.instance, negDurations);
+ }
+
+ @Test
+ public void testDynamicCompositeType()
+ {
+ DynamicCompositeType dynamicCompType = DynamicCompositeType.getInstance(new HashMap<>());
+ ImmutableList<String> allTypes = ImmutableList.of("org.apache.cassandra.db.marshal.BytesType",
+ "org.apache.cassandra.db.marshal.TimeUUIDType",
+ "org.apache.cassandra.db.marshal.IntegerType");
+ List<ByteBuffer> allValues = new ArrayList<>();
+ List<ByteBuffer> byteBuffers = new ArrayList<>();
+ Random prng = new Random();
+ for (int i = 0; i < 10; ++i)
+ {
+ String randomString = newRandomAlphanumeric(prng, 10);
+ allValues.add(ByteBufferUtil.bytes(randomString));
+ UUID randomUuid = TimeUUID.Generator.nextTimeAsUUID();
+ allValues.add(ByteBuffer.wrap(UUIDGen.decompose(randomUuid)));
+ byte randomByte = (byte) prng.nextInt();
+ allValues.add(ByteBuffer.allocate(1).put(randomByte));
+
+ // Three-component key with aliased and non-aliased types and end-of-component byte varying (0, 1, -1).
+ byteBuffers.add(DynamicCompositeType.build(allTypes, allValues));
+ byteBuffers.add(createStringUuidVarintDynamicCompositeKey(randomString, randomUuid, randomByte, (byte) 1));
+ byteBuffers.add(createStringUuidVarintDynamicCompositeKey(randomString, randomUuid, randomByte, (byte) -1));
+
+ // Two-component key with aliased and non-aliased types and end-of-component byte varying (0, 1, -1).
+ byteBuffers.add(DynamicCompositeType.build(allTypes.subList(0, 2), allValues.subList(0, 2)));
+ byteBuffers.add(createStringUuidVarintDynamicCompositeKey(randomString, randomUuid, -1, (byte) 1));
+ byteBuffers.add(createStringUuidVarintDynamicCompositeKey(randomString, randomUuid, -1, (byte) -1));
+
+ // One-component key with aliased and non-aliased type and end-of-component byte varying (0, 1, -1).
+ byteBuffers.add(DynamicCompositeType.build(allTypes.subList(0, 1), allValues.subList(0, 1)));
+ byteBuffers.add(createStringUuidVarintDynamicCompositeKey(randomString, null, -1, (byte) 1));
+ byteBuffers.add(createStringUuidVarintDynamicCompositeKey(randomString, null, -1, (byte) -1));
+
+ allValues.clear();
+ }
+ testValuesForType(dynamicCompType, byteBuffers.toArray(new ByteBuffer[0]));
+ }
+
+ // Similar to DynamicCompositeTypeTest.createDynamicCompositeKey(string, uuid, i, true, false), but not using any
+ // aliased types, in order to do an exact comparison of the unmarshalled DynamicCompositeType payload with the
+ // input one. If aliased types are used, due to DynamicCompositeType.build(List<String>, List<ByteBuffer>)
+ // always including the full type info in the newly constructed payload, an exact comparison won't work.
+ private static ByteBuffer createStringUuidVarintDynamicCompositeKey(String string, UUID uuid, int i, byte lastEocByte)
+ {
+ // 1. Calculate how many bytes do we need for a key of this DynamicCompositeType
+ String bytesType = "org.apache.cassandra.db.marshal.BytesType";
+ String timeUuidType = "org.apache.cassandra.db.marshal.TimeUUIDType";
+ String varintType = "org.apache.cassandra.db.marshal.IntegerType";
+ ByteBuffer bytes = ByteBufferUtil.bytes(string);
+ int totalSize = 0;
+ // Take into account the string component data (BytesType is aliased)
+ totalSize += 2 + bytesType.length() + 2 + bytes.remaining() + 1;
+ if (uuid != null)
+ {
+ // Take into account the UUID component data (TimeUUIDType is aliased)
+ totalSize += 2 + timeUuidType.length() + 2 + 16 + 1;
+ if (i != -1)
+ {
+ // Take into account the varint component data (IntegerType is _not_ aliased).
+ // Notice that we account for a single byte of varint data, so we'll downcast the int payload
+ // to byte and use only that as the actual varint payload.
+ totalSize += 2 + varintType.length() + 2 + 1 + 1;
+ }
+ }
+
+ // 2. Allocate a buffer with that many bytes
+ ByteBuffer bb = ByteBuffer.allocate(totalSize);
+
+ // 3. Write the key data for each component in the allocated buffer
+ bb.putShort((short) bytesType.length());
+ bb.put(ByteBufferUtil.bytes(bytesType));
+ bb.putShort((short) bytes.remaining());
+ bb.put(bytes);
+ // Make the end-of-component byte 1 if requested and the time-UUID component is null.
+ bb.put(uuid == null ? lastEocByte : (byte) 0);
+ if (uuid != null)
+ {
+ bb.putShort((short) timeUuidType.length());
+ bb.put(ByteBufferUtil.bytes(timeUuidType));
+ bb.putShort((short) 16);
+ bb.put(UUIDGen.decompose(uuid));
+ // Set the end-of-component byte if requested and the varint component is null.
+ bb.put(i == -1 ? lastEocByte : (byte) 0);
+ if (i != -1)
+ {
+ bb.putShort((short) varintType.length());
+ bb.put(ByteBufferUtil.bytes(varintType));
+ bb.putShort((short) 1);
+ bb.put((byte) i);
+ bb.put(lastEocByte);
+ }
+ }
+ bb.rewind();
+ return bb;
+ }
+
+ @Test
+ public void testFloatType()
+ {
+ Stream<Float> floats = Stream.of(null,
+ Float.NaN,
+ Float.POSITIVE_INFINITY,
+ Float.NEGATIVE_INFINITY,
+ Float.MAX_VALUE,
+ Float.MIN_VALUE,
+ +0.0F,
+ -0.0F,
+ +1.0F,
+ -1.0F,
+ +123456.7891011F,
+ -123456.7891011F);
+ testValuesForType(FloatType.instance, floats);
+
+ floats = new Random().ints(1000).mapToObj(Float::intBitsToFloat);
+ testValuesForType(FloatType.instance, floats);
+ }
+
+ @Test
+ public void testInetAddressType() throws UnknownHostException
+ {
+ Stream<InetAddress> inetAddresses = Stream.of(null,
+ InetAddress.getLocalHost(),
+ InetAddress.getLoopbackAddress(),
+ InetAddress.getByName("0.0.0.0"),
+ InetAddress.getByName("10.0.0.1"),
+ InetAddress.getByName("172.16.1.1"),
+ InetAddress.getByName("192.168.2.2"),
+ InetAddress.getByName("224.3.3.3"),
+ InetAddress.getByName("255.255.255.255"),
+ InetAddress.getByName("0000:0000:0000:0000:0000:0000:0000:0000"),
+ InetAddress.getByName("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"),
+ InetAddress.getByName("fe80:1:23:456:7890:1:23:456"));
+ testValuesForType(InetAddressType.instance, inetAddresses);
+
+ Random prng = new Random();
+ byte[] ipv4Bytes = new byte[4];
+ byte[] ipv6Bytes = new byte[16];
+ InetAddress[] addresses = new InetAddress[2000];
+ for (int i = 0; i < addresses.length / 2; ++i)
+ {
+ prng.nextBytes(ipv4Bytes);
+ addresses[2 * i] = InetAddress.getByAddress(ipv4Bytes);
+ addresses[2 * i + 1] = InetAddress.getByAddress(ipv6Bytes);
+ }
+ testValuesForType(InetAddressType.instance, addresses);
+
+ }
+
+ @Test
+ public void testInt32Type()
+ {
+ Stream<Integer> ints = Stream.of(null,
+ Integer.MIN_VALUE,
+ Integer.MIN_VALUE + 1,
+ -256, -255, -128, -127, -1,
+ 0,
+ 1, 127, 128, 255, 256,
+ Integer.MAX_VALUE - 1,
+ Integer.MAX_VALUE);
+ testValuesForType(Int32Type.instance, ints);
+
+ ints = new Random().ints(1000).boxed();
+ testValuesForType(Int32Type.instance, ints);
+ }
+
+ @Test
+ public void testIntegerType()
+ {
+ Stream<BigInteger> varints = IntStream.range(-1000000, 1000000).mapToObj(BigInteger::valueOf);
+ testValuesForType(IntegerType.instance, varints);
+
+ varints = Stream.of(null,
+ BigInteger.valueOf(12345678910111213L),
+ BigInteger.valueOf(12345678910111213L).negate(),
+ BigInteger.valueOf(Long.MAX_VALUE),
+ BigInteger.valueOf(Long.MAX_VALUE).negate(),
+ BigInteger.valueOf(Long.MAX_VALUE - 1).multiply(BigInteger.valueOf(Long.MAX_VALUE - 1)),
+ BigInteger.valueOf(Long.MAX_VALUE - 1).multiply(BigInteger.valueOf(Long.MAX_VALUE - 1)).negate());
+ testValuesForType(IntegerType.instance, varints);
+
+ List<BigInteger> varintList = new ArrayList<>();
+ for (int i = 0; i < 10000; ++i)
+ {
+ BigInteger initial = BigInteger.ONE.shiftLeft(i);
+ varintList.add(initial);
+ BigInteger plusOne = initial.add(BigInteger.ONE);
+ varintList.add(plusOne);
+ varintList.add(plusOne.negate());
+ BigInteger minusOne = initial.subtract(BigInteger.ONE);
+ varintList.add(minusOne);
+ varintList.add(minusOne.negate());
+ }
+ testValuesForType(IntegerType.instance, varintList.toArray(new BigInteger[0]));
+ }
+
+ @Test
+ public void testUuidTypes()
+ {
+ Random prng = new Random();
+ UUID[] testUuids = new UUID[3001];
+ for (int i = 0; i < testUuids.length / 3; ++i)
+ {
+ testUuids[3 * i] = UUID.randomUUID();
+ testUuids[3 * i + 1] = TimeUUID.Generator.nextTimeAsUUID();
+ testUuids[3 * i + 2] = TimeUUID.atUnixMicrosWithLsbAsUUID(prng.nextLong(), prng.nextLong());
+ }
+ testUuids[testUuids.length - 1] = null;
+ testValuesForType(UUIDType.instance, testUuids);
+ testValuesForType(LexicalUUIDType.instance, testUuids);
+ testValuesForType(TimeUUIDType.instance, Arrays.stream(testUuids)
+ .filter(u -> u == null || u.version() == 1)
+ .map(u -> u != null ? TimeUUID.fromUuid(u) : null));
+ }
+
+ private static <E, C extends Collection<E>> List<C> newRandomElementCollections(Supplier<? extends C> collectionProducer,
+ Supplier<? extends E> elementProducer,
+ int numCollections,
+ int numElementsInCollection)
+ {
+ List<C> result = new ArrayList<>();
+ for (int i = 0; i < numCollections; ++i)
+ {
+ C coll = collectionProducer.get();
+ for (int j = 0; j < numElementsInCollection; ++j)
+ {
+ coll.add(elementProducer.get());
+ }
+ result.add(coll);
+ }
+ return result;
+ }
+
+ @Test
+ public void testListType()
+ {
+ // Test lists with element components not having known/computable length (e.g. strings).
+ Random prng = new Random();
+ List<List<String>> stringLists = newRandomElementCollections(ArrayList::new,
+ () -> newRandomAlphanumeric(prng, 10),
+ 100,
+ 100);
+ testValuesForType(ListType.getInstance(UTF8Type.instance, false), stringLists);
+ testValuesForType(ListType.getInstance(UTF8Type.instance, true), stringLists);
+ // Test lists with element components with known/computable length (e.g. 128-bit UUIDs).
+ List<List<UUID>> uuidLists = newRandomElementCollections(ArrayList::new,
+ UUID::randomUUID,
+ 100,
+ 100);
+ testValuesForType(ListType.getInstance(UUIDType.instance, false), uuidLists);
+ testValuesForType(ListType.getInstance(UUIDType.instance, true), uuidLists);
+ }
+
+ @Test
+ public void testLongType()
+ {
+ Stream<Long> longs = Stream.of(null,
+ Long.MIN_VALUE,
+ Long.MIN_VALUE + 1,
+ (long) Integer.MIN_VALUE - 1,
+ -256L, -255L, -128L, -127L, -1L,
+ 0L,
+ 1L, 127L, 128L, 255L, 256L,
+ (long) Integer.MAX_VALUE + 1,
+ Long.MAX_VALUE - 1,
+ Long.MAX_VALUE);
+ testValuesForType(LongType.instance, longs);
+
+ longs = new Random().longs(1000).boxed();
+ testValuesForType(LongType.instance, longs);
+ }
+
+ private static <K, V> List<Map<K, V>> newRandomEntryMaps(Supplier<? extends K> keyProducer,
+ Supplier<? extends V> valueProducer,
+ int numMaps,
+ int numEntries)
+ {
+ List<Map<K, V>> result = new ArrayList<>();
+ for (int i = 0; i < numMaps; ++i)
+ {
+ Map<K, V> map = new HashMap<>();
+ for (int j = 0; j < numEntries; ++j)
+ {
+ K key = keyProducer.get();
+ V value = valueProducer.get();
+ map.put(key, value);
+ }
+ result.add(map);
+ }
+ return result;
+ }
+
+ @Test
+ public void testMapType()
+ {
+ Random prng = new Random();
+ List<Map<String, UUID>> stringToUuidMaps = newRandomEntryMaps(() -> newRandomAlphanumeric(prng, 10),
+ UUID::randomUUID,
+ 100,
+ 100);
+ testValuesForType(MapType.getInstance(UTF8Type.instance, UUIDType.instance, false), stringToUuidMaps);
+ testValuesForType(MapType.getInstance(UTF8Type.instance, UUIDType.instance, true), stringToUuidMaps);
+
+ List<Map<UUID, String>> uuidToStringMaps = newRandomEntryMaps(UUID::randomUUID,
+ () -> newRandomAlphanumeric(prng, 10),
+ 100,
+ 100);
+ testValuesForType(MapType.getInstance(UUIDType.instance, UTF8Type.instance, false), uuidToStringMaps);
+ testValuesForType(MapType.getInstance(UUIDType.instance, UTF8Type.instance, true), uuidToStringMaps);
+ }
+
+ @Test
+ public void testPartitionerDefinedOrder()
+ {
+ Random prng = new Random();
+ List<ByteBuffer> byteBuffers = new ArrayList<>();
+ byteBuffers.add(ByteBufferUtil.EMPTY_BYTE_BUFFER);
+ for (int i = 0; i < 1000; ++i)
+ {
+ String randomString = newRandomAlphanumeric(prng, 10);
+ byteBuffers.add(UTF8Type.instance.decompose(randomString));
+ int randomInt = prng.nextInt();
+ byteBuffers.add(Int32Type.instance.decompose(randomInt));
+ double randomDouble = prng.nextDouble();
+ byteBuffers.add(DoubleType.instance.decompose(randomDouble));
+ BigInteger randomishVarint = BigInteger.probablePrime(100, prng);
+ byteBuffers.add(IntegerType.instance.decompose(randomishVarint));
+ BigDecimal randomishDecimal = BigDecimal.valueOf(prng.nextLong(), prng.nextInt(100) - 50);
+ byteBuffers.add(DecimalType.instance.decompose(randomishDecimal));
+ }
+
+ byte[] bytes = new byte[100];
+ prng.nextBytes(bytes);
+ ByteBuffer exhausted = ByteBuffer.wrap(bytes);
+ ByteBufferUtil.readBytes(exhausted, 100);
+
+ List<IPartitioner> partitioners = Arrays.asList(
+ Murmur3Partitioner.instance,
+ RandomPartitioner.instance,
+ LengthPartitioner.instance
+ // NOTE LocalPartitioner, OrderPreservingPartitioner, and ByteOrderedPartitioner don't need a dedicated
+ // PartitionerDefinedOrder.
+ // 1) LocalPartitioner uses its inner AbstractType
+ // 2) OrderPreservingPartitioner uses UTF8Type
+ // 3) ByteOrderedPartitioner uses BytesType
+ );
+ for (IPartitioner partitioner : partitioners)
+ {
+ AbstractType<?> partitionOrdering = partitioner.partitionOrdering();
+ Assert.assertTrue(partitionOrdering instanceof PartitionerDefinedOrder);
+ for (ByteBuffer input : byteBuffers)
+ {
+ ByteSource byteSource = partitionOrdering.asComparableBytes(input, version);
+ ByteBuffer output = partitionOrdering.fromComparableBytes(ByteSource.peekable(byteSource), version);
+ Assert.assertEquals("For partitioner " + partitioner.getClass().getSimpleName(),
+ ByteBufferUtil.bytesToHex(input),
+ ByteBufferUtil.bytesToHex(output));
+ }
+ ByteSource byteSource = partitionOrdering.asComparableBytes(exhausted, version);
+ ByteBuffer output = partitionOrdering.fromComparableBytes(ByteSource.peekable(byteSource), version);
+ Assert.assertEquals(ByteBufferUtil.EMPTY_BYTE_BUFFER, output);
+ }
+ }
+
+ @Test
+ public void testReversedType()
+ {
+ // Test how ReversedType handles null ByteSource.Peekable - here the choice of base type is important, as
+ // the base type should also be able to handle null ByteSource.Peekable.
+ ReversedType<BigInteger> reversedVarintType = ReversedType.getInstance(IntegerType.instance);
+ ByteBuffer decodedNull = reversedVarintType.fromComparableBytes(null, ByteComparable.Version.OSS50);
+ Assert.assertEquals(ByteBufferUtil.EMPTY_BYTE_BUFFER, decodedNull);
+
+ // Test how ReversedType handles random data with some common and important base types.
+ Map<AbstractType<?>, BiFunction<Random, Integer, ByteBuffer>> bufferGeneratorByType = new HashMap<>();
+ bufferGeneratorByType.put(UTF8Type.instance, (prng, length) -> UTF8Type.instance.decompose(newRandomAlphanumeric(prng, length)));
+ bufferGeneratorByType.put(BytesType.instance, (prng, length) ->
+ {
+ byte[] randomBytes = new byte[length];
+ prng.nextBytes(randomBytes);
+ return ByteBuffer.wrap(randomBytes);
+ });
+ bufferGeneratorByType.put(IntegerType.instance, (prng, length) ->
+ {
+ BigInteger randomVarint = BigInteger.valueOf(prng.nextLong());
+ for (int i = 1; i < length / 8; ++i)
+ randomVarint = randomVarint.multiply(BigInteger.valueOf(prng.nextLong()));
+ return IntegerType.instance.decompose(randomVarint);
+ });
+ bufferGeneratorByType.put(DecimalType.instance, (prng, length) ->
+ {
+ BigInteger randomMantissa = BigInteger.valueOf(prng.nextLong());
+ for (int i = 1; i < length / 8; ++i)
+ randomMantissa = randomMantissa.multiply(BigInteger.valueOf(prng.nextLong()));
+ // Remove all trailing zeros from the mantissa and use an even scale, in order to have a "canonically
+ // represented" (in the context of DecimalType's encoding) decimal, i.e. one which wouldn't be re-scaled to
+ // conform with the "compacted mantissa between 0 and 1, scale as a power of 100" rule.
+ while (randomMantissa.remainder(BigInteger.TEN).equals(BigInteger.ZERO))
+ randomMantissa = randomMantissa.divide(BigInteger.TEN);
+ int randomScale = prng.nextInt() & -2;
+ BigDecimal randomDecimal = new BigDecimal(randomMantissa, randomScale);
+ return DecimalType.instance.decompose(randomDecimal);
+ });
+ Random prng = new Random();
+ for (Map.Entry<AbstractType<?>, BiFunction<Random, Integer, ByteBuffer>> entry : bufferGeneratorByType.entrySet())
+ {
+ ReversedType<?> reversedType = ReversedType.getInstance(entry.getKey());
+ for (int length = 32; length <= 512; length *= 4)
+ {
+ for (int i = 0; i < 100; ++i)
+ {
+ ByteBuffer initial = entry.getValue().apply(prng, length);
+ ByteSource.Peekable reversedPeekable = ByteSource.peekable(reversedType.asComparableBytes(initial, ByteComparable.Version.OSS50));
+ ByteBuffer decoded = reversedType.fromComparableBytes(reversedPeekable, ByteComparable.Version.OSS50);
+ Assert.assertEquals(initial, decoded);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testSetType()
+ {
+ // Test sets with element components not having known/computable length (e.g. strings).
+ Random prng = new Random();
+ List<Set<String>> stringSets = newRandomElementCollections(HashSet::new,
+ () -> newRandomAlphanumeric(prng, 10),
+ 100,
+ 100);
+ testValuesForType(SetType.getInstance(UTF8Type.instance, false), stringSets);
+ testValuesForType(SetType.getInstance(UTF8Type.instance, true), stringSets);
+ // Test sets with element components with known/computable length (e.g. 128-bit UUIDs).
+ List<Set<UUID>> uuidSets = newRandomElementCollections(HashSet::new,
+ UUID::randomUUID,
+ 100,
+ 100);
+ testValuesForType(SetType.getInstance(UUIDType.instance, false), uuidSets);
+ testValuesForType(SetType.getInstance(UUIDType.instance, true), uuidSets);
+ }
+
+ @Test
+ public void testShortType()
+ {
+ testValuesForType(ShortType.instance, new Short[] { null });
+
+ Stream<Short> allShorts = IntStream.range(Short.MIN_VALUE, Short.MAX_VALUE + 1)
+ .mapToObj(value -> (short) value);
+ testValuesForType(ShortType.instance, allShorts);
+ }
+
+ @Test
+ public void testSimpleDateType()
+ {
+ testValuesForType(SimpleDateType.instance, new Integer[] { null });
+
+ testValuesForType(SimpleDateType.instance, new Random().ints(1000).boxed());
+
+ // Test by manually creating and manually interpreting simple dates from random millis.
+ new Random().ints(1000).forEach(initialMillis ->
+ {
+ initialMillis = Math.abs(initialMillis);
+ Integer initialDays = SimpleDateSerializer.timeInMillisToDay(initialMillis);
+ ByteBuffer simpleDateBuffer = SimpleDateType.instance.fromTimeInMillis(initialMillis);
+ ByteSource byteSource = SimpleDateType.instance.asComparableBytes(simpleDateBuffer, version);
+ Integer decodedDays = SimpleDateType.instance.compose(SimpleDateType.instance.fromComparableBytes(ByteSource.peekable(byteSource), version));
+ Assert.assertEquals(initialDays, decodedDays);
+ });
+
+ // Test by manually creating and manually interpreting simple dates from strings.
+ String[] simpleDateStrings = new String[]
+ {
+ "1970-01-01",
+ "1970-01-02",
+ "1969-12-31",
+ "-0001-01-02",
+ "-5877521-01-02",
+ "2014-01-01",
+ "+5881580-01-10",
+ "1920-12-01",
+ "1582-10-19"
+ };
+ for (String simpleDate : simpleDateStrings)
+ {
+ ByteBuffer simpleDataBuffer = SimpleDateType.instance.fromString(simpleDate);
+ ByteSource byteSource = SimpleDateType.instance.asComparableBytes(simpleDataBuffer, version);
+ Integer decodedDays = SimpleDateType.instance.compose(SimpleDateType.instance.fromComparableBytes(ByteSource.peekable(byteSource), version));
+ String decodedDate = SimpleDateSerializer.instance.toString(decodedDays);
+ Assert.assertEquals(simpleDate, decodedDate);
+ }
+ }
+
+ @Test
+ public void testTimestampType()
+ {
+ Date[] dates = new Date[]
+ {
+ null,
+ new Date(),
+ new Date(0L),
+ new Date(-1L),
+ new Date(Long.MAX_VALUE),
+ new Date(Long.MIN_VALUE)
+ };
+ testValuesForType(TimestampType.instance, dates);
+ testValuesForType(TimestampType.instance, new Random().longs(1000).mapToObj(Date::new));
+ }
+
+ @Test
+ public void testTimeType()
+ {
+ testValuesForType(TimeType.instance, new Long[] { null });
+
+ testValuesForType(TimeType.instance, new Random().longs(1000).boxed());
+ }
+
+ @Test
+ public void testTupleType()
+ {
+ TupleType tt = new TupleType(Arrays.asList(UTF8Type.instance,
+ DecimalType.instance,
+ IntegerType.instance,
+ BytesType.instance));
+ Random prng = new Random();
+ List<ByteBuffer> tuplesData = new ArrayList<>();
+ String[] utf8Values = new String[]
+ {
+ "a",
+ "©",
+ newRandomAlphanumeric(prng, 10),
+ newRandomAlphanumeric(prng, 100)
+ };
+ BigDecimal[] decimalValues = new BigDecimal[]
+ {
+ null,
+ BigDecimal.ZERO,
+ BigDecimal.ONE,
+ BigDecimal.valueOf(1234567891011121314L, 50),
+ BigDecimal.valueOf(1234567891011121314L, 50).negate()
+ };
+ BigInteger[] varintValues = new BigInteger[]
+ {
+ null,
+ BigInteger.ZERO,
+ BigInteger.TEN.pow(1000),
+ BigInteger.TEN.pow(1000).negate()
+ };
+ byte[] oneByte = new byte[1];
+ byte[] tenBytes = new byte[10];
+ byte[] hundredBytes = new byte[100];
+ byte[] thousandBytes = new byte[1000];
+ prng.nextBytes(oneByte);
+ prng.nextBytes(tenBytes);
+ prng.nextBytes(hundredBytes);
+ prng.nextBytes(thousandBytes);
+ byte[][] bytesValues = new byte[][]
+ {
+ new byte[0],
+ oneByte,
+ tenBytes,
+ hundredBytes,
+ thousandBytes
+ };
+ for (String utf8 : utf8Values)
+ {
+ for (BigDecimal decimal : decimalValues)
+ {
+ for (BigInteger varint : varintValues)
+ {
+ for (byte[] bytes : bytesValues)
+ {
+ ByteBuffer tupleData = TupleType.buildValue(UTF8Type.instance.decompose(utf8),
+ decimal != null ? DecimalType.instance.decompose(decimal) : null,
+ varint != null ? IntegerType.instance.decompose(varint) : null,
+ // We could also use the wrapped bytes directly
+ BytesType.instance.decompose(ByteBuffer.wrap(bytes)));
+ tuplesData.add(tupleData);
+ }
+ }
+ }
+ }
+ testValuesForType(tt, tuplesData.toArray(new ByteBuffer[0]));
+ }
+
+ @Test
+ public void testUtf8Type()
+ {
+ Random prng = new Random();
+ testValuesForType(UTF8Type.instance, Stream.generate(() -> newRandomAlphanumeric(prng, 100)).limit(1000));
+ }
+
+ @Test
+ public void testTypeWithByteOrderedComparison()
+ {
+ Random prng = new Random();
+ byte[] singleByte = new byte[] { (byte) prng.nextInt() };
+ byte[] tenBytes = new byte[10];
+ prng.nextBytes(tenBytes);
+ byte[] hundredBytes = new byte[100];
+ prng.nextBytes(hundredBytes);
+ byte[] thousandBytes = new byte[1000];
+ prng.nextBytes(thousandBytes);
+ // No null here, as the default asComparableBytes(ByteBuffer, Version) implementation (and more specifically
+ // the ByteSource.of(ByteBuffer, Version) encoding) would throw then.
+ testValuesForType(ByteOrderedType.instance, Stream.of(ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ ByteBuffer.wrap(singleByte),
+ ByteBuffer.wrap(tenBytes),
+ ByteBuffer.wrap(hundredBytes),
+ ByteBuffer.wrap(thousandBytes)));
+ }
+
+ private static class ByteOrderedType extends AbstractType<ByteBuffer>
+ {
+ public static final ByteOrderedType instance = new ByteOrderedType();
+
+ private ByteOrderedType()
+ {
+ super(ComparisonType.BYTE_ORDER);
+ }
+
+ @Override
+ public ByteBuffer fromString(String source) throws MarshalException
+ {
+ return null;
+ }
+
+ @Override
+ public Term fromJSONObject(Object parsed) throws MarshalException
+ {
+ return null;
+ }
+
+ @Override
+ public TypeSerializer<ByteBuffer> getSerializer()
+ {
+ return ByteOrderedSerializer.instance;
+ }
+
+ static class ByteOrderedSerializer extends TypeSerializer<ByteBuffer>
+ {
+
+ static final ByteOrderedSerializer instance = new ByteOrderedSerializer();
+
+ @Override
+ public ByteBuffer serialize(ByteBuffer value)
+ {
+ return value != null ? value.duplicate() : null;
+ }
+
+ @Override
+ public <V> ByteBuffer deserialize(V bytes, ValueAccessor<V> accessor)
+ {
+ return accessor.toBuffer(bytes);
+ }
+
+ @Override
+ public <V> void validate(V bytes, ValueAccessor<V> accessor) throws MarshalException
+ {
+
+ }
+
+ @Override
+ public String toString(ByteBuffer value)
+ {
+ return ByteBufferUtil.bytesToHex(value);
+ }
+
+ @Override
+ public Class<ByteBuffer> getType()
+ {
+ return ByteBuffer.class;
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
new file mode 100644
index 0000000..56309e8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
@@ -0,0 +1,1179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.math.BigDecimal;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ordering;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.MurmurHash;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests forward conversion to ByteSource/ByteComparable and that the result compares correctly.
+ */
+public class ByteSourceComparisonTest extends ByteSourceTestBase
+{
+ private final static Logger logger = LoggerFactory.getLogger(ByteSourceComparisonTest.class);
+
+ @Rule
+ public final ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testStringsAscii()
+ {
+ testType(AsciiType.instance, testStrings);
+ }
+
+ @Test
+ public void testStringsUTF8()
+ {
+ testType(UTF8Type.instance, testStrings);
+ testDirect(x -> ByteSource.of(x, Version.OSS50), Ordering.<String>natural()::compare, testStrings);
+ }
+
+ @Test
+ public void testBooleans()
+ {
+ testType(BooleanType.instance, testBools);
+ }
+
+ @Test
+ public void testInts()
+ {
+ testType(Int32Type.instance, testInts);
+ testDirect(x -> ByteSource.of(x), Integer::compare, testInts);
+ }
+
+ @Test
+ public void randomTestInts()
+ {
+ Random rand = new Random();
+ for (int i=0; i<10000; ++i)
+ {
+ int i1 = rand.nextInt();
+ int i2 = rand.nextInt();
+ assertComparesSame(Int32Type.instance, i1, i2);
+ }
+
+ }
+
+ @Test
+ public void testLongs()
+ {
+ testType(LongType.instance, testLongs);
+ testDirect(x -> ByteSource.of(x), Long::compare, testLongs);
+ }
+
+ @Test
+ public void testShorts()
+ {
+ testType(ShortType.instance, testShorts);
+ }
+
+ @Test
+ public void testBytes()
+ {
+ testType(ByteType.instance, testBytes);
+ }
+
+ @Test
+ public void testDoubles()
+ {
+ testType(DoubleType.instance, testDoubles);
+ }
+
+ @Test
+ public void testFloats()
+ {
+ testType(FloatType.instance, testFloats);
+ }
+
+ @Test
+ public void testBigInts()
+ {
+ testType(IntegerType.instance, testBigInts);
+ }
+
+ @Test
+ public void testBigDecimals()
+ {
+ testType(DecimalType.instance, testBigDecimals);
+ }
+
+ @Test
+ public void testBigDecimalInCombination()
+ {
+ BigDecimal b1 = new BigDecimal("123456.78901201");
+ BigDecimal b2 = new BigDecimal("123456.789012");
+ Boolean b = false;
+
+ assertClusteringPairComparesSame(DecimalType.instance, BooleanType.instance, b1, b, b2, b);
+ assertClusteringPairComparesSame(BooleanType.instance, DecimalType.instance, b, b1, b, b2);
+
+ b1 = b1.negate();
+ b2 = b2.negate();
+
+ assertClusteringPairComparesSame(DecimalType.instance, BooleanType.instance, b1, b, b2, b);
+ assertClusteringPairComparesSame(BooleanType.instance, DecimalType.instance, b, b1, b, b2);
+
+ b1 = new BigDecimal("-123456.78901289");
+ b2 = new BigDecimal("-123456.789012");
+
+ assertClusteringPairComparesSame(DecimalType.instance, BooleanType.instance, b1, b, b2, b);
+ assertClusteringPairComparesSame(BooleanType.instance, DecimalType.instance, b, b1, b, b2);
+
+ b1 = new BigDecimal("1");
+ b2 = new BigDecimal("1.1");
+
+ assertClusteringPairComparesSame(DecimalType.instance, BooleanType.instance, b1, b, b2, b);
+ assertClusteringPairComparesSame(BooleanType.instance, DecimalType.instance, b, b1, b, b2);
+
+ b1 = b1.negate();
+ b2 = b2.negate();
+
+ assertClusteringPairComparesSame(DecimalType.instance, BooleanType.instance, b1, b, b2, b);
+ assertClusteringPairComparesSame(BooleanType.instance, DecimalType.instance, b, b1, b, b2);
+ }
+
+ @Test
+ public void testUUIDs()
+ {
+ testType(UUIDType.instance, testUUIDs);
+ }
+
+ @Test
+ public void testTimeUUIDs()
+ {
+ testType(TimeUUIDType.instance, Arrays.stream(testUUIDs)
+ .filter(x -> x == null || x.version() == 1)
+ .map(x -> x != null ? TimeUUID.fromUuid(x) : null)
+ .toArray());
+ }
+
+ @Test
+ public void testLexicalUUIDs()
+ {
+ testType(LexicalUUIDType.instance, testUUIDs);
+ }
+
+ @Test
+ public void testSimpleDate()
+ {
+ testType(SimpleDateType.instance, Arrays.stream(testInts).filter(x -> x != null).toArray());
+ }
+
+ @Test
+ public void testTimeType()
+ {
+ testType(TimeType.instance, Arrays.stream(testLongs).filter(x -> x != null && x >= 0 && x <= 24L * 60 * 60 * 1000 * 1000 * 1000).toArray());
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testDateType()
+ {
+ testType(DateType.instance, testDates);
+ }
+
+ @Test
+ public void testTimestampType()
+ {
+ testType(TimestampType.instance, testDates);
+ }
+
+ @Test
+ public void testBytesType()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < testValues.length; ++i)
+ for (Object o : testValues[i])
+ values.add(testTypes[i].decompose(o));
+
+ testType(BytesType.instance, values.toArray());
+ }
+
+ @Test
+ public void testInetAddressType() throws UnknownHostException
+ {
+ testType(InetAddressType.instance, testInets);
+ }
+
+ @Test
+ public void testEmptyType()
+ {
+ testType(EmptyType.instance, new Void[] { null });
+ }
+
+ @Test
+ public void testPatitionerDefinedOrder()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < testValues.length; ++i)
+ for (Object o : testValues[i])
+ values.add(testTypes[i].decompose(o));
+
+ testBuffers(new PartitionerDefinedOrder(Murmur3Partitioner.instance), values);
+ testBuffers(new PartitionerDefinedOrder(RandomPartitioner.instance), values);
+ testBuffers(new PartitionerDefinedOrder(ByteOrderedPartitioner.instance), values);
+ }
+
+ @Test
+ public void testPatitionerOrder()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < testValues.length; ++i)
+ for (Object o : testValues[i])
+ values.add(testTypes[i].decompose(o));
+
+ testDecoratedKeys(Murmur3Partitioner.instance, values);
+ testDecoratedKeys(RandomPartitioner.instance, values);
+ testDecoratedKeys(ByteOrderedPartitioner.instance, values);
+ }
+
+ @Test
+ public void testLocalPatitionerOrder()
+ {
+ for (int i = 0; i < testValues.length; ++i)
+ {
+ final AbstractType testType = testTypes[i];
+ testDecoratedKeys(new LocalPartitioner(testType), Lists.transform(Arrays.asList(testValues[i]),
+ v -> testType.decompose(v)));
+ }
+ }
+
+ interface PairTester
+ {
+ void test(AbstractType t1, AbstractType t2, Object o1, Object o2, Object o3, Object o4);
+ }
+
+ void testCombinationSampling(Random rand, PairTester tester)
+ {
+ for (int i=0;i<testTypes.length;++i)
+ for (int j=0;j<testTypes.length;++j)
+ {
+ Object[] tv1 = new Object[3];
+ Object[] tv2 = new Object[3];
+ for (int t=0; t<tv1.length; ++t)
+ {
+ tv1[t] = testValues[i][rand.nextInt(testValues[i].length)];
+ tv2[t] = testValues[j][rand.nextInt(testValues[j].length)];
+ }
+
+ for (Object o1 : tv1)
+ for (Object o2 : tv2)
+ for (Object o3 : tv1)
+ for (Object o4 : tv2)
+
+ {
+ tester.test(testTypes[i], testTypes[j], o1, o2, o3, o4);
+ }
+ }
+ }
+
+ @Test
+ public void testCombinations()
+ {
+ Random rand = new Random(0);
+ testCombinationSampling(rand, this::assertClusteringPairComparesSame);
+ }
+
+ @Test
+ public void testNullsInClustering()
+ {
+ ByteBuffer[][] inputs = new ByteBuffer[][]
+ {
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, "a"),
+ decomposeAndRandomPad(Int32Type.instance, 0)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, "a"),
+ decomposeAndRandomPad(Int32Type.instance, null)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, "a"),
+ null},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, ""),
+ decomposeAndRandomPad(Int32Type.instance, 0)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, ""),
+ decomposeAndRandomPad(Int32Type.instance, null)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, ""),
+ null},
+ new ByteBuffer[] {null,
+ decomposeAndRandomPad(Int32Type.instance, 0)},
+ new ByteBuffer[] {null,
+ decomposeAndRandomPad(Int32Type.instance, null)},
+ new ByteBuffer[] {null,
+ null}
+ };
+ for (ByteBuffer[] input1 : inputs)
+ for (ByteBuffer[] input2 : inputs)
+ {
+ assertClusteringPairComparesSame(UTF8Type.instance, Int32Type.instance,
+ input1[0], input1[1], input2[0], input2[1],
+ (t, v) -> (ByteBuffer) v,
+ input1[0] != null && input1[1] != null && input2[0] != null && input2[1] != null);
+ }
+ }
+
+ @Test
+ public void testNullsInClusteringLegacy()
+ {
+ // verify the legacy encoding treats null clustering the same as null value
+ ClusteringPrefix<ByteBuffer> aNull = makeBound(ClusteringPrefix.Kind.CLUSTERING,
+ decomposeAndRandomPad(UTF8Type.instance, "a"),
+ decomposeAndRandomPad(Int32Type.instance, null));
+ ClusteringPrefix<ByteBuffer> aEmpty = makeBound(ClusteringPrefix.Kind.CLUSTERING,
+ decomposeAndRandomPad(UTF8Type.instance, "a"),
+ null);
+ ClusteringComparator comp = new ClusteringComparator(UTF8Type.instance, Int32Type.instance);
+ assertEquals(0, ByteComparable.compare(comp.asByteComparable(aNull), comp.asByteComparable(aEmpty), Version.LEGACY));
+ ClusteringComparator compReversed = new ClusteringComparator(UTF8Type.instance, ReversedType.getInstance(Int32Type.instance));
+ assertEquals(0, ByteComparable.compare(compReversed.asByteComparable(aNull), compReversed.asByteComparable(aEmpty), Version.LEGACY));
+ }
+
+ @Test
+ public void testEmptyClustering()
+ {
+ assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.CLUSTERING, Version.OSS50);
+ assertEmptyComparedToStatic(0, ClusteringPrefix.Kind.STATIC_CLUSTERING, Version.OSS50);
+ assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_START_BOUND, Version.OSS50);
+ assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_END_BOUND, Version.OSS50);
+
+ assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.CLUSTERING, Version.LEGACY);
+ assertEmptyComparedToStatic(0, ClusteringPrefix.Kind.STATIC_CLUSTERING, Version.LEGACY);
+ assertEmptyComparedToStatic(-1, ClusteringPrefix.Kind.INCL_START_BOUND, Version.LEGACY);
+ assertEmptyComparedToStatic(1, ClusteringPrefix.Kind.INCL_END_BOUND, Version.LEGACY);
+ }
+
+ private void assertEmptyComparedToStatic(int expected, ClusteringPrefix.Kind kind, Version version)
+ {
+ ClusteringPrefix<ByteBuffer> empty = makeBound(kind);
+ ClusteringComparator compEmpty = new ClusteringComparator();
+ assertEquals(expected, Integer.signum(ByteComparable.compare(compEmpty.asByteComparable(empty),
+ compEmpty.asByteComparable(Clustering.STATIC_CLUSTERING),
+ version)));
+ }
+
+ void assertClusteringPairComparesSame(AbstractType<?> t1, AbstractType<?> t2, Object o1, Object o2, Object o3, Object o4)
+ {
+ assertClusteringPairComparesSame(t1, t2, o1, o2, o3, o4, AbstractType::decompose, true);
+ }
+
+ void assertClusteringPairComparesSame(AbstractType<?> t1, AbstractType<?> t2,
+ Object o1, Object o2, Object o3, Object o4,
+ BiFunction<AbstractType, Object, ByteBuffer> decompose,
+ boolean testLegacy)
+ {
+ EnumSet<ClusteringPrefix.Kind> skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND);
+ for (Version v : Version.values())
+ for (ClusteringPrefix.Kind k1 : EnumSet.complementOf(skippedKinds))
+ for (ClusteringPrefix.Kind k2 : EnumSet.complementOf(skippedKinds))
+ {
+ if (!testLegacy && v == Version.LEGACY)
+ continue;
+
+ ClusteringComparator comp = new ClusteringComparator(t1, t2);
+ ByteBuffer[] b = new ByteBuffer[2];
+ ByteBuffer[] d = new ByteBuffer[2];
+ b[0] = decompose.apply(t1, o1);
+ b[1] = decompose.apply(t2, o2);
+ d[0] = decompose.apply(t1, o3);
+ d[1] = decompose.apply(t2, o4);
+ ClusteringPrefix<ByteBuffer> c = makeBound(k1, b);
+ ClusteringPrefix<ByteBuffer> e = makeBound(k2, d);
+ final ByteComparable bsc = comp.asByteComparable(c);
+ final ByteComparable bse = comp.asByteComparable(e);
+ int expected = Integer.signum(comp.compare(c, e));
+ assertEquals(String.format("Failed comparing %s and %s, %s vs %s version %s",
+ safeStr(c.clusteringString(comp.subtypes())),
+ safeStr(e.clusteringString(comp.subtypes())), bsc, bse, v),
+ expected, Integer.signum(ByteComparable.compare(bsc, bse, v)));
+ maybeCheck41Properties(expected, bsc, bse, v);
+ maybeAssertNotPrefix(bsc, bse, v);
+
+ ClusteringComparator compR = new ClusteringComparator(ReversedType.getInstance(t1), ReversedType.getInstance(t2));
+ final ByteComparable bsrc = compR.asByteComparable(c);
+ final ByteComparable bsre = compR.asByteComparable(e);
+ int expectedR = Integer.signum(compR.compare(c, e));
+ assertEquals(String.format("Failed comparing reversed %s and %s, %s vs %s version %s",
+ safeStr(c.clusteringString(comp.subtypes())),
+ safeStr(e.clusteringString(comp.subtypes())), bsrc, bsre, v),
+ expectedR, Integer.signum(ByteComparable.compare(bsrc, bsre, v)));
+ maybeCheck41Properties(expectedR, bsrc, bsre, v);
+ maybeAssertNotPrefix(bsrc, bsre, v);
+ }
+ }
+
+ static ClusteringPrefix<ByteBuffer> makeBound(ClusteringPrefix.Kind k1, ByteBuffer... b)
+ {
+ return makeBound(ByteBufferAccessor.instance.factory(), k1, b);
+ }
+
+ static <T> ClusteringPrefix<T> makeBound(ValueAccessor.ObjectFactory<T> factory, ClusteringPrefix.Kind k1, T[] b)
+ {
+ switch (k1)
+ {
+ case INCL_END_EXCL_START_BOUNDARY:
+ case EXCL_END_INCL_START_BOUNDARY:
+ return factory.boundary(k1, b);
+
+ case INCL_END_BOUND:
+ case EXCL_END_BOUND:
+ case INCL_START_BOUND:
+ case EXCL_START_BOUND:
+ return factory.bound(k1, b);
+
+ case CLUSTERING:
+ return factory.clustering(b);
+
+ case STATIC_CLUSTERING:
+ return factory.staticClustering();
+
+ default:
+ throw new AssertionError(k1);
+ }
+ }
+
+ @Test
+ public void testTupleType()
+ {
+ Random rand = ThreadLocalRandom.current();
+ testCombinationSampling(rand, this::assertTupleComparesSame);
+ }
+
+ @Test
+ public void testTupleTypeNonFull()
+ {
+ TupleType tt = new TupleType(ImmutableList.of(UTF8Type.instance, Int32Type.instance));
+ List<ByteBuffer> tests = ImmutableList.of
+ (
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, ""),
+ decomposeAndRandomPad(Int32Type.instance, 0)),
+ // Note: a decomposed null (e.g. decomposeAndRandomPad(Int32Type.instance, null)) should not reach a tuple
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, ""),
+ null),
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ null,
+ decomposeAndRandomPad(Int32Type.instance, 0)),
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, "")),
+ TupleType.buildValue(ByteBufferAccessor.instance, (ByteBuffer) null),
+ TupleType.buildValue(ByteBufferAccessor.instance)
+ );
+ testBuffers(tt, tests);
+ }
+
+ @Test
+ public void testTupleNewField()
+ {
+ TupleType t1 = new TupleType(ImmutableList.of(UTF8Type.instance));
+ TupleType t2 = new TupleType(ImmutableList.of(UTF8Type.instance, Int32Type.instance));
+
+ ByteBuffer vOne = TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, "str"));
+ ByteBuffer vOneAndNull = TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, "str"),
+ null);
+
+ ByteComparable bOne1 = typeToComparable(t1, vOne);
+ ByteComparable bOne2 = typeToComparable(t2, vOne);
+ ByteComparable bOneAndNull2 = typeToComparable(t2, vOneAndNull);
+
+ assertEquals("The byte-comparable version of a one-field tuple must be the same as a two-field tuple with non-present second component.",
+ bOne1.byteComparableAsString(Version.OSS50),
+ bOne2.byteComparableAsString(Version.OSS50));
+ assertEquals("The byte-comparable version of a one-field tuple must be the same as a two-field tuple with null as second component.",
+ bOne1.byteComparableAsString(Version.OSS50),
+ bOneAndNull2.byteComparableAsString(Version.OSS50));
+ }
+
+
+ void assertTupleComparesSame(AbstractType t1, AbstractType t2, Object o1, Object o2, Object o3, Object o4)
+ {
+ TupleType tt = new TupleType(ImmutableList.of(t1, t2));
+ ByteBuffer b1 = TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeForTuple(t1, o1),
+ decomposeForTuple(t2, o2));
+ ByteBuffer b2 = TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeForTuple(t1, o3),
+ decomposeForTuple(t2, o4));
+ assertComparesSameBuffers(tt, b1, b2);
+ }
+
+ static <T> ByteBuffer decomposeForTuple(AbstractType<T> t, T o)
+ {
+ return o != null ? t.decompose(o) : null;
+ }
+
+ @Test
+ public void testCompositeType()
+ {
+ Random rand = new Random(0);
+ testCombinationSampling(rand, this::assertCompositeComparesSame);
+ }
+
+ @Test
+ public void testCompositeTypeNonFull()
+ {
+ CompositeType tt = CompositeType.getInstance(UTF8Type.instance, Int32Type.instance);
+ List<ByteBuffer> tests = ImmutableList.of
+ (
+ CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, ""), decomposeAndRandomPad(Int32Type.instance, 0)),
+ CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, ""), decomposeAndRandomPad(Int32Type.instance, null)),
+ CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, "")),
+ CompositeType.build(ByteBufferAccessor.instance),
+ CompositeType.build(ByteBufferAccessor.instance, true, decomposeAndRandomPad(UTF8Type.instance, "")),
+ CompositeType.build(ByteBufferAccessor.instance,true)
+ );
+ for (ByteBuffer b : tests)
+ tt.validate(b);
+ testBuffers(tt, tests);
+ }
+
+ void assertCompositeComparesSame(AbstractType t1, AbstractType t2, Object o1, Object o2, Object o3, Object o4)
+ {
+ CompositeType tt = CompositeType.getInstance(t1, t2);
+ ByteBuffer b1 = CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(t1, o1), decomposeAndRandomPad(t2, o2));
+ ByteBuffer b2 = CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(t1, o3), decomposeAndRandomPad(t2, o4));
+ assertComparesSameBuffers(tt, b1, b2);
+ }
+
+ @Test
+ public void testDynamicComposite()
+ {
+ DynamicCompositeType tt = DynamicCompositeType.getInstance(DynamicCompositeTypeTest.aliases);
+ UUID[] uuids = DynamicCompositeTypeTest.uuids;
+ List<ByteBuffer> tests = ImmutableList.of
+ (
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test1", null, -1, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test1", uuids[0], 24, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test1", uuids[0], 42, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test2", uuids[0], -1, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test2", uuids[1], 42, false, true)
+ );
+ for (ByteBuffer b : tests)
+ tt.validate(b);
+ testBuffers(tt, tests);
+ }
+
+ @Test
+ public void testListTypeString()
+ {
+ testCollection(ListType.getInstance(UTF8Type.instance, true), testStrings, () -> new ArrayList<>(), new Random());
+ }
+
+ @Test
+ public void testListTypeLong()
+ {
+ testCollection(ListType.getInstance(LongType.instance, true), testLongs, () -> new ArrayList<>(), new Random());
+ }
+
+ @Test
+ public void testSetTypeString()
+ {
+ testCollection(SetType.getInstance(UTF8Type.instance, true), testStrings, () -> new HashSet<>(), new Random());
+ }
+
+ @Test
+ public void testSetTypeLong()
+ {
+ testCollection(SetType.getInstance(LongType.instance, true), testLongs, () -> new HashSet<>(), new Random());
+ }
+
+ <T, CT extends Collection<T>> void testCollection(CollectionType<CT> tt, T[] values, Supplier<CT> gen, Random rand)
+ {
+ int cnt = 0;
+ List<CT> tests = new ArrayList<>();
+ tests.add(gen.get());
+ for (int c = 1; c <= 3; ++c)
+ for (int j = 0; j < 5; ++j)
+ {
+ CT l = gen.get();
+ for (int i = 0; i < c; ++i)
+ l.add(values[cnt++ % values.length]);
+
+ tests.add(l);
+ }
+ testType(tt, tests);
+ }
+
+ @Test
+ public void testMapTypeStringLong()
+ {
+ testMap(MapType.getInstance(UTF8Type.instance, LongType.instance, true), testStrings, testLongs, () -> new HashMap<>(), new Random());
+ }
+
+ @Test
+ public void testMapTypeStringLongTree()
+ {
+ testMap(MapType.getInstance(UTF8Type.instance, LongType.instance, true), testStrings, testLongs, () -> new TreeMap<>(), new Random());
+ }
+
+ @Test
+ public void testDecoratedKeyPrefixesVOSS50()
+ {
+ // This should pass with the OSS 4.1 encoding
+ testDecoratedKeyPrefixes(Version.OSS50);
+ }
+
+ @Test
+ public void testDecoratedKeyPrefixesVLegacy()
+ {
+ // ... and fail with the legacy encoding
+ try
+ {
+ testDecoratedKeyPrefixes(Version.LEGACY);
+ }
+ catch (AssertionError e)
+ {
+ // Correct path, test failing.
+ return;
+ }
+ Assert.fail("Test expected to fail.");
+ }
+
+ @Test
+ public void testFixedLengthWithOffset()
+ {
+ byte[] bytes = new byte[]{ 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+
+ ByteSource source = ByteSource.fixedLength(bytes, 0, 1);
+ assertEquals(1, source.next());
+ assertEquals(ByteSource.END_OF_STREAM, source.next());
+
+ source = ByteSource.fixedLength(bytes, 4, 5);
+ assertEquals(5, source.next());
+ assertEquals(6, source.next());
+ assertEquals(7, source.next());
+ assertEquals(8, source.next());
+ assertEquals(9, source.next());
+ assertEquals(ByteSource.END_OF_STREAM, source.next());
+
+ ByteSource.fixedLength(bytes, 9, 0);
+ assertEquals(ByteSource.END_OF_STREAM, source.next());
+ }
+
+ @Test
+ public void testFixedLengthNegativeLength()
+ {
+ byte[] bytes = new byte[]{ 1, 2, 3 };
+
+ expectedException.expect(IllegalArgumentException.class);
+ ByteSource.fixedLength(bytes, 0, -1);
+ }
+
+ @Test
+ public void testFixedLengthNegativeOffset()
+ {
+ byte[] bytes = new byte[]{ 1, 2, 3 };
+
+ expectedException.expect(IllegalArgumentException.class);
+ ByteSource.fixedLength(bytes, -1, 1);
+ }
+
+ @Test
+ public void testFixedLengthOutOfBounds()
+ {
+ byte[] bytes = new byte[]{ 1, 2, 3 };
+
+ expectedException.expect(IllegalArgumentException.class);
+ ByteSource.fixedLength(bytes, 0, 4);
+ }
+
+ @Test
+ public void testFixedOffsetOutOfBounds()
+ {
+ byte[] bytes = new byte[]{ 1, 2, 3 };
+
+ expectedException.expect(IllegalArgumentException.class);
+ ByteSource.fixedLength(bytes, 4, 1);
+ }
+
+ @Test
+ public void testSeparatorGT()
+ {
+ testSeparator(ByteComparable::separatorGt, testLongs, LongType.instance);
+ }
+
+ @Test
+ public void testSeparatorPrefix()
+ {
+ testSeparator(ByteComparable::separatorPrefix, testLongs, LongType.instance);
+ }
+
+ @Test
+ public void testSeparatorPrefixViaDiffPoint()
+ {
+ testSeparator((x, y) -> version -> ByteSource.cut(y.asComparableBytes(version),
+ ByteComparable.diffPoint(x, y, version)),
+ testLongs,
+ LongType.instance);
+ }
+ @Test
+ public void testSeparatorNext()
+ {
+ // Appending a 00 byte at the end gives the immediate next possible value after x.
+ testSeparator((x, y) -> version -> ByteSource.cutOrRightPad(x.asComparableBytes(version),
+ ByteComparable.length(x, version) + 1,
+ 0),
+ testLongs,
+ LongType.instance);
+ }
+
+ private <T> void testSeparator(BiFunction<ByteComparable, ByteComparable, ByteComparable> separatorMethod, T[] testValues, AbstractType<T> type)
+ {
+ for (T v1 : testValues)
+ for (T v2 : testValues)
+ {
+ if (v1 == null || v2 == null)
+ continue;
+ if (type.compare(type.decompose(v1), type.decompose(v2)) >= 0)
+ continue;
+ ByteComparable bc1 = getByteComparable(type, v1);
+ ByteComparable bc2 = getByteComparable(type, v2);
+ ByteComparable separator = separatorMethod.apply(bc1, bc2);
+
+ for (Version version : Version.values())
+ {
+ Assert.assertTrue("Sanity check failed", ByteComparable.compare(bc1, bc2, version) < 0);
+ Assert.assertTrue(String.format("Separator %s must be greater than left %s (for %s) (version %s)",
+ separator.byteComparableAsString(version),
+ bc1.byteComparableAsString(version),
+ v1,
+ version),
+ ByteComparable.compare(bc1, separator, version) < 0);
+ Assert.assertTrue(String.format("Separator %s must be less than or equal to right %s (for %s) (version %s)",
+ separator.byteComparableAsString(version),
+ bc2.byteComparableAsString(version),
+ v2,
+ version),
+ ByteComparable.compare(separator, bc2, version) <= 0);
+ }
+ }
+ }
+
+ private <T> ByteComparable getByteComparable(AbstractType<T> type, T v1)
+ {
+ return version -> type.asComparableBytes(type.decompose(v1), version);
+ }
+
+ public void testDecoratedKeyPrefixes(Version version)
+ {
+ testDecoratedKeyPrefixes("012345678BCDE\0", "", version);
+ testDecoratedKeyPrefixes("012345678ABCDE\0", "ABC", version);
+ testDecoratedKeyPrefixes("0123456789ABCDE\0", "\0AB", version);
+ testDecoratedKeyPrefixes("0123456789ABCDEF\0", "\0", version);
+
+ testDecoratedKeyPrefixes("0123456789ABCDEF0", "ABC", version);
+ testDecoratedKeyPrefixes("0123456789ABCDEF", "", version);
+ testDecoratedKeyPrefixes("0123456789ABCDE", "", version);
+ testDecoratedKeyPrefixes("0123456789ABCD", "\0AB", version);
+ testDecoratedKeyPrefixes("0123456789ABC", "\0", version);
+
+ }
+
+ public void testDecoratedKeyPrefixes(String key, String append, Version version)
+ {
+ logger.info("Testing {} + {}", safeStr(key), safeStr(append));
+ IPartitioner partitioner = Murmur3Partitioner.instance;
+ ByteBuffer original = ByteBufferUtil.bytes(key);
+ ByteBuffer collision = Util.generateMurmurCollision(original, append.getBytes(StandardCharsets.UTF_8));
+
+ long[] hash = new long[2];
+ MurmurHash.hash3_x64_128(original, 0, original.limit(), 0, hash);
+ logger.info(String.format("Original hash %016x,%016x", hash[0], hash[1]));
+ MurmurHash.hash3_x64_128(collision, 0, collision.limit(), 0, hash);
+ logger.info(String.format("Collision hash %016x,%016x", hash[0], hash[1]));
+
+ DecoratedKey kk1 = partitioner.decorateKey(original);
+ DecoratedKey kk2 = partitioner.decorateKey(collision);
+ logger.info("{}\n{}\n{}\n{}", kk1, kk2, kk1.byteComparableAsString(version), kk2.byteComparableAsString(version));
+
+ final ByteSource s1 = kk1.asComparableBytes(version);
+ final ByteSource s2 = kk2.asComparableBytes(version);
+ logger.info("{}\n{}", s1, s2);
+
+ // Check that the representations compare correctly
+ Assert.assertEquals(Long.signum(kk1.compareTo(kk2)), ByteComparable.compare(kk1, kk2, version));
+ // s1 must not be a prefix of s2
+ assertNotPrefix(s1, s2);
+ }
+
+ private void assertNotPrefix(ByteSource s1, ByteSource s2)
+ {
+ int c1, c2;
+ do
+ {
+ c1 = s1.next();
+ c2 = s2.next();
+ }
+ while (c1 == c2 && c1 != ByteSource.END_OF_STREAM);
+
+ // Equal is ok
+ if (c1 == c2)
+ return;
+
+ Assert.assertNotEquals("ByteComparable is a prefix of other", ByteSource.END_OF_STREAM, c1);
+ Assert.assertNotEquals("ByteComparable is a prefix of other", ByteSource.END_OF_STREAM, c2);
+ }
+
+ private int compare(ByteSource s1, ByteSource s2)
+ {
+ int c1, c2;
+ do
+ {
+ c1 = s1.next();
+ c2 = s2.next();
+ }
+ while (c1 == c2 && c1 != ByteSource.END_OF_STREAM);
+
+ return Integer.compare(c1, c2);
+ }
+
+ private void maybeAssertNotPrefix(ByteComparable s1, ByteComparable s2, Version version)
+ {
+ if (version == Version.OSS50)
+ assertNotPrefix(s1.asComparableBytes(version), s2.asComparableBytes(version));
+ }
+
+ private void maybeCheck41Properties(int expectedComparison, ByteComparable s1, ByteComparable s2, Version version)
+ {
+ if (version != Version.OSS50)
+ return;
+
+ if (s1 == null || s2 == null || 0 == expectedComparison)
+ return;
+ int b1 = randomTerminator();
+ int b2 = randomTerminator();
+ assertEquals(String.format("Comparison failed for %s(%s + %02x) and %s(%s + %02x)", s1, s1.byteComparableAsString(version), b1, s2, s2.byteComparableAsString(version), b2),
+ expectedComparison, Integer.signum(compare(ByteSource.withTerminator(b1, s1.asComparableBytes(version)), ByteSource.withTerminator(b2, s2.asComparableBytes(version)))));
+ assertNotPrefix(ByteSource.withTerminator(b1, s1.asComparableBytes(version)), ByteSource.withTerminator(b2, s2.asComparableBytes(version)));
+ }
+
+ private int randomTerminator()
+ {
+ int term;
+ do
+ {
+ term = ThreadLocalRandom.current().nextInt(ByteSource.MIN_SEPARATOR, ByteSource.MAX_SEPARATOR + 1);
+ }
+ while (term >= ByteSource.MIN_NEXT_COMPONENT && term <= ByteSource.MAX_NEXT_COMPONENT);
+ return term;
+ }
+
+ <K, V, M extends Map<K, V>> void testMap(MapType<K, V> tt, K[] keys, V[] values, Supplier<M> gen, Random rand)
+ {
+ List<M> tests = new ArrayList<>();
+ tests.add(gen.get());
+ for (int c = 1; c <= 3; ++c)
+ for (int j = 0; j < 5; ++j)
+ {
+ M l = gen.get();
+ for (int i = 0; i < c; ++i)
+ l.put(keys[rand.nextInt(keys.length)], values[rand.nextInt(values.length)]);
+
+ tests.add(l);
+ }
+ testType(tt, tests);
+ }
+
+ /*
+ * Convert type to a comparable.
+ */
+ private ByteComparable typeToComparable(AbstractType<?> type, ByteBuffer value)
+ {
+ return new ByteComparable()
+ {
+ @Override
+ public ByteSource asComparableBytes(Version v)
+ {
+ return type.asComparableBytes(value, v);
+ }
+
+ @Override
+ public String toString()
+ {
+ return type.getString(value);
+ }
+ };
+ }
+
+ public <T> void testType(AbstractType<T> type, Object[] values)
+ {
+ testType(type, Iterables.transform(Arrays.asList(values), x -> (T) x));
+ }
+
+ public <T> void testType(AbstractType<? super T> type, Iterable<T> values)
+ {
+ for (T i : values) {
+ ByteBuffer b = decomposeAndRandomPad(type, i);
+ logger.info("Value {} ({}) bytes {} ByteSource {}",
+ safeStr(i),
+ safeStr(type.getSerializer().toCQLLiteral(b)),
+ safeStr(ByteBufferUtil.bytesToHex(b)),
+ typeToComparable(type, b).byteComparableAsString(Version.OSS50));
+ }
+ for (T i : values)
+ for (T j : values)
+ assertComparesSame(type, i, j);
+ if (!type.isReversed())
+ testType(ReversedType.getInstance(type), values);
+ }
+
+ public void testBuffers(AbstractType<?> type, List<ByteBuffer> values)
+ {
+ try
+ {
+ for (ByteBuffer b : values) {
+ logger.info("Value {} bytes {} ByteSource {}",
+ safeStr(type.getSerializer().toCQLLiteral(b)),
+ safeStr(ByteBufferUtil.bytesToHex(b)),
+ typeToComparable(type, b).byteComparableAsString(Version.OSS50));
+ }
+ }
+ catch (UnsupportedOperationException e)
+ {
+ // Continue without listing values.
+ }
+
+ for (ByteBuffer i : values)
+ for (ByteBuffer j : values)
+ assertComparesSameBuffers(type, i, j);
+ }
+
+ void assertComparesSameBuffers(AbstractType<?> type, ByteBuffer b1, ByteBuffer b2)
+ {
+ int expected = Integer.signum(type.compare(b1, b2));
+ final ByteComparable bs1 = typeToComparable(type, b1);
+ final ByteComparable bs2 = typeToComparable(type, b2);
+
+ for (Version version : Version.values())
+ {
+ int actual = Integer.signum(ByteComparable.compare(bs1, bs2, version));
+ assertEquals(String.format("Failed comparing %s(%s) and %s(%s)", ByteBufferUtil.bytesToHex(b1), bs1.byteComparableAsString(version), ByteBufferUtil.bytesToHex(b2), bs2.byteComparableAsString(version)),
+ expected,
+ actual);
+ maybeCheck41Properties(expected, bs1, bs2, version);
+ }
+ }
+
+ public void testDecoratedKeys(IPartitioner type, List<ByteBuffer> values)
+ {
+ for (ByteBuffer i : values)
+ for (ByteBuffer j : values)
+ assertComparesSameDecoratedKeys(type, i, j);
+ for (ByteBuffer i : values)
+ assertDecoratedKeyBounds(type, i);
+ }
+
+ void assertComparesSameDecoratedKeys(IPartitioner type, ByteBuffer b1, ByteBuffer b2)
+ {
+ DecoratedKey k1 = type.decorateKey(b1);
+ DecoratedKey k2 = type.decorateKey(b2);
+ int expected = Integer.signum(k1.compareTo(k2));
+
+ for (Version version : Version.values())
+ {
+ int actual = Integer.signum(ByteComparable.compare(k1, k2, version));
+ assertEquals(String.format("Failed comparing %s[%s](%s) and %s[%s](%s)\npartitioner %s version %s",
+ ByteBufferUtil.bytesToHex(b1),
+ k1,
+ k1.byteComparableAsString(version),
+ ByteBufferUtil.bytesToHex(b2),
+ k2,
+ k2.byteComparableAsString(version),
+ type,
+ version),
+ expected,
+ actual);
+ maybeAssertNotPrefix(k1, k2, version);
+ }
+ }
+
+ void assertDecoratedKeyBounds(IPartitioner type, ByteBuffer b)
+ {
+ Version version = Version.OSS50;
+ DecoratedKey k = type.decorateKey(b);
+ final ByteComparable after = k.asComparableBound(false);
+ final ByteComparable before = k.asComparableBound(true);
+
+ int actual = Integer.signum(ByteComparable.compare(k, before, version));
+ assertEquals(String.format("Failed comparing bound before (%s) for %s[%s](%s)\npartitioner %s version %s",
+ before.byteComparableAsString(version),
+ ByteBufferUtil.bytesToHex(b),
+ k,
+ k.byteComparableAsString(version),
+ type,
+ version),
+ 1,
+ actual);
+ maybeAssertNotPrefix(k, before, version);
+
+ actual = Integer.signum(ByteComparable.compare(k, after, version));
+ assertEquals(String.format("Failed comparing bound after (%s) for %s[%s](%s)\npartitioner %s version %s",
+ after.byteComparableAsString(version),
+ ByteBufferUtil.bytesToHex(b),
+ k,
+ k.byteComparableAsString(version),
+ type,
+ version),
+ -1,
+ actual);
+ maybeAssertNotPrefix(k, after, version);
+
+ actual = Integer.signum(ByteComparable.compare(before, after, version));
+ assertEquals(String.format("Failed comparing bound before (%s) to after (%s) for %s[%s](%s)\npartitioner %s version %s",
+ before.byteComparableAsString(version),
+ after.byteComparableAsString(version),
+ ByteBufferUtil.bytesToHex(b),
+ k,
+ k.byteComparableAsString(version),
+ type,
+ version),
+ -1,
+ actual);
+ maybeAssertNotPrefix(after, before, version);
+ }
+
+ static Object safeStr(Object i)
+ {
+ if (i == null)
+ return null;
+ String s = i.toString();
+ if (s.length() > 100)
+ s = s.substring(0, 100) + "...";
+ return s.replaceAll("\0", "<0>");
+ }
+
+ public <T> void testDirect(Function<T, ByteSource> convertor, BiFunction<T, T, Integer> comparator, T[] values)
+ {
+ for (T i : values) {
+ if (i == null)
+ continue;
+
+ logger.info("Value {} ByteSource {}\n",
+ safeStr(i),
+ convertor.apply(i));
+ }
+ for (T i : values)
+ if (i != null)
+ for (T j : values)
+ if (j != null)
+ assertComparesSame(convertor, comparator, i, j);
+ }
+
+ <T> void assertComparesSame(Function<T, ByteSource> convertor, BiFunction<T, T, Integer> comparator, T v1, T v2)
+ {
+ ByteComparable b1 = v -> convertor.apply(v1);
+ ByteComparable b2 = v -> convertor.apply(v2);
+ int expected = Integer.signum(comparator.apply(v1, v2));
+ int actual = Integer.signum(ByteComparable.compare(b1, b2, null)); // version ignored above
+ assertEquals(String.format("Failed comparing %s and %s", v1, v2), expected, actual);
+ }
+
+ <T> void assertComparesSame(AbstractType<T> type, T v1, T v2)
+ {
+ ByteBuffer b1 = decomposeAndRandomPad(type, v1);
+ ByteBuffer b2 = decomposeAndRandomPad(type, v2);
+ int expected = Integer.signum(type.compare(b1, b2));
+ final ByteComparable bc1 = typeToComparable(type, b1);
+ final ByteComparable bc2 = typeToComparable(type, b2);
+
+ for (Version version : Version.values())
+ {
+ int actual = Integer.signum(ByteComparable.compare(bc1, bc2, version));
+ if (expected != actual)
+ {
+ if (type.isReversed())
+ {
+ // This can happen for reverse of nulls and prefixes. Check that it's ok within multi-component
+ ClusteringComparator cc = new ClusteringComparator(type);
+ ByteComparable c1 = cc.asByteComparable(Clustering.make(b1));
+ ByteComparable c2 = cc.asByteComparable(Clustering.make(b2));
+ int actualcc = Integer.signum(ByteComparable.compare(c1, c2, version));
+ if (actualcc == expected)
+ return;
+ assertEquals(String.format("Failed comparing reversed %s(%s, %s) and %s(%s, %s) direct (%d) and as clustering", safeStr(v1), ByteBufferUtil.bytesToHex(b1), c1, safeStr(v2), ByteBufferUtil.bytesToHex(b2), c2, actual), expected, actualcc);
+ }
+ else
+ assertEquals(String.format("Failed comparing %s(%s BC %s) and %s(%s BC %s) version %s",
+ safeStr(v1),
+ ByteBufferUtil.bytesToHex(b1),
+ bc1.byteComparableAsString(version),
+ safeStr(v2),
+ ByteBufferUtil.bytesToHex(b2),
+ bc2.byteComparableAsString(version),
+ version),
+ expected,
+ actual);
+ }
+ maybeCheck41Properties(expected, bc1, bc2, version);
+ }
+ }
+
+ <T> ByteBuffer decomposeAndRandomPad(AbstractType<T> type, T v)
+ {
+ ByteBuffer b = type.decompose(v);
+ Random rand = new Random(0);
+ int padBefore = rand.nextInt(16);
+ int padAfter = rand.nextInt(16);
+ int paddedCapacity = b.remaining() + padBefore + padAfter;
+ ByteBuffer padded = allocateBuffer(paddedCapacity);
+ rand.ints(padBefore).forEach(x -> padded.put((byte) x));
+ padded.put(b.duplicate());
+ rand.ints(padAfter).forEach(x -> padded.put((byte) x));
+ padded.clear().limit(padded.capacity() - padAfter).position(padBefore);
+ return padded;
+ }
+
+ protected ByteBuffer allocateBuffer(int paddedCapacity)
+ {
+ return ByteBuffer.allocate(paddedCapacity);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java
new file mode 100644
index 0000000..95ea614
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java
@@ -0,0 +1,786 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
+
+import static org.apache.cassandra.utils.bytecomparable.ByteSourceComparisonTest.decomposeForTuple;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests that the result of forward + backward ByteSource translation is the same as the original.
+ */
+public class ByteSourceConversionTest extends ByteSourceTestBase
+{
+ private final static Logger logger = LoggerFactory.getLogger(ByteSourceConversionTest.class);
+ public static final Version VERSION = Version.OSS50;
+
+ @Rule
+ public final ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testStringsAscii()
+ {
+ testType(AsciiType.instance, Arrays.stream(testStrings)
+ .filter(s -> s.equals(new String(s.getBytes(StandardCharsets.US_ASCII),
+ StandardCharsets.US_ASCII)))
+ .toArray());
+ }
+
+ @Test
+ public void testStringsUTF8()
+ {
+ testType(UTF8Type.instance, testStrings);
+ testDirect(x -> ByteSource.of(x, VERSION), ByteSourceInverse::getString, testStrings);
+ }
+
+ @Test
+ public void testBooleans()
+ {
+ testType(BooleanType.instance, testBools);
+ }
+
+ @Test
+ public void testInts()
+ {
+ testType(Int32Type.instance, testInts);
+ testDirect(ByteSource::of, ByteSourceInverse::getSignedInt, testInts);
+ }
+
+ @Test
+ public void randomTestInts()
+ {
+ Random rand = new Random();
+ for (int i=0; i<10000; ++i)
+ {
+ int i1 = rand.nextInt();
+ assertConvertsSame(Int32Type.instance, i1);
+ }
+
+ }
+
+ @Test
+ public void testLongs()
+ {
+ testType(LongType.instance, testLongs);
+ testDirect(ByteSource::of, ByteSourceInverse::getSignedLong, testLongs);
+ }
+
+ @Test
+ public void testShorts()
+ {
+ testType(ShortType.instance, testShorts);
+ }
+
+ @Test
+ public void testBytes()
+ {
+ testType(ByteType.instance, testBytes);
+ }
+
+ @Test
+ public void testDoubles()
+ {
+ testType(DoubleType.instance, testDoubles);
+ }
+
+ @Test
+ public void testFloats()
+ {
+ testType(FloatType.instance, testFloats);
+ }
+
+ @Test
+ public void testBigInts()
+ {
+ testType(IntegerType.instance, testBigInts);
+ }
+
+ @Test
+ public void testBigDecimals()
+ {
+ testTypeBuffers(DecimalType.instance, testBigDecimals);
+ }
+
+ @Test
+ public void testUUIDs()
+ {
+ testType(UUIDType.instance, testUUIDs);
+ }
+
+ @Test
+ public void testTimeUUIDs()
+ {
+ testType(TimeUUIDType.instance, Arrays.stream(testUUIDs)
+ .filter(x -> x == null || x.version() == 1)
+ .map(x -> x != null ? TimeUUID.fromUuid(x) : null)
+ .toArray());
+ }
+
+ @Test
+ public void testLexicalUUIDs()
+ {
+ testType(LexicalUUIDType.instance, testUUIDs);
+ }
+
+ @Test
+ public void testSimpleDate()
+ {
+ testType(SimpleDateType.instance, Arrays.stream(testInts).filter(x -> x != null).toArray());
+ }
+
+ @Test
+ public void testTimeType()
+ {
+ testType(TimeType.instance, Arrays.stream(testLongs).filter(x -> x != null && x >= 0 && x <= 24L * 60 * 60 * 1000 * 1000 * 1000).toArray());
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testDateType()
+ {
+ testType(DateType.instance, testDates);
+ }
+
+ @Test
+ public void testTimestampType()
+ {
+ testType(TimestampType.instance, testDates);
+ }
+
+ @Test
+ public void testBytesType()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < testValues.length; ++i)
+ for (Object o : testValues[i])
+ values.add(testTypes[i].decompose(o));
+
+ testType(BytesType.instance, values);
+ }
+
+ @Test
+ public void testInetAddressType() throws UnknownHostException
+ {
+ testType(InetAddressType.instance, testInets);
+ }
+
+ @Test
+ public void testEmptyType()
+ {
+ testType(EmptyType.instance, new Void[] { null });
+ }
+
+ @Test
+ public void testPatitionerDefinedOrder()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < testValues.length; ++i)
+ for (Object o : testValues[i])
+ values.add(testTypes[i].decompose(o));
+
+ testBuffers(new PartitionerDefinedOrder(Murmur3Partitioner.instance), values);
+ testBuffers(new PartitionerDefinedOrder(RandomPartitioner.instance), values);
+ testBuffers(new PartitionerDefinedOrder(ByteOrderedPartitioner.instance), values);
+ }
+
+ @Test
+ public void testPatitionerOrder()
+ {
+ List<ByteBuffer> values = new ArrayList<>();
+ for (int i = 0; i < testValues.length; ++i)
+ for (Object o : testValues[i])
+ values.add(testTypes[i].decompose(o));
+
+ testDecoratedKeys(Murmur3Partitioner.instance, values);
+ testDecoratedKeys(RandomPartitioner.instance, values);
+ testDecoratedKeys(ByteOrderedPartitioner.instance, values);
+ }
+
+ @Test
+ public void testLocalPatitionerOrder()
+ {
+ for (int i = 0; i < testValues.length; ++i)
+ {
+ final AbstractType testType = testTypes[i];
+ testDecoratedKeys(new LocalPartitioner(testType), Lists.transform(Arrays.asList(testValues[i]),
+ v -> testType.decompose(v)));
+ }
+ }
+
+ interface PairTester
+ {
+ void test(AbstractType t1, AbstractType t2, Object o1, Object o2);
+ }
+
+ void testCombinationSampling(Random rand, PairTester tester)
+ {
+ for (int i=0;i<testTypes.length;++i)
+ for (int j=0;j<testTypes.length;++j)
+ {
+ Object[] tv1 = new Object[3];
+ Object[] tv2 = new Object[3];
+ for (int t=0; t<tv1.length; ++t)
+ {
+ tv1[t] = testValues[i][rand.nextInt(testValues[i].length)];
+ tv2[t] = testValues[j][rand.nextInt(testValues[j].length)];
+ }
+
+ for (Object o1 : tv1)
+ for (Object o2 : tv2)
+
+ {
+ tester.test(testTypes[i], testTypes[j], o1, o2);
+ }
+ }
+ }
+
+ @Test
+ public void testCombinations()
+ {
+ Random rand = new Random(0);
+ testCombinationSampling(rand, this::assertClusteringPairConvertsSame);
+ }
+
+ @Test
+ public void testNullsInClustering()
+ {
+ ByteBuffer[][] inputs = new ByteBuffer[][]
+ {
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, "a"),
+ decomposeAndRandomPad(Int32Type.instance, 0)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, "a"),
+ decomposeAndRandomPad(Int32Type.instance, null)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, "a"),
+ null},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, ""),
+ decomposeAndRandomPad(Int32Type.instance, 0)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, ""),
+ decomposeAndRandomPad(Int32Type.instance, null)},
+ new ByteBuffer[] {decomposeAndRandomPad(UTF8Type.instance, ""),
+ null},
+ new ByteBuffer[] {null,
+ decomposeAndRandomPad(Int32Type.instance, 0)},
+ new ByteBuffer[] {null,
+ decomposeAndRandomPad(Int32Type.instance, null)},
+ new ByteBuffer[] {null,
+ null},
+ };
+ for (ByteBuffer[] input : inputs)
+ {
+ assertClusteringPairConvertsSame(ByteBufferAccessor.instance,
+ UTF8Type.instance,
+ Int32Type.instance,
+ input[0],
+ input[1],
+ (t, v) -> (ByteBuffer) v);
+ }
+ }
+
+ @Test
+ public void testEmptyClustering()
+ {
+ ValueAccessor<ByteBuffer> accessor = ByteBufferAccessor.instance;
+ ClusteringComparator comp = new ClusteringComparator();
+ EnumSet<ClusteringPrefix.Kind> skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND);
+ for (ClusteringPrefix.Kind kind : EnumSet.complementOf(skippedKinds))
+ {
+ if (kind.isBoundary())
+ continue;
+
+ ClusteringPrefix<ByteBuffer> empty = ByteSourceComparisonTest.makeBound(kind);
+ ClusteringPrefix<ByteBuffer> converted = getClusteringPrefix(accessor, kind, comp, comp.asByteComparable(empty));
+ assertEquals(empty, converted);
+ }
+ }
+
+ void assertClusteringPairConvertsSame(AbstractType t1, AbstractType t2, Object o1, Object o2)
+ {
+ for (ValueAccessor<?> accessor : ValueAccessors.ACCESSORS)
+ assertClusteringPairConvertsSame(accessor, t1, t2, o1, o2, AbstractType::decompose);
+ }
+
+ <V> void assertClusteringPairConvertsSame(ValueAccessor<V> accessor,
+ AbstractType<?> t1, AbstractType<?> t2,
+ Object o1, Object o2,
+ BiFunction<AbstractType, Object, ByteBuffer> decompose)
+ {
+ boolean checkEquals = t1 != DecimalType.instance && t2 != DecimalType.instance;
+ EnumSet<ClusteringPrefix.Kind> skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND);
+ for (ClusteringPrefix.Kind k1 : EnumSet.complementOf(skippedKinds))
+ {
+ ClusteringComparator comp = new ClusteringComparator(t1, t2);
+ V[] b = accessor.createArray(2);
+ b[0] = accessor.valueOf(decompose.apply(t1, o1));
+ b[1] = accessor.valueOf(decompose.apply(t2, o2));
+ ClusteringPrefix<V> c = ByteSourceComparisonTest.makeBound(accessor.factory(), k1, b);
+ final ByteComparable bsc = comp.asByteComparable(c);
+ logger.info("Clustering {} bytesource {}", c.clusteringString(comp.subtypes()), bsc.byteComparableAsString(VERSION));
+ ClusteringPrefix<V> converted = getClusteringPrefix(accessor, k1, comp, bsc);
+ assertEquals(String.format("Failed compare(%s, converted %s ByteSource %s) == 0\ntype %s",
+ safeStr(c.clusteringString(comp.subtypes())),
+ safeStr(converted.clusteringString(comp.subtypes())),
+ bsc.byteComparableAsString(VERSION),
+ comp),
+ 0, comp.compare(c, converted));
+ if (checkEquals)
+ assertEquals(String.format("Failed equals %s, got %s ByteSource %s\ntype %s",
+ safeStr(c.clusteringString(comp.subtypes())),
+ safeStr(converted.clusteringString(comp.subtypes())),
+ bsc.byteComparableAsString(VERSION),
+ comp),
+ c, converted);
+
+ ClusteringComparator compR = new ClusteringComparator(ReversedType.getInstance(t1), ReversedType.getInstance(t2));
+ final ByteComparable bsrc = compR.asByteComparable(c);
+ converted = getClusteringPrefix(accessor, k1, compR, bsrc);
+ assertEquals(String.format("Failed reverse compare(%s, converted %s ByteSource %s) == 0\ntype %s",
+ safeStr(c.clusteringString(compR.subtypes())),
+ safeStr(converted.clusteringString(compR.subtypes())),
+ bsrc.byteComparableAsString(VERSION),
+ compR),
+ 0, compR.compare(c, converted));
+ if (checkEquals)
+ assertEquals(String.format("Failed reverse equals %s, got %s ByteSource %s\ntype %s",
+ safeStr(c.clusteringString(compR.subtypes())),
+ safeStr(converted.clusteringString(compR.subtypes())),
+ bsrc.byteComparableAsString(VERSION),
+ compR),
+ c, converted);
+ }
+ }
+
+ private static <V> ClusteringPrefix<V> getClusteringPrefix(ValueAccessor<V> accessor,
+ ClusteringPrefix.Kind k1,
+ ClusteringComparator comp,
+ ByteComparable bsc)
+ {
+ switch (k1)
+ {
+ case STATIC_CLUSTERING:
+ case CLUSTERING:
+ return comp.clusteringFromByteComparable(accessor, bsc);
+ case EXCL_END_BOUND:
+ case INCL_END_BOUND:
+ return comp.boundFromByteComparable(accessor, bsc, true);
+ case INCL_START_BOUND:
+ case EXCL_START_BOUND:
+ return comp.boundFromByteComparable(accessor, bsc, false);
+ case EXCL_END_INCL_START_BOUNDARY:
+ case INCL_END_EXCL_START_BOUNDARY:
+ return comp.boundaryFromByteComparable(accessor, bsc);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private static ByteSource.Peekable source(ByteComparable bsc)
+ {
+ if (bsc == null)
+ return null;
+ return ByteSource.peekable(bsc.asComparableBytes(VERSION));
+ }
+
+ @Test
+ public void testTupleType()
+ {
+ Random rand = ThreadLocalRandom.current();
+ testCombinationSampling(rand, this::assertTupleConvertsSame);
+ }
+
+ @Test
+ public void testTupleTypeNonFull()
+ {
+ TupleType tt = new TupleType(ImmutableList.of(UTF8Type.instance, Int32Type.instance));
+ List<ByteBuffer> tests = ImmutableList.of
+ (
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, ""),
+ decomposeAndRandomPad(Int32Type.instance, 0)),
+ // Note: a decomposed null (e.g. decomposeAndRandomPad(Int32Type.instance, null)) should not reach a tuple
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeAndRandomPad(UTF8Type.instance, ""),
+ null),
+ TupleType.buildValue(ByteBufferAccessor.instance,
+ null,
+ decomposeAndRandomPad(Int32Type.instance, 0)),
+ TupleType.buildValue(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, "")),
+ TupleType.buildValue(ByteBufferAccessor.instance, (ByteBuffer) null),
+ TupleType.buildValue(ByteBufferAccessor.instance)
+ );
+ testBuffers(tt, tests);
+ }
+
+ void assertTupleConvertsSame(AbstractType t1, AbstractType t2, Object o1, Object o2)
+ {
+ TupleType tt = new TupleType(ImmutableList.of(t1, t2));
+ ByteBuffer b1 = TupleType.buildValue(ByteBufferAccessor.instance,
+ decomposeForTuple(t1, o1),
+ decomposeForTuple(t2, o2));
+ assertConvertsSameBuffers(tt, b1);
+ }
+
+ @Test
+ public void testCompositeType()
+ {
+ Random rand = new Random(0);
+ testCombinationSampling(rand, this::assertCompositeConvertsSame);
+ }
+
+ @Test
+ public void testCompositeTypeNonFull()
+ {
+ CompositeType tt = CompositeType.getInstance(UTF8Type.instance, Int32Type.instance);
+ List<ByteBuffer> tests = ImmutableList.of
+ (
+ CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, ""), decomposeAndRandomPad(Int32Type.instance, 0)),
+ CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, ""), decomposeAndRandomPad(Int32Type.instance, null)),
+ CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(UTF8Type.instance, "")),
+ CompositeType.build(ByteBufferAccessor.instance),
+ CompositeType.build(ByteBufferAccessor.instance, true, decomposeAndRandomPad(UTF8Type.instance, "")),
+ CompositeType.build(ByteBufferAccessor.instance,true)
+ );
+ for (ByteBuffer b : tests)
+ tt.validate(b);
+ testBuffers(tt, tests);
+ }
+
+ void assertCompositeConvertsSame(AbstractType t1, AbstractType t2, Object o1, Object o2)
+ {
+ CompositeType tt = CompositeType.getInstance(t1, t2);
+ ByteBuffer b1 = CompositeType.build(ByteBufferAccessor.instance, decomposeAndRandomPad(t1, o1), decomposeAndRandomPad(t2, o2));
+ assertConvertsSameBuffers(tt, b1);
+ }
+
+ @Test
+ public void testDynamicComposite()
+ {
+ DynamicCompositeType tt = DynamicCompositeType.getInstance(DynamicCompositeTypeTest.aliases);
+ UUID[] uuids = DynamicCompositeTypeTest.uuids;
+ List<ByteBuffer> tests = ImmutableList.of
+ (
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test1", null, -1, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test1", uuids[0], 24, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test1", uuids[0], 42, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test2", uuids[0], -1, false, true),
+ DynamicCompositeTypeTest.createDynamicCompositeKey("test2", uuids[1], 42, false, true)
+ );
+ for (ByteBuffer b : tests)
+ tt.validate(b);
+ testBuffers(tt, tests);
+ }
+
+ @Test
+ public void testListTypeString()
+ {
+ testCollection(ListType.getInstance(UTF8Type.instance, true), testStrings, () -> new ArrayList<>(), new Random());
+ }
+
+ @Test
+ public void testListTypeLong()
+ {
+ testCollection(ListType.getInstance(LongType.instance, true), testLongs, () -> new ArrayList<>(), new Random());
+ }
+
+ @Test
+ public void testSetTypeString()
+ {
+ testCollection(SetType.getInstance(UTF8Type.instance, true), testStrings, () -> new HashSet<>(), new Random());
+ }
+
+ @Test
+ public void testSetTypeLong()
+ {
+ testCollection(SetType.getInstance(LongType.instance, true), testLongs, () -> new HashSet<>(), new Random());
+ }
+
+ <T, CT extends Collection<T>> void testCollection(CollectionType<CT> tt, T[] values, Supplier<CT> gen, Random rand)
+ {
+ int cnt = 0;
+ List<CT> tests = new ArrayList<>();
+ tests.add(gen.get());
+ for (int c = 1; c <= 3; ++c)
+ for (int j = 0; j < 5; ++j)
+ {
+ CT l = gen.get();
+ for (int i = 0; i < c; ++i)
+ {
+ T value = values[cnt++ % values.length];
+ if (value != null)
+ l.add(value);
+ }
+
+ tests.add(l);
+ }
+ testType(tt, tests);
+ }
+
+ @Test
+ public void testMapTypeStringLong()
+ {
+ testMap(MapType.getInstance(UTF8Type.instance, LongType.instance, true), testStrings, testLongs, () -> new HashMap<>(), new Random());
+ }
+
+ @Test
+ public void testMapTypeStringLongTree()
+ {
+ testMap(MapType.getInstance(UTF8Type.instance, LongType.instance, true), testStrings, testLongs, () -> new TreeMap<>(), new Random());
+ }
+
+ <K, V, M extends Map<K, V>> void testMap(MapType<K, V> tt, K[] keys, V[] values, Supplier<M> gen, Random rand)
+ {
+ List<M> tests = new ArrayList<>();
+ tests.add(gen.get());
+ for (int c = 1; c <= 3; ++c)
+ for (int j = 0; j < 5; ++j)
+ {
+ M l = gen.get();
+ for (int i = 0; i < c; ++i)
+ {
+ V value = values[rand.nextInt(values.length)];
+ if (value != null)
+ l.put(keys[rand.nextInt(keys.length)], value);
+ }
+
+ tests.add(l);
+ }
+ testType(tt, tests);
+ }
+
+ /*
+ * Convert type to a comparable.
+ */
+ private ByteComparable typeToComparable(AbstractType<?> type, ByteBuffer value)
+ {
+ return new ByteComparable()
+ {
+ @Override
+ public ByteSource asComparableBytes(Version v)
+ {
+ return type.asComparableBytes(value, v);
+ }
+
+ @Override
+ public String toString()
+ {
+ return type.getString(value);
+ }
+ };
+ }
+
+ public <T> void testType(AbstractType<T> type, Object[] values)
+ {
+ testType(type, Iterables.transform(Arrays.asList(values), x -> (T) x));
+ }
+
+ public <T> void testType(AbstractType<? super T> type, Iterable<T> values)
+ {
+ for (T i : values) {
+ ByteBuffer b = decomposeAndRandomPad(type, i);
+ logger.info("Value {} ({}) bytes {} ByteSource {}",
+ safeStr(i),
+ safeStr(type.getSerializer().toCQLLiteral(b)),
+ safeStr(ByteBufferUtil.bytesToHex(b)),
+ typeToComparable(type, b).byteComparableAsString(VERSION));
+ assertConvertsSame(type, i);
+ }
+ if (!type.isReversed())
+ testType(ReversedType.getInstance(type), values);
+ }
+
+ public <T> void testTypeBuffers(AbstractType<T> type, Object[] values)
+ {
+ testTypeBuffers(type, Lists.transform(Arrays.asList(values), x -> (T) x));
+ }
+
+ public <T> void testTypeBuffers(AbstractType<T> type, List<T> values)
+ {
+ // Main difference with above is that we use type.compare instead of checking equals
+ testBuffers(type, Lists.transform(values, value -> decomposeAndRandomPad(type, value)));
+
+ }
+ public void testBuffers(AbstractType<?> type, List<ByteBuffer> values)
+ {
+ try
+ {
+ for (ByteBuffer b : values) {
+ logger.info("Value {} bytes {} ByteSource {}",
+ safeStr(type.getSerializer().toCQLLiteral(b)),
+ safeStr(ByteBufferUtil.bytesToHex(b)),
+ typeToComparable(type, b).byteComparableAsString(VERSION));
+ }
+ }
+ catch (UnsupportedOperationException e)
+ {
+ // Continue without listing values.
+ }
+
+ for (ByteBuffer i : values)
+ assertConvertsSameBuffers(type, i);
+ }
+
+ void assertConvertsSameBuffers(AbstractType<?> type, ByteBuffer b1)
+ {
+ final ByteComparable bs1 = typeToComparable(type, b1);
+
+ ByteBuffer actual = type.fromComparableBytes(source(bs1), VERSION);
+ assertEquals(String.format("Failed compare(%s, converted %s (bytesource %s))",
+ ByteBufferUtil.bytesToHex(b1),
+ ByteBufferUtil.bytesToHex(actual),
+ bs1.byteComparableAsString(VERSION)),
+ 0,
+ type.compare(b1, actual));
+ }
+
+ public void testDecoratedKeys(IPartitioner type, List<ByteBuffer> values)
+ {
+ for (ByteBuffer i : values)
+ assertConvertsSameDecoratedKeys(type, i);
+ }
+
+ void assertConvertsSameDecoratedKeys(IPartitioner type, ByteBuffer b1)
+ {
+ DecoratedKey k1 = type.decorateKey(b1);
+ DecoratedKey actual = BufferDecoratedKey.fromByteComparable(k1, VERSION, type);
+
+ assertEquals(String.format("Failed compare(%s[%s bs %s], %s[%s bs %s])\npartitioner %s",
+ k1,
+ ByteBufferUtil.bytesToHex(b1),
+ k1.byteComparableAsString(VERSION),
+ actual,
+ ByteBufferUtil.bytesToHex(actual.getKey()),
+ actual.byteComparableAsString(VERSION),
+ type),
+ 0,
+ k1.compareTo(actual));
+ assertEquals(String.format("Failed equals(%s[%s bs %s], %s[%s bs %s])\npartitioner %s",
+ k1,
+ ByteBufferUtil.bytesToHex(b1),
+ k1.byteComparableAsString(VERSION),
+ actual,
+ ByteBufferUtil.bytesToHex(actual.getKey()),
+ actual.byteComparableAsString(VERSION),
+ type),
+ k1,
+ actual);
+ }
+
+ static Object safeStr(Object i)
+ {
+ if (i == null)
+ return null;
+ if (i instanceof ByteBuffer)
+ {
+ ByteBuffer buf = (ByteBuffer) i;
+ i = ByteBufferUtil.bytesToHex(buf);
+ }
+ String s = i.toString();
+ if (s.length() > 100)
+ s = s.substring(0, 100) + "...";
+ return s.replaceAll("\0", "<0>");
+ }
+
+ public <T> void testDirect(Function<T, ByteSource> convertor, Function<ByteSource.Peekable, T> inverse, T[] values)
+ {
+ for (T i : values) {
+ if (i == null)
+ continue;
+
+ logger.info("Value {} ByteSource {}\n",
+ safeStr(i),
+ convertor.apply(i));
+
+ }
+ for (T i : values)
+ if (i != null)
+ assertConvertsSame(convertor, inverse, i);
+ }
+
+ <T> void assertConvertsSame(Function<T, ByteSource> convertor, Function<ByteSource.Peekable, T> inverse, T v1)
+ {
+ ByteComparable b1 = v -> convertor.apply(v1);
+ T actual = inverse.apply(source(b1));
+ assertEquals(String.format("ByteSource %s", b1.byteComparableAsString(VERSION)), v1, actual);
+ }
+
+ <T> void assertConvertsSame(AbstractType<T> type, T v1)
+ {
+ ByteBuffer b1 = decomposeAndRandomPad(type, v1);
+ final ByteComparable bc1 = typeToComparable(type, b1);
+ ByteBuffer convertedBuffer = type.fromComparableBytes(source(bc1), VERSION);
+ T actual = type.compose(convertedBuffer);
+
+ assertEquals(String.format("Failed equals %s(%s bs %s), got %s",
+ safeStr(v1),
+ ByteBufferUtil.bytesToHex(b1),
+ safeStr(bc1.byteComparableAsString(VERSION)),
+ safeStr(actual)),
+ v1,
+ actual);
+ }
+
+ <T> ByteBuffer decomposeAndRandomPad(AbstractType<T> type, T v)
+ {
+ ByteBuffer b = type.decompose(v);
+ Random rand = new Random(0);
+ int padBefore = rand.nextInt(16);
+ int padAfter = rand.nextInt(16);
+ int paddedCapacity = b.remaining() + padBefore + padAfter;
+ ByteBuffer padded = allocateBuffer(paddedCapacity);
+ rand.ints(padBefore).forEach(x -> padded.put((byte) x));
+ padded.put(b.duplicate());
+ rand.ints(padAfter).forEach(x -> padded.put((byte) x));
+ padded.clear().limit(padded.capacity() - padAfter).position(padBefore);
+ return padded;
+ }
+
+ protected ByteBuffer allocateBuffer(int paddedCapacity)
+ {
+ return ByteBuffer.allocate(paddedCapacity);
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java
new file mode 100644
index 0000000..1fd6cc2
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceInverseTest.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.IntConsumer;
+import java.util.function.LongConsumer;
+import java.util.stream.*;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+@RunWith(Parameterized.class)
+public class ByteSourceInverseTest
+{
+ private static final String ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()";
+
+ @Parameterized.Parameters(name = "version={0}")
+ public static Iterable<ByteComparable.Version> versions()
+ {
+ return ImmutableList.of(ByteComparable.Version.OSS50);
+ }
+
+ private final ByteComparable.Version version;
+
+ public ByteSourceInverseTest(ByteComparable.Version version)
+ {
+ this.version = version;
+ }
+
+ @Test
+ public void testGetSignedInt()
+ {
+ IntConsumer intConsumer = initial ->
+ {
+ ByteSource byteSource = ByteSource.of(initial);
+ int decoded = ByteSourceInverse.getSignedInt(byteSource);
+ Assert.assertEquals(initial, decoded);
+ };
+
+ IntStream.of(Integer.MIN_VALUE, Integer.MIN_VALUE + 1,
+ -256, -255, -128, -127, -1, 0, 1, 127, 128, 255, 256,
+ Integer.MAX_VALUE - 1, Integer.MAX_VALUE)
+ .forEach(intConsumer);
+ new Random().ints(1000)
+ .forEach(intConsumer);
+ }
+
+ @Test
+ public void testNextInt()
+ {
+ // The high and low 32 bits of this long differ only in the first and last bit (in the high 32 bits they are
+ // both 0s instead of 1s). The first bit difference will be negated by the bit flipping when writing down a
+ // fixed length signed number, so the only remaining difference will be in the last bit.
+ int hi = 0b0001_0010_0011_0100_0101_0110_0111_1000;
+ int lo = hi | 1 | 1 << 31;
+ long l1 = Integer.toUnsignedLong(hi) << 32 | Integer.toUnsignedLong(lo);
+
+ ByteSource byteSource = ByteSource.of(l1);
+ int i1 = ByteSourceInverse.getSignedInt(byteSource);
+ int i2 = ByteSourceInverse.getSignedInt(byteSource);
+ Assert.assertEquals(i1 + 1, i2);
+
+ try
+ {
+ ByteSourceInverse.getSignedInt(byteSource);
+ Assert.fail();
+ }
+ catch (IllegalArgumentException e)
+ {
+ // Expected.
+ }
+
+ byteSource = ByteSource.of(l1);
+ int iFirst = ByteSourceInverse.getSignedInt(byteSource);
+ Assert.assertEquals(i1, iFirst);
+ int iNext = ByteSourceInverse.getSignedInt(byteSource);
+ Assert.assertEquals(i2, iNext);
+ }
+
+ @Test
+ public void testGetSignedLong()
+ {
+ LongConsumer longConsumer = initial ->
+ {
+ ByteSource byteSource = ByteSource.of(initial);
+ long decoded = ByteSourceInverse.getSignedLong(byteSource);
+ Assert.assertEquals(initial, decoded);
+ };
+
+ LongStream.of(Long.MIN_VALUE, Long.MIN_VALUE + 1, Integer.MIN_VALUE - 1L,
+ -256L, -255L, -128L, -127L, -1L, 0L, 1L, 127L, 128L, 255L, 256L,
+ Integer.MAX_VALUE + 1L, Long.MAX_VALUE - 1, Long.MAX_VALUE)
+ .forEach(longConsumer);
+ new Random().longs(1000)
+ .forEach(longConsumer);
+ }
+
+ @Test
+ public void testGetSignedByte()
+ {
+ Consumer<Byte> byteConsumer = boxedByte ->
+ {
+ byte initial = boxedByte;
+ ByteBuffer byteBuffer = ByteType.instance.decompose(initial);
+ ByteSource byteSource = ByteType.instance.asComparableBytes(byteBuffer, version);
+ byte decoded = ByteSourceInverse.getSignedByte(byteSource);
+ Assert.assertEquals(initial, decoded);
+ };
+
+ IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE + 1)
+ .forEach(byteInteger -> byteConsumer.accept((byte) byteInteger));
+ }
+
+ @Test
+ public void testGetSignedShort()
+ {
+ Consumer<Short> shortConsumer = boxedShort ->
+ {
+ short initial = boxedShort;
+ ByteBuffer shortBuffer = ShortType.instance.decompose(initial);
+ ByteSource byteSource = ShortType.instance.asComparableBytes(shortBuffer, version);
+ short decoded = ByteSourceInverse.getSignedShort(byteSource);
+ Assert.assertEquals(initial, decoded);
+ };
+
+ IntStream.range(Short.MIN_VALUE, Short.MAX_VALUE + 1)
+ .forEach(shortInteger -> shortConsumer.accept((short) shortInteger));
+ }
+
+ @Test
+ public void testBadByteSourceForFixedLengthNumbers()
+ {
+ byte[] bytes = new byte[8];
+ new Random().nextBytes(bytes);
+ for (Map.Entry<String, Integer> entries : ImmutableMap.of("getSignedInt", 4,
+ "getSignedLong", 8,
+ "getSignedByte", 1,
+ "getSignedShort", 2).entrySet())
+ {
+ String methodName = entries.getKey();
+ int length = entries.getValue();
+ try
+ {
+ Method fixedLengthNumberMethod = ByteSourceInverse.class.getMethod(methodName, ByteSource.class);
+ ArrayList<ByteSource> sources = new ArrayList<>();
+ sources.add(null);
+ sources.add(ByteSource.EMPTY);
+ for (int i = 0; i < length; ++i)
+ sources.add(ByteSource.fixedLength(bytes, 0, i));
+ // Note: not testing invalid bytes (e.g. using the construction below) as they signify a programming
+ // error (throwing AssertionError) rather than something that could happen due to e.g. bad files.
+ // ByteSource.withTerminatorLegacy(257, ByteSource.fixedLength(bytes, 0, length - 1));
+ for (ByteSource badSource : sources)
+ {
+ try
+ {
+ fixedLengthNumberMethod.invoke(ByteSourceInverse.class, badSource);
+ Assert.fail("Expected exception not thrown");
+ }
+ catch (Throwable maybe)
+ {
+ maybe = Throwables.unwrapped(maybe);
+ final String message = "Unexpected throwable " + maybe + " with cause " + maybe.getCause();
+ if (badSource == null)
+ Assert.assertTrue(message,
+ maybe instanceof NullPointerException);
+ else
+ Assert.assertTrue(message,
+ maybe instanceof IllegalArgumentException);
+ }
+ }
+ }
+ catch (NoSuchMethodException e)
+ {
+ Assert.fail("Expected ByteSourceInverse to have method called " + methodName
+ + " with a single parameter of type ByteSource");
+ }
+ }
+ }
+
+ @Test
+ public void testBadByteSourceForVariableLengthNumbers()
+ {
+ for (long value : Arrays.asList(0L, 1L << 6, 1L << 13, 1L << 20, 1L << 27, 1L << 34, 1L << 41, 1L << 48, 1L << 55))
+ {
+ Assert.assertEquals(value, ByteSourceInverse.getVariableLengthInteger(ByteSource.variableLengthInteger(value)));
+
+ ArrayList<ByteSource> sources = new ArrayList<>();
+ sources.add(null);
+ sources.add(ByteSource.EMPTY);
+ int length = ByteComparable.length(version -> ByteSource.variableLengthInteger(value), ByteComparable.Version.OSS50);
+ for (int i = 0; i < length; ++i)
+ sources.add(ByteSource.cut(ByteSource.variableLengthInteger(value), i));
+
+ for (ByteSource badSource : sources)
+ {
+ try
+ {
+ ByteSourceInverse.getVariableLengthInteger(badSource);
+ Assert.fail("Expected exception not thrown");
+ }
+ catch (Throwable maybe)
+ {
+ maybe = Throwables.unwrapped(maybe);
+ final String message = "Unexpected throwable " + maybe + " with cause " + maybe.getCause();
+ if (badSource == null)
+ Assert.assertTrue(message,
+ maybe instanceof NullPointerException);
+ else
+ Assert.assertTrue(message,
+ maybe instanceof IllegalArgumentException);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testGetString()
+ {
+ Consumer<String> stringConsumer = initial ->
+ {
+ ByteSource.Peekable byteSource = initial == null ? null : ByteSource.peekable(ByteSource.of(initial, version));
+ String decoded = ByteSourceInverse.getString(byteSource);
+ Assert.assertEquals(initial, decoded);
+ };
+
+ Stream.of(null, "© 2018 DataStax", "", "\n", "\0", "\0\0", "\001", "0", "0\0", "00", "1")
+ .forEach(stringConsumer);
+
+ Random prng = new Random();
+ int stringLength = 10;
+ String random;
+ for (int i = 0; i < 1000; ++i)
+ {
+ random = newRandomAlphanumeric(prng, stringLength);
+ stringConsumer.accept(random);
+ }
+ }
+
+ private static String newRandomAlphanumeric(Random prng, int length)
+ {
+ StringBuilder random = new StringBuilder(length);
+ for (int i = 0; i < length; ++i)
+ random.append(ALPHABET.charAt(prng.nextInt(ALPHABET.length())));
+ return random.toString();
+ }
+
+ @Test
+ public void testGetByteBuffer()
+ {
+ for (Consumer<byte[]> byteArrayConsumer : Arrays.<Consumer<byte[]>>asList(initialBytes ->
+ {
+ ByteSource.Peekable byteSource = ByteSource.peekable(ByteSource.of(ByteBuffer.wrap(initialBytes), version));
+ byte[] decodedBytes = ByteSourceInverse.getUnescapedBytes(byteSource);
+ Assert.assertArrayEquals(initialBytes, decodedBytes);
+ },
+ initialBytes ->
+ {
+ ByteSource.Peekable byteSource = ByteSource.peekable(ByteSource.of(initialBytes, version));
+ byte[] decodedBytes = ByteSourceInverse.getUnescapedBytes(byteSource);
+ Assert.assertArrayEquals(initialBytes, decodedBytes);
+ },
+ initialBytes ->
+ {
+ long address = MemoryUtil.allocate(initialBytes.length);
+ try
+ {
+ MemoryUtil.setBytes(address, initialBytes, 0, initialBytes.length);
+ ByteSource.Peekable byteSource = ByteSource.peekable(ByteSource.ofMemory(address, initialBytes.length, version));
+ byte[] decodedBytes = ByteSourceInverse.getUnescapedBytes(byteSource);
+ Assert.assertArrayEquals(initialBytes, decodedBytes);
+ }
+ finally
+ {
+ MemoryUtil.free(address);
+ }
+ }
+ ))
+ {
+ for (byte[] tricky : Arrays.asList(
+ // ESCAPE - leading, in the middle, trailing
+ new byte[]{ 0, 2, 3, 4, 5 }, new byte[]{ 1, 2, 0, 4, 5 }, new byte[]{ 1, 2, 3, 4, 0 },
+ // END_OF_STREAM/ESCAPED_0_DONE - leading, in the middle, trailing
+ new byte[]{ -1, 2, 3, 4, 5 }, new byte[]{ 1, 2, -1, 4, 5 }, new byte[]{ 1, 2, 3, 4, -1 },
+ // ESCAPED_0_CONT - leading, in the middle, trailing
+ new byte[]{ -2, 2, 3, 4, 5 }, new byte[]{ 1, 2, -2, 4, 5 }, new byte[]{ 1, 2, 3, 4, -2 },
+ // ESCAPE + ESCAPED_0_DONE - leading, in the middle, trailing
+ new byte[]{ 0, -1, 3, 4, 5 }, new byte[]{ 1, 0, -1, 4, 5 }, new byte[]{ 1, 2, 3, 0, -1 },
+ // ESCAPE + ESCAPED_0_CONT + ESCAPED_0_DONE - leading, in the middle, trailing
+ new byte[]{ 0, -2, -1, 4, 5 }, new byte[]{ 1, 0, -2, -1, 5 }, new byte[]{ 1, 2, 0, -2, -1 }))
+ {
+ byteArrayConsumer.accept(tricky);
+ }
+
+ byte[] bytes = new byte[1000];
+ Random prng = new Random();
+ for (int i = 0; i < 1000; ++i)
+ {
+ prng.nextBytes(bytes);
+ byteArrayConsumer.accept(bytes);
+ }
+
+ int stringLength = 10;
+ String random;
+ for (int i = 0; i < 1000; ++i)
+ {
+ random = newRandomAlphanumeric(prng, stringLength);
+ byteArrayConsumer.accept(random.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ }
+
+ @Test
+ public void testReadBytes()
+ {
+ Map<Class<?>, Function<Object, ByteSource>> generatorPerType = new HashMap<>();
+ List<Object> originalValues = new ArrayList<>();
+ Random prng = new Random();
+
+ generatorPerType.put(String.class, s ->
+ {
+ String string = (String) s;
+ return ByteSource.of(string, version);
+ });
+ for (int i = 0; i < 100; ++i)
+ originalValues.add(newRandomAlphanumeric(prng, 10));
+
+ generatorPerType.put(Integer.class, i ->
+ {
+ Integer integer = (Integer) i;
+ return ByteSource.of(integer);
+ });
+ for (int i = 0; i < 100; ++i)
+ originalValues.add(prng.nextInt());
+
+ generatorPerType.put(Long.class, l ->
+ {
+ Long looong = (Long) l;
+ return ByteSource.of(looong);
+ });
+ for (int i = 0; i < 100; ++i)
+ originalValues.add(prng.nextLong());
+
+ generatorPerType.put(UUID.class, u ->
+ {
+ UUID uuid = (UUID) u;
+ ByteBuffer uuidBuffer = UUIDType.instance.decompose(uuid);
+ return UUIDType.instance.asComparableBytes(uuidBuffer, version);
+ });
+ for (int i = 0; i < 100; ++i)
+ originalValues.add(UUID.randomUUID());
+
+ for (Object value : originalValues)
+ {
+ Class<?> type = value.getClass();
+ Function<Object, ByteSource> generator = generatorPerType.get(type);
+ ByteSource originalSource = generator.apply(value);
+ ByteSource originalSourceCopy = generator.apply(value);
+ byte[] bytes = ByteSourceInverse.readBytes(originalSource);
+ // The best way to test the read bytes seems to be to assert that just directly using them as a
+ // ByteSource (using ByteSource.fixedLength(byte[])) they compare as equal to another ByteSource obtained
+ // from the same original value.
+ int compare = ByteComparable.compare(v -> originalSourceCopy, v -> ByteSource.fixedLength(bytes), version);
+ Assert.assertEquals(0, compare);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java
new file mode 100644
index 0000000..532bd1b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceSequenceTest.java
@@ -0,0 +1,784 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.BufferClusteringBound;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.CachedHashDecoratedKey;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+@RunWith(Parameterized.class)
+public class ByteSourceSequenceTest
+{
+
+ private static final String ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()";
+
+ @Parameterized.Parameters(name = "version={0}")
+ public static Iterable<ByteComparable.Version> versions()
+ {
+ return ImmutableList.of(ByteComparable.Version.OSS50);
+ }
+
+ private final ByteComparable.Version version;
+
+ public ByteSourceSequenceTest(ByteComparable.Version version)
+ {
+ this.version = version;
+ }
+
+ @Test
+ public void testNullsSequence()
+ {
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ null, null, null
+ ));
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+ }
+
+ @Test
+ public void testNullsAndUnknownLengthsSequence()
+ {
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ null, ByteSource.of("b", version), ByteSource.of("c", version)
+ ));
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "b");
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "c");
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of("a", version), null, ByteSource.of("c", version)
+ ));
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "a");
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "c");
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of("a", version), ByteSource.of("b", version), null
+ ));
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "a");
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "b");
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of("a", version), null, null
+ ));
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "a");
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ null, null, ByteSource.of("c", version)
+ ));
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getString, "c");
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+ }
+
+ private static void expectNextComponentNull(ByteSource.Peekable comparableBytes)
+ {
+ // We expect null-signifying separator, followed by a null ByteSource component
+ ByteSource.Peekable next = ByteSourceInverse.nextComponentSource(comparableBytes);
+ assertNull(next);
+ }
+
+ private static <T> void expectNextComponentValue(ByteSource.Peekable comparableBytes,
+ Function<ByteSource.Peekable, T> decoder,
+ T expected)
+ {
+ // We expect a regular separator, followed by a ByteSource component corresponding to the expected value
+ ByteSource.Peekable next = ByteSourceInverse.nextComponentSource(comparableBytes);
+ assertNotNull(next);
+ T decoded = decoder.apply(next);
+ assertEquals(expected, decoded);
+ }
+
+ @Test
+ public void testNullsAndKnownLengthsSequence()
+ {
+ int intValue = 42;
+ BigInteger varintValue = BigInteger.valueOf(2018L);
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ null, ByteSource.of(intValue), varintToByteSource(varintValue)
+ ));
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getSignedInt, intValue);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(intValue), null, varintToByteSource(varintValue)
+ ));
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getSignedInt, intValue);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(intValue), varintToByteSource(varintValue), null
+ ));
+ expectNextComponentValue(comparableBytes, ByteSourceInverse::getSignedInt, intValue);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ null, null, varintToByteSource(varintValue)
+ ));
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ null, varintToByteSource(varintValue), null
+ ));
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ varintToByteSource(varintValue), null, null
+ ));
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ Boolean boolValue = new Random().nextBoolean();
+ ByteSource boolSource = BooleanType.instance.asComparableBytes(BooleanType.instance.decompose(boolValue), version);
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ varintToByteSource(varintValue), boolSource, null
+ ));
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ expectNextComponentValue(comparableBytes, BooleanType.instance, boolValue);
+ expectNextComponentNull(comparableBytes);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+
+ boolSource = BooleanType.instance.asComparableBytes(BooleanType.instance.decompose(boolValue), version);
+ comparableBytes = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ varintToByteSource(varintValue), null, boolSource
+ ));
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, BooleanType.instance, boolValue);
+ assertEquals(ByteSource.TERMINATOR, comparableBytes.next());
+ }
+
+ @Test
+ public void testOptionalSignedFixedLengthTypesSequence()
+ {
+ Random prng = new Random();
+ String randomString = newRandomAlphanumeric(prng, 10);
+ byte randomByte = (byte) prng.nextInt();
+ short randomShort = (short) prng.nextInt();
+ int randomInt = prng.nextInt();
+ long randomLong = prng.nextLong();
+ BigInteger randomVarint = BigInteger.probablePrime(80, prng);
+
+ Map<AbstractType<?>, ByteBuffer> valuesByType = new HashMap<AbstractType<?>, ByteBuffer>()
+ {{
+ put(ByteType.instance, ByteType.instance.decompose(randomByte));
+ put(ShortType.instance, ShortType.instance.decompose(randomShort));
+ put(SimpleDateType.instance, SimpleDateType.instance.decompose(randomInt));
+ put(TimeType.instance, TimeType.instance.decompose(randomLong));
+ }};
+
+ for (Map.Entry<AbstractType<?>, ByteBuffer> entry : valuesByType.entrySet())
+ {
+ AbstractType<?> type = entry.getKey();
+ ByteBuffer value = entry.getValue();
+
+ ByteSource byteSource = type.asComparableBytes(value, version);
+ ByteSource.Peekable sequence = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(randomString, version), byteSource, varintToByteSource(randomVarint)
+ ));
+ expectNextComponentValue(sequence, ByteSourceInverse::getString, randomString);
+ expectNextComponentValue(sequence, type, value);
+ expectNextComponentValue(sequence, VARINT, randomVarint);
+ assertEquals(ByteSource.TERMINATOR, sequence.next());
+
+ byteSource = type.asComparableBytes(type.decompose(null), version);
+ sequence = ByteSource.peekable(ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(randomString, version), byteSource, varintToByteSource(randomVarint)
+ ));
+ expectNextComponentValue(sequence, ByteSourceInverse::getString, randomString);
+ expectNextComponentNull(sequence);
+ expectNextComponentValue(sequence, VARINT, randomVarint);
+ assertEquals(ByteSource.TERMINATOR, sequence.next());
+ }
+ }
+
+ private ByteSource varintToByteSource(BigInteger value)
+ {
+ ByteBuffer valueByteBuffer = VARINT.decompose(value);
+ return VARINT.asComparableBytes(valueByteBuffer, version);
+ }
+
+ private static final UTF8Type UTF8 = UTF8Type.instance;
+ private static final DecimalType DECIMAL = DecimalType.instance;
+ private static final IntegerType VARINT = IntegerType.instance;
+
+ // A regular comparator using the natural ordering for all types.
+ private static final ClusteringComparator COMP = new ClusteringComparator(Arrays.asList(
+ UTF8,
+ DECIMAL,
+ VARINT
+ ));
+ // A comparator that reverses the ordering for the first unknown length type
+ private static final ClusteringComparator COMP_REVERSED_UNKNOWN_LENGTH = new ClusteringComparator(Arrays.asList(
+ ReversedType.getInstance(UTF8),
+ DECIMAL,
+ VARINT
+ ));
+ // A comparator that reverses the ordering for the second unknown length type
+ private static final ClusteringComparator COMP_REVERSED_UNKNOWN_LENGTH_2 = new ClusteringComparator(Arrays.asList(
+ UTF8,
+ ReversedType.getInstance(DECIMAL),
+ VARINT
+ ));
+ // A comparator that reverses the ordering for the sole known/computable length type
+ private static final ClusteringComparator COMP_REVERSED_KNOWN_LENGTH = new ClusteringComparator(Arrays.asList(
+ UTF8,
+ DECIMAL,
+ ReversedType.getInstance(VARINT)
+ ));
+ // A comparator that reverses the ordering for all types
+ private static final ClusteringComparator COMP_ALL_REVERSED = new ClusteringComparator(Arrays.asList(
+ ReversedType.getInstance(UTF8),
+ ReversedType.getInstance(DECIMAL),
+ ReversedType.getInstance(VARINT)
+ ));
+
+ @Test
+ public void testClusteringPrefixBoundNormalAndReversed()
+ {
+ String stringValue = "Lorem ipsum dolor sit amet";
+ BigDecimal decimalValue = BigDecimal.valueOf(123456789, 20);
+ BigInteger varintValue = BigInteger.valueOf(2018L);
+
+ // Create some non-null clustering key values that will be encoded and decoded to byte-ordered representation
+ // with different types of clustering comparators (and in other tests with different types of prefixes).
+ ByteBuffer[] clusteringKeyValues = new ByteBuffer[] {
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue),
+ VARINT.decompose(varintValue)
+ };
+
+ for (ClusteringPrefix.Kind prefixKind : ClusteringPrefix.Kind.values())
+ {
+ if (prefixKind.isBoundary())
+ continue;
+
+ ClusteringPrefix prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ // Use the regular comparator.
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(COMP.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentValue(comparableBytes, DECIMAL, decimalValue);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+
+ prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ // Use the comparator reversing the ordering for the first unknown length type.
+ comparableBytes = ByteSource.peekable(COMP_REVERSED_UNKNOWN_LENGTH.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, ReversedType.getInstance(UTF8), stringValue);
+ expectNextComponentValue(comparableBytes, DECIMAL, decimalValue);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+
+ prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ // Use the comparator reversing the ordering for the second unknown length type.
+ comparableBytes = ByteSource.peekable(COMP_REVERSED_UNKNOWN_LENGTH_2.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentValue(comparableBytes, ReversedType.getInstance(DECIMAL), decimalValue);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+
+ prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ // Use the comparator reversing the ordering for the known/computable length type.
+ comparableBytes = ByteSource.peekable(COMP_REVERSED_KNOWN_LENGTH.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentValue(comparableBytes, DECIMAL, decimalValue);
+ expectNextComponentValue(comparableBytes, ReversedType.getInstance(VARINT), varintValue);
+
+ prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ // Use the all-reversing comparator.
+ comparableBytes = ByteSource.peekable(COMP_ALL_REVERSED.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, ReversedType.getInstance(UTF8), stringValue);
+ expectNextComponentValue(comparableBytes, ReversedType.getInstance(DECIMAL), decimalValue);
+ expectNextComponentValue(comparableBytes, ReversedType.getInstance(VARINT), varintValue);
+ }
+ }
+
+ @Test
+ public void testClusteringPrefixBoundNulls()
+ {
+ String stringValue = "Lorem ipsum dolor sit amet";
+ BigDecimal decimalValue = BigDecimal.valueOf(123456789, 20);
+ BigInteger varintValue = BigInteger.valueOf(2018L);
+
+ // Create clustering key values where the component for an unknown length type is null.
+ ByteBuffer[] unknownLengthNull = new ByteBuffer[] {
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(null),
+ VARINT.decompose(varintValue)
+ };
+ // Create clustering key values where the component for a known/computable length type is null.
+ ByteBuffer[] knownLengthNull = new ByteBuffer[] {
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue),
+ VARINT.decompose(null)
+ };
+
+ for (ClusteringPrefix.Kind prefixKind : ClusteringPrefix.Kind.values())
+ {
+ if (prefixKind.isBoundary())
+ continue;
+
+ // Test the decoding of a null component of a non-reversed unknown length type.
+ ClusteringPrefix prefix = BufferClusteringBound.create(prefixKind, unknownLengthNull);
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(COMP.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+ // Test the decoding of a null component of a reversed unknown length type.
+ prefix = BufferClusteringBound.create(prefixKind, unknownLengthNull);
+ comparableBytes = ByteSource.peekable(COMP_REVERSED_UNKNOWN_LENGTH_2.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentNull(comparableBytes);
+ expectNextComponentValue(comparableBytes, VARINT, varintValue);
+
+ // Test the decoding of a null component of a non-reversed known/computable length type.
+ prefix = BufferClusteringBound.create(prefixKind, knownLengthNull);
+ comparableBytes = ByteSource.peekable(COMP.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentValue(comparableBytes, DECIMAL, decimalValue);
+ expectNextComponentNull(comparableBytes);
+ // Test the decoding of a null component of a reversed known/computable length type.
+ prefix = BufferClusteringBound.create(prefixKind, knownLengthNull);
+ comparableBytes = ByteSource.peekable(COMP_REVERSED_KNOWN_LENGTH.asByteComparable(prefix).asComparableBytes(version));
+ expectNextComponentValue(comparableBytes, UTF8, stringValue);
+ expectNextComponentValue(comparableBytes, DECIMAL, decimalValue);
+ expectNextComponentNull(comparableBytes);
+ }
+ }
+
+ private <T> void expectNextComponentValue(ByteSource.Peekable comparableBytes,
+ AbstractType<T> type,
+ T expected)
+ {
+ // We expect a regular separator, followed by a ByteSource component corresponding to the expected value
+ ByteSource.Peekable next = ByteSourceInverse.nextComponentSource(comparableBytes);
+ T decoded = type.compose(type.fromComparableBytes(next, version));
+ assertEquals(expected, decoded);
+ }
+
+ private void expectNextComponentValue(ByteSource.Peekable comparableBytes,
+ AbstractType<?> type,
+ ByteBuffer expected)
+ {
+ // We expect a regular separator, followed by a ByteSource component corresponding to the expected value
+ ByteSource.Peekable next = ByteSourceInverse.nextComponentSource(comparableBytes);
+ assertEquals(expected, type.fromComparableBytes(next, version));
+ }
+
+ @Test
+ public void testGetBoundFromPrefixTerminator()
+ {
+ String stringValue = "Lorem ipsum dolor sit amet";
+ BigDecimal decimalValue = BigDecimal.valueOf(123456789, 20);
+ BigInteger varintValue = BigInteger.valueOf(2018L);
+
+ ByteBuffer[] clusteringKeyValues = new ByteBuffer[] {
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue),
+ VARINT.decompose(varintValue)
+ };
+ ByteBuffer[] nullValueBeforeTerminator = new ByteBuffer[] {
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue),
+ VARINT.decompose(null)
+ };
+
+ for (ClusteringPrefix.Kind prefixKind : ClusteringPrefix.Kind.values())
+ {
+ // NOTE dimitar.dimitrov I assume there's a sensible explanation why does STATIC_CLUSTERING use a custom
+ // terminator that's not one of the common separator values, but I haven't spent enough time to get it.
+ if (prefixKind.isBoundary())
+ continue;
+
+ // Test that the read terminator value is exactly the encoded value of this prefix' bound.
+ ClusteringPrefix prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(COMP.asByteComparable(prefix).asComparableBytes(version));
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ ByteSourceInverse.getString(comparableBytes);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ DECIMAL.fromComparableBytes(comparableBytes, version);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ VARINT.fromComparableBytes(comparableBytes, version);
+ // Expect the last separator (i.e. the terminator) to be the one specified by the prefix kind.
+ assertEquals(prefixKind.asByteComparableValue(version), comparableBytes.next());
+
+ // Test that the read terminator value is exactly the encoded value of this prefix' bound, when the
+ // terminator is preceded by a null value.
+ prefix = BufferClusteringBound.create(prefixKind, nullValueBeforeTerminator);
+ comparableBytes = ByteSource.peekable(COMP.asByteComparable(prefix).asComparableBytes(version));
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ ByteSourceInverse.getString(comparableBytes);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ DECIMAL.fromComparableBytes(comparableBytes, version);
+ // Expect null-signifying separator here.
+ assertEquals(ByteSource.NEXT_COMPONENT_EMPTY, comparableBytes.next());
+ // No varint to read
+ // Expect the last separator (i.e. the terminator) to be the one specified by the prefix kind.
+ assertEquals(prefixKind.asByteComparableValue(version), comparableBytes.next());
+
+ // Test that the read terminator value is exactly the encoded value of this prefix' bound, when the
+ // terminator is preceded by a reversed null value.
+ prefix = BufferClusteringBound.create(prefixKind, nullValueBeforeTerminator);
+ // That's the comparator that will reverse the ordering of the type of the last value in the prefix (the
+ // one before the terminator). In other tests we're more interested in the fact that values of this type
+ // have known/computable length, which is why we've named it so...
+ comparableBytes = ByteSource.peekable(COMP_REVERSED_KNOWN_LENGTH.asByteComparable(prefix).asComparableBytes(version));
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ ByteSourceInverse.getString(comparableBytes);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ DECIMAL.fromComparableBytes(comparableBytes, version);
+ // Expect reversed null-signifying separator here.
+ assertEquals(ByteSource.NEXT_COMPONENT_EMPTY_REVERSED, comparableBytes.next());
+ // No varint to read
+ // Expect the last separator (i.e. the terminator) to be the one specified by the prefix kind.
+ assertEquals(prefixKind.asByteComparableValue(version), comparableBytes.next());
+ }
+ }
+
+ @Test
+ public void testReversedTypesInClusteringKey()
+ {
+ String stringValue = "Lorem ipsum dolor sit amet";
+ BigDecimal decimalValue = BigDecimal.valueOf(123456789, 20);
+
+ AbstractType<String> reversedStringType = ReversedType.getInstance(UTF8);
+ AbstractType<BigDecimal> reversedDecimalType = ReversedType.getInstance(DECIMAL);
+
+ final ClusteringComparator comparator = new ClusteringComparator(Arrays.asList(
+ // unknown length type
+ UTF8,
+ // known length type
+ DECIMAL,
+ // reversed unknown length type
+ reversedStringType,
+ // reversed known length type
+ reversedDecimalType
+ ));
+ ByteBuffer[] clusteringKeyValues = new ByteBuffer[] {
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue),
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue)
+ };
+
+ final ClusteringComparator comparator2 = new ClusteringComparator(Arrays.asList(
+ // known length type
+ DECIMAL,
+ // unknown length type
+ UTF8,
+ // reversed known length type
+ reversedDecimalType,
+ // reversed unknown length type
+ reversedStringType
+ ));
+ ByteBuffer[] clusteringKeyValues2 = new ByteBuffer[] {
+ DECIMAL.decompose(decimalValue),
+ UTF8.decompose(stringValue),
+ DECIMAL.decompose(decimalValue),
+ UTF8.decompose(stringValue)
+ };
+
+ for (ClusteringPrefix.Kind prefixKind : ClusteringPrefix.Kind.values())
+ {
+ if (prefixKind.isBoundary())
+ continue;
+
+ ClusteringPrefix prefix = BufferClusteringBound.create(prefixKind, clusteringKeyValues);
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(comparator.asByteComparable(prefix).asComparableBytes(version));
+
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), stringValue);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(DECIMAL, comparableBytes), decimalValue);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(reversedStringType, comparableBytes), stringValue);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(reversedDecimalType, comparableBytes), decimalValue);
+
+ assertEquals(prefixKind.asByteComparableValue(version), comparableBytes.next());
+ assertEquals(ByteSource.END_OF_STREAM, comparableBytes.next());
+
+ ClusteringPrefix prefix2 = BufferClusteringBound.create(prefixKind, clusteringKeyValues2);
+ ByteSource.Peekable comparableBytes2 = ByteSource.peekable(comparator2.asByteComparable(prefix2).asComparableBytes(version));
+
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes2.next());
+ assertEquals(getComponentValue(DECIMAL, comparableBytes2), decimalValue);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes2.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes2), stringValue);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes2.next());
+ assertEquals(getComponentValue(reversedDecimalType, comparableBytes2), decimalValue);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes2.next());
+ assertEquals(getComponentValue(reversedStringType, comparableBytes2), stringValue);
+
+ assertEquals(prefixKind.asByteComparableValue(version), comparableBytes2.next());
+ assertEquals(ByteSource.END_OF_STREAM, comparableBytes2.next());
+ }
+ }
+
+ private <T extends AbstractType<E>, E> E getComponentValue(T type, ByteSource.Peekable comparableBytes)
+ {
+ return type.compose(type.fromComparableBytes(comparableBytes, version));
+ }
+
+ @Test
+ public void testReadingNestedSequence_Simple()
+ {
+ String padding1 = "A string";
+ String padding2 = "Another string";
+
+ BigInteger varint1 = BigInteger.valueOf(0b10000000);
+ BigInteger varint2 = BigInteger.valueOf(1 >> 30);
+ BigInteger varint3 = BigInteger.valueOf(0x10000000L);
+ BigInteger varint4 = BigInteger.valueOf(Long.MAX_VALUE);
+
+ String string1 = "Testing byte sources";
+ String string2 = "is neither easy nor fun;";
+ String string3 = "But do it we must.";
+ String string4 = "— DataStax, 2018";
+
+ MapType<BigInteger, String> varintStringMapType = MapType.getInstance(VARINT, UTF8, false);
+ Map<BigInteger, String> varintStringMap = new TreeMap<>();
+ varintStringMap.put(varint1, string1);
+ varintStringMap.put(varint2, string2);
+ varintStringMap.put(varint3, string3);
+ varintStringMap.put(varint4, string4);
+
+ ByteSource sequence = ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(padding1, version),
+ varintStringMapType.asComparableBytes(varintStringMapType.decompose(varintStringMap), version),
+ ByteSource.of(padding2, version)
+ );
+ ByteSource.Peekable comparableBytes = ByteSource.peekable(sequence);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding1);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(varintStringMapType, comparableBytes), varintStringMap);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding2);
+ sequence = ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ varintStringMapType.asComparableBytes(varintStringMapType.decompose(varintStringMap), version),
+ ByteSource.of(padding1, version),
+ ByteSource.of(padding2, version)
+ );
+ comparableBytes = ByteSource.peekable(sequence);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(varintStringMapType, comparableBytes), varintStringMap);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding1);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding2);
+ sequence = ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(padding1, version),
+ ByteSource.of(padding2, version),
+ varintStringMapType.asComparableBytes(varintStringMapType.decompose(varintStringMap), version)
+ );
+ comparableBytes = ByteSource.peekable(sequence);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding1);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding2);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(varintStringMapType, comparableBytes), varintStringMap);
+
+ MapType<String, BigInteger> stringVarintMapType = MapType.getInstance(UTF8, VARINT, false);
+ Map<String, BigInteger> stringVarintMap = new HashMap<>();
+ stringVarintMap.put(string1, varint1);
+ stringVarintMap.put(string2, varint2);
+ stringVarintMap.put(string3, varint3);
+ stringVarintMap.put(string4, varint4);
+
+ sequence = ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(padding1, version),
+ stringVarintMapType.asComparableBytes(stringVarintMapType.decompose(stringVarintMap), version),
+ ByteSource.of(padding2, version)
+ );
+ comparableBytes = ByteSource.peekable(sequence);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding1);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(stringVarintMapType, comparableBytes), stringVarintMap);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding2);
+
+ MapType<String, String> stringStringMapType = MapType.getInstance(UTF8, UTF8, false);
+ Map<String, String> stringStringMap = new HashMap<>();
+ stringStringMap.put(string1, string4);
+ stringStringMap.put(string2, string3);
+ stringStringMap.put(string3, string2);
+ stringStringMap.put(string4, string1);
+
+ sequence = ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(padding1, version),
+ stringStringMapType.asComparableBytes(stringStringMapType.decompose(stringStringMap), version),
+ ByteSource.of(padding2, version)
+ );
+ comparableBytes = ByteSource.peekable(sequence);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding1);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(stringStringMapType, comparableBytes), stringStringMap);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding2);
+
+ MapType<BigInteger, BigInteger> varintVarintMapType = MapType.getInstance(VARINT, VARINT, false);
+ Map<BigInteger, BigInteger> varintVarintMap = new HashMap<>();
+ varintVarintMap.put(varint1, varint4);
+ varintVarintMap.put(varint2, varint3);
+ varintVarintMap.put(varint3, varint2);
+ varintVarintMap.put(varint4, varint1);
+
+ sequence = ByteSource.withTerminator(
+ ByteSource.TERMINATOR,
+ ByteSource.of(padding1, version),
+ varintVarintMapType.asComparableBytes(varintVarintMapType.decompose(varintVarintMap), version),
+ ByteSource.of(padding2, version)
+ );
+ comparableBytes = ByteSource.peekable(sequence);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding1);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(varintVarintMapType, comparableBytes), varintVarintMap);
+ assertEquals(ByteSource.NEXT_COMPONENT, comparableBytes.next());
+ assertEquals(getComponentValue(UTF8, comparableBytes), padding2);
+ }
+
+ @Test
+ public void testReadingNestedSequence_DecoratedKey()
+ {
+ Random prng = new Random();
+
+ MapType<String, BigDecimal> stringDecimalMapType = MapType.getInstance(UTF8, DECIMAL, false);
+ Map<String, BigDecimal> stringDecimalMap = new HashMap<>();
+ for (int i = 0; i < 4; ++i)
+ stringDecimalMap.put(newRandomAlphanumeric(prng, 10), BigDecimal.valueOf(prng.nextDouble()));
+ ByteBuffer key = stringDecimalMapType.decompose(stringDecimalMap);
+ testDecodingKeyWithLocalPartitionerForType(key, stringDecimalMapType);
+
+ MapType<BigDecimal, String> decimalStringMapType = MapType.getInstance(DECIMAL, UTF8, false);
+ Map<BigDecimal, String> decimalStringMap = new HashMap<>();
+ for (int i = 0; i < 4; ++i)
+ decimalStringMap.put(BigDecimal.valueOf(prng.nextDouble()), newRandomAlphanumeric(prng, 10));
+ key = decimalStringMapType.decompose(decimalStringMap);
+ testDecodingKeyWithLocalPartitionerForType(key, decimalStringMapType);
+
+ if (version != ByteComparable.Version.LEGACY)
+ {
+ CompositeType stringDecimalCompType = CompositeType.getInstance(UTF8, DECIMAL);
+ key = stringDecimalCompType.decompose(newRandomAlphanumeric(prng, 10), BigDecimal.valueOf(prng.nextDouble()));
+ testDecodingKeyWithLocalPartitionerForType(key, stringDecimalCompType);
+
+ CompositeType decimalStringCompType = CompositeType.getInstance(DECIMAL, UTF8);
+ key = decimalStringCompType.decompose(BigDecimal.valueOf(prng.nextDouble()), newRandomAlphanumeric(prng, 10));
+ testDecodingKeyWithLocalPartitionerForType(key, decimalStringCompType);
+
+ DynamicCompositeType dynamicCompType = DynamicCompositeType.getInstance(DynamicCompositeTypeTest.aliases);
+ key = DynamicCompositeTypeTest.createDynamicCompositeKey(
+ newRandomAlphanumeric(prng, 10), TimeUUID.Generator.nextTimeAsUUID(), 42, true, false);
+ testDecodingKeyWithLocalPartitionerForType(key, dynamicCompType);
+
+ key = DynamicCompositeTypeTest.createDynamicCompositeKey(
+ newRandomAlphanumeric(prng, 10), TimeUUID.Generator.nextTimeAsUUID(), 42, true, true);
+ testDecodingKeyWithLocalPartitionerForType(key, dynamicCompType);
+ }
+ }
+
+ private static String newRandomAlphanumeric(Random prng, int length)
+ {
+ StringBuilder random = new StringBuilder(length);
+ for (int i = 0; i < length; ++i)
+ random.append(ALPHABET.charAt(prng.nextInt(ALPHABET.length())));
+ return random.toString();
+ }
+
+ private <T> void testDecodingKeyWithLocalPartitionerForType(ByteBuffer key, AbstractType<T> type)
+ {
+ IPartitioner partitioner = new LocalPartitioner(type);
+ CachedHashDecoratedKey initial = (CachedHashDecoratedKey) partitioner.decorateKey(key);
+ BufferDecoratedKey base = BufferDecoratedKey.fromByteComparable(initial, version, partitioner);
+ CachedHashDecoratedKey decoded = new CachedHashDecoratedKey(base.getToken(), base.getKey());
+ Assert.assertEquals(initial, decoded);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java
new file mode 100644
index 0000000..050bec5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceTestBase.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.utils.TimeUUID;
+
+public class ByteSourceTestBase
+{
+ String[] testStrings = new String[]{ "", "\0", "\0\0", "\001", "A\0\0B", "A\0B\0", "0", "0\0", "00", "1", "\377" };
+ Integer[] testInts = new Integer[]{ null,
+ Integer.MIN_VALUE,
+ Integer.MIN_VALUE + 1,
+ -256,
+ -255,
+ -128,
+ -127,
+ -64,
+ -63,
+ -1,
+ 0,
+ 1,
+ 63,
+ 64,
+ 127,
+ 128,
+ 255,
+ 256,
+ Integer.MAX_VALUE - 1,
+ Integer.MAX_VALUE };
+ Byte[] testBytes = new Byte[]{ -128, -127, -1, 0, 1, 127 };
+ Short[] testShorts = new Short[]{ Short.MIN_VALUE,
+ Short.MIN_VALUE + 1,
+ -256,
+ -255,
+ -128,
+ -127,
+ -65,
+ -64,
+ -63,
+ -1,
+ 0,
+ 1,
+ 127,
+ 128,
+ 255,
+ 256,
+ Short.MAX_VALUE - 1,
+ Short.MAX_VALUE };
+ Long[] testLongs = new Long[]{ null,
+ Long.MIN_VALUE,
+ Long.MIN_VALUE + 1,
+ Integer.MIN_VALUE - 1L,
+ -256L,
+ -255L,
+ -128L,
+ -127L,
+ -65L,
+ -64L,
+ -63L,
+ -1L,
+ 0L,
+ 1L,
+ Integer.MAX_VALUE + 1L,
+ Long.MAX_VALUE - 1,
+ Long.MAX_VALUE,
+
+ (1L << 1) - 1,
+ (1L << 1),
+ (1L << 2) - 1,
+ (1L << 2),
+ (1L << 3) - 1,
+ (1L << 3),
+ (1L << 4) - 1,
+ (1L << 4),
+ (1L << 5) - 1,
+ (1L << 5),
+ (1L << 6) - 1,
+ (1L << 6),
+ (1L << 7) - 1,
+ (1L << 7),
+ (1L << 8) - 1,
+ (1L << 8),
+ (1L << 9) - 1,
+ (1L << 9),
+ (1L << 10) - 1,
+ (1L << 10),
+ (1L << 11) - 1,
+ (1L << 11),
+ (1L << 12) - 1,
+ (1L << 12),
+ (1L << 13) - 1,
+ (1L << 13),
+ (1L << 14) - 1,
+ (1L << 14),
+ (1L << 15) - 1,
+ (1L << 15),
+ (1L << 16) - 1,
+ (1L << 16),
+ (1L << 17) - 1,
+ (1L << 17),
+ (1L << 18) - 1,
+ (1L << 18),
+ (1L << 19) - 1,
+ (1L << 19),
+ (1L << 20) - 1,
+ (1L << 20),
+ (1L << 21) - 1,
+ (1L << 21),
+ (1L << 22) - 1,
+ (1L << 22),
+ (1L << 23) - 1,
+ (1L << 23),
+ (1L << 24) - 1,
+ (1L << 24),
+ (1L << 25) - 1,
+ (1L << 25),
+ (1L << 26) - 1,
+ (1L << 26),
+ (1L << 27) - 1,
+ (1L << 27),
+ (1L << 28) - 1,
+ (1L << 28),
+ (1L << 29) - 1,
+ (1L << 29),
+ (1L << 30) - 1,
+ (1L << 30),
+ (1L << 31) - 1,
+ (1L << 31),
+ (1L << 32) - 1,
+ (1L << 32),
+ (1L << 33) - 1,
+ (1L << 33),
+ (1L << 34) - 1,
+ (1L << 34),
+ (1L << 35) - 1,
+ (1L << 35),
+ (1L << 36) - 1,
+ (1L << 36),
+ (1L << 37) - 1,
+ (1L << 37),
+ (1L << 38) - 1,
+ (1L << 38),
+ (1L << 39) - 1,
+ (1L << 39),
+ (1L << 40) - 1,
+ (1L << 40),
+ (1L << 41) - 1,
+ (1L << 41),
+ (1L << 42) - 1,
+ (1L << 42),
+ (1L << 43) - 1,
+ (1L << 43),
+ (1L << 44) - 1,
+ (1L << 44),
+ (1L << 45) - 1,
+ (1L << 45),
+ (1L << 46) - 1,
+ (1L << 46),
+ (1L << 47) - 1,
+ (1L << 47),
+ (1L << 48) - 1,
+ (1L << 48),
+ (1L << 49) - 1,
+ (1L << 49),
+ (1L << 50) - 1,
+ (1L << 50),
+ (1L << 51) - 1,
+ (1L << 51),
+ (1L << 52) - 1,
+ (1L << 52),
+ (1L << 53) - 1,
+ (1L << 53),
+ (1L << 54) - 1,
+ (1L << 54),
+ (1L << 55) - 1,
+ (1L << 55),
+ (1L << 56) - 1,
+ (1L << 56),
+ (1L << 57) - 1,
+ (1L << 57),
+ (1L << 58) - 1,
+ (1L << 58),
+ (1L << 59) - 1,
+ (1L << 59),
+ (1L << 60) - 1,
+ (1L << 60),
+ (1L << 61) - 1,
+ (1L << 61),
+ (1L << 62) - 1,
+ (1L << 62),
+ (1L << 63) - 1,
+
+ ~((1L << 1) - 1),
+ ~((1L << 1)),
+ ~((1L << 2) - 1),
+ ~((1L << 2)),
+ ~((1L << 3) - 1),
+ ~((1L << 3)),
+ ~((1L << 4) - 1),
+ ~((1L << 4)),
+ ~((1L << 5) - 1),
+ ~((1L << 5)),
+ ~((1L << 6) - 1),
+ ~((1L << 6)),
+ ~((1L << 7) - 1),
+ ~((1L << 7)),
+ ~((1L << 8) - 1),
+ ~((1L << 8)),
+ ~((1L << 9) - 1),
+ ~((1L << 9)),
+ ~((1L << 10) - 1),
+ ~((1L << 10)),
+ ~((1L << 11) - 1),
+ ~((1L << 11)),
+ ~((1L << 12) - 1),
+ ~((1L << 12)),
+ ~((1L << 13) - 1),
+ ~((1L << 13)),
+ ~((1L << 14) - 1),
+ ~((1L << 14)),
+ ~((1L << 15) - 1),
+ ~((1L << 15)),
+ ~((1L << 16) - 1),
+ ~((1L << 16)),
+ ~((1L << 17) - 1),
+ ~((1L << 17)),
+ ~((1L << 18) - 1),
+ ~((1L << 18)),
+ ~((1L << 19) - 1),
+ ~((1L << 19)),
+ ~((1L << 20) - 1),
+ ~((1L << 20)),
+ ~((1L << 21) - 1),
+ ~((1L << 21)),
+ ~((1L << 22) - 1),
+ ~((1L << 22)),
+ ~((1L << 23) - 1),
+ ~((1L << 23)),
+ ~((1L << 24) - 1),
+ ~((1L << 24)),
+ ~((1L << 25) - 1),
+ ~((1L << 25)),
+ ~((1L << 26) - 1),
+ ~((1L << 26)),
+ ~((1L << 27) - 1),
+ ~((1L << 27)),
+ ~((1L << 28) - 1),
+ ~((1L << 28)),
+ ~((1L << 29) - 1),
+ ~((1L << 29)),
+ ~((1L << 30) - 1),
+ ~((1L << 30)),
+ ~((1L << 31) - 1),
+ ~((1L << 31)),
+ ~((1L << 32) - 1),
+ ~((1L << 32)),
+ ~((1L << 33) - 1),
+ ~((1L << 33)),
+ ~((1L << 34) - 1),
+ ~((1L << 34)),
+ ~((1L << 35) - 1),
+ ~((1L << 35)),
+ ~((1L << 36) - 1),
+ ~((1L << 36)),
+ ~((1L << 37) - 1),
+ ~((1L << 37)),
+ ~((1L << 38) - 1),
+ ~((1L << 38)),
+ ~((1L << 39) - 1),
+ ~((1L << 39)),
+ ~((1L << 40) - 1),
+ ~((1L << 40)),
+ ~((1L << 41) - 1),
+ ~((1L << 41)),
+ ~((1L << 42) - 1),
+ ~((1L << 42)),
+ ~((1L << 43) - 1),
+ ~((1L << 43)),
+ ~((1L << 44) - 1),
+ ~((1L << 44)),
+ ~((1L << 45) - 1),
+ ~((1L << 45)),
+ ~((1L << 46) - 1),
+ ~((1L << 46)),
+ ~((1L << 47) - 1),
+ ~((1L << 47)),
+ ~((1L << 48) - 1),
+ ~((1L << 48)),
+ ~((1L << 49) - 1),
+ ~((1L << 49)),
+ ~((1L << 50) - 1),
+ ~((1L << 50)),
+ ~((1L << 51) - 1),
+ ~((1L << 51)),
+ ~((1L << 52) - 1),
+ ~((1L << 52)),
+ ~((1L << 53) - 1),
+ ~((1L << 53)),
+ ~((1L << 54) - 1),
+ ~((1L << 54)),
+ ~((1L << 55) - 1),
+ ~((1L << 55)),
+ ~((1L << 56) - 1),
+ ~((1L << 56)),
+ ~((1L << 57) - 1),
+ ~((1L << 57)),
+ ~((1L << 58) - 1),
+ ~((1L << 58)),
+ ~((1L << 59) - 1),
+ ~((1L << 59)),
+ ~((1L << 60) - 1),
+ ~((1L << 60)),
+ ~((1L << 61) - 1),
+ ~((1L << 61)),
+ ~((1L << 62) - 1),
+ ~((1L << 62)),
+ ~((1L << 63) - 1),
+ };
+ Double[] testDoubles = new Double[]{ null,
+ Double.NEGATIVE_INFINITY,
+ -Double.MAX_VALUE,
+ -1e+200,
+ -1e3,
+ -1e0,
+ -1e-3,
+ -1e-200,
+ -Double.MIN_VALUE,
+ -0.0,
+ 0.0,
+ Double.MIN_VALUE,
+ 1e-200,
+ 1e-3,
+ 1e0,
+ 1e3,
+ 1e+200,
+ Double.MAX_VALUE,
+ Double.POSITIVE_INFINITY,
+ Double.NaN };
+ Float[] testFloats = new Float[]{ null,
+ Float.NEGATIVE_INFINITY,
+ -Float.MAX_VALUE,
+ -1e+30f,
+ -1e3f,
+ -1e0f,
+ -1e-3f,
+ -1e-30f,
+ -Float.MIN_VALUE,
+ -0.0f,
+ 0.0f,
+ Float.MIN_VALUE,
+ 1e-30f,
+ 1e-3f,
+ 1e0f,
+ 1e3f,
+ 1e+30f,
+ Float.MAX_VALUE,
+ Float.POSITIVE_INFINITY,
+ Float.NaN };
+ Boolean[] testBools = new Boolean[]{ null, false, true };
+ UUID[] testUUIDs = new UUID[]{ null,
+ TimeUUID.Generator.nextTimeAsUUID(),
+ UUID.randomUUID(),
+ UUID.randomUUID(),
+ UUID.randomUUID(),
+ TimeUUID.Generator.atUnixMillis(123, 234).asUUID(),
+ TimeUUID.Generator.atUnixMillis(123, 234).asUUID(),
+ TimeUUID.Generator.atUnixMillis(123).asUUID(),
+ UUID.fromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8"),
+ UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"),
+ UUID.fromString("e902893a-9d22-3c7e-a7b8-d6e313b71d9f"),
+ UUID.fromString("74738ff5-5367-5958-9aee-98fffdcd1876"),
+ UUID.fromString("52df1bb0-6a2f-11e6-b6e4-a6dea7a01b67"),
+ UUID.fromString("52df1bb0-6a2f-11e6-362d-aff2143498ea"),
+ UUID.fromString("52df1bb0-6a2f-11e6-b62d-aff2143498ea") };
+ // Instant.MIN/MAX fail Date.from.
+ Date[] testDates = new Date[]{ null,
+ Date.from(Instant.ofEpochSecond(Integer.MIN_VALUE)),
+ Date.from(Instant.ofEpochSecond(Short.MIN_VALUE)),
+ Date.from(Instant.ofEpochMilli(-2000)),
+ Date.from(Instant.EPOCH),
+ Date.from(Instant.ofEpochMilli(2000)),
+ Date.from(Instant.ofEpochSecond(Integer.MAX_VALUE)),
+ Date.from(Instant.now()) };
+ InetAddress[] testInets;
+ {
+ try
+ {
+ testInets = new InetAddress[]{ null,
+ InetAddress.getLocalHost(),
+ InetAddress.getLoopbackAddress(),
+ InetAddress.getByName("192.168.0.1"),
+ InetAddress.getByName("fe80::428d:5cff:fe53:1dc9"),
+ InetAddress.getByName("2001:610:3:200a:192:87:36:2"),
+ InetAddress.getByName("10.0.0.1"),
+ InetAddress.getByName("0a00:0001::"),
+ InetAddress.getByName("::10.0.0.1") };
+ }
+ catch (UnknownHostException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ BigInteger[] testBigInts;
+
+ {
+ Set<BigInteger> bigs = new TreeSet<>();
+ for (Long l : testLongs)
+ if (l != null)
+ bigs.add(BigInteger.valueOf(l));
+ for (int i = 0; i < 11; ++i)
+ {
+ bigs.add(BigInteger.valueOf(i));
+ bigs.add(BigInteger.valueOf(-i));
+
+ bigs.add(BigInteger.valueOf((1L << 4 * i) - 1));
+ bigs.add(BigInteger.valueOf((1L << 4 * i)));
+ bigs.add(BigInteger.valueOf(-(1L << 4 * i) - 1));
+ bigs.add(BigInteger.valueOf(-(1L << 4 * i)));
+ String p = exp10(i);
+ bigs.add(new BigInteger(p));
+ bigs.add(new BigInteger("-" + p));
+ p = exp10(1 << i);
+ bigs.add(new BigInteger(p));
+ bigs.add(new BigInteger("-" + p));
+
+ BigInteger base = BigInteger.ONE.shiftLeft(512 * i);
+ bigs.add(base);
+ bigs.add(base.add(BigInteger.ONE));
+ bigs.add(base.subtract(BigInteger.ONE));
+ base = base.negate();
+ bigs.add(base);
+ bigs.add(base.add(BigInteger.ONE));
+ bigs.add(base.subtract(BigInteger.ONE));
+ }
+ testBigInts = bigs.toArray(new BigInteger[0]);
+ }
+
+ static String exp10(int pow)
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append('1');
+ for (int i=0; i<pow; ++i)
+ builder.append('0');
+ return builder.toString();
+ }
+
+ BigDecimal[] testBigDecimals;
+ {
+ String vals = "0, 1, 1.1, 21, 98.9, 99, 99.9, 100, 100.1, 101, 331, 0.4, 0.07, 0.0700, 0.005, " +
+ "6e4, 7e200, 6e-300, 8.1e2000, 8.1e-2000, 9e2000000000, " +
+ "123456789012.34567890e-1000000000, 123456.78901234, 1234.56789012e2, " +
+ "1.0000, 0.01e2, 100e-2, 00, 0.000, 0E-18, 0E+18";
+ List<BigDecimal> decs = new ArrayList<>();
+ for (String s : vals.split(", "))
+ {
+ decs.add(new BigDecimal(s));
+ decs.add(new BigDecimal("-" + s));
+ }
+ testBigDecimals = decs.toArray(new BigDecimal[0]);
+ }
+
+ Object[][] testValues = new Object[][]{ testStrings,
+ testInts,
+ testBools,
+ testDoubles,
+ testBigInts,
+ testBigDecimals };
+
+ AbstractType[] testTypes = new AbstractType[]{ UTF8Type.instance,
+ Int32Type.instance,
+ BooleanType.instance,
+ DoubleType.instance,
+ IntegerType.instance,
+ DecimalType.instance };
+}
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java
new file mode 100644
index 0000000..6c3e72e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/DecoratedKeyByteSourceTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils.bytecomparable;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+
+@RunWith(Parameterized.class)
+public class DecoratedKeyByteSourceTest
+{
+ private static final int NUM_ITERATIONS = 100;
+ private static final int RANDOM_BYTES_LENGTH = 100;
+
+ @Parameterized.Parameters(name = "version={0}")
+ public static Iterable<ByteComparable.Version> versions()
+ {
+ return ImmutableList.of(ByteComparable.Version.OSS50);
+ }
+
+ private final ByteComparable.Version version;
+
+ public DecoratedKeyByteSourceTest(ByteComparable.Version version)
+ {
+ this.version = version;
+ }
+
+ @Test
+ public void testDecodeBufferDecoratedKey()
+ {
+ for (int i = 0; i < NUM_ITERATIONS; ++i)
+ {
+ BufferDecoratedKey initialBuffer =
+ (BufferDecoratedKey) ByteOrderedPartitioner.instance.decorateKey(newRandomBytesBuffer());
+ BufferDecoratedKey decodedBuffer = BufferDecoratedKey.fromByteComparable(
+ initialBuffer, version, ByteOrderedPartitioner.instance);
+ Assert.assertEquals(initialBuffer, decodedBuffer);
+ }
+ }
+
+ @Test
+ public void testDecodeKeyBytes()
+ {
+ for (int i = 0; i < NUM_ITERATIONS; ++i)
+ {
+ BufferDecoratedKey initialBuffer =
+ (BufferDecoratedKey) ByteOrderedPartitioner.instance.decorateKey(newRandomBytesBuffer());
+ ByteSource.Peekable src = ByteSource.peekable(initialBuffer.asComparableBytes(version));
+ byte[] keyBytes = DecoratedKey.keyFromByteSource(src, version, ByteOrderedPartitioner.instance);
+ Assert.assertEquals(ByteSource.END_OF_STREAM, src.next());
+ Assert.assertArrayEquals(initialBuffer.getKey().array(), keyBytes);
+ }
+ }
+
+ private static ByteBuffer newRandomBytesBuffer()
+ {
+ byte[] randomBytes = new byte[RANDOM_BYTES_LENGTH];
+ new Random().nextBytes(randomBytes);
+ return ByteBuffer.wrap(randomBytes);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java b/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java
index e99d368..d50e408 100644
--- a/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java
+++ b/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java
@@ -18,28 +18,41 @@
*/
package org.apache.cassandra.utils.concurrent;
-import org.junit.Test;
-
-import org.junit.Assert;
-
-import org.apache.cassandra.io.util.File;
import java.lang.ref.WeakReference;
+import java.time.Duration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Ref.Visitor;
+import org.awaitility.Awaitility;
+
+import static org.assertj.core.api.Assertions.assertThat;
@SuppressWarnings({"unused", "unchecked", "rawtypes"})
public class RefCountedTest
@@ -406,4 +419,134 @@
Assert.assertTrue(visitor.haveLoops.isEmpty());
}
+
+ static class LambdaTestClassTidier implements RefCounted.Tidy
+ {
+ Runnable runOnClose;
+
+ @Override
+ public void tidy() throws Exception
+ {
+ runOnClose.run();
+ }
+
+ @Override
+ public String name()
+ {
+ return "42";
+ }
+ }
+
+ static class LambdaTestClass
+ {
+ String a = "x";
+ Ref<Object> ref;
+
+ Runnable getRunOnCloseLambda()
+ {
+ return () -> System.out.println("aaa");
+ }
+
+ Runnable getRunOnCloseInner()
+ {
+ return new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ System.out.println("aaa");
+ }
+ };
+ }
+
+ Runnable getRunOnCloseLambdaWithThis()
+ {
+ return () -> System.out.println(a);
+ }
+ }
+
+ private Set<Ref.GlobalState> testCycles(Function<LambdaTestClass, Runnable> runOnCloseSupplier)
+ {
+ LambdaTestClass test = new LambdaTestClass();
+ Runnable weakRef = runOnCloseSupplier.apply(test);
+ RefCounted.Tidy tidier = new RefCounted.Tidy()
+ {
+ Runnable ref = weakRef;
+
+ public void tidy()
+ {
+ }
+
+ public String name()
+ {
+ return "42";
+ }
+ };
+
+ Ref<Object> ref = new Ref(test, tidier);
+ test.ref = ref;
+
+ Visitor visitor = new Visitor();
+ visitor.haveLoops = new HashSet<>();
+ visitor.run();
+ ref.close();
+
+ return visitor.haveLoops;
+ }
+
+ /**
+ * The intention of this test is to confirm that lambda without a reference to `this`, does not implicitly
+ * include a reference to the enclosing class. If it did, we would detect cycles, as we do when we deal with
+ * anonymous inner classes or lamba which references `this` explicitly (see the sanity checks).
+ * <p>
+ * This test aims to confirm that JVM works as we assumed because we couldn't find that in the specification.
+ */
+ @Test
+ public void testCycles()
+ {
+ assertThat(testCycles(LambdaTestClass::getRunOnCloseLambdaWithThis)).isNotEmpty(); // sanity test
+ assertThat(testCycles(LambdaTestClass::getRunOnCloseInner)).isNotEmpty(); // sanity test
+
+ assertThat(testCycles(LambdaTestClass::getRunOnCloseLambda)).isEmpty();
+ }
+
+ @Test
+ public void testSSTableReaderLeakIsDetected()
+ {
+ DatabaseDescriptor.clientInitialization();
+ DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+ Descriptor descriptor = Descriptor.fromFileWithComponent(new File("test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-1-big-Data.db"), false).left;
+ TableMetadata tm = TableMetadata.builder("legacy_tables", "legacy_nb_simple").addPartitionKeyColumn("pk", UTF8Type.instance).addRegularColumn("val", UTF8Type.instance).build();
+ AtomicBoolean leakDetected = new AtomicBoolean();
+ AtomicBoolean runOnCloseExecuted1 = new AtomicBoolean();
+ AtomicBoolean runOnCloseExecuted2 = new AtomicBoolean();
+ Ref.OnLeak prevOnLeak = Ref.ON_LEAK;
+
+ try
+ {
+ Ref.ON_LEAK = state -> {
+ leakDetected.set(true);
+ };
+ {
+ SSTableReader reader = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(tm));
+ reader.runOnClose(() -> runOnCloseExecuted1.set(true));
+ reader.runOnClose(() -> runOnCloseExecuted2.set(true)); // second time to actually create lambda referencing to lambda, see runOnClose impl
+ //noinspection UnusedAssignment
+ reader = null; // this is required, otherwise GC will not attempt to collect the created reader
+ }
+ Awaitility.await().atMost(Duration.ofSeconds(30)).pollDelay(Duration.ofSeconds(1)).untilAsserted(() -> {
+ System.gc();
+ System.gc();
+ System.gc();
+ System.gc();
+ assertThat(leakDetected.get()).isTrue();
+ });
+ assertThat(runOnCloseExecuted1.get()).isTrue();
+ assertThat(runOnCloseExecuted2.get()).isTrue();
+ }
+ finally
+ {
+ Ref.ON_LEAK = prevOnLeak;
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java
index 15f9cdc..3397426 100644
--- a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java
+++ b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java
@@ -18,21 +18,19 @@
*/
package org.apache.cassandra.utils.vint;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
+import com.google.common.primitives.UnsignedInteger;
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
-import org.junit.Test;
-
-import org.junit.Assert;
-
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class VIntCodingTest
@@ -51,21 +49,21 @@
assertEncodedAtExpectedSize((1L << 7 * size) - 1, size);
assertEncodedAtExpectedSize(1L << 7 * size, size + 1);
}
- Assert.assertEquals(9, VIntCoding.computeUnsignedVIntSize(Long.MAX_VALUE));
+ assertEquals(9, VIntCoding.computeUnsignedVIntSize(Long.MAX_VALUE));
}
private void assertEncodedAtExpectedSize(long value, int expectedSize) throws Exception
{
- Assert.assertEquals(expectedSize, VIntCoding.computeUnsignedVIntSize(value));
+ assertEquals(expectedSize, VIntCoding.computeUnsignedVIntSize(value));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
WrappedDataOutputStreamPlus out = new WrappedDataOutputStreamPlus(baos);
VIntCoding.writeUnsignedVInt(value, out);
out.flush();
- Assert.assertEquals( expectedSize, baos.toByteArray().length);
+ assertEquals( expectedSize, baos.toByteArray().length);
DataOutputBuffer dob = new DataOutputBuffer();
dob.writeUnsignedVInt(value);
- Assert.assertEquals( expectedSize, dob.buffer().remaining());
+ assertEquals( expectedSize, dob.buffer().remaining());
dob.close();
}
@@ -73,7 +71,7 @@
public void testReadExtraBytesCount()
{
for (int i = 1 ; i < 8 ; i++)
- Assert.assertEquals(i, VIntCoding.numberOfExtraBytesToRead((byte) ((0xFF << (8 - i)) & 0xFF)));
+ assertEquals(i, VIntCoding.numberOfExtraBytesToRead((byte) ((0xFF << (8 - i)) & 0xFF)));
}
/*
@@ -85,13 +83,13 @@
ByteArrayOutputStream baos = new ByteArrayOutputStream();
WrappedDataOutputStreamPlus out = new WrappedDataOutputStreamPlus(baos);
- VIntCoding.writeUnsignedVInt(biggestOneByte, out);
+ VIntCoding.writeUnsignedVInt32(biggestOneByte, out);
out.flush();
- Assert.assertEquals( 1, baos.toByteArray().length);
+ assertEquals( 1, baos.toByteArray().length);
DataOutputBuffer dob = new DataOutputBuffer();
- dob.writeUnsignedVInt(biggestOneByte);
- Assert.assertEquals( 1, dob.buffer().remaining());
+ dob.writeUnsignedVInt32(biggestOneByte);
+ assertEquals( 1, dob.buffer().remaining());
dob.close();
}
@@ -101,9 +99,9 @@
int i = -1231238694;
try (DataOutputBuffer out = new DataOutputBuffer())
{
- VIntCoding.writeUnsignedVInt(i, out);
+ VIntCoding.writeUnsignedVInt32(i, out);
long result = VIntCoding.getUnsignedVInt(out.buffer(), 0);
- Assert.assertEquals(i, result);
+ assertEquals(i, result);
}
}
@@ -113,15 +111,14 @@
for (int i = 0; i < VIntCoding.MAX_SIZE - 1; i++)
{
long val = LONGS[i];
- Assert.assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val));
+ assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val));
try (DataOutputBuffer out = new DataOutputBuffer())
{
VIntCoding.writeUnsignedVInt(val, out);
// read as ByteBuffer
- Assert.assertEquals(val, VIntCoding.getUnsignedVInt(out.buffer(), 0));
+ assertEquals(val, VIntCoding.getUnsignedVInt(out.buffer(), 0));
// read as DataInput
- InputStream is = new ByteArrayInputStream(out.toByteArray());
- Assert.assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is)));
+ assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(out.toByteArray())));
}
}
}
@@ -132,18 +129,17 @@
for (int i = 0; i < VIntCoding.MAX_SIZE - 1; i++)
{
long val = LONGS[i];
- Assert.assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val));
+ assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (WrappedDataOutputStreamPlus out = new WrappedDataOutputStreamPlus(baos))
{
VIntCoding.writeUnsignedVInt(val, out);
out.flush();
- Assert.assertEquals( i + 1, baos.toByteArray().length);
+ assertEquals( i + 1, baos.toByteArray().length);
// read as ByteBuffer
- Assert.assertEquals(val, VIntCoding.getUnsignedVInt(ByteBuffer.wrap(baos.toByteArray()), 0));
+ assertEquals(val, VIntCoding.getUnsignedVInt(ByteBuffer.wrap(baos.toByteArray()), 0));
// read as DataInput
- InputStream is = new ByteArrayInputStream(baos.toByteArray());
- Assert.assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is)));
+ assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(baos.toByteArray())));
}
}
}
@@ -154,14 +150,13 @@
for (int i = 0; i < VIntCoding.MAX_SIZE - 1; i++)
{
long val = LONGS[i];
- Assert.assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val));
+ assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val));
ByteBuffer bb = ByteBuffer.allocate(VIntCoding.MAX_SIZE);
VIntCoding.writeUnsignedVInt(val, bb);
// read as ByteBuffer
- Assert.assertEquals(val, VIntCoding.getUnsignedVInt(bb, 0));
+ assertEquals(val, VIntCoding.getUnsignedVInt(bb, 0));
// read as DataInput
- InputStream is = new ByteArrayInputStream(bb.array());
- Assert.assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is)));
+ assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(bb.array())));
}
}
@@ -169,26 +164,25 @@
public void testWriteUnsignedVIntBBLessThan8Bytes() throws IOException
{
long val = 10201L;
- Assert.assertEquals(2, VIntCoding.computeUnsignedVIntSize(val));
+ assertEquals(2, VIntCoding.computeUnsignedVIntSize(val));
ByteBuffer bb = ByteBuffer.allocate(2);
VIntCoding.writeUnsignedVInt(val, bb);
// read as ByteBuffer
- Assert.assertEquals(val, VIntCoding.getUnsignedVInt(bb, 0));
+ assertEquals(val, VIntCoding.getUnsignedVInt(bb, 0));
// read as DataInput
- InputStream is = new ByteArrayInputStream(bb.array());
- Assert.assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is)));
+ assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(bb.array())));
}
@Test
public void testWriteUnsignedVIntBBHasLessThan8BytesLeft()
{
long val = 10201L;
- Assert.assertEquals(2, VIntCoding.computeUnsignedVIntSize(val));
+ assertEquals(2, VIntCoding.computeUnsignedVIntSize(val));
ByteBuffer bb = ByteBuffer.allocate(3);
bb.position(1);
VIntCoding.writeUnsignedVInt(val, bb);
// read as ByteBuffer
- Assert.assertEquals(val, VIntCoding.getUnsignedVInt(bb, 1));
+ assertEquals(val, VIntCoding.getUnsignedVInt(bb, 1));
}
@Test
@@ -201,4 +195,66 @@
fail();
} catch (BufferOverflowException e) {}
}
+
+ static int[] roundtripTestValues = new int[] {
+ UnsignedInteger.MAX_VALUE.intValue(),
+ Integer.MAX_VALUE + 1,
+ Integer.MAX_VALUE,
+ Integer.MAX_VALUE - 1,
+ Integer.MIN_VALUE,
+ Integer.MIN_VALUE + 1,
+ Integer.MIN_VALUE - 1,
+ 0,
+ -1,
+ 1
+ };
+
+ @Test
+ public void testRoundtripUnsignedVInt32() throws Throwable
+ {
+ for (int value : roundtripTestValues)
+ testRoundtripUnsignedVInt32(value);
+ }
+
+ private static void testRoundtripUnsignedVInt32(int value) throws Throwable
+ {
+ ByteBuffer bb = ByteBuffer.allocate(9);
+ VIntCoding.writeUnsignedVInt32(value, bb);
+ bb.flip();
+ assertEquals(value, VIntCoding.getUnsignedVInt32(bb, 0));
+
+ try (DataOutputBuffer dob = new DataOutputBuffer())
+ {
+ dob.writeUnsignedVInt32(value);
+ try (DataInputBuffer dib = new DataInputBuffer(dob.buffer(), false))
+ {
+ assertEquals(value, dib.readUnsignedVInt32());
+ }
+ }
+ }
+
+ @Test
+ public void testRoundtripVInt32() throws Throwable
+ {
+ for (int value : roundtripTestValues)
+ testRoundtripVInt32(value);
+ }
+
+ private static void testRoundtripVInt32(int value) throws Throwable
+ {
+ ByteBuffer bb = ByteBuffer.allocate(9);
+
+ VIntCoding.writeVInt32(value, bb);
+ bb.flip();
+ assertEquals(value, VIntCoding.getVInt32(bb, 0));
+
+ try (DataOutputBuffer dob = new DataOutputBuffer())
+ {
+ dob.writeVInt32(value);
+ try (DataInputBuffer dib = new DataInputBuffer(dob.buffer(), false))
+ {
+ assertEquals(value, dib.readVInt32());
+ }
+ }
+ }
}
diff --git a/tools/bin/cassandra.in.sh b/tools/bin/cassandra.in.sh
index e1d1fe3..864b87e 100644
--- a/tools/bin/cassandra.in.sh
+++ b/tools/bin/cassandra.in.sh
@@ -80,8 +80,9 @@
java_ver_output=`"${JAVA:-java}" -version 2>&1`
jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
JVM_VERSION=${jvmver%_*}
+short=$(echo "${jvmver}" | cut -c1-2)
-JAVA_VERSION=11
+JAVA_VERSION=17
if [ "$JVM_VERSION" = "1.8.0" ] ; then
JVM_PATCH_VERSION=${jvmver#*_}
if [ "$JVM_VERSION" \< "1.8" ] || [ "$JVM_VERSION" \> "1.8.2" ] ; then
@@ -96,6 +97,10 @@
elif [ "$JVM_VERSION" \< "11" ] ; then
echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
exit 1;
+elif [ "$short" = "11" ] ; then
+ JAVA_VERSION=11
+elif [ "$JVM_VERSION" \< "17" ] ; then
+ echo "Cassandra 5.0 requires Java 11 or Java 17(or newer)."
fi
jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
@@ -119,7 +124,9 @@
# Read user-defined JVM options from jvm-server.options file
JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
-if [ $JAVA_VERSION -ge 11 ] ; then
+if [ $JAVA_VERSION -ge 17 ] ; then
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm17${jvmoptions_variant:--clients}.options
+elif [ $JAVA_VERSION -ge 11 ] ; then
JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
else
JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm8${jvmoptions_variant:--clients}.options
diff --git a/tools/bin/sstablepartitions b/tools/bin/sstablepartitions
new file mode 100755
index 0000000..a5f3248
--- /dev/null
+++ b/tools/bin/sstablepartitions
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ # Locations (in order) to use when searching for an include file.
+ for include in "`dirname "$0"`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r "$include" ]; then
+ . "$include"
+ break
+ fi
+ done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+ . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+ MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+ -Dcassandra.storagedir="$cassandra_storagedir" \
+ -Dlogback.configurationFile=logback-tools.xml \
+ org.apache.cassandra.tools.SSTablePartitions "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et
diff --git a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
index 8aa9867..cdd05ec 100644
--- a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
+++ b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java
@@ -21,22 +21,22 @@
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.antlr.runtime.RecognitionException;
-import org.apache.cassandra.cql3.CQLStatement;
-import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
-import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaTransformations;
import org.apache.cassandra.cql3.CQLFragmentParser;
+import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.CqlParser;
import org.apache.cassandra.cql3.QueryOptions;
@@ -44,7 +44,12 @@
import org.apache.cassandra.cql3.functions.UDHelper;
import org.apache.cassandra.cql3.functions.types.TypeCodec;
import org.apache.cassandra.cql3.statements.UpdateStatement;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -53,6 +58,10 @@
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTransformations;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.service.ClientState;
@@ -345,7 +354,7 @@
private final List<File> directoryList;
private ColumnFamilyStore cfs;
- protected SSTableFormat.Type formatType = null;
+ protected SSTableFormat<?, ?> format = null;
private Boolean makeRangeAware = false;
@@ -581,8 +590,8 @@
? new SSTableSimpleWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.updatedColumns())
: new SSTableSimpleUnsortedWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.updatedColumns(), bufferSizeInMiB);
- if (formatType != null)
- writer.setSSTableFormatType(formatType);
+ if (format != null)
+ writer.setSSTableFormatType(format);
writer.setRangeAwareWriting(makeRangeAware);
diff --git a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
index 0e7eec5..9829d56 100644
--- a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
+++ b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
@@ -21,7 +21,12 @@
import java.io.File;
import java.io.IOError;
import java.net.URI;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -31,7 +36,11 @@
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Uninterruptibles;
-import io.airlift.airline.*;
+import io.airlift.airline.Cli;
+import io.airlift.airline.Command;
+import io.airlift.airline.Help;
+import io.airlift.airline.HelpOption;
+import io.airlift.airline.Option;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -41,9 +50,10 @@
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.StressCQLSSTableWriter;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.StressCQLSSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -131,7 +141,7 @@
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
{
Set<Component> components = entry.getValue();
- if (!components.contains(Component.DATA))
+ if (!components.contains(Components.DATA))
continue;
try
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressGraph.java b/tools/stress/src/org/apache/cassandra/stress/StressGraph.java
index 76a86db..9b38bb8 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressGraph.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressGraph.java
@@ -28,18 +28,17 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.io.ByteStreams;
import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.cassandra.stress.report.StressMetrics;
import org.apache.cassandra.stress.settings.StressSettings;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
+import org.apache.cassandra.utils.JsonUtils;
public class StressGraph
{
@@ -62,7 +61,7 @@
public void generateGraph()
{
File htmlFile = new File(stressSettings.graph.file);
- JSONObject stats;
+ ObjectNode stats;
if (htmlFile.isFile())
{
try
@@ -84,7 +83,7 @@
try
{
PrintWriter out = new PrintWriter(htmlFile);
- String statsBlock = "/* stats start */\nstats = " + stats.toJSONString() + ";\n/* stats end */\n";
+ String statsBlock = "/* stats start */\nstats = " + JsonUtils.writeAsJsonString(stats) + ";\n/* stats end */\n";
String html = getGraphHTML().replaceFirst("/\\* stats start \\*/\n\n/\\* stats end \\*/\n", statsBlock);
out.write(html);
out.close();
@@ -95,16 +94,19 @@
}
}
- private JSONObject parseExistingStats(String html)
+ private ObjectNode parseExistingStats(String html)
{
- JSONObject stats;
-
Pattern pattern = Pattern.compile("(?s).*/\\* stats start \\*/\\nstats = (.*);\\n/\\* stats end \\*/.*");
Matcher matcher = pattern.matcher(html);
matcher.matches();
- stats = (JSONObject) JSONValue.parse(matcher.group(1));
-
- return stats;
+ try
+ {
+ return (ObjectNode) JsonUtils.JSON_OBJECT_MAPPER.readTree(matcher.group(1));
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Couldn't parser stats json: "+e.getMessage(), e);
+ }
}
private String getGraphHTML()
@@ -120,10 +122,10 @@
}
/** Parse log and append to stats array */
- private JSONArray parseLogStats(InputStream log, JSONArray stats) {
+ private ArrayNode parseLogStats(InputStream log, ArrayNode stats) {
BufferedReader reader = new BufferedReader(new InputStreamReader(log));
- JSONObject json = new JSONObject();
- JSONArray intervals = new JSONArray();
+ ObjectNode json = JsonUtils.JSON_OBJECT_MAPPER.createObjectNode();
+ ArrayNode intervals = JsonUtils.JSON_OBJECT_MAPPER.createArrayNode();
boolean runningMultipleThreadCounts = false;
String currentThreadCount = null;
Pattern threadCountMessage = Pattern.compile("Running ([A-Z]+) with ([0-9]+) threads .*");
@@ -171,7 +173,7 @@
// Process lines
if (mode == ReadingMode.METRICS)
{
- JSONArray metrics = new JSONArray();
+ ArrayNode metrics = JsonUtils.JSON_OBJECT_MAPPER.createArrayNode();
String[] parts = line.split(",");
if (parts.length != StressMetrics.HEADMETRICS.length)
{
@@ -185,7 +187,7 @@
}
catch (NumberFormatException e)
{
- metrics.add(null);
+ metrics.addNull();
}
}
intervals.add(metrics);
@@ -203,7 +205,10 @@
else if (mode == ReadingMode.NEXTITERATION)
{
//Wrap up the results of this test and append to the array.
- json.put("metrics", Arrays.asList(StressMetrics.HEADMETRICS));
+ ArrayNode metrics = json.putArray("metrics");
+ for (String metric : StressMetrics.HEADMETRICS) {
+ metrics.add(metric);
+ }
json.put("test", stressSettings.graph.operation);
if (currentThreadCount == null)
json.put("revision", stressSettings.graph.revision);
@@ -211,12 +216,12 @@
json.put("revision", String.format("%s - %s threads", stressSettings.graph.revision, currentThreadCount));
String command = StringUtils.join(stressArguments, " ").replaceAll("password=.*? ", "password=******* ");
json.put("command", command);
- json.put("intervals", intervals);
+ json.set("intervals", intervals);
stats.add(json);
//Start fresh for next iteration:
- json = new JSONObject();
- intervals = new JSONArray();
+ json = JsonUtils.JSON_OBJECT_MAPPER.createObjectNode();
+ intervals = JsonUtils.JSON_OBJECT_MAPPER.createArrayNode();
mode = ReadingMode.START;
}
}
@@ -229,25 +234,25 @@
return stats;
}
- private JSONObject createJSONStats(JSONObject json)
+ private ObjectNode createJSONStats(ObjectNode json)
{
try (InputStream logStream = Files.newInputStream(stressSettings.graph.temporaryLogFile.toPath()))
{
- JSONArray stats;
+ ArrayNode stats;
if (json == null)
{
- json = new JSONObject();
- stats = new JSONArray();
+ json = JsonUtils.JSON_OBJECT_MAPPER.createObjectNode();
+ stats = JsonUtils.JSON_OBJECT_MAPPER.createArrayNode();
}
else
{
- stats = (JSONArray) json.get("stats");
+ stats = (ArrayNode) json.get("stats");
}
stats = parseLogStats(logStream, stats);
json.put("title", stressSettings.graph.title);
- json.put("stats", stats);
+ json.set("stats", stats);
return json;
}
catch (IOException e)
diff --git a/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java b/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java
index b50dfd2..8579bbf 100644
--- a/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java
+++ b/tools/stress/src/org/apache/cassandra/stress/report/StressMetrics.java
@@ -83,11 +83,14 @@
private final Queue<OpMeasurement> leftovers = new ArrayDeque<>();
private final TimingInterval totalCurrentInterval;
private final TimingInterval totalSummaryInterval;
+ private final int outputFrequencyInSeconds;
+ private final int headerFrequencyInSeconds;
+ private int outputLines = 0;
public StressMetrics(ResultLogger output, final long logIntervalMillis, StressSettings settings)
{
this.output = output;
- if(settings.log.hdrFile != null)
+ if (settings.log.hdrFile != null)
{
try
{
@@ -114,7 +117,8 @@
try
{
gcStatsCollector = new JmxCollector(toJmxNodes(settings.node.resolveAllPermitted(settings)),
- settings.port.jmxPort);
+ settings.port.jmxPort,
+ settings.jmx);
}
catch (Throwable t)
{
@@ -133,6 +137,8 @@
reportingLoop(logIntervalMillis);
});
thread.setName("StressMetrics");
+ headerFrequencyInSeconds = settings.reporting.headerFrequency;
+ outputFrequencyInSeconds = settings.reporting.outputFrequency;
}
public void start()
{
@@ -262,7 +268,12 @@
opInterval.reset();
}
- printRow("", "total", totalCurrentInterval, totalSummaryInterval, gcStats, rowRateUncertainty, output);
+ ++outputLines;
+ if (outputFrequencyInSeconds == 0 || outputLines % outputFrequencyInSeconds == 0)
+ printRow("", "total", totalCurrentInterval, totalSummaryInterval, gcStats, rowRateUncertainty, output);
+ if (headerFrequencyInSeconds != 0 && outputLines % headerFrequencyInSeconds == 0)
+ printHeader("\n", output);
+
totalCurrentInterval.reset();
}
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java b/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java
index eba276e..5504671 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java
@@ -37,8 +37,11 @@
LOG("Where to log progress to, and the interval at which to do it", SettingsLog.helpPrinter()),
TRANSPORT("Custom transport factories", SettingsTransport.helpPrinter()),
PORT("The port to connect to cassandra nodes on", SettingsPort.helpPrinter()),
+ JMX("JMX credentials", SettingsJMX.helpPrinter()),
GRAPH("-graph", "Graph recorded metrics", SettingsGraph.helpPrinter()),
- TOKENRANGE("Token range settings", SettingsTokenRange.helpPrinter())
+ TOKENRANGE("Token range settings", SettingsTokenRange.helpPrinter()),
+ CREDENTIALS_FILE("Credentials file for CQL, JMX and transport", SettingsCredentials.helpPrinter()),
+ REPORTING("Frequency of printing statistics and header for stress output", SettingsReporting.helpPrinter());
;
private static final Map<String, CliOption> LOOKUP;
@@ -63,11 +66,12 @@
public final String description;
private final Runnable helpPrinter;
- private CliOption(String description, Runnable helpPrinter)
+ CliOption(String description, Runnable helpPrinter)
{
this(null, description, helpPrinter);
}
- private CliOption(String extraName, String description, Runnable helpPrinter)
+
+ CliOption(String extraName, String description, Runnable helpPrinter)
{
this.extraName = extraName;
this.description = description;
@@ -79,4 +83,8 @@
helpPrinter.run();
}
+ public String toString()
+ {
+ return name().replaceAll("_", "-");
+ }
}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCredentials.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCredentials.java
new file mode 100644
index 0000000..76513b7
--- /dev/null
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCredentials.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress.settings;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.stress.util.ResultLogger;
+
+public class SettingsCredentials implements Serializable
+{
+ public static final String CQL_USERNAME_PROPERTY_KEY = "cql.username";
+ public static final String CQL_PASSWORD_PROPERTY_KEY = "cql.password";
+ public static final String JMX_USERNAME_PROPERTY_KEY = "jmx.username";
+ public static final String JMX_PASSWORD_PROPERTY_KEY = "jmx.password";
+ public static final String TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY = "transport.truststore.password";
+ public static final String TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY = "transport.keystore.password";
+
+ private final String file;
+
+ public final String cqlUsername;
+ public final String cqlPassword;
+ public final String jmxUsername;
+ public final String jmxPassword;
+ public final String transportTruststorePassword;
+ public final String transportKeystorePassword;
+
+ public SettingsCredentials(String file)
+ {
+ this.file = file;
+ if (file == null)
+ {
+ cqlUsername = null;
+ cqlPassword = null;
+ jmxUsername = null;
+ jmxPassword = null;
+ transportTruststorePassword = null;
+ transportKeystorePassword = null;
+ return;
+ }
+
+ try
+ {
+ Properties properties = new Properties();
+ try (InputStream is = new FileInputStream(new File(file).toJavaIOFile()))
+ {
+ properties.load(is);
+
+ cqlUsername = properties.getProperty(CQL_USERNAME_PROPERTY_KEY);
+ cqlPassword = properties.getProperty(CQL_PASSWORD_PROPERTY_KEY);
+ jmxUsername = properties.getProperty(JMX_USERNAME_PROPERTY_KEY);
+ jmxPassword = properties.getProperty(JMX_PASSWORD_PROPERTY_KEY);
+ transportTruststorePassword = properties.getProperty(TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY);
+ transportKeystorePassword = properties.getProperty(TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY);
+ }
+ }
+ catch (IOException ioe)
+ {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ // CLI Utility Methods
+ public void printSettings(ResultLogger out)
+ {
+ out.printf(" File: %s%n", file == null ? "*not set*" : file);
+ out.printf(" CQL username: %s%n", cqlUsername == null ? "*not set*" : cqlUsername);
+ out.printf(" CQL password: %s%n", cqlPassword == null ? "*not set*" : "*suppressed*");
+ out.printf(" JMX username: %s%n", jmxUsername == null ? "*not set*" : jmxUsername);
+ out.printf(" JMX password: %s%n", jmxPassword == null ? "*not set*" : "*suppressed*");
+ out.printf(" Transport truststore password: %s%n", transportTruststorePassword == null ? "*not set*" : "*suppressed*");
+ out.printf(" Transport keystore password: %s%n", transportKeystorePassword == null ? "*not set*" : "*suppressed*");
+ }
+
+ public static SettingsCredentials get(Map<String, String[]> clArgs)
+ {
+ String[] params = clArgs.remove("-credentials-file");
+ if (params == null)
+ return new SettingsCredentials(null);
+
+ if (params.length != 1)
+ {
+ printHelp();
+ System.out.println("Invalid -credentials-file option provided, see output for valid options");
+ System.exit(1);
+ }
+
+ return new SettingsCredentials(params[0]);
+ }
+
+ public static void printHelp()
+ {
+ System.out.println("Usage: -credentials-file <file> ");
+ System.out.printf("File is supposed to be a standard property file with '%s', '%s', '%s', '%s', '%s', and '%s' as keys. " +
+ "The values for these keys will be overriden by their command-line counterparts when specified.%n",
+ CQL_USERNAME_PROPERTY_KEY,
+ CQL_PASSWORD_PROPERTY_KEY,
+ JMX_USERNAME_PROPERTY_KEY,
+ JMX_PASSWORD_PROPERTY_KEY,
+ TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY,
+ TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY);
+ }
+
+ public static Runnable helpPrinter()
+ {
+ return SettingsCredentials::printHelp;
+ }
+}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsJMX.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsJMX.java
new file mode 100644
index 0000000..af202be
--- /dev/null
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsJMX.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress.settings;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.stress.util.ResultLogger;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.JMX_PASSWORD_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.JMX_USERNAME_PROPERTY_KEY;
+
+public class SettingsJMX implements Serializable
+{
+ public final String user;
+ public final String password;
+
+ public SettingsJMX(Options options, SettingsCredentials credentials)
+ {
+ this.user = options.user.setByUser() ? options.user.value() : credentials.jmxUsername;
+ this.password = options.password.setByUser() ? options.password.value() : credentials.jmxPassword;
+ }
+
+ // Option Declarations
+
+ public static final class Options extends GroupedOptions
+ {
+ final OptionSimple user = new OptionSimple("user=",
+ ".*",
+ null,
+ format("Username for JMX connection, when specified, it will override the value in credentials file for key '%s'", JMX_USERNAME_PROPERTY_KEY),
+ false);
+
+ final OptionSimple password = new OptionSimple("password=",
+ ".*",
+ null,
+ format("Password for JMX connection, when specified, it will override the value in credentials file for key '%s'", JMX_PASSWORD_PROPERTY_KEY),
+ false);
+
+ @Override
+ public List<? extends Option> options()
+ {
+ return Arrays.asList(user, password);
+ }
+ }
+
+ // CLI Utility Methods
+ public void printSettings(ResultLogger out)
+ {
+ out.printf(" Username: %s%n", user);
+ out.printf(" Password: %s%n", (password == null ? "*not set*" : "*suppressed*"));
+ }
+
+ public static SettingsJMX get(Map<String, String[]> clArgs, SettingsCredentials credentials)
+ {
+ String[] params = clArgs.remove("-jmx");
+ if (params == null)
+ return new SettingsJMX(new SettingsJMX.Options(), credentials);
+
+ GroupedOptions options = GroupedOptions.select(params, new SettingsJMX.Options());
+ if (options == null)
+ {
+ printHelp();
+ System.out.println("Invalid -jmx options provided, see output for valid options");
+ System.exit(1);
+ }
+ return new SettingsJMX((SettingsJMX.Options) options, credentials);
+ }
+
+ public static void printHelp()
+ {
+ GroupedOptions.printOptions(System.out, "-jmx", new SettingsJMX.Options());
+ }
+
+ public static Runnable helpPrinter()
+ {
+ return SettingsJMX::printHelp;
+ }
+}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java
index 7e53547..b009d04 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java
@@ -32,6 +32,10 @@
import com.datastax.driver.core.ProtocolVersion;
import org.apache.cassandra.stress.util.ResultLogger;
+import static java.lang.String.format;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.CQL_PASSWORD_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.CQL_USERNAME_PROPERTY_KEY;
+
public class SettingsMode implements Serializable
{
@@ -51,7 +55,7 @@
private final String compression;
- public SettingsMode(GroupedOptions options)
+ public SettingsMode(GroupedOptions options, SettingsCredentials credentials)
{
if (options instanceof Cql3Options)
{
@@ -63,8 +67,8 @@
api = ConnectionAPI.JAVA_DRIVER_NATIVE;
style = opts.useUnPrepared.setByUser() ? ConnectionStyle.CQL : ConnectionStyle.CQL_PREPARED;
compression = ProtocolOptions.Compression.valueOf(opts.useCompression.value().toUpperCase()).name();
- username = opts.user.value();
- password = opts.password.value();
+ username = opts.user.setByUser() ? opts.user.value() : credentials.cqlUsername;
+ password = opts.password.setByUser() ? opts.password.value() : credentials.cqlPassword;
maxPendingPerConnection = opts.maxPendingPerConnection.value().isEmpty() ? null : Integer.valueOf(opts.maxPendingPerConnection.value());
connectionsPerHost = opts.connectionsPerHost.value().isEmpty() ? null : Integer.valueOf(opts.connectionsPerHost.value());
authProviderClassname = opts.authProvider.value();
@@ -137,8 +141,12 @@
final OptionSimple useUnPrepared = new OptionSimple("unprepared", "", null, "force use of unprepared statements", false);
final OptionSimple useCompression = new OptionSimple("compression=", "none|lz4|snappy", "none", "", false);
final OptionSimple port = new OptionSimple("port=", "[0-9]+", "9046", "", false);
- final OptionSimple user = new OptionSimple("user=", ".+", null, "username", false);
- final OptionSimple password = new OptionSimple("password=", ".+", null, "password", false);
+ final OptionSimple user = new OptionSimple("user=", ".+", null,
+ format("CQL user, when specified, it will override the value in credentials file for key '%s'", CQL_USERNAME_PROPERTY_KEY),
+ false);
+ final OptionSimple password = new OptionSimple("password=", ".+", null,
+ format("CQL password, when specified, it will override the value in credentials file for key '%s'", CQL_PASSWORD_PROPERTY_KEY),
+ false);
final OptionSimple authProvider = new OptionSimple("auth-provider=", ".*", null, "Fully qualified implementation of com.datastax.driver.core.AuthProvider", false);
final OptionSimple maxPendingPerConnection = new OptionSimple("maxPending=", "[0-9]+", "128", "Maximum pending requests per connection", false);
final OptionSimple connectionsPerHost = new OptionSimple("connectionsPerHost=", "[0-9]+", "8", "Number of connections per host", false);
@@ -179,11 +187,9 @@
out.printf(" Max Pending Per Connection: %d%n", maxPendingPerConnection);
out.printf(" Connections Per Host: %d%n", connectionsPerHost);
out.printf(" Compression: %s%n", compression);
-
}
-
- public static SettingsMode get(Map<String, String[]> clArgs)
+ public static SettingsMode get(Map<String, String[]> clArgs, SettingsCredentials credentials)
{
String[] params = clArgs.remove("-mode");
if (params == null)
@@ -192,7 +198,7 @@
opts.accept("cql3");
opts.accept("native");
opts.accept("prepared");
- return new SettingsMode(opts);
+ return new SettingsMode(opts, credentials);
}
GroupedOptions options = GroupedOptions.select(params, new Cql3NativeOptions(), new Cql3SimpleNativeOptions());
@@ -202,7 +208,7 @@
System.out.println("Invalid -mode options provided, see output for valid options");
System.exit(1);
}
- return new SettingsMode(options);
+ return new SettingsMode(options, credentials);
}
public static void printHelp()
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsReporting.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsReporting.java
new file mode 100644
index 0000000..1fa2154
--- /dev/null
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsReporting.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress.settings;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.stress.util.ResultLogger;
+
+public class SettingsReporting implements Serializable
+{
+ public final int outputFrequency;
+ private final String outputFrequencyString;
+ public final int headerFrequency;
+ private final String headerFrequencyString;
+
+ public SettingsReporting(SettingsReporting.Options reporting)
+ {
+ if (reporting.headerFrequency.present())
+ {
+ headerFrequencyString = reporting.headerFrequency.value();
+ headerFrequency = new DurationSpec.IntSecondsBound(headerFrequencyString).toSeconds();
+ }
+ else
+ {
+ headerFrequency = 0;
+ headerFrequencyString = "*not set*";
+ }
+
+ if (reporting.outputFrequency.present())
+ {
+ outputFrequencyString = reporting.outputFrequency.value();
+ outputFrequency = new DurationSpec.IntSecondsBound(outputFrequencyString).toSeconds();
+ }
+ else
+ {
+ outputFrequency = 0;
+ outputFrequencyString = "*not set*";
+ }
+ }
+
+ // Option Declarations
+
+ public static final class Options extends GroupedOptions
+ {
+ final OptionSimple outputFrequency = new OptionSimple("output-frequency=",
+ ".*",
+ "1s",
+ "Frequency each line of output will be printed out when running a stress test, defaults to '1s'.",
+ false);
+
+ final OptionSimple headerFrequency = new OptionSimple("header-frequency=",
+ ".*",
+ null,
+ "Frequency the header for the statistics will be printed out. " +
+ "If not specified, the header will be printed at the beginning of the test only.",
+ false);
+
+ @Override
+ public List<? extends Option> options()
+ {
+ return Arrays.asList(outputFrequency, headerFrequency);
+ }
+ }
+
+ public static SettingsReporting get(Map<String, String[]> clArgs)
+ {
+ String[] params = clArgs.remove("-reporting");
+ if (params == null)
+ return new SettingsReporting(new SettingsReporting.Options());
+
+ GroupedOptions options = GroupedOptions.select(params, new SettingsReporting.Options());
+ if (options == null)
+ {
+ printHelp();
+ System.out.println("Invalid -reporting options provided, see output for valid options");
+ System.exit(1);
+ }
+ return new SettingsReporting((SettingsReporting.Options) options);
+ }
+
+ public void printSettings(ResultLogger out)
+ {
+ out.printf(" Output frequency: %s%n", outputFrequencyString);
+ out.printf(" Header frequency: %s%n", headerFrequencyString);
+ }
+
+ public static void printHelp()
+ {
+ GroupedOptions.printOptions(System.out, "-reporting", new SettingsReporting.Options());
+ }
+
+ public static Runnable helpPrinter()
+ {
+ return SettingsReporting::printHelp;
+ }
+}
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java
index 5f22a7b..cf62999 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java
@@ -29,13 +29,19 @@
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.stress.util.ResultLogger;
+import static java.lang.String.format;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY;
+
public class SettingsTransport implements Serializable
{
private final TOptions options;
+ private final SettingsCredentials credentials;
- public SettingsTransport(TOptions options)
+ public SettingsTransport(TOptions options, SettingsCredentials credentials)
{
this.options = options;
+ this.credentials = credentials;
}
public EncryptionOptions getEncryptionOptions()
@@ -46,7 +52,7 @@
encOptions = encOptions
.withEnabled(true)
.withTrustStore(options.trustStore.value())
- .withTrustStorePassword(options.trustStorePw.value())
+ .withTrustStorePassword(options.trustStorePw.setByUser() ? options.trustStorePw.value() : credentials.transportTruststorePassword)
.withAlgorithm(options.alg.value())
.withProtocol(options.protocol.value())
.withCipherSuites(options.ciphers.value().split(","));
@@ -54,14 +60,14 @@
{
encOptions = encOptions
.withKeyStore(options.keyStore.value())
- .withKeyStorePassword(options.keyStorePw.value());
+ .withKeyStorePassword(options.keyStorePw.setByUser() ? options.keyStorePw.value() : credentials.transportKeystorePassword);
}
else
{
// mandatory for SSLFactory.createSSLContext(), see CASSANDRA-9325
encOptions = encOptions
.withKeyStore(encOptions.truststore)
- .withKeyStorePassword(encOptions.truststore_password);
+ .withKeyStorePassword(encOptions.truststore_password != null ? encOptions.truststore_password : credentials.transportTruststorePassword);
}
}
return encOptions;
@@ -72,12 +78,18 @@
static class TOptions extends GroupedOptions implements Serializable
{
final OptionSimple trustStore = new OptionSimple("truststore=", ".*", null, "SSL: full path to truststore", false);
- final OptionSimple trustStorePw = new OptionSimple("truststore-password=", ".*", null, "SSL: truststore password", false);
+ final OptionSimple trustStorePw = new OptionSimple("truststore-password=", ".*", null,
+ format("SSL: truststore password, when specified, it will override the value in credentials file of key '%s'",
+ TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY), false);
final OptionSimple keyStore = new OptionSimple("keystore=", ".*", null, "SSL: full path to keystore", false);
- final OptionSimple keyStorePw = new OptionSimple("keystore-password=", ".*", null, "SSL: keystore password", false);
+ final OptionSimple keyStorePw = new OptionSimple("keystore-password=", ".*", null,
+ format("SSL: keystore password, when specified, it will override the value in credentials file for key '%s'",
+ TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY), false);
final OptionSimple protocol = new OptionSimple("ssl-protocol=", ".*", "TLS", "SSL: connection protocol to use", false);
final OptionSimple alg = new OptionSimple("ssl-alg=", ".*", null, "SSL: algorithm", false);
- final OptionSimple ciphers = new OptionSimple("ssl-ciphers=", ".*", "TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA", "SSL: comma delimited list of encryption suites to use", false);
+ final OptionSimple ciphers = new OptionSimple("ssl-ciphers=", ".*",
+ "TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA",
+ "SSL: comma delimited list of encryption suites to use", false);
@Override
public List<? extends Option> options()
@@ -89,14 +101,26 @@
// CLI Utility Methods
public void printSettings(ResultLogger out)
{
- out.println(" " + options.getOptionAsString());
+ String tPassword = options.trustStorePw.setByUser() ? options.trustStorePw.value() : credentials.transportTruststorePassword;
+ tPassword = tPassword != null ? "*suppressed*" : tPassword;
+
+ String kPassword = options.keyStorePw.setByUser() ? options.keyStore.value() : credentials.transportKeystorePassword;
+ kPassword = kPassword != null ? "*suppressed*" : kPassword;
+
+ out.printf(" Truststore: %s%n", options.trustStore.value());
+ out.printf(" Truststore Password: %s%n", tPassword);
+ out.printf(" Keystore: %s%n", options.keyStore.value());
+ out.printf(" Keystore Password: %s%n", kPassword);
+ out.printf(" SSL Protocol: %s%n", options.protocol.value());
+ out.printf(" SSL Algorithm: %s%n", options.alg.value());
+ out.printf(" SSL Ciphers: %s%n", options.ciphers.value());
}
- public static SettingsTransport get(Map<String, String[]> clArgs)
+ public static SettingsTransport get(Map<String, String[]> clArgs, SettingsCredentials credentials)
{
String[] params = clArgs.remove("-transport");
if (params == null)
- return new SettingsTransport(new TOptions());
+ return new SettingsTransport(new TOptions(), credentials);
GroupedOptions options = GroupedOptions.select(params, new TOptions());
if (options == null)
@@ -105,7 +129,7 @@
System.out.println("Invalid -transport options provided, see output for valid options");
System.exit(1);
}
- return new SettingsTransport((TOptions) options);
+ return new SettingsTransport((TOptions) options, credentials);
}
public static void printHelp()
diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java b/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java
index 2f76dfb..12d731e 100644
--- a/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java
+++ b/tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java
@@ -38,13 +38,16 @@
public final SettingsColumn columns;
public final SettingsErrors errors;
public final SettingsLog log;
+ public final SettingsCredentials credentials;
public final SettingsMode mode;
public final SettingsNode node;
public final SettingsSchema schema;
public final SettingsTransport transport;
public final SettingsPort port;
+ public final SettingsJMX jmx;
public final SettingsGraph graph;
public final SettingsTokenRange tokenRange;
+ public final SettingsReporting reporting;
public StressSettings(SettingsCommand command,
SettingsRate rate,
@@ -53,13 +56,16 @@
SettingsColumn columns,
SettingsErrors errors,
SettingsLog log,
+ SettingsCredentials credentials,
SettingsMode mode,
SettingsNode node,
SettingsSchema schema,
SettingsTransport transport,
SettingsPort port,
+ SettingsJMX jmx,
SettingsGraph graph,
- SettingsTokenRange tokenRange)
+ SettingsTokenRange tokenRange,
+ SettingsReporting reporting)
{
this.command = command;
this.rate = rate;
@@ -68,13 +74,16 @@
this.columns = columns;
this.errors = errors;
this.log = log;
+ this.credentials = credentials;
this.mode = mode;
this.node = node;
this.schema = schema;
this.transport = transport;
this.port = port;
+ this.jmx = jmx;
this.graph = graph;
this.tokenRange = tokenRange;
+ this.reporting = reporting;
}
public SimpleClient getSimpleNativeClient()
@@ -194,11 +203,14 @@
SettingsColumn columns = SettingsColumn.get(clArgs);
SettingsErrors errors = SettingsErrors.get(clArgs);
SettingsLog log = SettingsLog.get(clArgs);
- SettingsMode mode = SettingsMode.get(clArgs);
+ SettingsCredentials credentials = SettingsCredentials.get(clArgs);
+ SettingsMode mode = SettingsMode.get(clArgs, credentials);
SettingsNode node = SettingsNode.get(clArgs);
SettingsSchema schema = SettingsSchema.get(clArgs, command);
- SettingsTransport transport = SettingsTransport.get(clArgs);
+ SettingsTransport transport = SettingsTransport.get(clArgs, credentials);
+ SettingsJMX jmx = SettingsJMX.get(clArgs, credentials);
SettingsGraph graph = SettingsGraph.get(clArgs, command);
+ SettingsReporting reporting = SettingsReporting.get(clArgs);
if (!clArgs.isEmpty())
{
printHelp();
@@ -216,7 +228,7 @@
System.exit(1);
}
- return new StressSettings(command, rate, generate, insert, columns, errors, log, mode, node, schema, transport, port, graph, tokenRange);
+ return new StressSettings(command, rate, generate, insert, columns, errors, log, credentials, mode, node, schema, transport, port, jmx, graph, tokenRange, reporting);
}
private static Map<String, String[]> parseMap(String[] args)
@@ -290,10 +302,16 @@
transport.printSettings(out);
out.println("Port:");
port.printSettings(out);
+ out.println("JMX:");
+ jmx.printSettings(out);
out.println("Graph:");
graph.printSettings(out);
out.println("TokenRange:");
tokenRange.printSettings(out);
+ out.println("Credentials file:");
+ credentials.printSettings(out);
+ out.println("Reporting:");
+ reporting.printSettings(out);
if (command.type == Command.USER)
{
diff --git a/tools/stress/src/org/apache/cassandra/stress/util/JmxCollector.java b/tools/stress/src/org/apache/cassandra/stress/util/JmxCollector.java
index 8cfbebb..24cb4c7 100644
--- a/tools/stress/src/org/apache/cassandra/stress/util/JmxCollector.java
+++ b/tools/stress/src/org/apache/cassandra/stress/util/JmxCollector.java
@@ -28,6 +28,7 @@
import java.util.concurrent.Future;
import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.stress.settings.SettingsJMX;
import org.apache.cassandra.tools.NodeProbe;
public class JmxCollector implements Callable<JmxCollector.GcStats>
@@ -76,23 +77,26 @@
final NodeProbe[] probes;
// TODO: should expand to whole cluster
- public JmxCollector(Collection<String> hosts, int port)
+ public JmxCollector(Collection<String> hosts, int port, SettingsJMX jmx)
{
probes = new NodeProbe[hosts.size()];
int i = 0;
for (String host : hosts)
{
- probes[i] = connect(host, port);
+ probes[i] = connect(host, port, jmx);
probes[i].getAndResetGCStats();
i++;
}
}
- private static NodeProbe connect(String host, int port)
+ private static NodeProbe connect(String host, int port, SettingsJMX jmx)
{
try
{
- return new NodeProbe(host, port);
+ if (jmx.user != null && jmx.password != null)
+ return new NodeProbe(host, port, jmx.user, jmx.password);
+ else
+ return new NodeProbe(host, port);
}
catch (IOException e)
{
diff --git a/tools/stress/test/unit/org/apache/cassandra/stress/settings/SettingsCredentialsTest.java b/tools/stress/test/unit/org/apache/cassandra/stress/settings/SettingsCredentialsTest.java
new file mode 100644
index 0000000..a1a07c4
--- /dev/null
+++ b/tools/stress/test/unit/org/apache/cassandra/stress/settings/SettingsCredentialsTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress.settings;
+
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.Test;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
+
+import static org.apache.cassandra.io.util.File.WriteMode.OVERWRITE;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.CQL_PASSWORD_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.CQL_USERNAME_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.JMX_PASSWORD_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.JMX_USERNAME_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY;
+import static org.apache.cassandra.stress.settings.SettingsCredentials.TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY;
+import static org.junit.Assert.assertEquals;
+
+public class SettingsCredentialsTest
+{
+ @Test
+ public void testReadCredentialsFromFileMixed() throws Exception
+ {
+ Properties properties = new Properties();
+ properties.setProperty(CQL_USERNAME_PROPERTY_KEY, "cqluserfromfile");
+ properties.setProperty(CQL_PASSWORD_PROPERTY_KEY, "cqlpasswordfromfile");
+ properties.setProperty(JMX_USERNAME_PROPERTY_KEY, "jmxuserfromfile");
+ properties.setProperty(JMX_PASSWORD_PROPERTY_KEY, "jmxpasswordfromfile");
+ properties.setProperty(TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY, "keystorestorepasswordfromfile");
+ properties.setProperty(TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY, "truststorepasswordfromfile");
+
+ File tempFile = FileUtils.createTempFile("cassandra-stress-credentials-test", "properties");
+
+ try (Writer w = tempFile.newWriter(OVERWRITE))
+ {
+ properties.store(w, null);
+ }
+
+ Map<String, String[]> args = new HashMap<>();
+ args.put("write", new String[]{});
+ args.put("-mode", new String[]{ "cql3", "native", "password=cqlpasswordoncommandline" });
+ args.put("-transport", new String[]{ "truststore=sometruststore", "keystore=somekeystore" });
+ args.put("-jmx", new String[]{ "password=jmxpasswordoncommandline" });
+ args.put("-credentials-file", new String[]{ tempFile.absolutePath() });
+ StressSettings settings = StressSettings.get(args);
+
+ assertEquals("cqluserfromfile", settings.credentials.cqlUsername);
+ assertEquals("cqlpasswordfromfile", settings.credentials.cqlPassword);
+ assertEquals("jmxuserfromfile", settings.credentials.jmxUsername);
+ assertEquals("jmxpasswordfromfile", settings.credentials.jmxPassword);
+ assertEquals("keystorestorepasswordfromfile", settings.credentials.transportKeystorePassword);
+ assertEquals("truststorepasswordfromfile", settings.credentials.transportTruststorePassword);
+
+ assertEquals("cqluserfromfile", settings.mode.username);
+ assertEquals("cqlpasswordoncommandline", settings.mode.password);
+ assertEquals("jmxuserfromfile", settings.jmx.user);
+ assertEquals("jmxpasswordoncommandline", settings.jmx.password);
+ assertEquals("keystorestorepasswordfromfile", settings.transport.getEncryptionOptions().keystore_password);
+ assertEquals("truststorepasswordfromfile", settings.transport.getEncryptionOptions().truststore_password);
+ }
+
+ @Test
+ public void testReadCredentialsFromFileOverridenByCommandLine() throws Exception
+ {
+ Properties properties = new Properties();
+ properties.setProperty(CQL_USERNAME_PROPERTY_KEY, "cqluserfromfile");
+ properties.setProperty(CQL_PASSWORD_PROPERTY_KEY, "cqlpasswordfromfile");
+ properties.setProperty(JMX_USERNAME_PROPERTY_KEY, "jmxuserfromfile");
+ properties.setProperty(JMX_PASSWORD_PROPERTY_KEY, "jmxpasswordfromfile");
+ properties.setProperty(TRANSPORT_KEYSTORE_PASSWORD_PROPERTY_KEY, "keystorestorepasswordfromfile");
+ properties.setProperty(TRANSPORT_TRUSTSTORE_PASSWORD_PROPERTY_KEY, "truststorepasswordfromfile");
+
+ File tempFile = FileUtils.createTempFile("cassandra-stress-credentials-test", "properties");
+
+ try (Writer w = tempFile.newWriter(OVERWRITE))
+ {
+ properties.store(w, null);
+ }
+
+ Map<String, String[]> args = new HashMap<>();
+ args.put("write", new String[]{});
+ args.put("-mode", new String[]{ "cql3", "native", "password=cqlpasswordoncommandline", "user=cqluseroncommandline" });
+ args.put("-jmx", new String[]{ "password=jmxpasswordoncommandline", "user=jmxuseroncommandline" });
+ args.put("-transport", new String[]{ "truststore=sometruststore",
+ "keystore=somekeystore",
+ "truststore-password=truststorepasswordfromcommandline",
+ "keystore-password=keystorepasswordfromcommandline" });
+ args.put("-credentials-file", new String[]{ tempFile.absolutePath() });
+ StressSettings settings = StressSettings.get(args);
+
+ assertEquals("cqluserfromfile", settings.credentials.cqlUsername);
+ assertEquals("cqlpasswordfromfile", settings.credentials.cqlPassword);
+ assertEquals("jmxuserfromfile", settings.credentials.jmxUsername);
+ assertEquals("jmxpasswordfromfile", settings.credentials.jmxPassword);
+ assertEquals("keystorestorepasswordfromfile", settings.credentials.transportKeystorePassword);
+ assertEquals("truststorepasswordfromfile", settings.credentials.transportTruststorePassword);
+
+ assertEquals("cqluseroncommandline", settings.mode.username);
+ assertEquals("cqlpasswordoncommandline", settings.mode.password);
+ assertEquals("jmxuseroncommandline", settings.jmx.user);
+ assertEquals("jmxpasswordoncommandline", settings.jmx.password);
+ assertEquals("truststorepasswordfromcommandline", settings.transport.getEncryptionOptions().truststore_password);
+ assertEquals("keystorepasswordfromcommandline", settings.transport.getEncryptionOptions().keystore_password);
+ }
+}
